Merge branch 'release/v4.3.4'

The problem happens when value has type "unicode" that shouldn't be decoded

.appveyor.yml

@@ -1,29 +0,0 @@
-build: off
-
-platform:
-  - x64
-
-environment:
-  matrix:
-    - TOXENV: "py27"
-      PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P2_{build}
-
-    - TOXENV: "py36"
-      PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P3_{build}
-
-install:
-  - cmd: git submodule update --init --recursive
-  - cmd: SET PATH=C:\MinGW\bin;%PATH%
-  - cmd: SET PLATFORMIO_CORE_DIR=C:\.pio
-  - cmd: pip install --force-reinstall tox
-
-test_script:
-  - cmd: tox
-
-notifications:
-  - provider: Slack
-    incoming_webhook:
-      secure: E9H0SU0Ju7WLDvgxsV8cs3J62T3nTTX7QkEjsczN0Sto/c9hWkVfhc5gGWUkxhlD975cokHByKGJIdwYwCewqOI+7BrcT8U+nlga4Uau7J8=
-    on_build_success: false
-    on_build_failure: true
-    on_build_status_changed: true

.github/workflows/core.yml

@@ -0,0 +1,43 @@
+name: Core
+
+on: [push]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [2.7, 3.7]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+
+      - name: Python Lint
+        run: |
+          tox -e lint
+      - name: Integration Tests
+        env:
+          PLATFORMIO_TEST_ACCOUNT_LOGIN: ${{ secrets.PLATFORMIO_TEST_ACCOUNT_LOGIN }}
+          PLATFORMIO_TEST_ACCOUNT_PASSWORD: ${{ secrets.PLATFORMIO_TEST_ACCOUNT_PASSWORD }}
+        run: |
+          tox -e testcore
+
+      - name: Slack Notification
+        uses: homoluctus/slatify@master
+        if: failure()
+        with:
+          type: ${{ job.status }}
+          job_name: '*Core*'
+          commit: true
+          url: ${{ secrets.SLACK_BUILD_WEBHOOK }}

.github/workflows/docs.yml

@@ -0,0 +1,32 @@
+name: Docs
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+
+      - name: Build docs
+        run: |
+          tox -e docs
+
+      - name: Slack Notification
+        uses: homoluctus/slatify@master
+        if: failure()
+        with:
+          type: ${{ job.status }}
+          job_name: '*Docs*'
+          commit: true
+          url: ${{ secrets.SLACK_BUILD_WEBHOOK }}

.github/workflows/examples.yml

@@ -0,0 +1,63 @@
+name: Examples
+
+on: [push]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-16.04, windows-latest, macos-latest]
+        python-version: [2.7, 3.7]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+
+      - name: Run on Linux
+        if: startsWith(matrix.os, 'ubuntu')
+        env:
+          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,intel_mcs51,aceinna_imu"
+        run: |
+          # ChipKIT issue: install 32-bit support for GCC PIC32
+          sudo apt-get install libc6-i386
+          # Free space
+          sudo apt clean
+          docker rmi $(docker image ls -aq)
+          df -h
+          # Run
+          tox -e testexamples
+
+      - name: Run on macOS
+        if: startsWith(matrix.os, 'macos')
+        env:
+          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,microchippic32,gd32v,nuclei"
+        run: |
+          df -h
+          tox -e testexamples
+
+      - name: Run on Windows
+        if: startsWith(matrix.os, 'windows')
+        env:
+          PLATFORMIO_CORE_DIR: C:/pio
+          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap"
+        run: |
+          tox -e testexamples
+
+      - name: Slack Notification
+        uses: homoluctus/slatify@master
+        if: failure()
+        with:
+          type: ${{ job.status }}
+          job_name: '*Examples*'
+          commit: true
+          url: ${{ secrets.SLACK_BUILD_WEBHOOK }}

.isort.cfg

@@ -1,3 +1,3 @@
 [settings]
 line_length=88
-known_third_party=SCons, twisted, autobahn, jsonrpc
+known_third_party=OpenSSL, SCons, autobahn, jsonrpc, twisted, zope

.style.yapf

@@ -1,3 +0,0 @@
-[style]
-blank_line_before_nested_class_or_def = true
-allow_multiline_lambdas = true

.travis.yml

@@ -1,39 +0,0 @@
-language: python
-
-matrix:
-  include:
-    - os: linux
-      sudo: false
-      python: 2.7
-      env: TOX_ENV=docs
-    - os: linux
-      sudo: required
-      python: 2.7
-      env: TOX_ENV=py27 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d)
-    - os: linux
-      sudo: required
-      python: 3.6
-      env: TOX_ENV=py36 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d)
-    - os: osx
-      language: generic
-      env: TOX_ENV=skipexamples
-
-install:
-  - git submodule update --init --recursive
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl -fsSL https://bootstrap.pypa.io/get-pip.py | sudo python; fi
-  - pip install -U tox
-
-  # ChipKIT issue: install 32-bit support for GCC PIC32
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi
-
-script:
-  - tox -e $TOX_ENV
-
-notifications:
-  email: false
-
-  slack:
-    rooms:
-      secure: JD6VGfN4+SLU2CwDdiIOr1VgwD+zbYUCE/srwyGuHavnjIkPItkl6T6Bn8Y4VrU6ysbuKotfdV2TAJJ82ivFbY8BvZBc7FBcYp/AGQ4FaCCV5ySv8RDAcQgdE12oaGzMdODiLqsB85f65zOlAFa+htaXyEiRTcotn6Y2hupatrI=
-    on_failure: always
-    on_success: change

CONTRIBUTING.md

@@ -1,20 +1,20 @@
 Contributing
 ------------
 
-To get started, <a href="https://www.clahub.com/agreements/platformio/platformio-core">sign the Contributor License Agreement</a>.
+To get started, <a href="https://cla-assistant.io/platformio/platformio-core">sign the Contributor License Agreement</a>.
 
 1. Fork the repository on GitHub.
 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git`
 3. Run `pip install tox`
-4. Go to the root of project where is located `tox.ini` and run `tox -e py27`
+4. Go to the root of project where is located `tox.ini` and run `tox -e py37`
 5. Activate current development environment:
 
-   * Windows: `.tox\py27\Scripts\activate`
-   * Bash/ZSH: `source .tox/py27/bin/activate`
-   * Fish: `source .tox/py27/bin/activate.fish`
+   * Windows: `.tox\py37\Scripts\activate`
+   * Bash/ZSH: `source .tox/py37/bin/activate`
+   * Fish: `source .tox/py37/bin/activate.fish`
 
 6. Make changes to code, documentation, etc.
-7. Lint source code `make lint`
+7. Lint source code `make before-commit`
 8. Run the tests `make test`
 9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html)
 10. Commit changes to your forked repository

HISTORY.rst

@@ -6,6 +6,70 @@ Release Notes
 PlatformIO Core 4
 -----------------
 
+4.3.4 (2020-05-23)
+~~~~~~~~~~~~~~~~~~
+
+* Added `PlatformIO CLI Shell Completion <https://docs.platformio.org/page/core/userguide/system/completion/index.html>`__ for Fish, Zsh, Bash, and PowerShell (`issue #3435 <https://github.com/platformio/platformio-core/issues/3435>`_)
+* Automatically build ``contrib-pysite`` package on a target machine when pre-built package is not compatible (`issue #3482 <https://github.com/platformio/platformio-core/issues/3482>`_)
+* Fixed an issue on Windows when installing a library dependency from Git repository (`issue #2844 <https://github.com/platformio/platformio-core/issues/2844>`_, `issue #3328 <https://github.com/platformio/platformio-core/issues/3328>`_)
+
+4.3.3 (2020-04-28)
+~~~~~~~~~~~~~~~~~~
+
+* Fixed "UnicodeDecodeError: 'utf-8' codec can't decode byte" when non-Latin chars are used in project path (`issue #3481 <https://github.com/platformio/platformio-core/issues/3481>`_)
+
+4.3.2 (2020-04-28)
+~~~~~~~~~~~~~~~~~~
+
+* New `Account Management System <https://docs.platformio.org/page/plus/pio-account.html>`__ (preview)
+* Open source `PIO Remote <http://docs.platformio.org/page/plus/pio-remote.html>`__ client
+* Improved `PIO Check <http://docs.platformio.org/page/plus/pio-check.html>`__ with more accurate project processing
+* Echo what is typed when ``send_on_enter`` `device monitor filter <https://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-filters>`__ is used (`issue #3452 <https://github.com/platformio/platformio-core/issues/3452>`_)
+* Fixed PIO Unit Testing for Zephyr RTOS
+* Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
+* Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 <https://github.com/platformio/platformio-core/issues/3442>`_)
+* Fixed an incorrect node path used for pattern matching when processing middleware nodes
+* Fixed an issue with missing ``lib_extra_dirs`` option in SRC_LIST for CLion (`issue #3460 <https://github.com/platformio/platformio-core/issues/3460>`_)
+
+4.3.1 (2020-03-20)
+~~~~~~~~~~~~~~~~~~
+
+* Fixed a SyntaxError "'return' with argument inside generator" for PIO Unified Debugger when Python 2.7 is used
+* Fixed an issue when ``lib_archive = no`` was not honored in `"platformio.ini" <https://docs.platformio.org/page/projectconf.html>`__
+* Fixed a TypeError "super(type, obj): obj must be an instance or subtype of type" when device monitor is used with a custom dev-platform filter (`issue #3431 <https://github.com/platformio/platformio-core/issues/3431>`_)
+
+4.3.0 (2020-03-19)
+~~~~~~~~~~~~~~~~~~
+
+* Initial support for an official `PlatformIO for CLion IDE <https://docs.platformio.org/page/integration/ide/clion.html>`__ plugin:
+
+  - Smart C and C++ editor
+  - Code refactoring
+  - On-the-fly code analysis
+  - "New PlatformIO Project" wizard
+  - Building, Uploading, Testing
+  - Integrated debugger (inline variable view, conditional breakpoints, expressions, watchpoints, peripheral registers, multi-thread support, etc.)
+
+* `Device Monitor 2.0 <https://docs.platformio.org/page/core/userguide/device/cmd_monitor.html>`__
+
+  - Added **PlatformIO Device Monitor Filter API** (dev-platforms can extend base device monitor with a custom functionality, such as exception decoding) (`pull #3383 <https://github.com/platformio/platformio-core/pull/3383>`_)
+  - Configure project device monitor with `monitor_filters <https://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-filters>`__ option
+  - `Capture device monitor output to a file <https://docs.platformio.org/page/core/userguide/device/cmd_monitor.html#capture-output-to-a-file>`__ with ``log2file`` filter (`issue #670 <https://github.com/platformio/platformio-core/issues/670>`_)
+  - Show a timestamp for each new line with ``time`` filter (`issue #981 <https://github.com/platformio/platformio-core/issues/981>`_)
+  - Send a text to device on ENTER with ``send_on_enter`` filter (`issue #926 <https://github.com/platformio/platformio-core/issues/926>`_)
+  - Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter
+
+* New standalone (1-script) `PlatformIO Core Installer <https://github.com/platformio/platformio-core-installer>`_
+* Initial support for `Renode <https://docs.platformio.org/page/plus/debug-tools/renode.html>`__ simulation framework (`issue #3401 <https://github.com/platformio/platformio-core/issues/3401>`_)
+* Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 <https://github.com/platformio/platformio-core/issues/3400>`_)
+* Improved support for Arduino "library.properties" ``depends`` field
+* Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server
+* Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 <https://github.com/platformio/platformio-core/issues/3396>`_)
+* Fixed an issue when Python 2 does not keep encoding when converting ".ino" (`issue #3393 <https://github.com/platformio/platformio-core/issues/3393>`_)
+* Fixed an issue when ``"libArchive": false`` in "library.json" does not work (`issue #3403 <https://github.com/platformio/platformio-core/issues/3403>`_)
+* Fixed an issue when not all commands in `compilation database "compile_commands.json" <https://docs.platformio.org/page/integration/compile_commands.html>`__ use absolute paths (`pull #3415 <https://github.com/platformio/platformio-core/pull/3415>`_)
+* Fixed an issue when unknown transport is used for `PIO Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html>`__ engine (`issue #3422 <https://github.com/platformio/platformio-core/issues/3422>`_)
+
 4.2.1 (2020-02-17)
 ~~~~~~~~~~~~~~~~~~
 
@@ -34,7 +98,7 @@ PlatformIO Core 4
   - Show computed project configuration with a new `platformio project config <https://docs.platformio.org/page/userguide/project/cmd_config.html>`_ command or dump to JSON with ``platformio project config --json-output`` (`issue #3335 <https://github.com/platformio/platformio-core/issues/3335>`_)
   - Moved ``platformio init`` command to `platformio project init <https://docs.platformio.org/page/userguide/project/cmd_init.html>`_
 
-* Generate `compilation database "compile_commands.json" <https://docs.platformio.org/page/faq.html#compilation-database-compile-commands-json>`_ (`issue #2990 <https://github.com/platformio/platformio-core/issues/2990>`_)
+* Generate `compilation database "compile_commands.json" <https://docs.platformio.org/page/integration/compile_commands.html>`__ (`issue #2990 <https://github.com/platformio/platformio-core/issues/2990>`_)
 * Control debug flags and optimization level with a new `debug_build_flags <https://docs.platformio.org/page/projectconf/section_env_debug.html#debug-build-flags>`__ option
 * Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install <https://docs.platformio.org/page/userguide/platforms/cmd_install.html>`__ command  (`issue #3345 <https://github.com/platformio/platformio-core/issues/3345>`_)
 * Added support for "pythonPackages" in `platform.json <https://docs.platformio.org/page/platforms/creating_platform.html#manifest-file-platform-json>`__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed)

Makefile

@@ -12,7 +12,7 @@ format:
 test:
 	py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
 
-before-commit: isort format lint test
+before-commit: isort format lint
 
 clean-docs:
 	rm -rf docs/_build

README.rst

@@ -1,12 +1,15 @@
 PlatformIO
 ==========
 
-.. image:: https://travis-ci.org/platformio/platformio-core.svg?branch=develop
-    :target: https://travis-ci.org/platformio/platformio-core
-    :alt: Travis.CI Build Status
-.. image:: https://ci.appveyor.com/api/projects/status/unnpw0n3c5k14btn/branch/develop?svg=true
-    :target: https://ci.appveyor.com/project/ivankravets/platformio-core
-    :alt: AppVeyor.CI Build Status
+.. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg
+    :target: https://docs.platformio.org/page/core/index.html
+    :alt:  CI Build for PlatformIO Core
+.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
+    :target: https://github.com/platformio/platformio-examples
+    :alt:  CI Build for dev-platform examples
+.. image:: https://github.com/platformio/platformio-core/workflows/Docs/badge.svg
+    :target: https://docs.platformio.org?utm_source=github&utm_medium=core
+    :alt:  CI Build for Docs
 .. image:: https://img.shields.io/pypi/v/platformio.svg
     :target: https://pypi.python.org/pypi/platformio/
     :alt: Latest Version
@@ -45,26 +48,26 @@ PlatformIO
 Get Started
 -----------
 
-* `What is PlatformIO? <https://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
+* `What is PlatformIO? <https://docs.platformio.org/page/what-is-platformio.html?utm_source=github&utm_medium=core>`_
 
 Instruments
 -----------
 
 * `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_
-* `PlatformIO Core (CLI) <https://docs.platformio.org/en/latest/core.html?utm_source=github&utm_medium=core>`_
+* `PlatformIO Core (CLI) <https://docs.platformio.org/page/core.html?utm_source=github&utm_medium=core>`_
 * `Library Management <https://docs.platformio.org/page/librarymanager/index.html?utm_source=github&utm_medium=core>`_
 * `Project Examples <https://github.com/platformio/platformio-examples?utm_source=github&utm_medium=core>`__
 * `Desktop IDEs Integration <https://docs.platformio.org/page/ide.html?utm_source=github&utm_medium=core>`_
 * `Continuous Integration <https://docs.platformio.org/page/ci/index.html?utm_source=github&utm_medium=core>`_
 * `Advanced Scripting API <https://docs.platformio.org/page/projectconf/advanced_scripting.html?utm_source=github&utm_medium=core>`_
 
-PIO Plus
---------
+Professional
+------------
 
 * `PIO Check <https://docs.platformio.org/page/plus/pio-check.html?utm_source=github&utm_medium=core>`_
 * `PIO Remote <https://docs.platformio.org/page/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
 * `PIO Unified Debugger <https://docs.platformio.org/page/plus/debugging.html?utm_source=github&utm_medium=core>`_
-* `PIO Unit Testing <https://docs.platformio.org/en/latest/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
+* `PIO Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
 
 Registry
 --------
@@ -140,8 +143,8 @@ Telemetry / Privacy Policy
 Share minimal diagnostics and usage information to help us make PlatformIO better.
 It is enabled by default. For more information see:
 
-* `Telemetry Setting <https://docs.platformio.org/en/latest/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
-* `SSL Setting <https://docs.platformio.org/en/latest/userguide/cmd_settings.html?utm_source=github&utm_medium=core#strict-ssl>`_
+* `Telemetry Setting <https://docs.platformio.org/page/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
+* `SSL Setting <https://docs.platformio.org/page/userguide/cmd_settings.html?utm_source=github&utm_medium=core#strict-ssl>`_
 
 License
 -------

platformio/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 2, 1)
+VERSION = (4, 3, 4)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -34,3 +34,5 @@ __license__ = "Apache Software License"
 __copyright__ = "Copyright 2014-present PlatformIO"
 
 __apiurl__ = "https://api.platformio.org"
+__pioaccount_api__ = "https://api.accounts.platformio.org"
+__pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"

platformio/__main__.py

@@ -22,6 +22,13 @@ from platformio import __version__, exception, maintenance, util
 from platformio.commands import PlatformioCLI
 from platformio.compat import CYGWIN
 
+try:
+    import click_completion  # pylint: disable=import-error
+
+    click_completion.init()
+except:  # pylint: disable=bare-except
+    pass
+
 
 @click.command(
     cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])

platformio/app.py

@@ -13,8 +13,11 @@
 # limitations under the License.
 
 import codecs
+import getpass
 import hashlib
 import os
+import platform
+import socket
 import uuid
 from os import environ, getenv, listdir, remove
 from os.path import dirname, isdir, isfile, join, realpath
@@ -22,7 +25,7 @@ from time import time
 
 import requests
 
-from platformio import exception, fs, lockfile
+from platformio import __version__, exception, fs, lockfile
 from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data
 from platformio.proc import is_ci
 from platformio.project.helpers import (
@@ -414,3 +417,28 @@ def get_cid():
     if WINDOWS or os.getuid() > 0:  # pylint: disable=no-member
         set_state_item("cid", cid)
     return cid
+
+
+def get_user_agent():
+    data = ["PlatformIO/%s" % __version__, "CI/%d" % int(is_ci())]
+    if get_session_var("caller_id"):
+        data.append("Caller/%s" % get_session_var("caller_id"))
+    if os.getenv("PLATFORMIO_IDE"):
+        data.append("IDE/%s" % os.getenv("PLATFORMIO_IDE"))
+    data.append("Python/%s" % platform.python_version())
+    data.append("Platform/%s" % platform.platform())
+    return " ".join(data)
+
+
+def get_host_id():
+    h = hashlib.sha1(hashlib_encode_data(get_cid()))
+    try:
+        username = getpass.getuser()
+        h.update(hashlib_encode_data(username))
+    except:  # pylint: disable=bare-except
+        pass
+    return h.hexdigest()
+
+
+def get_host_name():
+    return str(socket.gethostname())[:255]

platformio/builder/main.py

@@ -28,7 +28,7 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
-from platformio import fs
+from platformio import compat, fs
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformBase
 from platformio.proc import get_pythonexe_path
@@ -120,6 +120,18 @@ env.Replace(
     ],
 )
 
+if (
+    compat.WINDOWS
+    and sys.version_info >= (3, 8)
+    and env["PROJECT_DIR"].startswith("\\\\")
+):
+    click.secho(
+        "There is a known issue with Python 3.8+ and mapped network drives on "
+        "Windows.\nPlease downgrade Python to the latest 3.7. More details at:\n"
+        "https://github.com/platformio/platformio-core/issues/3417",
+        fg="yellow",
+    )
+
 if env.subst("$BUILD_CACHE_DIR"):
     if not isdir(env.subst("$BUILD_CACHE_DIR")):
         makedirs(env.subst("$BUILD_CACHE_DIR"))
@@ -147,7 +159,7 @@ env.LoadPioPlatform()
 
 env.SConscriptChdir(0)
 env.SConsignFile(
-    join("$BUILD_DIR", ".sconsign.py%d%d" % (sys.version_info[0], sys.version_info[1]))
+    join("$BUILD_DIR", ".sconsign%d%d.db" % (sys.version_info[0], sys.version_info[1]))
 )
 
 for item in env.GetExtraScripts("pre"):

platformio/builder/tools/compilation_db.py

@@ -1,17 +1,24 @@
 # Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-# Copyright 2015 MongoDB Inc.
+# Copyright 2020 MongoDB Inc.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 # pylint: disable=unused-argument, protected-access, unused-variable, import-error
 # Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py
@@ -76,6 +83,16 @@ def makeEmitCompilationDbEntry(comstr):
         :return: target(s), source(s)
         """
 
+        # Resolve absolute path of toolchain
+        for cmd in ("CC", "CXX", "AS"):
+            if cmd not in env:
+                continue
+            if os.path.isabs(env[cmd]):
+                continue
+            env[cmd] = where_is_program(
+                env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")
+            )
+
         dbtarget = __CompilationDbNode(source)
 
         entry = env.__COMPILATIONDB_Entry(
@@ -195,14 +212,6 @@ def generate(env, **kwargs):
     )
 
     def CompilationDatabase(env, target):
-        # Resolve absolute path of toolchain
-        for cmd in ("CC", "CXX", "AS"):
-            if cmd not in env:
-                continue
-            env[cmd] = where_is_program(
-                env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")
-            )
-
         result = env.__COMPILATIONDB_Database(target=target, source=[])
 
         env.AlwaysBuild(result)

platformio/builder/tools/pioide.py

@@ -25,44 +25,45 @@ from platformio.proc import exec_command, where_is_program
 
 
 def _dump_includes(env):
-    includes = []
+    includes = {}
 
-    for item in env.get("CPPPATH", []):
-        includes.append(env.subst(item))
+    includes["build"] = [
+        env.subst("$PROJECT_INCLUDE_DIR"),
+        env.subst("$PROJECT_SRC_DIR"),
+    ]
+    includes["build"].extend(
+        [os.path.realpath(env.subst(item)) for item in env.get("CPPPATH", [])]
+    )
 
     # installed libs
+    includes["compatlib"] = []
     for lb in env.GetLibBuilders():
-        includes.extend(lb.get_include_dirs())
+        includes["compatlib"].extend(
+            [os.path.realpath(inc) for inc in lb.get_include_dirs()]
+        )
 
     # includes from toolchains
     p = env.PioPlatform()
+    includes["toolchain"] = []
     for name in p.get_installed_packages():
         if p.get_package_type(name) != "toolchain":
             continue
         toolchain_dir = glob_escape(p.get_package_dir(name))
         toolchain_incglobs = [
-            os.path.join(toolchain_dir, "*", "include*"),
             os.path.join(toolchain_dir, "*", "include", "c++", "*"),
             os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"),
             os.path.join(toolchain_dir, "lib", "gcc", "*", "*", "include*"),
+            os.path.join(toolchain_dir, "*", "include*"),
         ]
         for g in toolchain_incglobs:
-            includes.extend(glob(g))
+            includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)])
 
+    includes["unity"] = []
     unity_dir = get_core_package_dir("tool-unity")
     if unity_dir:
-        includes.append(unity_dir)
+        includes["unity"].append(unity_dir)
 
-    includes.extend([env.subst("$PROJECT_INCLUDE_DIR"), env.subst("$PROJECT_SRC_DIR")])
-
-    # remove duplicates
-    result = []
-    for item in includes:
-        item = os.path.realpath(item)
-        if item not in result:
-            result.append(item)
-
-    return result
+    return includes
 
 
 def _get_gcc_defines(env):
@@ -158,8 +159,6 @@ def DumpIDEData(env):
         "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()],
         "defines": _dump_defines(env),
         "includes": _dump_includes(env),
-        "cc_flags": env.subst(LINTCCOM),
-        "cxx_flags": env.subst(LINTCXXCOM),
         "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
         "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
         "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),

platformio/builder/tools/piolib.py

@@ -14,10 +14,12 @@
 
 # pylint: disable=no-member, no-self-use, unused-argument, too-many-lines
 # pylint: disable=too-many-instance-attributes, too-many-public-methods
+# pylint: disable=assignment-from-no-return
 
 from __future__ import absolute_import
 
 import hashlib
+import io
 import os
 import re
 import sys
@@ -82,7 +84,8 @@ class LibBuilderFactory(object):
                     fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT
                 ):
                     continue
-                content = fs.get_file_contents(join(root, fname))
+                with io.open(join(root, fname), errors="ignore") as fp:
+                    content = fp.read()
                 if not content:
                     continue
                 if "Arduino.h" in content and include_re.search(content):
@@ -716,10 +719,14 @@ class PlatformIOLibBuilder(LibBuilderBase):
 
     @property
     def lib_archive(self):
-        unique_value = "_not_declared_%s" % id(self)
-        global_value = self.env.GetProjectOption("lib_archive", unique_value)
-        if global_value != unique_value:
-            return global_value
+        missing = object()
+        global_value = self.env.GetProjectConfig().getraw(
+            "env:" + self.env["PIOENV"], "lib_archive", missing
+        )
+        if global_value != missing:
+            return self.env.GetProjectConfig().get(
+                "env:" + self.env["PIOENV"], "lib_archive"
+            )
         return self._manifest.get("build", {}).get(
             "libArchive", LibBuilderBase.lib_archive.fget(self)
         )

platformio/builder/tools/piomaxlen.py

@@ -18,7 +18,6 @@ from hashlib import md5
 from os import makedirs
 from os.path import isdir, isfile, join
 
-from platformio import fs
 from platformio.compat import WINDOWS, hashlib_encode_data
 
 # Windows CLI has limit with command length to 8192
@@ -67,7 +66,8 @@ def _file_long_data(env, data):
     )
     if isfile(tmp_file):
         return tmp_file
-    fs.write_file_contents(tmp_file, data)
+    with open(tmp_file, "w") as fp:
+        fp.write(data)
     return tmp_file
 
 

platformio/builder/tools/piomisc.py

@@ -15,17 +15,19 @@
 from __future__ import absolute_import
 
 import atexit
+import io
 import re
 import sys
 from os import environ, remove, walk
 from os.path import basename, isdir, isfile, join, realpath, relpath, sep
 from tempfile import mkstemp
 
+import click
 from SCons.Action import Action  # pylint: disable=import-error
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 
 from platformio import fs, util
-from platformio.compat import glob_escape
+from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape
 from platformio.managers.core import get_core_package_dir
 from platformio.proc import exec_command
 
@@ -48,6 +50,39 @@ class InoToCPPConverter(object):
     def __init__(self, env):
         self.env = env
         self._main_ino = None
+        self._safe_encoding = None
+
+    def read_safe_contents(self, path):
+        error_reported = False
+        for encoding in (
+            "utf-8",
+            None,
+            get_filesystem_encoding(),
+            get_locale_encoding(),
+            "latin-1",
+        ):
+            try:
+                with io.open(path, encoding=encoding) as fp:
+                    contents = fp.read()
+                    self._safe_encoding = encoding
+                    return contents
+            except UnicodeDecodeError:
+                if not error_reported:
+                    error_reported = True
+                    click.secho(
+                        "Unicode decode error has occurred, please remove invalid "
+                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
+                        % path,
+                        fg="yellow",
+                        err=True,
+                    )
+        return ""
+
+    def write_safe_contents(self, path, contents):
+        with io.open(
+            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
+        ) as fp:
+            return fp.write(contents)
 
     def is_main_node(self, contents):
         return self.DETECTMAIN_RE.search(contents)
@@ -62,7 +97,7 @@ class InoToCPPConverter(object):
         assert nodes
         lines = []
         for node in nodes:
-            contents = fs.get_file_contents(node.get_path())
+            contents = self.read_safe_contents(node.get_path())
             _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
             if self.is_main_node(contents):
                 lines = _lines + lines
@@ -78,16 +113,14 @@ class InoToCPPConverter(object):
     def process(self, contents):
         out_file = self._main_ino + ".cpp"
         assert self._gcc_preprocess(contents, out_file)
-        contents = fs.get_file_contents(out_file)
+        contents = self.read_safe_contents(out_file)
         contents = self._join_multiline_strings(contents)
-        fs.write_file_contents(
-            out_file, self.append_prototypes(contents), errors="backslashreplace"
-        )
+        self.write_safe_contents(out_file, self.append_prototypes(contents))
         return out_file
 
     def _gcc_preprocess(self, contents, out_file):
         tmp_path = mkstemp()[1]
-        fs.write_file_contents(tmp_path, contents, errors="backslashreplace")
+        self.write_safe_contents(tmp_path, contents)
         self.env.Execute(
             self.env.VerboseAction(
                 '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(

platformio/builder/tools/pioplatform.py

@@ -140,13 +140,10 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
 
     def _get_plaform_data():
         data = ["PLATFORM: %s %s" % (platform.title, platform.version)]
-        src_manifest_path = platform.pm.get_src_manifest_path(platform.get_dir())
-        if src_manifest_path:
-            src_manifest = fs.load_json(src_manifest_path)
-            if "version" in src_manifest:
-                data.append("#" + src_manifest["version"])
-            if int(ARGUMENTS.get("PIOVERBOSE", 0)):
-                data.append("(%s)" % src_manifest["url"])
+        if platform.src_version:
+            data.append("#" + platform.src_version)
+        if int(ARGUMENTS.get("PIOVERBOSE", 0)) and platform.src_url:
+            data.append("(%s)" % platform.src_url)
         if board_config:
             data.extend([">", board_config.get("name")])
         return data
@@ -196,20 +193,14 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
 
     def _get_packages_data():
         data = []
-        for name, options in platform.packages.items():
-            if options.get("optional"):
-                continue
-            pkg_dir = platform.get_package_dir(name)
-            if not pkg_dir:
-                continue
-            manifest = platform.pm.load_manifest(pkg_dir)
-            original_version = util.get_original_version(manifest["version"])
-            info = "%s %s" % (manifest["name"], manifest["version"])
+        for item in platform.dump_used_packages():
+            original_version = util.get_original_version(item["version"])
+            info = "%s %s" % (item["name"], item["version"])
             extra = []
             if original_version:
                 extra.append(original_version)
-            if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)):
-                extra.append(manifest["__src_url"])
+            if "src_url" in item and int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                extra.append(item["src_url"])
             if extra:
                 info += " (%s)" % ", ".join(extra)
             data.append(info)

platformio/builder/tools/pioproject.py

@@ -14,7 +14,7 @@
 
 from __future__ import absolute_import
 
-from platformio.project.config import ProjectConfig, ProjectOptions
+from platformio.project.config import MISSING, ProjectConfig, ProjectOptions
 
 
 def GetProjectConfig(env):
@@ -25,7 +25,7 @@ def GetProjectOptions(env, as_dict=False):
     return env.GetProjectConfig().items(env=env["PIOENV"], as_dict=as_dict)
 
 
-def GetProjectOption(env, option, default=None):
+def GetProjectOption(env, option, default=MISSING):
     return env.GetProjectConfig().get("env:" + env["PIOENV"], option, default)
 
 

platformio/builder/tools/platformio.py

@@ -285,7 +285,7 @@ def CollectBuildFiles(
     for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []):
         tmp = []
         for node in sources:
-            if pattern and not fnmatch.fnmatch(node.get_path(), pattern):
+            if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern):
                 tmp.append(node)
                 continue
             n = callback(node)

platformio/commands/account.py

@@ -1,72 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import sys
-
-import click
-
-from platformio.managers.core import pioplus_call
-
-
-@click.group("account", short_help="Manage PIO Account")
-def cli():
-    pass
-
-
-@cli.command("register", short_help="Create new PIO Account")
-@click.option("-u", "--username")
-def account_register(**kwargs):
-    pioplus_call(sys.argv[1:])
-
-
-@cli.command("login", short_help="Log in to PIO Account")
-@click.option("-u", "--username")
-@click.option("-p", "--password")
-def account_login(**kwargs):
-    pioplus_call(sys.argv[1:])
-
-
-@cli.command("logout", short_help="Log out of PIO Account")
-def account_logout():
-    pioplus_call(sys.argv[1:])
-
-
-@cli.command("password", short_help="Change password")
-@click.option("--old-password")
-@click.option("--new-password")
-def account_password(**kwargs):
-    pioplus_call(sys.argv[1:])
-
-
-@cli.command("token", short_help="Get or regenerate Authentication Token")
-@click.option("-p", "--password")
-@click.option("--regenerate", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_token(**kwargs):
-    pioplus_call(sys.argv[1:])
-
-
-@cli.command("forgot", short_help="Forgot password")
-@click.option("-u", "--username")
-def account_forgot(**kwargs):
-    pioplus_call(sys.argv[1:])
-
-
-@cli.command("show", short_help="PIO Account information")
-@click.option("--offline", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_show(**kwargs):
-    pioplus_call(sys.argv[1:])

platformio/commands/account/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/account/client.py

@@ -0,0 +1,262 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=unused-argument
+
+import os
+import time
+
+import requests.adapters
+from requests.packages.urllib3.util.retry import Retry  # pylint:disable=import-error
+
+from platformio import __pioaccount_api__, app
+from platformio.commands.account import exception
+from platformio.exception import InternetIsOffline
+
+
+class AccountClient(object):
+
+    SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
+
+    def __init__(
+        self, api_base_url=__pioaccount_api__, retries=3,
+    ):
+        if api_base_url.endswith("/"):
+            api_base_url = api_base_url[:-1]
+        self.api_base_url = api_base_url
+        self._session = requests.Session()
+        self._session.headers.update({"User-Agent": app.get_user_agent()})
+        retry = Retry(
+            total=retries,
+            read=retries,
+            connect=retries,
+            backoff_factor=2,
+            method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"],
+        )
+        adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+        self._session.mount(api_base_url, adapter)
+
+    @staticmethod
+    def get_refresh_token():
+        try:
+            return app.get_state_item("account").get("auth").get("refresh_token")
+        except:  # pylint:disable=bare-except
+            raise exception.AccountNotAuthorized()
+
+    @staticmethod
+    def delete_local_session():
+        app.delete_state_item("account")
+
+    @staticmethod
+    def delete_local_state(key):
+        account = app.get_state_item("account")
+        if not account or key not in account:
+            return
+        del account[key]
+        app.set_state_item("account", account)
+
+    def login(self, username, password):
+        try:
+            self.fetch_authentication_token()
+        except:  # pylint:disable=bare-except
+            pass
+        else:
+            raise exception.AccountAlreadyAuthorized(
+                app.get_state_item("account", {}).get("email", "")
+            )
+
+        result = self.send_request(
+            "post",
+            self.api_base_url + "/v1/login",
+            data={"username": username, "password": password},
+        )
+        app.set_state_item("account", result)
+        return result
+
+    def login_with_code(self, client_id, code, redirect_uri):
+        try:
+            self.fetch_authentication_token()
+        except:  # pylint:disable=bare-except
+            pass
+        else:
+            raise exception.AccountAlreadyAuthorized(
+                app.get_state_item("account", {}).get("email", "")
+            )
+
+        result = self.send_request(
+            "post",
+            self.api_base_url + "/v1/login/code",
+            data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri},
+        )
+        app.set_state_item("account", result)
+        return result
+
+    def logout(self):
+        refresh_token = self.get_refresh_token()
+        self.delete_local_session()
+        try:
+            self.send_request(
+                "post",
+                self.api_base_url + "/v1/logout",
+                data={"refresh_token": refresh_token},
+            )
+        except exception.AccountError:
+            pass
+        return True
+
+    def change_password(self, old_password, new_password):
+        token = self.fetch_authentication_token()
+        self.send_request(
+            "post",
+            self.api_base_url + "/v1/password",
+            headers={"Authorization": "Bearer %s" % token},
+            data={"old_password": old_password, "new_password": new_password},
+        )
+        return True
+
+    def registration(
+        self, username, email, password, firstname, lastname
+    ):  # pylint:disable=too-many-arguments
+        try:
+            self.fetch_authentication_token()
+        except:  # pylint:disable=bare-except
+            pass
+        else:
+            raise exception.AccountAlreadyAuthorized(
+                app.get_state_item("account", {}).get("email", "")
+            )
+
+        return self.send_request(
+            "post",
+            self.api_base_url + "/v1/registration",
+            data={
+                "username": username,
+                "email": email,
+                "password": password,
+                "firstname": firstname,
+                "lastname": lastname,
+            },
+        )
+
+    def auth_token(self, password, regenerate):
+        token = self.fetch_authentication_token()
+        result = self.send_request(
+            "post",
+            self.api_base_url + "/v1/token",
+            headers={"Authorization": "Bearer %s" % token},
+            data={"password": password, "regenerate": 1 if regenerate else 0},
+        )
+        return result.get("auth_token")
+
+    def forgot_password(self, username):
+        return self.send_request(
+            "post", self.api_base_url + "/v1/forgot", data={"username": username},
+        )
+
+    def get_profile(self):
+        token = self.fetch_authentication_token()
+        return self.send_request(
+            "get",
+            self.api_base_url + "/v1/profile",
+            headers={"Authorization": "Bearer %s" % token},
+        )
+
+    def update_profile(self, profile, current_password):
+        token = self.fetch_authentication_token()
+        profile["current_password"] = current_password
+        self.delete_local_state("summary")
+        response = self.send_request(
+            "put",
+            self.api_base_url + "/v1/profile",
+            headers={"Authorization": "Bearer %s" % token},
+            data=profile,
+        )
+        return response
+
+    def get_account_info(self, offline):
+        account = app.get_state_item("account")
+        if not account:
+            raise exception.AccountNotAuthorized()
+        if (
+            account.get("summary")
+            and account["summary"].get("expire_at", 0) > time.time()
+        ):
+            return account["summary"]
+        if offline:
+            return {
+                "profile": {
+                    "email": account.get("email"),
+                    "username": account.get("username"),
+                }
+            }
+        token = self.fetch_authentication_token()
+        result = self.send_request(
+            "get",
+            self.api_base_url + "/v1/summary",
+            headers={"Authorization": "Bearer %s" % token},
+        )
+        account["summary"] = dict(
+            profile=result.get("profile"),
+            packages=result.get("packages"),
+            subscriptions=result.get("subscriptions"),
+            user_id=result.get("user_id"),
+            expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL,
+        )
+        app.set_state_item("account", account)
+        return result
+
+    def fetch_authentication_token(self):
+        if "PLATFORMIO_AUTH_TOKEN" in os.environ:
+            return os.environ["PLATFORMIO_AUTH_TOKEN"]
+        auth = app.get_state_item("account", {}).get("auth", {})
+        if auth.get("access_token") and auth.get("access_token_expire"):
+            if auth.get("access_token_expire") > time.time():
+                return auth.get("access_token")
+            if auth.get("refresh_token"):
+                try:
+                    result = self.send_request(
+                        "post",
+                        self.api_base_url + "/v1/login",
+                        headers={
+                            "Authorization": "Bearer %s" % auth.get("refresh_token")
+                        },
+                    )
+                    app.set_state_item("account", result)
+                    return result.get("auth").get("access_token")
+                except exception.AccountError:
+                    self.delete_local_session()
+        raise exception.AccountNotAuthorized()
+
+    def send_request(self, method, url, headers=None, data=None):
+        try:
+            response = getattr(self._session, method)(
+                url, headers=headers or {}, data=data or {}
+            )
+        except requests.exceptions.ConnectionError:
+            raise InternetIsOffline()
+        return self.raise_error_from_response(response)
+
+    def raise_error_from_response(self, response, expected_codes=(200, 201, 202)):
+        if response.status_code in expected_codes:
+            try:
+                return response.json()
+            except ValueError:
+                pass
+        try:
+            message = response.json()["message"]
+        except (KeyError, ValueError):
+            message = response.text
+        if "Authorization session has been expired" in message:
+            self.delete_local_session()
+        raise exception.AccountError(message)

platformio/commands/account/command.py

@@ -0,0 +1,278 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=unused-argument
+
+import datetime
+import json
+import re
+
+import click
+from tabulate import tabulate
+
+from platformio.commands.account import exception
+from platformio.commands.account.client import AccountClient
+
+
+@click.group("account", short_help="Manage PIO Account")
+def cli():
+    pass
+
+
+def validate_username(value):
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){3,38}$", value, flags=re.I):
+        raise click.BadParameter(
+            "Invalid username format. "
+            "Username must contain at least 4 characters including single hyphens,"
+            " and cannot begin or end with a hyphen"
+        )
+    return value
+
+
+def validate_email(value):
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d_.+-]+@[a-z\d\-]+\.[a-z\d\-.]+$", value, flags=re.I):
+        raise click.BadParameter("Invalid email address")
+    return value
+
+
+def validate_password(value):
+    value = str(value).strip()
+    if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
+        raise click.BadParameter(
+            "Invalid password format. "
+            "Password must contain at least 8 characters"
+            " including a number and a lowercase letter"
+        )
+    return value
+
+
+@cli.command("register", short_help="Create new PIO Account")
+@click.option(
+    "-u",
+    "--username",
+    prompt=True,
+    callback=lambda _, __, value: validate_username(value),
+)
+@click.option(
+    "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value)
+)
+@click.option(
+    "-p",
+    "--password",
+    prompt=True,
+    hide_input=True,
+    confirmation_prompt=True,
+    callback=lambda _, __, value: validate_password(value),
+)
+@click.option("--firstname", prompt=True)
+@click.option("--lastname", prompt=True)
+def account_register(username, email, password, firstname, lastname):
+    client = AccountClient()
+    client.registration(username, email, password, firstname, lastname)
+    return click.secho(
+        "An account has been successfully created. "
+        "Please check your mail to activate your account and verify your email address.",
+        fg="green",
+    )
+
+
+@cli.command("login", short_help="Log in to PIO Account")
+@click.option("-u", "--username", prompt="Username or email")
+@click.option("-p", "--password", prompt=True, hide_input=True)
+def account_login(username, password):
+    client = AccountClient()
+    client.login(username, password)
+    return click.secho("Successfully logged in!", fg="green")
+
+
+@cli.command("logout", short_help="Log out of PIO Account")
+def account_logout():
+    client = AccountClient()
+    client.logout()
+    return click.secho("Successfully logged out!", fg="green")
+
+
+@cli.command("password", short_help="Change password")
+@click.option("--old-password", prompt=True, hide_input=True)
+@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
+def account_password(old_password, new_password):
+    client = AccountClient()
+    client.change_password(old_password, new_password)
+    return click.secho("Password successfully changed!", fg="green")
+
+
+@cli.command("token", short_help="Get or regenerate Authentication Token")
+@click.option("-p", "--password", prompt=True, hide_input=True)
+@click.option("--regenerate", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_token(password, regenerate, json_output):
+    client = AccountClient()
+    auth_token = client.auth_token(password, regenerate)
+    if json_output:
+        return click.echo(json.dumps({"status": "success", "result": auth_token}))
+    return click.secho("Personal Authentication Token: %s" % auth_token, fg="green")
+
+
+@cli.command("forgot", short_help="Forgot password")
+@click.option("--username", prompt="Username or email")
+def account_forgot(username):
+    client = AccountClient()
+    client.forgot_password(username)
+    return click.secho(
+        "If this account is registered, we will send the "
+        "further instructions to your email.",
+        fg="green",
+    )
+
+
+@cli.command("update", short_help="Update profile information")
+@click.option("--current-password", prompt=True, hide_input=True)
+@click.option("--username")
+@click.option("--email")
+@click.option("--firstname")
+@click.option("--lastname")
+def account_update(current_password, **kwargs):
+    client = AccountClient()
+    profile = client.get_profile()
+    new_profile = profile.copy()
+    if not any(kwargs.values()):
+        for field in profile:
+            new_profile[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=profile[field]
+            )
+            if field == "email":
+                validate_email(new_profile[field])
+            if field == "username":
+                validate_username(new_profile[field])
+    else:
+        new_profile.update({key: value for key, value in kwargs.items() if value})
+    client.update_profile(new_profile, current_password)
+    click.secho("Profile successfully updated!", fg="green")
+    username_changed = new_profile["username"] != profile["username"]
+    email_changed = new_profile["email"] != profile["email"]
+    if not username_changed and not email_changed:
+        return None
+    try:
+        client.logout()
+    except exception.AccountNotAuthorized:
+        pass
+    if email_changed:
+        return click.secho(
+            "Please check your mail to verify your new email address and re-login. ",
+            fg="yellow",
+        )
+    return click.secho("Please re-login.", fg="yellow")
+
+
+@cli.command("show", short_help="PIO Account information")
+@click.option("--offline", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_show(offline, json_output):
+    client = AccountClient()
+    info = client.get_account_info(offline)
+    if json_output:
+        return click.echo(json.dumps(info))
+    click.echo()
+    if info.get("profile"):
+        print_profile(info["profile"])
+    if info.get("packages"):
+        print_packages(info["packages"])
+    if info.get("subscriptions"):
+        print_subscriptions(info["subscriptions"])
+    return click.echo()
+
+
+def print_profile(profile):
+    click.secho("Profile", fg="cyan", bold=True)
+    click.echo("=" * len("Profile"))
+    data = []
+    if profile.get("username"):
+        data.append(("Username:", profile["username"]))
+    if profile.get("email"):
+        data.append(("Email:", profile["email"]))
+    if profile.get("firstname"):
+        data.append(("First name:", profile["firstname"]))
+    if profile.get("lastname"):
+        data.append(("Last name:", profile["lastname"]))
+    click.echo(tabulate(data, tablefmt="plain"))
+
+
+def print_packages(packages):
+    click.echo()
+    click.secho("Packages", fg="cyan")
+    click.echo("=" * len("Packages"))
+    for package in packages:
+        click.echo()
+        click.secho(package.get("name"), bold=True)
+        click.echo("-" * len(package.get("name")))
+        if package.get("description"):
+            click.echo(package.get("description"))
+        data = []
+        expire = "-"
+        if "subscription" in package:
+            expire = datetime.datetime.strptime(
+                (
+                    package["subscription"].get("end_at")
+                    or package["subscription"].get("next_bill_at")
+                ),
+                "%Y-%m-%dT%H:%M:%SZ",
+            ).strftime("%Y-%m-%d")
+        data.append(("Expire:", expire))
+        services = []
+        for key in package:
+            if not key.startswith("service."):
+                continue
+            if isinstance(package[key], dict):
+                services.append(package[key].get("title"))
+            else:
+                services.append(package[key])
+        if services:
+            data.append(("Services:", ", ".join(services)))
+        click.echo(tabulate(data, tablefmt="plain"))
+
+
+def print_subscriptions(subscriptions):
+    click.echo()
+    click.secho("Subscriptions", fg="cyan")
+    click.echo("=" * len("Subscriptions"))
+    for subscription in subscriptions:
+        click.echo()
+        click.secho(subscription.get("product_name"), bold=True)
+        click.echo("-" * len(subscription.get("product_name")))
+        data = [("State:", subscription.get("status"))]
+        begin_at = datetime.datetime.strptime(
+            subscription.get("begin_at"), "%Y-%m-%dT%H:%M:%SZ"
+        ).strftime("%Y-%m-%d %H:%M:%S")
+        data.append(("Start date:", begin_at or "-"))
+        end_at = subscription.get("end_at")
+        if end_at:
+            end_at = datetime.datetime.strptime(
+                subscription.get("end_at"), "%Y-%m-%dT%H:%M:%SZ"
+            ).strftime("%Y-%m-%d %H:%M:%S")
+        data.append(("End date:", end_at or "-"))
+        next_bill_at = subscription.get("next_bill_at")
+        if next_bill_at:
+            next_bill_at = datetime.datetime.strptime(
+                subscription.get("next_bill_at"), "%Y-%m-%dT%H:%M:%SZ"
+            ).strftime("%Y-%m-%d %H:%M:%S")
+        data.append(("Next payment:", next_bill_at or "-"))
+        data.append(
+            ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")
+        )
+        data.append(
+            ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-")
+        )
+        click.echo(tabulate(data, tablefmt="plain"))

platformio/commands/account/exception.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.exception import PlatformioException
+
+
+class AccountError(PlatformioException):
+
+    MESSAGE = "{0}"
+
+
+class AccountNotAuthorized(AccountError):
+
+    MESSAGE = "You are not authorized! Please log in to PIO Account."
+
+
+class AccountAlreadyAuthorized(AccountError):
+
+    MESSAGE = "You are already authorized with {0} account."

platformio/commands/check/command.py

@@ -61,6 +61,7 @@ from platformio.project.helpers import find_project_dir_above, get_project_dir
     multiple=True,
     type=click.Choice(DefectItem.SEVERITY_LABELS.values()),
 )
+@click.option("--skip-packages", is_flag=True)
 def cli(
     environment,
     project_dir,
@@ -72,6 +73,7 @@ def cli(
     verbose,
     json_output,
     fail_on_defect,
+    skip_packages,
 ):
     app.set_session_var("custom_project_conf", project_conf)
 
@@ -114,6 +116,7 @@ def cli(
                 severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]]
                 if silent
                 else severity or config.get("env:" + envname, "check_severity"),
+                skip_packages=skip_packages or env_options.get("check_skip_packages"),
             )
 
             for tool in config.get("env:" + envname, "check_tool"):
@@ -222,7 +225,7 @@ def collect_component_stats(result):
         component = dirname(defect.file) or defect.file
         _append_defect(component, defect)
 
-        if component.startswith(get_project_dir()):
+        if component.lower().startswith(get_project_dir().lower()):
             while os.sep in component:
                 component = dirname(component)
                 _append_defect(component, defect)

platformio/commands/check/defect.py

@@ -51,7 +51,7 @@ class DefectItem(object):
         self.cwe = cwe
         self.id = id
         self.file = file
-        if file.startswith(get_project_dir()):
+        if file.lower().startswith(get_project_dir().lower()):
             self.file = os.path.relpath(file, get_project_dir())
 
     def __repr__(self):

platformio/commands/check/tools/base.py

@@ -14,12 +14,13 @@
 
 import glob
 import os
+from tempfile import NamedTemporaryFile
 
 import click
 
 from platformio import fs, proc
 from platformio.commands.check.defect import DefectItem
-from platformio.project.helpers import get_project_dir, load_project_ide_data
+from platformio.project.helpers import load_project_ide_data
 
 
 class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
@@ -32,12 +33,13 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         self.cpp_includes = []
         self.cpp_defines = []
         self.toolchain_defines = []
+        self._tmp_files = []
         self.cc_path = None
         self.cxx_path = None
         self._defects = []
         self._on_defect_callback = None
         self._bad_input = False
-        self._load_cpp_data(project_dir, envname)
+        self._load_cpp_data(project_dir)
 
         # detect all defects by default
         if not self.options.get("severity"):
@@ -52,17 +54,17 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
             for s in self.options["severity"]
         ]
 
-    def _load_cpp_data(self, project_dir, envname):
-        data = load_project_ide_data(project_dir, envname)
+    def _load_cpp_data(self, project_dir):
+        data = load_project_ide_data(project_dir, self.envname)
         if not data:
             return
-        self.cc_flags = data.get("cc_flags", "").split(" ")
-        self.cxx_flags = data.get("cxx_flags", "").split(" ")
-        self.cpp_includes = data.get("includes", [])
+        self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", ""))
+        self.cxx_flags = click.parser.split_arg_string(data.get("cxx_flags", ""))
+        self.cpp_includes = self._dump_includes(data.get("includes", {}))
         self.cpp_defines = data.get("defines", [])
         self.cc_path = data.get("cc_path")
         self.cxx_path = data.get("cxx_path")
-        self.toolchain_defines = self._get_toolchain_defines(self.cc_path)
+        self.toolchain_defines = self._get_toolchain_defines()
 
     def get_flags(self, tool):
         result = []
@@ -75,21 +77,52 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
 
         return result
 
+    def _get_toolchain_defines(self):
+        def _extract_defines(language, includes_file):
+            build_flags = self.cxx_flags if language == "c++" else self.cc_flags
+            defines = []
+            cmd = "echo | %s -x %s %s %s -dM -E -" % (
+                self.cc_path,
+                language,
+                " ".join([f for f in build_flags if f.startswith(("-m", "-f"))]),
+                includes_file,
+            )
+            result = proc.exec_command(cmd, shell=True)
+            for line in result["out"].split("\n"):
+                tokens = line.strip().split(" ", 2)
+                if not tokens or tokens[0] != "#define":
+                    continue
+                if len(tokens) > 2:
+                    defines.append("%s=%s" % (tokens[1], tokens[2]))
+                else:
+                    defines.append(tokens[1])
+
+            return defines
+
+        incflags_file = self._long_includes_hook(self.cpp_includes)
+        return {lang: _extract_defines(lang, incflags_file) for lang in ("c", "c++")}
+
+    def _create_tmp_file(self, data):
+        with NamedTemporaryFile("w", delete=False) as fp:
+            fp.write(data)
+            self._tmp_files.append(fp.name)
+            return fp.name
+
+    def _long_includes_hook(self, includes):
+        data = []
+        for inc in includes:
+            data.append('-I"%s"' % fs.to_unix_path(inc))
+
+        return '@"%s"' % self._create_tmp_file(" ".join(data))
+
     @staticmethod
-    def _get_toolchain_defines(cc_path):
-        defines = []
-        result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path, shell=True)
-
-        for line in result["out"].split("\n"):
-            tokens = line.strip().split(" ", 2)
-            if not tokens or tokens[0] != "#define":
-                continue
-            if len(tokens) > 2:
-                defines.append("%s=%s" % (tokens[1], tokens[2]))
-            else:
-                defines.append(tokens[1])
-
-        return defines
+    def _dump_includes(includes_map):
+        result = []
+        for includes in includes_map.values():
+            for include in includes:
+                if include not in result:
+                    result.append(include)
+        return result
 
     @staticmethod
     def is_flag_set(flag, flags):
@@ -129,18 +162,27 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         return raw_line
 
     def clean_up(self):
-        pass
+        for f in self._tmp_files:
+            if os.path.isfile(f):
+                os.remove(f)
 
-    def get_project_target_files(self):
-        allowed_extensions = (".h", ".hpp", ".c", ".cc", ".cpp", ".ino")
-        result = []
+    @staticmethod
+    def get_project_target_files(patterns):
+        c_extension = (".c",)
+        cpp_extensions = (".cc", ".cpp", ".cxx", ".ino")
+        header_extensions = (".h", ".hh", ".hpp", ".hxx")
+
+        result = {"c": [], "c++": [], "headers": []}
 
         def _add_file(path):
-            if not path.endswith(allowed_extensions):
-                return
-            result.append(os.path.realpath(path))
+            if path.endswith(header_extensions):
+                result["headers"].append(os.path.realpath(path))
+            elif path.endswith(c_extension):
+                result["c"].append(os.path.realpath(path))
+            elif path.endswith(cpp_extensions):
+                result["c++"].append(os.path.realpath(path))
 
-        for pattern in self.options["patterns"]:
+        for pattern in patterns:
             for item in glob.glob(pattern):
                 if not os.path.isdir(item):
                     _add_file(item)
@@ -150,27 +192,23 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
 
         return result
 
-    def get_source_language(self):
-        with fs.cd(get_project_dir()):
-            for _, __, files in os.walk(self.config.get_optional_dir("src")):
-                for name in files:
-                    if "." not in name:
-                        continue
-                    if os.path.splitext(name)[1].lower() in (".cpp", ".cxx", ".ino"):
-                        return "c++"
-            return "c"
-
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
         cmd = self.configure_command()
-        if self.options.get("verbose"):
-            click.echo(" ".join(cmd))
+        if cmd:
+            if self.options.get("verbose"):
+                click.echo(" ".join(cmd))
 
-        proc.exec_command(
-            cmd,
-            stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
-            stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
-        )
+            proc.exec_command(
+                cmd,
+                stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
+                stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
+            )
+
+        else:
+            if self.options.get("verbose"):
+                click.echo("Error: Couldn't configure command")
+            self._bad_input = True
 
         self.clean_up()
 

platformio/commands/check/tools/clangtidy.py

@@ -57,11 +57,28 @@ class ClangtidyCheckTool(CheckToolBase):
         if not self.is_flag_set("--checks", flags):
             cmd.append("--checks=*")
 
+        project_files = self.get_project_target_files(self.options["patterns"])
+
+        src_files = []
+        for scope in project_files:
+            src_files.extend(project_files[scope])
+
         cmd.extend(flags)
-        cmd.extend(self.get_project_target_files())
+        cmd.extend(src_files)
         cmd.append("--")
 
-        cmd.extend(["-D%s" % d for d in self.cpp_defines + self.toolchain_defines])
-        cmd.extend(["-I%s" % inc for inc in self.cpp_includes])
+        cmd.extend(
+            ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines["c++"]]
+        )
+
+        includes = []
+        for inc in self.cpp_includes:
+            if self.options.get("skip_packages") and inc.lower().startswith(
+                self.config.get_optional_dir("packages").lower()
+            ):
+                continue
+            includes.append(inc)
+
+        cmd.append("--extra-arg=" + self._long_includes_hook(includes))
 
         return cmd

platformio/commands/check/tools/cppcheck.py

@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from os import remove
-from os.path import isfile, join
-from tempfile import NamedTemporaryFile
+import os
 
+import click
+
+from platformio import proc
 from platformio.commands.check.defect import DefectItem
 from platformio.commands.check.tools.base import CheckToolBase
 from platformio.managers.core import get_core_package_dir
@@ -23,7 +24,6 @@ from platformio.managers.core import get_core_package_dir
 
 class CppcheckCheckTool(CheckToolBase):
     def __init__(self, *args, **kwargs):
-        self._tmp_files = []
         self.defect_fields = [
             "severity",
             "message",
@@ -74,10 +74,32 @@ class CppcheckCheckTool(CheckToolBase):
         else:
             args["severity"] = DefectItem.SEVERITY_LOW
 
+        # Skip defects found in third-party software, but keep in mind that such defects
+        # might break checking process so defects from project files are not reported
+        breaking_defect_ids = ("preprocessorErrorDirective", "syntaxError")
+        if (
+            args.get("file", "")
+            .lower()
+            .startswith(self.config.get_optional_dir("packages").lower())
+        ):
+            if args["id"] in breaking_defect_ids:
+                if self.options.get("verbose"):
+                    click.echo(
+                        "Error: Found a breaking defect '%s' in %s:%s\n"
+                        "Please note: check results might not be valid!\n"
+                        "Try adding --skip-packages"
+                        % (args.get("message"), args.get("file"), args.get("line"))
+                    )
+                    click.echo()
+                self._bad_input = True
+            return None
+
         return DefectItem(**args)
 
-    def configure_command(self):
-        tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck")
+    def configure_command(
+        self, language, src_files
+    ):  # pylint: disable=arguments-differ
+        tool_path = os.path.join(get_core_package_dir("tool-cppcheck"), "cppcheck")
 
         cmd = [
             tool_path,
@@ -108,51 +130,112 @@ class CppcheckCheckTool(CheckToolBase):
             cmd.append("--enable=%s" % ",".join(enabled_checks))
 
         if not self.is_flag_set("--language", flags):
-            if self.get_source_language() == "c++":
-                cmd.append("--language=c++")
+            cmd.append("--language=" + language)
 
-                if not self.is_flag_set("--std", flags):
-                    for f in self.cxx_flags + self.cc_flags:
-                        if "-std" in f:
-                            # Standards with GNU extensions are not allowed
-                            cmd.append("-" + f.replace("gnu", "c"))
+        build_flags = self.cxx_flags if language == "c++" else self.cc_flags
+
+        for flag in build_flags:
+            if "-std" in flag:
+                # Standards with GNU extensions are not allowed
+                cmd.append("-" + flag.replace("gnu", "c"))
+
+        cmd.extend(
+            ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines[language]]
+        )
 
-        cmd.extend(["-D%s" % d for d in self.cpp_defines + self.toolchain_defines])
         cmd.extend(flags)
 
-        cmd.append("--file-list=%s" % self._generate_src_file())
+        cmd.extend(
+            "--include=" + inc
+            for inc in self.get_forced_includes(build_flags, self.cpp_includes)
+        )
+        cmd.append("--file-list=%s" % self._generate_src_file(src_files))
         cmd.append("--includes-file=%s" % self._generate_inc_file())
 
-        core_dir = self.config.get_optional_dir("packages")
-        cmd.append("--suppress=*:%s*" % core_dir)
-        cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir)
-
         return cmd
 
-    def _create_tmp_file(self, data):
-        with NamedTemporaryFile("w", delete=False) as fp:
-            fp.write(data)
-            self._tmp_files.append(fp.name)
-            return fp.name
+    @staticmethod
+    def get_forced_includes(build_flags, includes):
+        def _extract_filepath(flag, include_options, build_flags):
+            path = ""
+            for option in include_options:
+                if not flag.startswith(option):
+                    continue
+                if flag.split(option)[1].strip():
+                    path = flag.split(option)[1].strip()
+                elif build_flags.index(flag) + 1 < len(build_flags):
+                    path = build_flags[build_flags.index(flag) + 1]
+            return path
 
-    def _generate_src_file(self):
-        src_files = [
-            f for f in self.get_project_target_files() if not f.endswith((".h", ".hpp"))
-        ]
+        def _search_include_dir(filepath, include_paths):
+            for inc_path in include_paths:
+                path = os.path.join(inc_path, filepath)
+                if os.path.isfile(path):
+                    return path
+            return ""
+
+        result = []
+        include_options = ("-include", "-imacros")
+        for f in build_flags:
+            if f.startswith(include_options):
+                filepath = _extract_filepath(f, include_options, build_flags)
+                if not os.path.isabs(filepath):
+                    filepath = _search_include_dir(filepath, includes)
+                if os.path.isfile(filepath):
+                    result.append(filepath)
+
+        return result
+
+    def _generate_src_file(self, src_files):
         return self._create_tmp_file("\n".join(src_files))
 
     def _generate_inc_file(self):
-        return self._create_tmp_file("\n".join(self.cpp_includes))
+        result = []
+        for inc in self.cpp_includes:
+            if self.options.get("skip_packages") and inc.lower().startswith(
+                self.config.get_optional_dir("packages").lower()
+            ):
+                continue
+            result.append(inc)
+        return self._create_tmp_file("\n".join(result))
 
     def clean_up(self):
-        for f in self._tmp_files:
-            if isfile(f):
-                remove(f)
+        super(CppcheckCheckTool, self).clean_up()
 
         # delete temporary dump files generated by addons
         if not self.is_flag_set("--addon", self.get_flags("cppcheck")):
             return
-        for f in self.get_project_target_files():
-            dump_file = f + ".dump"
-            if isfile(dump_file):
-                remove(dump_file)
+
+        for files in self.get_project_target_files(self.options["patterns"]).values():
+            for f in files:
+                dump_file = f + ".dump"
+                if os.path.isfile(dump_file):
+                    os.remove(dump_file)
+
+    def check(self, on_defect_callback=None):
+        self._on_defect_callback = on_defect_callback
+        project_files = self.get_project_target_files(self.options["patterns"])
+
+        languages = ("c", "c++")
+        if not any([project_files[t] for t in languages]):
+            click.echo("Error: Nothing to check.")
+            return True
+        for language in languages:
+            if not project_files[language]:
+                continue
+            cmd = self.configure_command(language, project_files[language])
+            if not cmd:
+                self._bad_input = True
+                continue
+            if self.options.get("verbose"):
+                click.echo(" ".join(cmd))
+
+            proc.exec_command(
+                cmd,
+                stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
+                stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
+            )
+
+        self.clean_up()
+
+        return self._bad_input

platformio/commands/check/tools/pvsstudio.py

@@ -140,9 +140,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
             os.remove(self._tmp_output_file)
 
         if not os.path.isfile(self._tmp_preprocessed_file):
-            click.echo(
-                "Error: Missing preprocessed file '%s'" % (self._tmp_preprocessed_file)
-            )
+            click.echo("Error: Missing preprocessed file for '%s'" % src_file)
             return ""
 
         cmd = [
@@ -175,6 +173,9 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
         return os.path.join(self._tmp_dir, next(tempfile._get_candidate_names()))
 
     def _prepare_preprocessed_file(self, src_file):
+        if os.path.isfile(self._tmp_preprocessed_file):
+            os.remove(self._tmp_preprocessed_file)
+
         flags = self.cxx_flags
         compiler = self.cxx_path
         if src_file.endswith(".c"):
@@ -186,40 +187,46 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
         cmd.extend(["-D%s" % d for d in self.cpp_defines])
         cmd.append('@"%s"' % self._tmp_cmd_file)
 
+        # Explicitly specify C++ as the language used in .ino files
+        if src_file.endswith(".ino"):
+            cmd.insert(1, "-xc++")
+
         result = proc.exec_command(" ".join(cmd), shell=True)
-        if result["returncode"] != 0:
+        if result["returncode"] != 0 or result["err"]:
             if self.options.get("verbose"):
                 click.echo(" ".join(cmd))
             click.echo(result["err"])
             self._bad_input = True
 
     def clean_up(self):
+        super(PvsStudioCheckTool, self).clean_up()
         if os.path.isdir(self._tmp_dir):
             shutil.rmtree(self._tmp_dir)
 
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
-        src_files = [
-            f for f in self.get_project_target_files() if not f.endswith((".h", ".hpp"))
-        ]
-
-        for src_file in src_files:
-            self._prepare_preprocessed_file(src_file)
-            cmd = self.configure_command(src_file)
-            if self.options.get("verbose"):
-                click.echo(" ".join(cmd))
-            if not cmd:
-                self._bad_input = True
+        for scope, files in self.get_project_target_files(
+            self.options["patterns"]
+        ).items():
+            if scope not in ("c", "c++"):
                 continue
+            for src_file in files:
+                self._prepare_preprocessed_file(src_file)
+                cmd = self.configure_command(src_file)
+                if self.options.get("verbose"):
+                    click.echo(" ".join(cmd))
+                if not cmd:
+                    self._bad_input = True
+                    continue
 
-            result = proc.exec_command(cmd)
-            # pylint: disable=unsupported-membership-test
-            if result["returncode"] != 0 or "License was not entered" in result["err"]:
-                self._bad_input = True
-                click.echo(result["err"])
-                continue
+                result = proc.exec_command(cmd)
+                # pylint: disable=unsupported-membership-test
+                if result["returncode"] != 0 or "license" in result["err"].lower():
+                    self._bad_input = True
+                    click.echo(result["err"])
+                    continue
 
-            self._process_defects(self.parse_defects(self._tmp_output_file))
+                self._process_defects(self.parse_defects(self._tmp_output_file))
 
         self.clean_up()
 

platformio/commands/debug/command.py

@@ -148,7 +148,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro
     inject_contrib_pysite()
 
     # pylint: disable=import-outside-toplevel
-    from platformio.commands.debug.client import GDBClient, reactor
+    from platformio.commands.debug.process.client import GDBClient, reactor
 
     client = GDBClient(project_dir, __unprocessed, debug_options, env_options)
     client.spawn(configuration["gdb_path"], configuration["prog_path"])

platformio/commands/debug/helpers.py

@@ -125,7 +125,8 @@ def validate_debug_options(cmd_ctx, env_options):
         server_options["executable"] = server_options["arguments"][0]
         server_options["arguments"] = server_options["arguments"][1:]
     elif "server" in tool_settings:
-        server_package = tool_settings["server"].get("package")
+        server_options = tool_settings["server"]
+        server_package = server_options.get("package")
         server_package_dir = (
             platform.get_package_dir(server_package) if server_package else None
         )
@@ -134,15 +135,17 @@ def validate_debug_options(cmd_ctx, env_options):
                 with_packages=[server_package], skip_default_package=True, silent=True
             )
             server_package_dir = platform.get_package_dir(server_package)
-        server_options = dict(
-            cwd=server_package_dir if server_package else None,
-            executable=tool_settings["server"].get("executable"),
-            arguments=[
-                a.replace("$PACKAGE_DIR", server_package_dir)
-                if server_package_dir
-                else a
-                for a in tool_settings["server"].get("arguments", [])
-            ],
+        server_options.update(
+            dict(
+                cwd=server_package_dir if server_package else None,
+                executable=server_options.get("executable"),
+                arguments=[
+                    a.replace("$PACKAGE_DIR", server_package_dir)
+                    if server_package_dir
+                    else a
+                    for a in server_options.get("arguments", [])
+                ],
+            )
         )
 
     extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds"))
@@ -252,12 +255,14 @@ def is_prog_obsolete(prog_path):
                 break
             shasum.update(data)
     new_digest = shasum.hexdigest()
-    old_digest = (
-        fs.get_file_contents(prog_hash_path) if isfile(prog_hash_path) else None
-    )
+    old_digest = None
+    if isfile(prog_hash_path):
+        with open(prog_hash_path) as fp:
+            old_digest = fp.read()
     if new_digest == old_digest:
         return False
-    fs.write_file_contents(prog_hash_path, new_digest)
+    with open(prog_hash_path, "w") as fp:
+        fp.write(new_digest)
     return True
 
 

platformio/commands/debug/initcfgs.py

@@ -123,3 +123,39 @@ $LOAD_CMDS
 pio_reset_halt_target
 $INIT_BREAK
 """
+
+GDB_RENODE_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor machine Reset
+    $LOAD_CMDS
+    monitor start
+end
+
+define pio_reset_run_target
+    pio_reset_halt_target
+end
+
+target extended-remote $DEBUG_PORT
+$LOAD_CMDS
+$INIT_BREAK
+monitor start
+"""
+
+
+TOOL_TO_CONFIG = {
+    "jlink": GDB_JLINK_INIT_CONFIG,
+    "mspdebug": GDB_MSPDEBUG_INIT_CONFIG,
+    "qemu": GDB_QEMU_INIT_CONFIG,
+    "blackmagic": GDB_BLACKMAGIC_INIT_CONFIG,
+    "renode": GDB_RENODE_INIT_CONFIG,
+}
+
+
+def get_gdb_init_config(debug_options):
+    tool = debug_options.get("tool")
+    if tool and tool in TOOL_TO_CONFIG:
+        return TOOL_TO_CONFIG[tool]
+    server_exe = (debug_options.get("server") or {}).get("executable", "").lower()
+    if "st-util" in server_exe:
+        return GDB_STUTIL_INIT_CONFIG
+    return GDB_DEFAULT_INIT_CONFIG

platformio/commands/debug/process/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/debug/process/base.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import signal
+import time
 
 import click
 from twisted.internet import protocol  # pylint: disable=import-error
@@ -22,12 +23,11 @@ from platformio.compat import string_types
 from platformio.proc import get_pythonexe_path
 from platformio.project.helpers import get_project_core_dir
 
-LOG_FILE = None
-
 
 class BaseProcess(protocol.ProcessProtocol, object):
 
     STDOUT_CHUNK_SIZE = 2048
+    LOG_FILE = None
 
     COMMON_PATTERNS = {
         "PLATFORMIO_HOME_DIR": get_project_core_dir(),
@@ -35,6 +35,9 @@ class BaseProcess(protocol.ProcessProtocol, object):
         "PYTHONEXE": get_pythonexe_path(),
     }
 
+    def __init__(self):
+        self._last_activity = 0
+
     def apply_patterns(self, source, patterns=None):
         _patterns = self.COMMON_PATTERNS.copy()
         _patterns.update(patterns or {})
@@ -61,23 +64,30 @@ class BaseProcess(protocol.ProcessProtocol, object):
 
         return source
 
+    def onStdInData(self, data):
+        self._last_activity = time.time()
+        if self.LOG_FILE:
+            with open(self.LOG_FILE, "ab") as fp:
+                fp.write(data)
+
     def outReceived(self, data):
-        if LOG_FILE:
-            with open(LOG_FILE, "ab") as fp:
+        self._last_activity = time.time()
+        if self.LOG_FILE:
+            with open(self.LOG_FILE, "ab") as fp:
                 fp.write(data)
         while data:
             chunk = data[: self.STDOUT_CHUNK_SIZE]
             click.echo(chunk, nl=False)
             data = data[self.STDOUT_CHUNK_SIZE :]
 
-    @staticmethod
-    def errReceived(data):
-        if LOG_FILE:
-            with open(LOG_FILE, "ab") as fp:
+    def errReceived(self, data):
+        self._last_activity = time.time()
+        if self.LOG_FILE:
+            with open(self.LOG_FILE, "ab") as fp:
                 fp.write(data)
         click.echo(data, nl=False, err=True)
 
-    @staticmethod
-    def processEnded(_):
+    def processEnded(self, _):
+        self._last_activity = time.time()
         # Allow terminating via SIGINT/CTRL+C
         signal.signal(signal.SIGINT, signal.default_int_handler)

platformio/commands/debug/process/client.py

@@ -20,21 +20,21 @@ from hashlib import sha1
 from os.path import basename, dirname, isdir, join, realpath, splitext
 from tempfile import mkdtemp
 
+from twisted.internet import defer  # pylint: disable=import-error
 from twisted.internet import protocol  # pylint: disable=import-error
 from twisted.internet import reactor  # pylint: disable=import-error
 from twisted.internet import stdio  # pylint: disable=import-error
 from twisted.internet import task  # pylint: disable=import-error
 
 from platformio import app, fs, proc, telemetry, util
-from platformio.commands.debug import helpers, initcfgs
+from platformio.commands.debug import helpers
 from platformio.commands.debug.exception import DebugInvalidOptionsError
-from platformio.commands.debug.process import BaseProcess
-from platformio.commands.debug.server import DebugServer
+from platformio.commands.debug.initcfgs import get_gdb_init_config
+from platformio.commands.debug.process.base import BaseProcess
+from platformio.commands.debug.process.server import DebugServer
 from platformio.compat import hashlib_encode_data, is_bytes
 from platformio.project.helpers import get_project_cache_dir
 
-LOG_FILE = None
-
 
 class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
 
@@ -42,6 +42,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
     INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed"
 
     def __init__(self, project_dir, args, debug_options, env_options):
+        super(GDBClient, self).__init__()
         self.project_dir = project_dir
         self.args = list(args)
         self.debug_options = debug_options
@@ -55,10 +56,10 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), prefix=".piodebug-")
 
         self._target_is_run = False
-        self._last_server_activity = 0
         self._auto_continue_timer = None
         self._errors_buffer = b""
 
+    @defer.inlineCallbacks
     def spawn(self, gdb_path, prog_path):
         session_hash = gdb_path + prog_path
         self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest()
@@ -75,10 +76,10 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
             "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []),
         }
 
-        self._debug_server.spawn(patterns)
-
+        yield self._debug_server.spawn(patterns)
         if not patterns["DEBUG_PORT"]:
             patterns["DEBUG_PORT"] = self._debug_server.get_debug_port()
+
         self.generate_pioinit(self._gdbsrc_dir, patterns)
 
         # start GDB client
@@ -100,9 +101,10 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
             args.extend(["--data-directory", gdb_data_dir])
         args.append(patterns["PROG_PATH"])
 
-        return reactor.spawnProcess(
+        transport = reactor.spawnProcess(
             self, gdb_path, args, path=self.project_dir, env=os.environ
         )
+        defer.returnValue(transport)
 
     @staticmethod
     def _get_data_dir(gdb_path):
@@ -112,22 +114,8 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         return gdb_data_dir if isdir(gdb_data_dir) else None
 
     def generate_pioinit(self, dst_dir, patterns):
-        server_exe = (
-            (self.debug_options.get("server") or {}).get("executable", "").lower()
-        )
-        if "jlink" in server_exe:
-            cfg = initcfgs.GDB_JLINK_INIT_CONFIG
-        elif "st-util" in server_exe:
-            cfg = initcfgs.GDB_STUTIL_INIT_CONFIG
-        elif "mspdebug" in server_exe:
-            cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG
-        elif "qemu" in server_exe:
-            cfg = initcfgs.GDB_QEMU_INIT_CONFIG
-        elif self.debug_options["require_debug_port"]:
-            cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG
-        else:
-            cfg = initcfgs.GDB_DEFAULT_INIT_CONFIG
-        commands = cfg.split("\n")
+        # default GDB init commands depending on debug tool
+        commands = get_gdb_init_config(self.debug_options).split("\n")
 
         if self.debug_options["init_cmds"]:
             commands = self.debug_options["init_cmds"]
@@ -175,12 +163,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         stdio.StandardIO(p)
 
     def onStdInData(self, data):
-        if LOG_FILE:
-            with open(LOG_FILE, "ab") as fp:
-                fp.write(data)
-
-        self._last_server_activity = time.time()
-
+        super(GDBClient, self).onStdInData(data)
         if b"-exec-run" in data:
             if self._target_is_run:
                 token, _ = data.split(b"-", 1)
@@ -206,17 +189,12 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         reactor.stop()
 
     def outReceived(self, data):
-        if LOG_FILE:
-            with open(LOG_FILE, "ab") as fp:
-                fp.write(data)
-
-        self._last_server_activity = time.time()
         super(GDBClient, self).outReceived(data)
         self._handle_error(data)
         # go to init break automatically
         if self.INIT_COMPLETED_BANNER.encode() in data:
             telemetry.send_event(
-                "Debug", "Started", telemetry.encode_run_environment(self.env_options)
+                "Debug", "Started", telemetry.dump_run_environment(self.env_options)
             )
             self._auto_continue_timer = task.LoopingCall(self._auto_exec_continue)
             self._auto_continue_timer.start(0.1)
@@ -232,7 +210,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
 
     def _auto_exec_continue(self):
         auto_exec_delay = 0.5  # in seconds
-        if self._last_server_activity > (time.time() - auto_exec_delay):
+        if self._last_activity > (time.time() - auto_exec_delay):
             return
         if self._auto_continue_timer:
             self._auto_continue_timer.stop()
@@ -253,8 +231,11 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         self._target_is_run = True
 
     def _handle_error(self, data):
-        self._errors_buffer += data
-        if self.PIO_SRC_NAME.encode() not in data or b"Error in sourced" not in data:
+        self._errors_buffer = (self._errors_buffer + data)[-8192:]  # keep last 8 KBytes
+        if not (
+            self.PIO_SRC_NAME.encode() in self._errors_buffer
+            and b"Error in sourced" in self._errors_buffer
+        ):
             return
 
         last_erros = self._errors_buffer.decode()
@@ -262,7 +243,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M)
 
         err = "%s -> %s" % (
-            telemetry.encode_run_environment(self.env_options),
+            telemetry.dump_run_environment(self.env_options),
             last_erros,
         )
         telemetry.send_exception("DebugInitError: %s" % err)

platformio/commands/debug/process/server.py

@@ -13,35 +13,40 @@
 # limitations under the License.
 
 import os
+import time
 from os.path import isdir, isfile, join
 
+from twisted.internet import defer  # pylint: disable=import-error
 from twisted.internet import reactor  # pylint: disable=import-error
 
 from platformio import fs, util
 from platformio.commands.debug.exception import DebugInvalidOptionsError
 from platformio.commands.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode
-from platformio.commands.debug.process import BaseProcess
+from platformio.commands.debug.process.base import BaseProcess
 from platformio.proc import where_is_program
 
 
 class DebugServer(BaseProcess):
     def __init__(self, debug_options, env_options):
+        super(DebugServer, self).__init__()
         self.debug_options = debug_options
         self.env_options = env_options
 
-        self._debug_port = None
+        self._debug_port = ":3333"
         self._transport = None
         self._process_ended = False
+        self._ready = False
 
+    @defer.inlineCallbacks
     def spawn(self, patterns):  # pylint: disable=too-many-branches
         systype = util.get_systype()
         server = self.debug_options.get("server")
         if not server:
-            return None
+            defer.returnValue(None)
         server = self.apply_patterns(server, patterns)
         server_executable = server["executable"]
         if not server_executable:
-            return None
+            defer.returnValue(None)
         if server["cwd"]:
             server_executable = join(server["cwd"], server_executable)
         if (
@@ -62,7 +67,6 @@ class DebugServer(BaseProcess):
                 % server_executable
             )
 
-        self._debug_port = ":3333"
         openocd_pipe_allowed = all(
             [not self.debug_options["port"], "openocd" in server_executable]
         )
@@ -79,43 +83,62 @@ class DebugServer(BaseProcess):
             )
             self._debug_port = '| "%s" %s' % (server_executable, str_args)
             self._debug_port = fs.to_unix_path(self._debug_port)
-        else:
-            env = os.environ.copy()
-            # prepend server "lib" folder to LD path
-            if (
-                "windows" not in systype
-                and server["cwd"]
-                and isdir(join(server["cwd"], "lib"))
-            ):
-                ld_key = (
-                    "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH"
-                )
-                env[ld_key] = join(server["cwd"], "lib")
-                if os.environ.get(ld_key):
-                    env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key))
-            # prepend BIN to PATH
-            if server["cwd"] and isdir(join(server["cwd"], "bin")):
-                env["PATH"] = "%s%s%s" % (
-                    join(server["cwd"], "bin"),
-                    os.pathsep,
-                    os.environ.get("PATH", os.environ.get("Path", "")),
-                )
+            defer.returnValue(self._debug_port)
 
-            self._transport = reactor.spawnProcess(
-                self,
-                server_executable,
-                [server_executable] + server["arguments"],
-                path=server["cwd"],
-                env=env,
+        env = os.environ.copy()
+        # prepend server "lib" folder to LD path
+        if (
+            "windows" not in systype
+            and server["cwd"]
+            and isdir(join(server["cwd"], "lib"))
+        ):
+            ld_key = "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH"
+            env[ld_key] = join(server["cwd"], "lib")
+            if os.environ.get(ld_key):
+                env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key))
+        # prepend BIN to PATH
+        if server["cwd"] and isdir(join(server["cwd"], "bin")):
+            env["PATH"] = "%s%s%s" % (
+                join(server["cwd"], "bin"),
+                os.pathsep,
+                os.environ.get("PATH", os.environ.get("Path", "")),
             )
-            if "mspdebug" in server_executable.lower():
-                self._debug_port = ":2000"
-            elif "jlink" in server_executable.lower():
-                self._debug_port = ":2331"
-            elif "qemu" in server_executable.lower():
-                self._debug_port = ":1234"
 
-        return self._transport
+        self._transport = reactor.spawnProcess(
+            self,
+            server_executable,
+            [server_executable] + server["arguments"],
+            path=server["cwd"],
+            env=env,
+        )
+        if "mspdebug" in server_executable.lower():
+            self._debug_port = ":2000"
+        elif "jlink" in server_executable.lower():
+            self._debug_port = ":2331"
+        elif "qemu" in server_executable.lower():
+            self._debug_port = ":1234"
+
+        yield self._wait_until_ready()
+
+        defer.returnValue(self._debug_port)
+
+    @defer.inlineCallbacks
+    def _wait_until_ready(self):
+        timeout = 10
+        elapsed = 0
+        delay = 0.5
+        auto_ready_delay = 0.5
+        while not self._ready and not self._process_ended and elapsed < timeout:
+            yield self.async_sleep(delay)
+            if not self.debug_options.get("server", {}).get("ready_pattern"):
+                self._ready = self._last_activity < (time.time() - auto_ready_delay)
+            elapsed += delay
+
+    @staticmethod
+    def async_sleep(secs):
+        d = defer.Deferred()
+        reactor.callLater(secs, d.callback, None)
+        return d
 
     def get_debug_port(self):
         return self._debug_port
@@ -124,6 +147,11 @@ class DebugServer(BaseProcess):
         super(DebugServer, self).outReceived(
             escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data
         )
+        if self._ready:
+            return
+        ready_pattern = self.debug_options.get("server", {}).get("ready_pattern")
+        if ready_pattern:
+            self._ready = ready_pattern.encode() in data
 
     def processEnded(self, reason):
         self._process_ended = True

platformio/commands/device/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.device.filters.base import DeviceMonitorFilter

platformio/commands/device/command.py

@@ -12,17 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import sys
 from fnmatch import fnmatch
-from os import getcwd
 
 import click
 from serial.tools import miniterm
 
 from platformio import exception, fs, util
+from platformio.commands.device import helpers as device_helpers
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformFactory
-from platformio.project.config import ProjectConfig
 from platformio.project.exception import NotPlatformIOProjectError
 
 
@@ -135,7 +135,7 @@ def device_list(  # pylint: disable=too-many-branches
     help="Set the encoding for the serial port (e.g. hexlify, "
     "Latin1, UTF-8), default: UTF-8",
 )
-@click.option("--filter", "-f", multiple=True, help="Add text transformation")
+@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations")
 @click.option(
     "--eol",
     default="CRLF",
@@ -165,7 +165,7 @@ def device_list(  # pylint: disable=too-many-branches
 @click.option(
     "-d",
     "--project-dir",
-    default=getcwd,
+    default=os.getcwd,
     type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
 )
 @click.option(
@@ -174,27 +174,36 @@ def device_list(  # pylint: disable=too-many-branches
     help="Load configuration from `platformio.ini` and specified environment",
 )
 def device_monitor(**kwargs):  # pylint: disable=too-many-branches
-    click.echo(
-        "Looking for advanced Serial Monitor with UI? "
-        "Check http://bit.ly/pio-advanced-monitor"
-    )
+    # load default monitor filters
+    filters_dir = os.path.join(fs.get_source_dir(), "commands", "device", "filters")
+    for name in os.listdir(filters_dir):
+        if not name.endswith(".py"):
+            continue
+        device_helpers.load_monitor_filter(os.path.join(filters_dir, name))
+
     project_options = {}
     try:
         with fs.cd(kwargs["project_dir"]):
-            project_options = get_project_options(kwargs["environment"])
-        kwargs = apply_project_monitor_options(kwargs, project_options)
+            project_options = device_helpers.get_project_options(kwargs["environment"])
+        kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options)
     except NotPlatformIOProjectError:
         pass
 
+    platform = None
+    if "platform" in project_options:
+        with fs.cd(kwargs["project_dir"]):
+            platform = PlatformFactory.newPlatform(project_options["platform"])
+            device_helpers.register_platform_filters(
+                platform, kwargs["project_dir"], kwargs["environment"]
+            )
+
     if not kwargs["port"]:
         ports = util.get_serial_ports(filter_hwid=True)
         if len(ports) == 1:
             kwargs["port"] = ports[0]["port"]
         elif "platform" in project_options and "board" in project_options:
-            board_hwids = get_board_hwids(
-                kwargs["project_dir"],
-                project_options["platform"],
-                project_options["board"],
+            board_hwids = device_helpers.get_board_hwids(
+                kwargs["project_dir"], platform, project_options["board"],
             )
             for item in ports:
                 for hwid in board_hwids:
@@ -211,12 +220,18 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
                 break
 
     # override system argv with patched options
-    sys.argv = ["monitor"] + options_to_argv(
+    sys.argv = ["monitor"] + device_helpers.options_to_argv(
         kwargs,
         project_options,
         ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"),
     )
 
+    if not kwargs["quiet"]:
+        click.echo(
+            "--- Available filters and text transformations: %s"
+            % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys()))
+        )
+        click.echo("--- More details at http://bit.ly/pio-monitor-filters")
     try:
         miniterm.main(
             default_port=kwargs["port"],
@@ -226,55 +241,3 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
         )
     except Exception as e:
         raise exception.MinitermException(e)
-
-
-def apply_project_monitor_options(cli_options, project_options):
-    for k in ("port", "speed", "rts", "dtr"):
-        k2 = "monitor_%s" % k
-        if k == "speed":
-            k = "baud"
-        if cli_options[k] is None and k2 in project_options:
-            cli_options[k] = project_options[k2]
-            if k != "port":
-                cli_options[k] = int(cli_options[k])
-    return cli_options
-
-
-def options_to_argv(cli_options, project_options, ignore=None):
-    result = project_options.get("monitor_flags", [])
-    for k, v in cli_options.items():
-        if v is None or (ignore and k in ignore):
-            continue
-        k = "--" + k.replace("_", "-")
-        if k in project_options.get("monitor_flags", []):
-            continue
-        if isinstance(v, bool):
-            if v:
-                result.append(k)
-        elif isinstance(v, tuple):
-            for i in v:
-                result.extend([k, i])
-        else:
-            result.extend([k, str(v)])
-    return result
-
-
-def get_project_options(environment=None):
-    config = ProjectConfig.get_instance()
-    config.validate(envs=[environment] if environment else None)
-    if not environment:
-        default_envs = config.default_envs()
-        if default_envs:
-            environment = default_envs[0]
-        else:
-            environment = config.envs()[0]
-    return config.items(env=environment, as_dict=True)
-
-
-def get_board_hwids(project_dir, platform, board):
-    with fs.cd(project_dir):
-        return (
-            PlatformFactory.newPlatform(platform)
-            .board_config(board)
-            .get("build.hwids", [])
-        )

platformio/commands/device/filters/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/device/filters/base.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from serial.tools import miniterm
+
+from platformio.project.config import ProjectConfig
+
+
+class DeviceMonitorFilter(miniterm.Transform):
+    def __init__(self, project_dir=None, environment=None):
+        """ Called by PlatformIO to pass context """
+        miniterm.Transform.__init__(self)
+
+        self.project_dir = project_dir
+        self.environment = environment
+
+        self.config = ProjectConfig.get_instance()
+        if not self.environment:
+            default_envs = self.config.default_envs()
+            if default_envs:
+                self.environment = default_envs[0]
+            elif self.config.envs():
+                self.environment = self.config.envs()[0]
+
+    def __call__(self):
+        """ Called by the miniterm library when the filter is actually used """
+        return self
+
+    @property
+    def NAME(self):
+        raise NotImplementedError("Please declare NAME attribute for the filter class")

platformio/commands/device/filters/hexlify.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import serial
+
+from platformio.commands.device import DeviceMonitorFilter
+
+
+class Hexlify(DeviceMonitorFilter):
+    NAME = "hexlify"
+
+    def __init__(self, *args, **kwargs):
+        super(Hexlify, self).__init__(*args, **kwargs)
+        self._counter = 0
+
+    def rx(self, text):
+        result = ""
+        for b in serial.iterbytes(text):
+            if (self._counter % 16) == 0:
+                result += "\n{:04X} | ".format(self._counter)
+            asciicode = ord(b)
+            if asciicode <= 255:
+                result += "{:02X} ".format(asciicode)
+            else:
+                result += "?? "
+            self._counter += 1
+        return result

platformio/commands/device/filters/log2file.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import os.path
+from datetime import datetime
+
+from platformio.commands.device import DeviceMonitorFilter
+
+
+class LogToFile(DeviceMonitorFilter):
+    NAME = "log2file"
+
+    def __init__(self, *args, **kwargs):
+        super(LogToFile, self).__init__(*args, **kwargs)
+        self._log_fp = None
+
+    def __call__(self):
+        log_file_name = "platformio-device-monitor-%s.log" % datetime.now().strftime(
+            "%y%m%d-%H%M%S"
+        )
+        print("--- Logging an output to %s" % os.path.abspath(log_file_name))
+        self._log_fp = io.open(log_file_name, "w", encoding="utf-8")
+        return self
+
+    def __del__(self):
+        if self._log_fp:
+            self._log_fp.close()
+
+    def rx(self, text):
+        self._log_fp.write(text)
+        self._log_fp.flush()
+        return text

platformio/commands/device/filters/send_on_enter.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.device import DeviceMonitorFilter
+
+
+class SendOnEnter(DeviceMonitorFilter):
+    NAME = "send_on_enter"
+
+    def __init__(self, *args, **kwargs):
+        super(SendOnEnter, self).__init__(*args, **kwargs)
+        self._buffer = ""
+
+    def tx(self, text):
+        self._buffer += text
+        if self._buffer.endswith("\r\n"):
+            text = self._buffer[:-2]
+            self._buffer = ""
+            return text
+        return ""

platformio/commands/device/filters/time.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+
+from platformio.commands.device import DeviceMonitorFilter
+
+
+class Timestamp(DeviceMonitorFilter):
+    NAME = "time"
+
+    def __init__(self, *args, **kwargs):
+        super(Timestamp, self).__init__(*args, **kwargs)
+        self._first_text_received = False
+
+    def rx(self, text):
+        if self._first_text_received and "\n" not in text:
+            return text
+        timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
+        if not self._first_text_received:
+            self._first_text_received = True
+            return "%s > %s" % (timestamp, text)
+        return text.replace("\n", "\n%s > " % timestamp)

platformio/commands/device/helpers.py

@@ -0,0 +1,106 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import os
+
+from serial.tools import miniterm
+
+from platformio import fs
+from platformio.commands.device import DeviceMonitorFilter
+from platformio.compat import get_object_members, load_python_module
+from platformio.project.config import ProjectConfig
+
+
+def apply_project_monitor_options(cli_options, project_options):
+    for k in ("port", "speed", "rts", "dtr"):
+        k2 = "monitor_%s" % k
+        if k == "speed":
+            k = "baud"
+        if cli_options[k] is None and k2 in project_options:
+            cli_options[k] = project_options[k2]
+            if k != "port":
+                cli_options[k] = int(cli_options[k])
+    return cli_options
+
+
+def options_to_argv(cli_options, project_options, ignore=None):
+    confmon_flags = project_options.get("monitor_flags", [])
+    result = confmon_flags[::]
+
+    for f in project_options.get("monitor_filters", []):
+        result.extend(["--filter", f])
+
+    for k, v in cli_options.items():
+        if v is None or (ignore and k in ignore):
+            continue
+        k = "--" + k.replace("_", "-")
+        if k in confmon_flags:
+            continue
+        if isinstance(v, bool):
+            if v:
+                result.append(k)
+        elif isinstance(v, tuple):
+            for i in v:
+                result.extend([k, i])
+        else:
+            result.extend([k, str(v)])
+    return result
+
+
+def get_project_options(environment=None):
+    config = ProjectConfig.get_instance()
+    config.validate(envs=[environment] if environment else None)
+    if not environment:
+        default_envs = config.default_envs()
+        if default_envs:
+            environment = default_envs[0]
+        else:
+            environment = config.envs()[0]
+    return config.items(env=environment, as_dict=True)
+
+
+def get_board_hwids(project_dir, platform, board):
+    with fs.cd(project_dir):
+        return platform.board_config(board).get("build.hwids", [])
+
+
+def load_monitor_filter(path, project_dir=None, environment=None):
+    name = os.path.basename(path)
+    name = name[: name.find(".")]
+    module = load_python_module("platformio.commands.device.filters.%s" % name, path)
+    for cls in get_object_members(module).values():
+        if (
+            not inspect.isclass(cls)
+            or not issubclass(cls, DeviceMonitorFilter)
+            or cls == DeviceMonitorFilter
+        ):
+            continue
+        obj = cls(project_dir, environment)
+        miniterm.TRANSFORMATIONS[obj.NAME] = obj
+    return True
+
+
+def register_platform_filters(platform, project_dir, environment):
+    monitor_dir = os.path.join(platform.get_dir(), "monitor")
+    if not os.path.isdir(monitor_dir):
+        return
+
+    for name in os.listdir(monitor_dir):
+        if not name.startswith("filter_") or not name.endswith(".py"):
+            continue
+        path = os.path.join(monitor_dir, name)
+        if not os.path.isfile(path):
+            continue
+        load_monitor_filter(path, project_dir, environment)

platformio/commands/home/command.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=too-many-locals
+# pylint: disable=too-many-locals,too-many-statements
 
 import mimetypes
 import socket
@@ -22,11 +22,7 @@ import click
 
 from platformio import exception
 from platformio.compat import WINDOWS
-from platformio.managers.core import (
-    build_contrib_pysite_deps,
-    get_core_package_dir,
-    inject_contrib_pysite,
-)
+from platformio.managers.core import get_core_package_dir, inject_contrib_pysite
 
 
 @click.command("home", short_help="PIO Home")
@@ -55,14 +51,10 @@ def cli(port, host, no_open, shutdown_timeout):
     # import contrib modules
     inject_contrib_pysite()
 
-    try:
-        from autobahn.twisted.resource import WebSocketResource
-    except:  # pylint: disable=bare-except
-        build_contrib_pysite_deps(get_core_package_dir("contrib-pysite"))
-        from autobahn.twisted.resource import WebSocketResource
-
+    from autobahn.twisted.resource import WebSocketResource
     from twisted.internet import reactor
     from twisted.web import server
+    from twisted.internet.error import CannotListenError
 
     from platformio.commands.home.rpc.handlers.app import AppRPC
     from platformio.commands.home.rpc.handlers.ide import IDERPC
@@ -70,6 +62,7 @@ def cli(port, host, no_open, shutdown_timeout):
     from platformio.commands.home.rpc.handlers.os import OSRPC
     from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
     from platformio.commands.home.rpc.handlers.project import ProjectRPC
+    from platformio.commands.home.rpc.handlers.account import AccountRPC
     from platformio.commands.home.rpc.server import JSONRPCServerFactory
     from platformio.commands.home.web import WebRoot
 
@@ -80,6 +73,7 @@ def cli(port, host, no_open, shutdown_timeout):
     factory.addHandler(OSRPC(), namespace="os")
     factory.addHandler(PIOCoreRPC(), namespace="core")
     factory.addHandler(ProjectRPC(), namespace="project")
+    factory.addHandler(AccountRPC(), namespace="account")
 
     contrib_dir = get_core_package_dir("contrib-piohome")
     if not isdir(contrib_dir):
@@ -121,6 +115,12 @@ def cli(port, host, no_open, shutdown_timeout):
     click.echo("")
     click.echo("Open PlatformIO Home in your browser by this URL => %s" % home_url)
 
+    try:
+        reactor.listenTCP(port, site, interface=host)
+    except CannotListenError as e:
+        click.secho(str(e), fg="red", err=True)
+        already_started = True
+
     if already_started:
         click.secho(
             "PlatformIO Home server is already started in another process.", fg="yellow"
@@ -129,7 +129,6 @@ def cli(port, host, no_open, shutdown_timeout):
 
     click.echo("PIO Home has been started. Press Ctrl+C to shutdown.")
 
-    reactor.listenTCP(port, site, interface=host)
     reactor.run()
 
 

platformio/commands/home/helpers.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=keyword-arg-before-vararg, arguments-differ
+# pylint: disable=keyword-arg-before-vararg,arguments-differ,signature-differs
 
 import os
 import socket

platformio/commands/home/rpc/handlers/account.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import jsonrpc  # pylint: disable=import-error
+
+from platformio.commands.account.client import AccountClient
+
+
+class AccountRPC(object):
+    @staticmethod
+    def call_client(method, *args, **kwargs):
+        try:
+            client = AccountClient()
+            return getattr(client, method)(*args, **kwargs)
+        except Exception as e:  # pylint: disable=bare-except
+            raise jsonrpc.exceptions.JSONRPCDispatchException(
+                code=4003, message="PIO Account Call Error", data=str(e)
+            )

platformio/commands/home/rpc/handlers/os.py

@@ -15,6 +15,7 @@
 from __future__ import absolute_import
 
 import glob
+import io
 import os
 import shutil
 from functools import cmp_to_key
@@ -66,7 +67,8 @@ class OSRPC(object):
         if uri.startswith("http"):
             return self.fetch_content(uri, data, headers, cache_valid)
         if os.path.isfile(uri):
-            return fs.get_file_contents(uri, encoding="utf8")
+            with io.open(uri, encoding="utf-8") as fp:
+                return fp.read()
         return None
 
     @staticmethod
@@ -105,7 +107,7 @@ class OSRPC(object):
 
     @staticmethod
     def copy(src, dst):
-        return shutil.copytree(src, dst)
+        return shutil.copytree(src, dst, symlinks=True)
 
     @staticmethod
     def glob(pathnames, root=None):

platformio/commands/home/rpc/handlers/piocore.py

@@ -51,6 +51,8 @@ class MultiThreadingStdStream(object):
     def write(self, value):
         thread_id = thread_get_ident()
         self._ensure_thread_buffer(thread_id)
+        if PY2 and isinstance(value, unicode):  # pylint: disable=undefined-variable
+            value = value.encode()
         return self._buffers[thread_id].write(
             value.decode() if is_bytes(value) else value
         )
@@ -59,8 +61,8 @@ class MultiThreadingStdStream(object):
         result = ""
         try:
             result = self.getvalue()
-            self.truncate(0)
             self.seek(0)
+            self.truncate(0)
         except AttributeError:
             pass
         return result
@@ -96,7 +98,7 @@ class PIOCoreRPC(object):
         to_json = "--json-output" in args
 
         try:
-            if args and args[0] in ("account", "remote"):
+            if args and args[0] == "remote":
                 result = yield PIOCoreRPC._call_subprocess(args, options)
                 defer.returnValue(PIOCoreRPC._process_result(result, to_json))
             else:
@@ -146,6 +148,8 @@ class PIOCoreRPC(object):
             raise Exception(text)
         if not to_json:
             return text
+        if is_bytes(out):
+            out = out.decode()
         try:
             return json.loads(out)
         except ValueError as e:

platformio/commands/home/rpc/handlers/project.py

@@ -244,7 +244,8 @@ class ProjectRPC(object):
                 return project_dir
             if not os.path.isdir(src_dir):
                 os.makedirs(src_dir)
-            fs.write_file_contents(main_path, main_content.strip())
+            with open(main_path, "w") as fp:
+                fp.write(main_content.strip())
         return project_dir
 
     def import_arduino(self, board, use_arduino_libs, arduino_project_dir):
@@ -299,7 +300,7 @@ class ProjectRPC(object):
             src_dir = config.get_optional_dir("src")
             if os.path.isdir(src_dir):
                 fs.rmtree(src_dir)
-            shutil.copytree(arduino_project_dir, src_dir)
+            shutil.copytree(arduino_project_dir, src_dir, symlinks=True)
         return project_dir
 
     @staticmethod
@@ -312,7 +313,7 @@ class ProjectRPC(object):
             AppRPC.load_state()["storage"]["projectsDir"],
             time.strftime("%y%m%d-%H%M%S-") + os.path.basename(project_dir),
         )
-        shutil.copytree(project_dir, new_project_dir)
+        shutil.copytree(project_dir, new_project_dir, symlinks=True)
 
         state = AppRPC.load_state()
         args = ["init"]

platformio/commands/home/web.py

@@ -18,7 +18,7 @@ from twisted.web import static  # pylint: disable=import-error
 
 class WebRoot(static.File):
     def render_GET(self, request):
-        if request.args.get("__shutdown__", False):
+        if request.args.get(b"__shutdown__", False):
             reactor.stop()
             return "Server has been stopped"
 

platformio/commands/lib.py

@@ -21,7 +21,7 @@ import click
 import semantic_version
 from tabulate import tabulate
 
-from platformio import exception, util
+from platformio import exception, fs, util
 from platformio.commands import PlatformioCLI
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
@@ -29,6 +29,7 @@ from platformio.package.manifest.parser import ManifestParserFactory
 from platformio.package.manifest.schema import ManifestSchema
 from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
+from platformio.project.exception import InvalidProjectConfError
 from platformio.project.helpers import get_project_dir, is_platformio_project
 
 try:
@@ -106,17 +107,20 @@ def cli(ctx, **options):
         if not is_platformio_project(storage_dir):
             ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
             continue
-        config = ProjectConfig.get_instance(os.path.join(storage_dir, "platformio.ini"))
-        config.validate(options["environment"], silent=in_silence)
-        libdeps_dir = config.get_optional_dir("libdeps")
-        for env in config.envs():
-            if options["environment"] and env not in options["environment"]:
-                continue
-            storage_dir = os.path.join(libdeps_dir, env)
-            ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
-            ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get(
-                "env:" + env, "lib_deps", []
+        with fs.cd(storage_dir):
+            config = ProjectConfig.get_instance(
+                os.path.join(storage_dir, "platformio.ini")
             )
+            config.validate(options["environment"], silent=in_silence)
+            libdeps_dir = config.get_optional_dir("libdeps")
+            for env in config.envs():
+                if options["environment"] and env not in options["environment"]:
+                    continue
+                storage_dir = os.path.join(libdeps_dir, env)
+                ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
+                ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get(
+                    "env:" + env, "lib_deps", []
+                )
 
 
 @cli.command("install", short_help="Install library")
@@ -177,7 +181,10 @@ def lib_install(  # pylint: disable=too-many-arguments
             if project_environments and env not in project_environments:
                 continue
             config.expand_interpolations = False
-            lib_deps = config.get("env:" + env, "lib_deps", [])
+            try:
+                lib_deps = config.get("env:" + env, "lib_deps")
+            except InvalidProjectConfError:
+                lib_deps = []
             for library in libraries:
                 if library in lib_deps:
                     continue

platformio/commands/project.py

@@ -187,9 +187,9 @@ def init_base_project(project_dir):
 
 
 def init_include_readme(include_dir):
-    fs.write_file_contents(
-        os.path.join(include_dir, "README"),
-        """
+    with open(os.path.join(include_dir, "README"), "w") as fp:
+        fp.write(
+            """
 This directory is intended for project header files.
 
 A header file is a file containing C declarations and macro definitions
@@ -229,14 +229,14 @@ Read more about using header files in official GCC documentation:
 
 https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html
 """,
-    )
+        )
 
 
 def init_lib_readme(lib_dir):
     # pylint: disable=line-too-long
-    fs.write_file_contents(
-        os.path.join(lib_dir, "README"),
-        """
+    with open(os.path.join(lib_dir, "README"), "w") as fp:
+        fp.write(
+            """
 This directory is intended for project specific (private) libraries.
 PlatformIO will compile them to static libraries and link into executable file.
 
@@ -283,13 +283,13 @@ libraries scanning project source files.
 More information about PlatformIO Library Dependency Finder
 - https://docs.platformio.org/page/librarymanager/ldf.html
 """,
-    )
+        )
 
 
 def init_test_readme(test_dir):
-    fs.write_file_contents(
-        os.path.join(test_dir, "README"),
-        """
+    with open(os.path.join(test_dir, "README"), "w") as fp:
+        fp.write(
+            """
 This directory is intended for PIO Unit Testing and project tests.
 
 Unit Testing is a software testing method by which individual units of
@@ -301,16 +301,16 @@ in the development cycle.
 More information about PIO Unit Testing:
 - https://docs.platformio.org/page/plus/unit-testing.html
 """,
-    )
+        )
 
 
 def init_ci_conf(project_dir):
     conf_path = os.path.join(project_dir, ".travis.yml")
     if os.path.isfile(conf_path):
         return
-    fs.write_file_contents(
-        conf_path,
-        """# Continuous Integration (CI) is the practice, in software
+    with open(conf_path, "w") as fp:
+        fp.write(
+            """# Continuous Integration (CI) is the practice, in software
 # engineering, of merging all developer working copies with a shared mainline
 # several times a day < https://docs.platformio.org/page/ci/index.html >
 #
@@ -378,14 +378,15 @@ def init_ci_conf(project_dir):
 # script:
 #     - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N
 """,
-    )
+        )
 
 
 def init_cvs_ignore(project_dir):
     conf_path = os.path.join(project_dir, ".gitignore")
     if os.path.isfile(conf_path):
         return
-    fs.write_file_contents(conf_path, ".pio\n")
+    with open(conf_path, "w") as fp:
+        fp.write(".pio\n")
 
 
 def fill_project_envs(

platformio/commands/remote/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/remote/ac/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/remote/ac/base.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer  # pylint: disable=import-error
+from twisted.spread import pb  # pylint: disable=import-error
+
+
+class AsyncCommandBase(object):
+
+    MAX_BUFFER_SIZE = 1024 * 1024  # 1Mb
+
+    def __init__(self, options=None, on_end_callback=None):
+        self.options = options or {}
+        self.on_end_callback = on_end_callback
+        self._buffer = b""
+        self._return_code = None
+        self._d = None
+        self._paused = False
+
+        try:
+            self.start()
+        except Exception as e:
+            raise pb.Error(str(e))
+
+    @property
+    def id(self):
+        return id(self)
+
+    def pause(self):
+        self._paused = True
+        self.stop()
+
+    def unpause(self):
+        self._paused = False
+        self.start()
+
+    def start(self):
+        raise NotImplementedError
+
+    def stop(self):
+        self.transport.loseConnection()  # pylint: disable=no-member
+
+    def _ac_ended(self):
+        if self.on_end_callback:
+            self.on_end_callback()
+        if not self._d or self._d.called:
+            self._d = None
+            return
+        if self._buffer:
+            self._d.callback(self._buffer)
+        else:
+            self._d.callback(None)
+
+    def _ac_ondata(self, data):
+        self._buffer += data
+        if len(self._buffer) > self.MAX_BUFFER_SIZE:
+            self._buffer = self._buffer[-1 * self.MAX_BUFFER_SIZE :]
+        if self._paused:
+            return
+        if self._d and not self._d.called:
+            self._d.callback(self._buffer)
+            self._buffer = b""
+
+    def ac_read(self):
+        if self._buffer:
+            result = self._buffer
+            self._buffer = b""
+            return result
+        if self._return_code is None:
+            self._d = defer.Deferred()
+            return self._d
+        return None
+
+    def ac_write(self, data):
+        self.transport.write(data)  # pylint: disable=no-member
+        return len(data)
+
+    def ac_close(self):
+        self.stop()
+        return self._return_code

platformio/commands/remote/ac/process.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from twisted.internet import protocol, reactor  # pylint: disable=import-error
+
+from platformio.commands.remote.ac.base import AsyncCommandBase
+
+
+class ProcessAsyncCmd(protocol.ProcessProtocol, AsyncCommandBase):
+    def start(self):
+        env = dict(os.environ).copy()
+        env.update({"PLATFORMIO_FORCE_ANSI": "true"})
+        reactor.spawnProcess(
+            self, self.options["executable"], self.options["args"], env
+        )
+
+    def outReceived(self, data):
+        self._ac_ondata(data)
+
+    def errReceived(self, data):
+        self._ac_ondata(data)
+
+    def processExited(self, reason):
+        self._return_code = reason.value.exitCode
+
+    def processEnded(self, reason):
+        if self._return_code is None:
+            self._return_code = reason.value.exitCode
+        self._ac_ended()

platformio/commands/remote/ac/psync.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import zlib
+from io import BytesIO
+
+from platformio.commands.remote.ac.base import AsyncCommandBase
+from platformio.commands.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync
+
+
+class ProjectSyncAsyncCmd(AsyncCommandBase):
+    def __init__(self, *args, **kwargs):
+        self.psync = None
+        self._upstream = None
+        super(ProjectSyncAsyncCmd, self).__init__(*args, **kwargs)
+
+    def start(self):
+        project_dir = os.path.join(
+            self.options["agent_working_dir"], "projects", self.options["id"]
+        )
+        self.psync = ProjectSync(project_dir)
+        for name in self.options["items"]:
+            self.psync.add_item(os.path.join(project_dir, name), name)
+
+    def stop(self):
+        self.psync = None
+        self._upstream = None
+        self._return_code = PROJECT_SYNC_STAGE.COMPLETED.value
+
+    def ac_write(self, data):
+        stage = PROJECT_SYNC_STAGE.lookupByValue(data.get("stage"))
+
+        if stage is PROJECT_SYNC_STAGE.DBINDEX:
+            self.psync.rebuild_dbindex()
+            return zlib.compress(json.dumps(self.psync.get_dbindex()).encode())
+
+        if stage is PROJECT_SYNC_STAGE.DELETE:
+            return self.psync.delete_dbindex(
+                json.loads(zlib.decompress(data["dbindex"]))
+            )
+
+        if stage is PROJECT_SYNC_STAGE.UPLOAD:
+            if not self._upstream:
+                self._upstream = BytesIO()
+            self._upstream.write(data["chunk"])
+            if self._upstream.tell() == data["total"]:
+                self.psync.decompress_items(self._upstream)
+                self._upstream = None
+                return PROJECT_SYNC_STAGE.EXTRACTED.value
+
+            return PROJECT_SYNC_STAGE.UPLOAD.value
+
+        return None

platformio/commands/remote/ac/serial.py

@@ -0,0 +1,60 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from time import sleep
+
+from twisted.internet import protocol, reactor  # pylint: disable=import-error
+from twisted.internet.serialport import SerialPort  # pylint: disable=import-error
+
+from platformio.commands.remote.ac.base import AsyncCommandBase
+
+
+class SerialPortAsyncCmd(protocol.Protocol, AsyncCommandBase):
+    def start(self):
+        SerialPort(
+            self,
+            reactor=reactor,
+            **{
+                "deviceNameOrPortNumber": self.options["port"],
+                "baudrate": self.options["baud"],
+                "parity": self.options["parity"],
+                "rtscts": 1 if self.options["rtscts"] else 0,
+                "xonxoff": 1 if self.options["xonxoff"] else 0,
+            }
+        )
+
+    def connectionMade(self):
+        self.reset_device()
+        if self.options.get("rts", None) is not None:
+            self.transport.setRTS(self.options.get("rts"))
+        if self.options.get("dtr", None) is not None:
+            self.transport.setDTR(self.options.get("dtr"))
+
+    def reset_device(self):
+        self.transport.flushInput()
+        self.transport.setDTR(False)
+        self.transport.setRTS(False)
+        sleep(0.1)
+        self.transport.setDTR(True)
+        self.transport.setRTS(True)
+        sleep(0.1)
+
+    def dataReceived(self, data):
+        self._ac_ondata(data)
+
+    def connectionLost(self, reason):  # pylint: disable=unused-argument
+        if self._paused:
+            return
+        self._return_code = 0
+        self._ac_ended()

platformio/commands/remote/client/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/remote/client/agent_list.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+
+import click
+
+from platformio.commands.remote.client.base import RemoteClientBase
+
+
+class AgentListClient(RemoteClientBase):
+    def agent_pool_ready(self):
+        d = self.agentpool.callRemote("list", True)
+        d.addCallback(self._cbResult)
+        d.addErrback(self.cb_global_error)
+
+    def _cbResult(self, result):
+        for item in result:
+            click.secho(item["name"], fg="cyan")
+            click.echo("-" * len(item["name"]))
+            click.echo("ID: %s" % item["id"])
+            click.echo(
+                "Started: %s"
+                % datetime.fromtimestamp(item["started"]).strftime("%Y-%m-%d %H:%M:%S")
+            )
+            click.echo("")
+        self.disconnect()

platformio/commands/remote/client/agent_service.py

@@ -0,0 +1,222 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from os.path import getatime, getmtime, isdir, isfile, join
+
+from twisted.logger import LogLevel  # pylint: disable=import-error
+from twisted.spread import pb  # pylint: disable=import-error
+
+from platformio import proc, util
+from platformio.commands.remote.ac.process import ProcessAsyncCmd
+from platformio.commands.remote.ac.psync import ProjectSyncAsyncCmd
+from platformio.commands.remote.ac.serial import SerialPortAsyncCmd
+from platformio.commands.remote.client.base import RemoteClientBase
+from platformio.project.config import ProjectConfig
+from platformio.project.exception import NotPlatformIOProjectError
+from platformio.project.helpers import get_project_core_dir
+
+
+class RemoteAgentService(RemoteClientBase):
+    def __init__(self, name, share, working_dir=None):
+        RemoteClientBase.__init__(self)
+        self.log_level = LogLevel.info
+        self.working_dir = working_dir or join(get_project_core_dir(), "remote")
+        if not isdir(self.working_dir):
+            os.makedirs(self.working_dir)
+        if name:
+            self.name = str(name)[:50]
+        self.join_options.update(
+            {"agent": True, "share": [s.lower().strip()[:50] for s in share]}
+        )
+
+        self._acs = {}
+
+    def agent_pool_ready(self):
+        pass
+
+    def cb_disconnected(self, reason):
+        for ac in self._acs.values():
+            ac.ac_close()
+        RemoteClientBase.cb_disconnected(self, reason)
+
+    def remote_acread(self, ac_id):
+        self.log.debug("Async Read: {id}", id=ac_id)
+        if ac_id not in self._acs:
+            raise pb.Error("Invalid Async Identifier")
+        return self._acs[ac_id].ac_read()
+
+    def remote_acwrite(self, ac_id, data):
+        self.log.debug("Async Write: {id}", id=ac_id)
+        if ac_id not in self._acs:
+            raise pb.Error("Invalid Async Identifier")
+        return self._acs[ac_id].ac_write(data)
+
+    def remote_acclose(self, ac_id):
+        self.log.debug("Async Close: {id}", id=ac_id)
+        if ac_id not in self._acs:
+            raise pb.Error("Invalid Async Identifier")
+        return_code = self._acs[ac_id].ac_close()
+        del self._acs[ac_id]
+        return return_code
+
+    def remote_cmd(self, cmd, options):
+        self.log.info("Remote command received: {cmd}", cmd=cmd)
+        self.log.debug("Command options: {options!r}", options=options)
+        callback = "_process_cmd_%s" % cmd.replace(".", "_")
+        return getattr(self, callback)(options)
+
+    def _defer_async_cmd(self, ac, pass_agent_name=True):
+        self._acs[ac.id] = ac
+        if pass_agent_name:
+            return (self.id, ac.id, self.name)
+        return (self.id, ac.id)
+
+    def _process_cmd_device_list(self, _):
+        return (self.name, util.get_serialports())
+
+    def _process_cmd_device_monitor(self, options):
+        if not options["port"]:
+            for item in util.get_serialports():
+                if "VID:PID" in item["hwid"]:
+                    options["port"] = item["port"]
+                    break
+
+        # terminate opened monitors
+        if options["port"]:
+            for ac in list(self._acs.values()):
+                if (
+                    isinstance(ac, SerialPortAsyncCmd)
+                    and ac.options["port"] == options["port"]
+                ):
+                    self.log.info(
+                        "Terminate previously opened monitor at {port}",
+                        port=options["port"],
+                    )
+                    ac.ac_close()
+                    del self._acs[ac.id]
+
+        if not options["port"]:
+            raise pb.Error("Please specify serial port using `--port` option")
+        self.log.info("Starting serial monitor at {port}", port=options["port"])
+
+        return self._defer_async_cmd(SerialPortAsyncCmd(options), pass_agent_name=False)
+
+    def _process_cmd_psync(self, options):
+        for ac in list(self._acs.values()):
+            if (
+                isinstance(ac, ProjectSyncAsyncCmd)
+                and ac.options["id"] == options["id"]
+            ):
+                self.log.info("Terminate previous Project Sync process")
+                ac.ac_close()
+                del self._acs[ac.id]
+
+        options["agent_working_dir"] = self.working_dir
+        return self._defer_async_cmd(
+            ProjectSyncAsyncCmd(options), pass_agent_name=False
+        )
+
+    def _process_cmd_run(self, options):
+        return self._process_cmd_run_or_test("run", options)
+
+    def _process_cmd_test(self, options):
+        return self._process_cmd_run_or_test("test", options)
+
+    def _process_cmd_run_or_test(  # pylint: disable=too-many-locals,too-many-branches
+        self, command, options
+    ):
+        assert options and "project_id" in options
+        project_dir = join(self.working_dir, "projects", options["project_id"])
+        origin_pio_ini = join(project_dir, "platformio.ini")
+        back_pio_ini = join(project_dir, "platformio.ini.bak")
+
+        # remove insecure project options
+        try:
+            conf = ProjectConfig(origin_pio_ini)
+            if isfile(back_pio_ini):
+                os.remove(back_pio_ini)
+            os.rename(origin_pio_ini, back_pio_ini)
+            # cleanup
+            if conf.has_section("platformio"):
+                for opt in conf.options("platformio"):
+                    if opt.endswith("_dir"):
+                        conf.remove_option("platformio", opt)
+            else:
+                conf.add_section("platformio")
+            conf.set("platformio", "build_dir", ".pio/build")
+            conf.save(origin_pio_ini)
+
+            # restore A/M times
+            os.utime(origin_pio_ini, (getatime(back_pio_ini), getmtime(back_pio_ini)))
+        except NotPlatformIOProjectError as e:
+            raise pb.Error(str(e))
+
+        cmd_args = ["platformio", "--force", command, "-d", project_dir]
+        for env in options.get("environment", []):
+            cmd_args.extend(["-e", env])
+        for target in options.get("target", []):
+            cmd_args.extend(["-t", target])
+        for ignore in options.get("ignore", []):
+            cmd_args.extend(["-i", ignore])
+        if options.get("upload_port", False):
+            cmd_args.extend(["--upload-port", options.get("upload_port")])
+        if options.get("test_port", False):
+            cmd_args.extend(["--test-port", options.get("test_port")])
+        if options.get("disable_auto_clean", False):
+            cmd_args.append("--disable-auto-clean")
+        if options.get("without_building", False):
+            cmd_args.append("--without-building")
+        if options.get("without_uploading", False):
+            cmd_args.append("--without-uploading")
+        if options.get("silent", False):
+            cmd_args.append("-s")
+        if options.get("verbose", False):
+            cmd_args.append("-v")
+
+        paused_acs = []
+        for ac in self._acs.values():
+            if not isinstance(ac, SerialPortAsyncCmd):
+                continue
+            self.log.info("Pause active monitor at {port}", port=ac.options["port"])
+            ac.pause()
+            paused_acs.append(ac)
+
+        def _cb_on_end():
+            if isfile(back_pio_ini):
+                if isfile(origin_pio_ini):
+                    os.remove(origin_pio_ini)
+                os.rename(back_pio_ini, origin_pio_ini)
+            for ac in paused_acs:
+                ac.unpause()
+                self.log.info(
+                    "Unpause active monitor at {port}", port=ac.options["port"]
+                )
+
+        return self._defer_async_cmd(
+            ProcessAsyncCmd(
+                {"executable": proc.where_is_program("platformio"), "args": cmd_args},
+                on_end_callback=_cb_on_end,
+            )
+        )
+
+    def _process_cmd_update(self, options):
+        cmd_args = ["platformio", "--force", "update"]
+        if options.get("only_check"):
+            cmd_args.append("--only-check")
+        return self._defer_async_cmd(
+            ProcessAsyncCmd(
+                {"executable": proc.where_is_program("platformio"), "args": cmd_args}
+            )
+        )

platformio/commands/remote/client/async_base.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+from twisted.spread import pb  # pylint: disable=import-error
+
+from platformio.commands.remote.client.base import RemoteClientBase
+
+
+class AsyncClientBase(RemoteClientBase):
+    def __init__(self, command, agents, options):
+        RemoteClientBase.__init__(self)
+        self.command = command
+        self.agents = agents
+        self.options = options
+
+        self._acs_total = 0
+        self._acs_ended = 0
+
+    def agent_pool_ready(self):
+        pass
+
+    def cb_async_result(self, result):
+        if self._acs_total == 0:
+            self._acs_total = len(result)
+        for (success, value) in result:
+            if not success:
+                raise pb.Error(value)
+            self.acread_data(*value)
+
+    def acread_data(self, agent_id, ac_id, agent_name=None):
+        d = self.agentpool.callRemote("acread", agent_id, ac_id)
+        d.addCallback(self.cb_acread_result, agent_id, ac_id, agent_name)
+        d.addErrback(self.cb_global_error)
+
+    def cb_acread_result(self, result, agent_id, ac_id, agent_name):
+        if result is None:
+            self.acclose(agent_id, ac_id)
+        else:
+            if self._acs_total > 1 and agent_name:
+                click.echo("[%s] " % agent_name, nl=False)
+            click.echo(result, nl=False)
+            self.acread_data(agent_id, ac_id, agent_name)
+
+    def acclose(self, agent_id, ac_id):
+        d = self.agentpool.callRemote("acclose", agent_id, ac_id)
+        d.addCallback(self.cb_acclose_result)
+        d.addErrback(self.cb_global_error)
+
+    def cb_acclose_result(self, exit_code):
+        self._acs_ended += 1
+        if self._acs_ended != self._acs_total:
+            return
+        self.disconnect(exit_code)

platformio/commands/remote/client/base.py

@@ -0,0 +1,193 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+from time import time
+
+import click
+from twisted.internet import defer, endpoints, reactor  # pylint: disable=import-error
+from twisted.logger import ILogObserver  # pylint: disable=import-error
+from twisted.logger import Logger  # pylint: disable=import-error
+from twisted.logger import LogLevel  # pylint: disable=import-error
+from twisted.logger import formatEvent  # pylint: disable=import-error
+from twisted.python import failure  # pylint: disable=import-error
+from twisted.spread import pb  # pylint: disable=import-error
+from zope.interface import provider  # pylint: disable=import-error
+
+from platformio import __pioremote_endpoint__, __version__, app, exception, maintenance
+from platformio.commands.remote.factory.client import RemoteClientFactory
+from platformio.commands.remote.factory.ssl import SSLContextFactory
+
+
+class RemoteClientBase(  # pylint: disable=too-many-instance-attributes
+    pb.Referenceable
+):
+
+    PING_DELAY = 60
+    PING_MAX_FAILURES = 3
+    DEBUG = False
+
+    def __init__(self):
+        self.log_level = LogLevel.warn
+        self.log = Logger(namespace="remote", observer=self._log_observer)
+        self.id = app.get_host_id()
+        self.name = app.get_host_name()
+        self.join_options = {"corever": __version__}
+        self.perspective = None
+        self.agentpool = None
+
+        self._ping_id = 0
+        self._ping_caller = None
+        self._ping_counter = 0
+        self._reactor_stopped = False
+        self._exit_code = 0
+
+    @provider(ILogObserver)
+    def _log_observer(self, event):
+        if not self.DEBUG and (
+            event["log_namespace"] != self.log.namespace
+            or self.log_level > event["log_level"]
+        ):
+            return
+        msg = formatEvent(event)
+        click.echo(
+            "%s [%s] %s"
+            % (
+                datetime.fromtimestamp(event["log_time"]).strftime("%Y-%m-%d %H:%M:%S"),
+                event["log_level"].name,
+                msg,
+            )
+        )
+
+    def connect(self):
+        self.log.info("Name: {name}", name=self.name)
+        self.log.info("Connecting to PIO Remote Cloud")
+
+        # pylint: disable=protected-access
+        proto, options = endpoints._parse(__pioremote_endpoint__)
+        proto = proto[0]
+
+        factory = RemoteClientFactory()
+        factory.remote_client = self
+        factory.sslContextFactory = None
+        if proto == "ssl":
+            factory.sslContextFactory = SSLContextFactory(options["host"])
+            reactor.connectSSL(
+                options["host"],
+                int(options["port"]),
+                factory,
+                factory.sslContextFactory,
+            )
+        elif proto == "tcp":
+            reactor.connectTCP(options["host"], int(options["port"]), factory)
+        else:
+            raise exception.PlatformioException("Unknown PIO Remote Cloud protocol")
+        reactor.run()
+
+        if self._exit_code != 0:
+            raise exception.ReturnErrorCode(self._exit_code)
+
+    def cb_client_authorization_failed(self, err):
+        msg = "Bad account credentials"
+        if err.check(pb.Error):
+            msg = err.getErrorMessage()
+        self.log.error(msg)
+        self.disconnect(exit_code=1)
+
+    def cb_client_authorization_made(self, perspective):
+        self.log.info("Successfully authorized")
+        self.perspective = perspective
+        d = perspective.callRemote("join", self.id, self.name, self.join_options)
+        d.addCallback(self._cb_client_join_made)
+        d.addErrback(self.cb_global_error)
+
+    def _cb_client_join_made(self, result):
+        code = result[0]
+        if code == 1:
+            self.agentpool = result[1]
+            self.agent_pool_ready()
+            self.restart_ping()
+        elif code == 2:
+            self.remote_service(*result[1:])
+
+    def remote_service(self, command, options):
+        if command == "disconnect":
+            self.log.error(
+                "PIO Remote Cloud disconnected: {msg}", msg=options.get("message")
+            )
+            self.disconnect()
+
+    def restart_ping(self, reset_counter=True):
+        # stop previous ping callers
+        self.stop_ping(reset_counter)
+        self._ping_caller = reactor.callLater(self.PING_DELAY, self._do_ping)
+
+    def _do_ping(self):
+        self._ping_counter += 1
+        self._ping_id = int(time())
+        d = self.perspective.callRemote("service", "ping", {"id": self._ping_id})
+        d.addCallback(self._cb_pong)
+        d.addErrback(self._cb_pong)
+
+    def stop_ping(self, reset_counter=True):
+        if reset_counter:
+            self._ping_counter = 0
+        if not self._ping_caller or not self._ping_caller.active():
+            return
+        self._ping_caller.cancel()
+        self._ping_caller = None
+
+    def _cb_pong(self, result):
+        if not isinstance(result, failure.Failure) and self._ping_id == result:
+            self.restart_ping()
+            return
+        if self._ping_counter >= self.PING_MAX_FAILURES:
+            self.stop_ping()
+            self.perspective.broker.transport.loseConnection()
+        else:
+            self.restart_ping(reset_counter=False)
+
+    def agent_pool_ready(self):
+        raise NotImplementedError
+
+    def disconnect(self, exit_code=None):
+        self.stop_ping()
+        if exit_code is not None:
+            self._exit_code = exit_code
+        if reactor.running and not self._reactor_stopped:
+            self._reactor_stopped = True
+            reactor.stop()
+
+    def cb_disconnected(self, _):
+        self.stop_ping()
+        self.perspective = None
+        self.agentpool = None
+
+    def cb_global_error(self, err):
+        if err.check(pb.PBConnectionLost, defer.CancelledError):
+            return
+
+        msg = err.getErrorMessage()
+        if err.check(pb.DeadReferenceError):
+            msg = "Remote Client has been terminated"
+        elif "PioAgentNotStartedError" in str(err.type):
+            msg = (
+                "Could not find active agents. Please start it before on "
+                "a remote machine using `pio remote agent start` command.\n"
+                "See http://docs.platformio.org/page/plus/pio-remote.html"
+            )
+        else:
+            maintenance.on_platformio_exception(Exception(err.type))
+        click.secho(msg, fg="red", err=True)
+        self.disconnect(exit_code=1)

platformio/commands/remote/client/device_list.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+
+from platformio.commands.remote.client.base import RemoteClientBase
+
+
+class DeviceListClient(RemoteClientBase):
+    def __init__(self, agents, json_output):
+        RemoteClientBase.__init__(self)
+        self.agents = agents
+        self.json_output = json_output
+
+    def agent_pool_ready(self):
+        d = self.agentpool.callRemote("cmd", self.agents, "device.list")
+        d.addCallback(self._cbResult)
+        d.addErrback(self.cb_global_error)
+
+    def _cbResult(self, result):
+        data = {}
+        for (success, value) in result:
+            if not success:
+                click.secho(value, fg="red", err=True)
+                continue
+            (agent_name, devlist) = value
+            data[agent_name] = devlist
+
+        if self.json_output:
+            click.echo(json.dumps(data))
+        else:
+            for agent_name, devlist in data.items():
+                click.echo("Agent %s" % click.style(agent_name, fg="cyan", bold=True))
+                click.echo("=" * (6 + len(agent_name)))
+                for item in devlist:
+                    click.secho(item["port"], fg="cyan")
+                    click.echo("-" * len(item["port"]))
+                    click.echo("Hardware ID: %s" % item["hwid"])
+                    click.echo("Description: %s" % item["description"])
+                    click.echo("")
+        self.disconnect()

platformio/commands/remote/client/device_monitor.py

@@ -0,0 +1,236 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from fnmatch import fnmatch
+
+import click
+from twisted.internet import protocol, reactor, task  # pylint: disable=import-error
+from twisted.spread import pb  # pylint: disable=import-error
+
+from platformio.commands.remote.client.base import RemoteClientBase
+
+
+class SMBridgeProtocol(protocol.Protocol):  # pylint: disable=no-init
+    def connectionMade(self):
+        self.factory.add_client(self)
+
+    def connectionLost(self, reason):  # pylint: disable=unused-argument
+        self.factory.remove_client(self)
+
+    def dataReceived(self, data):
+        self.factory.send_to_server(data)
+
+
+class SMBridgeFactory(protocol.ServerFactory):
+    def __init__(self, cdm):
+        self.cdm = cdm
+        self._clients = []
+
+    def buildProtocol(self, addr):  # pylint: disable=unused-argument
+        p = SMBridgeProtocol()
+        p.factory = self  # pylint: disable=attribute-defined-outside-init
+        return p
+
+    def add_client(self, client):
+        self.cdm.log.debug("SMBridge: Client connected")
+        self._clients.append(client)
+        self.cdm.acread_data()
+
+    def remove_client(self, client):
+        self.cdm.log.debug("SMBridge: Client disconnected")
+        self._clients.remove(client)
+        if not self._clients:
+            self.cdm.client_terminal_stopped()
+
+    def has_clients(self):
+        return len(self._clients)
+
+    def send_to_clients(self, data):
+        if not self._clients:
+            return None
+        for client in self._clients:
+            client.transport.write(data)
+        return len(data)
+
+    def send_to_server(self, data):
+        self.cdm.acwrite_data(data)
+
+
+class DeviceMonitorClient(  # pylint: disable=too-many-instance-attributes
+    RemoteClientBase
+):
+
+    MAX_BUFFER_SIZE = 1024 * 1024
+
+    def __init__(self, agents, **kwargs):
+        RemoteClientBase.__init__(self)
+        self.agents = agents
+        self.cmd_options = kwargs
+
+        self._bridge_factory = SMBridgeFactory(self)
+        self._agent_id = None
+        self._ac_id = None
+        self._d_acread = None
+        self._d_acwrite = None
+        self._acwrite_buffer = ""
+
+    def agent_pool_ready(self):
+        d = task.deferLater(
+            reactor, 1, self.agentpool.callRemote, "cmd", self.agents, "device.list"
+        )
+        d.addCallback(self._cb_device_list)
+        d.addErrback(self.cb_global_error)
+
+    def _cb_device_list(self, result):
+        devices = []
+        hwid_devindexes = []
+        for (success, value) in result:
+            if not success:
+                click.secho(value, fg="red", err=True)
+                continue
+            (agent_name, ports) = value
+            for item in ports:
+                if "VID:PID" in item["hwid"]:
+                    hwid_devindexes.append(len(devices))
+                devices.append((agent_name, item))
+
+        if len(result) == 1 and self.cmd_options["port"]:
+            if set(["*", "?", "[", "]"]) & set(self.cmd_options["port"]):
+                for agent, item in devices:
+                    if fnmatch(item["port"], self.cmd_options["port"]):
+                        return self.start_remote_monitor(agent, item["port"])
+            return self.start_remote_monitor(result[0][1][0], self.cmd_options["port"])
+
+        device = None
+        if len(hwid_devindexes) == 1:
+            device = devices[hwid_devindexes[0]]
+        else:
+            click.echo("Available ports:")
+            for i, device in enumerate(devices):
+                click.echo(
+                    "{index}. {host}{port} \t{description}".format(
+                        index=i + 1,
+                        host=device[0] + ":" if len(result) > 1 else "",
+                        port=device[1]["port"],
+                        description=device[1]["description"]
+                        if device[1]["description"] != "n/a"
+                        else "",
+                    )
+                )
+            device_index = click.prompt(
+                "Please choose a port (number in the list above)",
+                type=click.Choice([str(i + 1) for i, _ in enumerate(devices)]),
+            )
+            device = devices[int(device_index) - 1]
+
+        self.start_remote_monitor(device[0], device[1]["port"])
+
+        return None
+
+    def start_remote_monitor(self, agent, port):
+        options = {"port": port}
+        for key in ("baud", "parity", "rtscts", "xonxoff", "rts", "dtr"):
+            options[key] = self.cmd_options[key]
+
+        click.echo(
+            "Starting Serial Monitor on {host}:{port}".format(
+                host=agent, port=options["port"]
+            )
+        )
+        d = self.agentpool.callRemote("cmd", [agent], "device.monitor", options)
+        d.addCallback(self.cb_async_result)
+        d.addErrback(self.cb_global_error)
+
+    def cb_async_result(self, result):
+        if len(result) != 1:
+            raise pb.Error("Invalid response from Remote Cloud")
+        success, value = result[0]
+        if not success:
+            raise pb.Error(value)
+
+        reconnected = self._agent_id is not None
+        self._agent_id, self._ac_id = value
+
+        if reconnected:
+            self.acread_data(force=True)
+            self.acwrite_data("", force=True)
+            return
+
+        # start bridge
+        port = reactor.listenTCP(0, self._bridge_factory)
+        address = port.getHost()
+        self.log.debug("Serial Bridge is started on {address!r}", address=address)
+        if "sock" in self.cmd_options:
+            with open(os.path.join(self.cmd_options["sock"], "sock"), "w") as fp:
+                fp.write("socket://localhost:%d" % address.port)
+
+    def client_terminal_stopped(self):
+        try:
+            d = self.agentpool.callRemote("acclose", self._agent_id, self._ac_id)
+            d.addCallback(lambda r: self.disconnect())
+            d.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+    def acread_data(self, force=False):
+        if force and self._d_acread:
+            self._d_acread.cancel()
+            self._d_acread = None
+
+        if (
+            self._d_acread and not self._d_acread.called
+        ) or not self._bridge_factory.has_clients():
+            return
+
+        try:
+            self._d_acread = self.agentpool.callRemote(
+                "acread", self._agent_id, self._ac_id
+            )
+            self._d_acread.addCallback(self.cb_acread_result)
+            self._d_acread.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+    def cb_acread_result(self, result):
+        if result is None:
+            self.disconnect(exit_code=1)
+        else:
+            self._bridge_factory.send_to_clients(result)
+            self.acread_data()
+
+    def acwrite_data(self, data, force=False):
+        if force and self._d_acwrite:
+            self._d_acwrite.cancel()
+            self._d_acwrite = None
+
+        self._acwrite_buffer += data
+        if len(self._acwrite_buffer) > self.MAX_BUFFER_SIZE:
+            self._acwrite_buffer = self._acwrite_buffer[-1 * self.MAX_BUFFER_SIZE :]
+        if (self._d_acwrite and not self._d_acwrite.called) or not self._acwrite_buffer:
+            return
+
+        data = self._acwrite_buffer
+        self._acwrite_buffer = ""
+        try:
+            d = self.agentpool.callRemote("acwrite", self._agent_id, self._ac_id, data)
+            d.addCallback(self.cb_acwrite_result)
+            d.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+    def cb_acwrite_result(self, result):
+        assert result > 0
+        if self._acwrite_buffer:
+            self.acwrite_data("")

platformio/commands/remote/client/run_or_test.py

@@ -0,0 +1,272 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import json
+import os
+import zlib
+from io import BytesIO
+
+from twisted.spread import pb  # pylint: disable=import-error
+
+from platformio import util
+from platformio.commands.remote.client.async_base import AsyncClientBase
+from platformio.commands.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync
+from platformio.compat import hashlib_encode_data
+from platformio.project.config import ProjectConfig
+
+
+class RunOrTestClient(AsyncClientBase):
+
+    MAX_ARCHIVE_SIZE = 50 * 1024 * 1024  # 50Mb
+    UPLOAD_CHUNK_SIZE = 256 * 1024  # 256Kb
+
+    PSYNC_SRC_EXTS = [
+        "c",
+        "cpp",
+        "S",
+        "spp",
+        "SPP",
+        "sx",
+        "s",
+        "asm",
+        "ASM",
+        "h",
+        "hpp",
+        "ipp",
+        "ino",
+        "pde",
+        "json",
+        "properties",
+    ]
+
+    PSYNC_SKIP_DIRS = (".git", ".svn", ".hg", "example", "examples", "test", "tests")
+
+    def __init__(self, *args, **kwargs):
+        AsyncClientBase.__init__(self, *args, **kwargs)
+        self.project_id = self.generate_project_id(self.options["project_dir"])
+        self.psync = ProjectSync(self.options["project_dir"])
+
+    def generate_project_id(self, path):
+        h = hashlib.sha1(hashlib_encode_data(self.id))
+        h.update(hashlib_encode_data(path))
+        return "%s-%s" % (os.path.basename(path), h.hexdigest())
+
+    def add_project_items(self, psync):
+        with util.cd(self.options["project_dir"]):
+            cfg = ProjectConfig.get_instance(
+                os.path.join(self.options["project_dir"], "platformio.ini")
+            )
+            psync.add_item(cfg.path, "platformio.ini")
+            psync.add_item(cfg.get_optional_dir("shared"), "shared")
+            psync.add_item(cfg.get_optional_dir("boards"), "boards")
+
+            if self.options["force_remote"]:
+                self._add_project_source_items(cfg, psync)
+            else:
+                self._add_project_binary_items(cfg, psync)
+
+            if self.command == "test":
+                psync.add_item(cfg.get_optional_dir("test"), "test")
+
+    def _add_project_source_items(self, cfg, psync):
+        psync.add_item(cfg.get_optional_dir("lib"), "lib")
+        psync.add_item(
+            cfg.get_optional_dir("include"),
+            "include",
+            cb_filter=self._cb_tarfile_filter,
+        )
+        psync.add_item(
+            cfg.get_optional_dir("src"), "src", cb_filter=self._cb_tarfile_filter
+        )
+        if set(["buildfs", "uploadfs", "uploadfsota"]) & set(
+            self.options.get("target", [])
+        ):
+            psync.add_item(cfg.get_optional_dir("data"), "data")
+
+    @staticmethod
+    def _add_project_binary_items(cfg, psync):
+        build_dir = cfg.get_optional_dir("build")
+        for env_name in os.listdir(build_dir):
+            env_dir = os.path.join(build_dir, env_name)
+            if not os.path.isdir(env_dir):
+                continue
+            for fname in os.listdir(env_dir):
+                bin_file = os.path.join(env_dir, fname)
+                bin_exts = (".elf", ".bin", ".hex", ".eep", "program")
+                if os.path.isfile(bin_file) and fname.endswith(bin_exts):
+                    psync.add_item(
+                        bin_file, os.path.join(".pio", "build", env_name, fname)
+                    )
+
+    def _cb_tarfile_filter(self, path):
+        if (
+            os.path.isdir(path)
+            and os.path.basename(path).lower() in self.PSYNC_SKIP_DIRS
+        ):
+            return None
+        if os.path.isfile(path) and not self.is_file_with_exts(
+            path, self.PSYNC_SRC_EXTS
+        ):
+            return None
+        return path
+
+    @staticmethod
+    def is_file_with_exts(path, exts):
+        if path.endswith(tuple(".%s" % e for e in exts)):
+            return True
+        return False
+
+    def agent_pool_ready(self):
+        self.psync_init()
+
+    def psync_init(self):
+        self.add_project_items(self.psync)
+        d = self.agentpool.callRemote(
+            "cmd",
+            self.agents,
+            "psync",
+            dict(id=self.project_id, items=[i[1] for i in self.psync.get_items()]),
+        )
+        d.addCallback(self.cb_psync_init_result)
+        d.addErrback(self.cb_global_error)
+
+        # build db index while wait for result from agent
+        self.psync.rebuild_dbindex()
+
+    def cb_psync_init_result(self, result):
+        self._acs_total = len(result)
+        for (success, value) in result:
+            if not success:
+                raise pb.Error(value)
+            agent_id, ac_id = value
+            try:
+                d = self.agentpool.callRemote(
+                    "acwrite",
+                    agent_id,
+                    ac_id,
+                    dict(stage=PROJECT_SYNC_STAGE.DBINDEX.value),
+                )
+                d.addCallback(self.cb_psync_dbindex_result, agent_id, ac_id)
+                d.addErrback(self.cb_global_error)
+            except (AttributeError, pb.DeadReferenceError):
+                self.disconnect(exit_code=1)
+
+    def cb_psync_dbindex_result(self, result, agent_id, ac_id):
+        result = set(json.loads(zlib.decompress(result)))
+        dbindex = set(self.psync.get_dbindex())
+        delete = list(result - dbindex)
+        delta = list(dbindex - result)
+
+        self.log.debug(
+            "PSync: stats, total={total}, delete={delete}, delta={delta}",
+            total=len(dbindex),
+            delete=len(delete),
+            delta=len(delta),
+        )
+
+        if not delete and not delta:
+            return self.psync_finalize(agent_id, ac_id)
+        if not delete:
+            return self.psync_upload(agent_id, ac_id, delta)
+
+        try:
+            d = self.agentpool.callRemote(
+                "acwrite",
+                agent_id,
+                ac_id,
+                dict(
+                    stage=PROJECT_SYNC_STAGE.DELETE.value,
+                    dbindex=zlib.compress(json.dumps(delete).encode()),
+                ),
+            )
+            d.addCallback(self.cb_psync_delete_result, agent_id, ac_id, delta)
+            d.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+        return None
+
+    def cb_psync_delete_result(self, result, agent_id, ac_id, dbindex):
+        assert result
+        self.psync_upload(agent_id, ac_id, dbindex)
+
+    def psync_upload(self, agent_id, ac_id, dbindex):
+        assert dbindex
+        fileobj = BytesIO()
+        compressed = self.psync.compress_items(fileobj, dbindex, self.MAX_ARCHIVE_SIZE)
+        fileobj.seek(0)
+        self.log.debug(
+            "PSync: upload project, size={size}", size=len(fileobj.getvalue())
+        )
+        self.psync_upload_chunk(
+            agent_id, ac_id, list(set(dbindex) - set(compressed)), fileobj
+        )
+
+    def psync_upload_chunk(self, agent_id, ac_id, dbindex, fileobj):
+        offset = fileobj.tell()
+        total = fileobj.seek(0, os.SEEK_END)
+        # unwind
+        fileobj.seek(offset)
+        chunk = fileobj.read(self.UPLOAD_CHUNK_SIZE)
+        assert chunk
+        try:
+            d = self.agentpool.callRemote(
+                "acwrite",
+                agent_id,
+                ac_id,
+                dict(
+                    stage=PROJECT_SYNC_STAGE.UPLOAD.value,
+                    chunk=chunk,
+                    length=len(chunk),
+                    total=total,
+                ),
+            )
+            d.addCallback(
+                self.cb_psync_upload_chunk_result, agent_id, ac_id, dbindex, fileobj
+            )
+            d.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+    def cb_psync_upload_chunk_result(  # pylint: disable=too-many-arguments
+        self, result, agent_id, ac_id, dbindex, fileobj
+    ):
+        result = PROJECT_SYNC_STAGE.lookupByValue(result)
+        self.log.debug("PSync: upload chunk result {r}", r=str(result))
+        assert result & (PROJECT_SYNC_STAGE.UPLOAD | PROJECT_SYNC_STAGE.EXTRACTED)
+        if result is PROJECT_SYNC_STAGE.EXTRACTED:
+            if dbindex:
+                self.psync_upload(agent_id, ac_id, dbindex)
+            else:
+                self.psync_finalize(agent_id, ac_id)
+        else:
+            self.psync_upload_chunk(agent_id, ac_id, dbindex, fileobj)
+
+    def psync_finalize(self, agent_id, ac_id):
+        try:
+            d = self.agentpool.callRemote("acclose", agent_id, ac_id)
+            d.addCallback(self.cb_psync_completed_result, agent_id)
+            d.addErrback(self.cb_global_error)
+        except (AttributeError, pb.DeadReferenceError):
+            self.disconnect(exit_code=1)
+
+    def cb_psync_completed_result(self, result, agent_id):
+        assert PROJECT_SYNC_STAGE.lookupByValue(result)
+        options = self.options.copy()
+        del options["project_dir"]
+        options["project_id"] = self.project_id
+        d = self.agentpool.callRemote("cmd", [agent_id], self.command, options)
+        d.addCallback(self.cb_async_result)
+        d.addErrback(self.cb_global_error)

platformio/commands/remote/client/update_core.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.remote.client.async_base import AsyncClientBase
+
+
+class UpdateCoreClient(AsyncClientBase):
+    def agent_pool_ready(self):
+        d = self.agentpool.callRemote("cmd", self.agents, self.command, self.options)
+        d.addCallback(self.cb_async_result)
+        d.addErrback(self.cb_global_error)

platformio/commands/remote/command.py

@@ -12,29 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=too-many-arguments, import-outside-toplevel
+# pylint: disable=inconsistent-return-statements
+
 import os
-import sys
+import subprocess
 import threading
 from tempfile import mkdtemp
 from time import sleep
 
 import click
 
-from platformio import exception, fs
-from platformio.commands import device
-from platformio.managers.core import pioplus_call
+from platformio import exception, fs, proc
+from platformio.commands.device import helpers as device_helpers
+from platformio.commands.device.command import device_monitor as cmd_device_monitor
+from platformio.commands.run.command import cli as cmd_run
+from platformio.commands.test.command import cli as cmd_test
+from platformio.compat import PY2
+from platformio.managers.core import inject_contrib_pysite
 from platformio.project.exception import NotPlatformIOProjectError
 
-# pylint: disable=unused-argument
-
 
 @click.group("remote", short_help="PIO Remote")
 @click.option("-a", "--agent", multiple=True)
-def cli(**kwargs):
-    pass
+@click.pass_context
+def cli(ctx, agent):
+    if PY2:
+        raise exception.UserSideException(
+            "PIO Remote requires Python 3.5 or above. \nPlease install the latest "
+            "Python 3 and reinstall PlatformIO Core using installation script:\n"
+            "https://docs.platformio.org/page/core/installation.html"
+        )
+    ctx.obj = agent
+    inject_contrib_pysite(verify_openssl=True)
 
 
-@cli.group("agent", short_help="Start new agent or list active")
+@cli.group("agent", short_help="Start a new agent or list active")
 def remote_agent():
     pass
 
@@ -48,18 +61,17 @@ def remote_agent():
     envvar="PLATFORMIO_REMOTE_AGENT_DIR",
     type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
 )
-def remote_agent_start(**kwargs):
-    pioplus_call(sys.argv[1:])
+def remote_agent_start(name, share, working_dir):
+    from platformio.commands.remote.client.agent_service import RemoteAgentService
 
-
-@remote_agent.command("reload", short_help="Reload agents")
-def remote_agent_reload():
-    pioplus_call(sys.argv[1:])
+    RemoteAgentService(name, share, working_dir).connect()
 
 
 @remote_agent.command("list", short_help="List active agents")
 def remote_agent_list():
-    pioplus_call(sys.argv[1:])
+    from platformio.commands.remote.client.agent_list import AgentListClient
+
+    AgentListClient().connect()
 
 
 @cli.command("update", short_help="Update installed Platforms, Packages and Libraries")
@@ -72,8 +84,11 @@ def remote_agent_list():
 @click.option(
     "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
 )
-def remote_update(only_check, dry_run):
-    pioplus_call(sys.argv[1:])
+@click.pass_obj
+def remote_update(agents, only_check, dry_run):
+    from platformio.commands.remote.client.update_core import UpdateCoreClient
+
+    UpdateCoreClient("update", agents, dict(only_check=only_check or dry_run)).connect()
 
 
 @cli.command("run", short_help="Process project environments remotely")
@@ -92,8 +107,65 @@ def remote_update(only_check, dry_run):
 @click.option("-r", "--force-remote", is_flag=True)
 @click.option("-s", "--silent", is_flag=True)
 @click.option("-v", "--verbose", is_flag=True)
-def remote_run(**kwargs):
-    pioplus_call(sys.argv[1:])
+@click.pass_obj
+@click.pass_context
+def remote_run(
+    ctx,
+    agents,
+    environment,
+    target,
+    upload_port,
+    project_dir,
+    disable_auto_clean,
+    force_remote,
+    silent,
+    verbose,
+):
+
+    from platformio.commands.remote.client.run_or_test import RunOrTestClient
+
+    cr = RunOrTestClient(
+        "run",
+        agents,
+        dict(
+            environment=environment,
+            target=target,
+            upload_port=upload_port,
+            project_dir=project_dir,
+            disable_auto_clean=disable_auto_clean,
+            force_remote=force_remote,
+            silent=silent,
+            verbose=verbose,
+        ),
+    )
+    if force_remote:
+        return cr.connect()
+
+    click.secho("Building project locally", bold=True)
+    local_targets = []
+    if "clean" in target:
+        local_targets = ["clean"]
+    elif set(["buildfs", "uploadfs", "uploadfsota"]) & set(target):
+        local_targets = ["buildfs"]
+    else:
+        local_targets = ["checkprogsize", "buildprog"]
+    ctx.invoke(
+        cmd_run,
+        environment=environment,
+        target=local_targets,
+        project_dir=project_dir,
+        # disable_auto_clean=True,
+        silent=silent,
+        verbose=verbose,
+    )
+
+    if any(["upload" in t for t in target] + ["program" in target]):
+        click.secho("Uploading firmware remotely", bold=True)
+        cr.options["target"] += ("nobuild",)
+        cr.options["disable_auto_clean"] = True
+        cr.connect()
+
+    return True
 
 
 @cli.command("test", short_help="Remote Unit Testing")
@@ -113,8 +185,59 @@ def remote_run(**kwargs):
 @click.option("--without-building", is_flag=True)
 @click.option("--without-uploading", is_flag=True)
 @click.option("--verbose", "-v", is_flag=True)
-def remote_test(**kwargs):
-    pioplus_call(sys.argv[1:])
+@click.pass_obj
+@click.pass_context
+def remote_test(
+    ctx,
+    agents,
+    environment,
+    ignore,
+    upload_port,
+    test_port,
+    project_dir,
+    force_remote,
+    without_building,
+    without_uploading,
+    verbose,
+):
+
+    from platformio.commands.remote.client.run_or_test import RunOrTestClient
+
+    cr = RunOrTestClient(
+        "test",
+        agents,
+        dict(
+            environment=environment,
+            ignore=ignore,
+            upload_port=upload_port,
+            test_port=test_port,
+            project_dir=project_dir,
+            force_remote=force_remote,
+            without_building=without_building,
+            without_uploading=without_uploading,
+            verbose=verbose,
+        ),
+    )
+    if force_remote:
+        return cr.connect()
+
+    click.secho("Building project locally", bold=True)
+
+    ctx.invoke(
+        cmd_test,
+        environment=environment,
+        ignore=ignore,
+        project_dir=project_dir,
+        without_uploading=True,
+        without_testing=True,
+        verbose=verbose,
+    )
+
+    click.secho("Testing project remotely", bold=True)
+    cr.options["without_building"] = True
+    cr.connect()
+
+    return True
 
 
 @cli.group("device", short_help="Monitor remote device or list existing")
@@ -124,8 +247,11 @@ def remote_device():
 
 @remote_device.command("list", short_help="List remote devices")
 @click.option("--json-output", is_flag=True)
-def device_list(json_output):
-    pioplus_call(sys.argv[1:])
+@click.pass_obj
+def device_list(agents, json_output):
+    from platformio.commands.remote.client.device_list import DeviceListClient
+
+    DeviceListClient(agents, json_output).connect()
 
 
 @remote_device.command("monitor", short_help="Monitor remote device")
@@ -192,28 +318,37 @@ def device_list(json_output):
     "--environment",
     help="Load configuration from `platformio.ini` and specified environment",
 )
+@click.option(
+    "--sock",
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.pass_obj
 @click.pass_context
-def device_monitor(ctx, **kwargs):
+def device_monitor(ctx, agents, **kwargs):
+    from platformio.commands.remote.client.device_monitor import DeviceMonitorClient
+
+    if kwargs["sock"]:
+        return DeviceMonitorClient(agents, **kwargs).connect()
+
     project_options = {}
     try:
         with fs.cd(kwargs["project_dir"]):
-            project_options = device.get_project_options(kwargs["environment"])
-        kwargs = device.apply_project_monitor_options(kwargs, project_options)
+            project_options = device_helpers.get_project_options(kwargs["environment"])
+        kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options)
     except NotPlatformIOProjectError:
         pass
 
     kwargs["baud"] = kwargs["baud"] or 9600
 
     def _tx_target(sock_dir):
-        pioplus_argv = ["remote", "device", "monitor"]
-        pioplus_argv.extend(device.options_to_argv(kwargs, project_options))
-        pioplus_argv.extend(["--sock", sock_dir])
-        try:
-            pioplus_call(pioplus_argv)
-        except exception.ReturnErrorCode:
-            pass
+        subcmd_argv = ["remote", "device", "monitor"]
+        subcmd_argv.extend(device_helpers.options_to_argv(kwargs, project_options))
+        subcmd_argv.extend(["--sock", sock_dir])
+        subprocess.call([proc.where_is_program("platformio")] + subcmd_argv)
 
-    sock_dir = mkdtemp(suffix="pioplus")
+    sock_dir = mkdtemp(suffix="pio")
     sock_file = os.path.join(sock_dir, "sock")
     try:
         t = threading.Thread(target=_tx_target, args=(sock_dir,))
@@ -222,8 +357,11 @@ def device_monitor(ctx, **kwargs):
             sleep(0.1)
         if not t.is_alive():
             return
-        kwargs["port"] = fs.get_file_contents(sock_file)
-        ctx.invoke(device.device_monitor, **kwargs)
+        with open(sock_file) as fp:
+            kwargs["port"] = fp.read()
+        ctx.invoke(cmd_device_monitor, **kwargs)
         t.join(2)
     finally:
         fs.rmtree(sock_dir)
+
+    return True

platformio/commands/remote/factory/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/remote/factory/client.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.cred import credentials  # pylint: disable=import-error
+from twisted.internet import defer, protocol, reactor  # pylint: disable=import-error
+from twisted.spread import pb  # pylint: disable=import-error
+
+from platformio.app import get_host_id
+from platformio.commands.account.client import AccountClient
+
+
+class RemoteClientFactory(pb.PBClientFactory, protocol.ReconnectingClientFactory):
+    def clientConnectionMade(self, broker):
+        if self.sslContextFactory and not self.sslContextFactory.certificate_verified:
+            self.remote_client.log.error(
+                "A remote cloud could not prove that its security certificate is "
+                "from {host}. This may cause a misconfiguration or an attacker "
+                "intercepting your connection.",
+                host=self.sslContextFactory.host,
+            )
+            return self.remote_client.disconnect()
+        pb.PBClientFactory.clientConnectionMade(self, broker)
+        protocol.ReconnectingClientFactory.resetDelay(self)
+        self.remote_client.log.info("Successfully connected")
+        self.remote_client.log.info("Authenticating")
+
+        auth_token = None
+        try:
+            auth_token = AccountClient().fetch_authentication_token()
+        except Exception as e:  # pylint:disable=broad-except
+            d = defer.Deferred()
+            d.addErrback(self.clientAuthorizationFailed)
+            d.errback(pb.Error(e))
+            return d
+
+        d = self.login(
+            credentials.UsernamePassword(auth_token.encode(), get_host_id().encode(),),
+            client=self.remote_client,
+        )
+        d.addCallback(self.remote_client.cb_client_authorization_made)
+        d.addErrback(self.clientAuthorizationFailed)
+        return d
+
+    def clientAuthorizationFailed(self, err):
+        AccountClient.delete_local_session()
+        self.remote_client.cb_client_authorization_failed(err)
+
+    def clientConnectionFailed(self, connector, reason):
+        self.remote_client.log.warn(
+            "Could not connect to PIO Remote Cloud. Reconnecting..."
+        )
+        self.remote_client.cb_disconnected(reason)
+        protocol.ReconnectingClientFactory.clientConnectionFailed(
+            self, connector, reason
+        )
+
+    def clientConnectionLost(  # pylint: disable=arguments-differ
+        self, connector, unused_reason
+    ):
+        if not reactor.running:
+            self.remote_client.log.info("Successfully disconnected")
+            return
+        self.remote_client.log.warn(
+            "Connection is lost to PIO Remote Cloud. Reconnecting"
+        )
+        pb.PBClientFactory.clientConnectionLost(
+            self, connector, unused_reason, reconnecting=1
+        )
+        self.remote_client.cb_disconnected(unused_reason)
+        protocol.ReconnectingClientFactory.clientConnectionLost(
+            self, connector, unused_reason
+        )

platformio/commands/remote/factory/ssl.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import certifi
+from OpenSSL import SSL  # pylint: disable=import-error
+from twisted.internet import ssl  # pylint: disable=import-error
+
+
+class SSLContextFactory(ssl.ClientContextFactory):
+    def __init__(self, host):
+        self.host = host
+        self.certificate_verified = False
+
+    def getContext(self):
+        ctx = super(SSLContextFactory, self).getContext()
+        ctx.set_verify(
+            SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, self.verifyHostname
+        )
+        ctx.load_verify_locations(certifi.where())
+        return ctx
+
+    def verifyHostname(  # pylint: disable=unused-argument,too-many-arguments
+        self, connection, x509, errno, depth, status
+    ):
+        cn = x509.get_subject().commonName
+        if cn.startswith("*"):
+            cn = cn[1:]
+        if self.host.endswith(cn):
+            self.certificate_verified = True
+        return status

platformio/commands/remote/projectsync.py

@@ -0,0 +1,117 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tarfile
+from binascii import crc32
+from os.path import getmtime, getsize, isdir, isfile, join
+
+from twisted.python import constants  # pylint: disable=import-error
+
+from platformio.compat import hashlib_encode_data
+
+
+class PROJECT_SYNC_STAGE(constants.Flags):
+    INIT = constants.FlagConstant()
+    DBINDEX = constants.FlagConstant()
+    DELETE = constants.FlagConstant()
+    UPLOAD = constants.FlagConstant()
+    EXTRACTED = constants.FlagConstant()
+    COMPLETED = constants.FlagConstant()
+
+
+class ProjectSync(object):
+    def __init__(self, path):
+        self.path = path
+        if not isdir(self.path):
+            os.makedirs(self.path)
+        self.items = []
+        self._db = {}
+
+    def add_item(self, path, relpath, cb_filter=None):
+        self.items.append((path, relpath, cb_filter))
+
+    def get_items(self):
+        return self.items
+
+    def rebuild_dbindex(self):
+        self._db = {}
+        for (path, relpath, cb_filter) in self.items:
+            if cb_filter and not cb_filter(path):
+                continue
+            self._insert_to_db(path, relpath)
+            if not isdir(path):
+                continue
+            for (root, _, files) in os.walk(path, followlinks=True):
+                for name in files:
+                    self._insert_to_db(
+                        join(root, name), join(relpath, root[len(path) + 1 :], name)
+                    )
+
+    def _insert_to_db(self, path, relpath):
+        if not isfile(path):
+            return
+        index_hash = "%s-%s-%s" % (relpath, getmtime(path), getsize(path))
+        index = crc32(hashlib_encode_data(index_hash))
+        self._db[index] = (path, relpath)
+
+    def get_dbindex(self):
+        return list(self._db.keys())
+
+    def delete_dbindex(self, dbindex):
+        for index in dbindex:
+            if index not in self._db:
+                continue
+            path = self._db[index][0]
+            if isfile(path):
+                os.remove(path)
+            del self._db[index]
+        self.delete_empty_folders()
+        return True
+
+    def delete_empty_folders(self):
+        deleted = False
+        for item in self.items:
+            if not isdir(item[0]):
+                continue
+            for root, dirs, files in os.walk(item[0]):
+                if not dirs and not files and root != item[0]:
+                    deleted = True
+                    os.rmdir(root)
+        if deleted:
+            return self.delete_empty_folders()
+
+        return True
+
+    def compress_items(self, fileobj, dbindex, max_size):
+        compressed = []
+        total_size = 0
+        tar_opts = dict(fileobj=fileobj, mode="w:gz", bufsize=0, dereference=True)
+        with tarfile.open(**tar_opts) as tgz:
+            for index in dbindex:
+                compressed.append(index)
+                if index not in self._db:
+                    continue
+                path, relpath = self._db[index]
+                tgz.add(path, relpath)
+                total_size += getsize(path)
+                if total_size > max_size:
+                    break
+        return compressed
+
+    def decompress_items(self, fileobj):
+        fileobj.seek(0)
+        with tarfile.open(fileobj=fileobj, mode="r:gz") as tgz:
+            tgz.extractall(self.path)
+        return True

platformio/commands/run/command.py

@@ -21,7 +21,7 @@ import click
 from tabulate import tabulate
 
 from platformio import app, exception, fs, util
-from platformio.commands.device import device_monitor as cmd_device_monitor
+from platformio.commands.device.command import device_monitor as cmd_device_monitor
 from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libdeps
 from platformio.commands.run.processor import EnvironmentProcessor
 from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING

platformio/commands/run/helpers.py

@@ -53,9 +53,12 @@ def clean_build_dir(build_dir, config):
 
     if isdir(build_dir):
         # check project structure
-        if isfile(checksum_file) and fs.get_file_contents(checksum_file) == checksum:
-            return
+        if isfile(checksum_file):
+            with open(checksum_file) as fp:
+                if fp.read() == checksum:
+                    return
         fs.rmtree(build_dir)
 
     makedirs(build_dir)
-    fs.write_file_contents(checksum_file, checksum)
+    with open(checksum_file, "w") as fp:
+        fp.write(checksum)

platformio/commands/run/processor.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio import exception, telemetry
+from platformio import exception
 from platformio.commands.platform import platform_install as cmd_platform_install
 from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME
 from platformio.managers.platform import PlatformFactory
@@ -62,8 +62,6 @@ class EnvironmentProcessor(object):
         build_vars = self.get_build_variables()
         build_targets = list(self.get_build_targets())
 
-        telemetry.send_run_environment(self.options, build_targets)
-
         # skip monitor target, we call it above
         if "monitor" in build_targets:
             build_targets.remove("monitor")

platformio/commands/system/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/system/command.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import subprocess
+
+import click
+
+from platformio import proc
+from platformio.commands.system.completion import (
+    get_completion_install_path,
+    install_completion_code,
+    uninstall_completion_code,
+)
+
+
+@click.group("system", short_help="Miscellaneous system commands")
+def cli():
+    pass
+
+
+@cli.group("completion", short_help="Shell completion support")
+def completion():
+    # pylint: disable=import-error,import-outside-toplevel
+    try:
+        import click_completion  # pylint: disable=unused-import,unused-variable
+    except ImportError:
+        click.echo("Installing dependent packages...")
+        subprocess.check_call(
+            [proc.get_pythonexe_path(), "-m", "pip", "install", "click-completion"],
+        )
+
+
+@completion.command("install", short_help="Install shell completion files/code")
+@click.option(
+    "--shell",
+    default=None,
+    type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]),
+    help="The shell type, default=auto",
+)
+@click.option(
+    "--path",
+    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
+    help="Custom installation path of the code to be evaluated by the shell. "
+    "The standard installation path is used by default.",
+)
+def completion_install(shell, path):
+
+    import click_completion  # pylint: disable=import-outside-toplevel,import-error
+
+    shell = shell or click_completion.get_auto_shell()
+    path = path or get_completion_install_path(shell)
+    install_completion_code(shell, path)
+    click.echo(
+        "PlatformIO CLI completion has been installed for %s shell to %s \n"
+        "Please restart a current shell session."
+        % (click.style(shell, fg="cyan"), click.style(path, fg="blue"))
+    )
+
+
+@completion.command("uninstall", short_help="Uninstall shell completion files/code")
+@click.option(
+    "--shell",
+    default=None,
+    type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]),
+    help="The shell type, default=auto",
+)
+@click.option(
+    "--path",
+    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
+    help="Custom installation path of the code to be evaluated by the shell. "
+    "The standard installation path is used by default.",
+)
+def completion_uninstall(shell, path):
+
+    import click_completion  # pylint: disable=import-outside-toplevel,import-error
+
+    shell = shell or click_completion.get_auto_shell()
+    path = path or get_completion_install_path(shell)
+    uninstall_completion_code(shell, path)
+    click.echo(
+        "PlatformIO CLI completion has been uninstalled for %s shell from %s \n"
+        "Please restart a current shell session."
+        % (click.style(shell, fg="cyan"), click.style(path, fg="blue"))
+    )

platformio/commands/system/completion.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+import click
+
+
+def get_completion_install_path(shell):
+    home_dir = os.path.expanduser("~")
+    prog_name = click.get_current_context().find_root().info_name
+    if shell == "fish":
+        return os.path.join(
+            home_dir, ".config", "fish", "completions", "%s.fish" % prog_name
+        )
+    if shell == "bash":
+        return os.path.join(home_dir, ".bash_completion")
+    if shell == "zsh":
+        return os.path.join(home_dir, ".zshrc")
+    if shell == "powershell":
+        return subprocess.check_output(
+            ["powershell", "-NoProfile", "echo $profile"]
+        ).strip()
+    raise click.ClickException("%s is not supported." % shell)
+
+
+def is_completion_code_installed(shell, path):
+    if shell == "fish" or not os.path.exists(path):
+        return False
+
+    import click_completion  # pylint: disable=import-error,import-outside-toplevel
+
+    with open(path) as fp:
+        return click_completion.get_code(shell=shell) in fp.read()
+
+
+def install_completion_code(shell, path):
+    import click_completion  # pylint: disable=import-error,import-outside-toplevel
+
+    if is_completion_code_installed(shell, path):
+        return None
+
+    return click_completion.install(shell=shell, path=path, append=shell != "fish")
+
+
+def uninstall_completion_code(shell, path):
+    if not os.path.exists(path):
+        return True
+    if shell == "fish":
+        os.remove(path)
+        return True
+
+    import click_completion  # pylint: disable=import-error,import-outside-toplevel
+
+    with open(path, "r+") as fp:
+        contents = fp.read()
+        fp.seek(0)
+        fp.truncate()
+        fp.write(contents.replace(click_completion.get_code(shell=shell), ""))
+
+    return True

platformio/commands/test/processor.py

@@ -19,7 +19,7 @@ from string import Template
 
 import click
 
-from platformio import exception, fs
+from platformio import exception
 
 TRANSPORT_OPTIONS = {
     "arduino": {
@@ -29,6 +29,7 @@ TRANSPORT_OPTIONS = {
         "flush": "Serial.flush()",
         "begin": "Serial.begin($baudrate)",
         "end": "Serial.end()",
+        "language": "cpp",
     },
     "mbed": {
         "include": "#include <mbed.h>",
@@ -37,6 +38,7 @@ TRANSPORT_OPTIONS = {
         "flush": "",
         "begin": "pc.baud($baudrate)",
         "end": "",
+        "language": "cpp",
     },
     "espidf": {
         "include": "#include <stdio.h>",
@@ -46,6 +48,14 @@ TRANSPORT_OPTIONS = {
         "begin": "",
         "end": "",
     },
+    "zephyr": {
+        "include": "#include <sys/printk.h>",
+        "object": "",
+        "putchar": 'printk("%c", c)',
+        "flush": "",
+        "begin": "",
+        "end": "",
+    },
     "native": {
         "include": "#include <stdio.h>",
         "object": "",
@@ -61,6 +71,7 @@ TRANSPORT_OPTIONS = {
         "flush": "unittest_uart_flush()",
         "begin": "unittest_uart_begin()",
         "end": "unittest_uart_end()",
+        "language": "cpp",
     },
 }
 
@@ -80,9 +91,10 @@ class TestProcessorBase(object):
         self.env_name = envname
         self.env_options = options["project_config"].items(env=envname, as_dict=True)
         self._run_failed = False
-        self._outputcpp_generated = False
+        self._output_file_generated = False
 
     def get_transport(self):
+        transport = None
         if self.env_options.get("platform") == "native":
             transport = "native"
         elif "framework" in self.env_options:
@@ -91,7 +103,9 @@ class TestProcessorBase(object):
             transport = self.env_options["test_transport"]
         if transport not in TRANSPORT_OPTIONS:
             raise exception.PlatformioException(
-                "Unknown Unit Test transport `%s`" % transport
+                "Unknown Unit Test transport `%s`. Please check a documentation how "
+                "to create an own 'Test Transport':\n"
+                "- https://docs.platformio.org/page/plus/unit-testing.html" % transport
             )
         return transport.lower()
 
@@ -102,11 +116,11 @@ class TestProcessorBase(object):
         click.secho(text, bold=self.options.get("verbose"))
 
     def build_or_upload(self, target):
-        if not self._outputcpp_generated:
-            self.generate_outputcpp(
+        if not self._output_file_generated:
+            self.generate_output_file(
                 self.options["project_config"].get_optional_dir("test")
             )
-            self._outputcpp_generated = True
+            self._output_file_generated = True
 
         if self.test_name != "*":
             self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name
@@ -144,10 +158,10 @@ class TestProcessorBase(object):
         else:
             click.echo(line)
 
-    def generate_outputcpp(self, test_dir):
+    def generate_output_file(self, test_dir):
         assert isdir(test_dir)
 
-        cpp_tpl = "\n".join(
+        file_tpl = "\n".join(
             [
                 "$include",
                 "#include <output_export.h>",
@@ -191,10 +205,13 @@ class TestProcessorBase(object):
                         fg="yellow",
                     )
 
-        tpl = Template(cpp_tpl).substitute(TRANSPORT_OPTIONS[self.get_transport()])
+        transport_options = TRANSPORT_OPTIONS[self.get_transport()]
+        tpl = Template(file_tpl).substitute(transport_options)
         data = Template(tpl).substitute(baudrate=self.get_baudrate())
-
-        tmp_file = join(test_dir, "output_export.cpp")
-        fs.write_file_contents(tmp_file, data)
+        tmp_file = join(
+            test_dir, "output_export." + transport_options.get("language", "c")
+        )
+        with open(tmp_file, "w") as fp:
+            fp.write(data)
 
         atexit.register(delete_tmptest_file, tmp_file)

platformio/commands/upgrade.py

@@ -19,7 +19,7 @@ from zipfile import ZipFile
 import click
 import requests
 
-from platformio import VERSION, __version__, app, exception, util
+from platformio import VERSION, __version__, app, exception
 from platformio.compat import WINDOWS
 from platformio.proc import exec_command, get_pythonexe_path
 from platformio.project.helpers import get_project_cache_dir
@@ -133,7 +133,7 @@ def get_develop_latest_version():
     r = requests.get(
         "https://raw.githubusercontent.com/platformio/platformio"
         "/develop/platformio/__init__.py",
-        headers=util.get_request_defheaders(),
+        headers={"User-Agent": app.get_user_agent()},
     )
     r.raise_for_status()
     for line in r.text.split("\n"):
@@ -153,7 +153,8 @@ def get_develop_latest_version():
 
 def get_pypi_latest_version():
     r = requests.get(
-        "https://pypi.org/pypi/platformio/json", headers=util.get_request_defheaders()
+        "https://pypi.org/pypi/platformio/json",
+        headers={"User-Agent": app.get_user_agent()},
     )
     r.raise_for_status()
     return r.json()["info"]["version"]

platformio/compat.py

@@ -38,12 +38,14 @@ def get_locale_encoding():
         return None
 
 
-def get_class_attributes(cls):
-    attributes = inspect.getmembers(cls, lambda a: not inspect.isroutine(a))
+def get_object_members(obj, ignore_private=True):
+    members = inspect.getmembers(obj, lambda a: not inspect.isroutine(a))
+    if not ignore_private:
+        return members
     return {
-        a[0]: a[1]
-        for a in attributes
-        if not (a[0].startswith("__") and a[0].endswith("__"))
+        item[0]: item[1]
+        for item in members
+        if not (item[0].startswith("__") and item[0].endswith("__"))
     }
 
 
@@ -58,7 +60,7 @@ if PY2:
     def path_to_unicode(path):
         if isinstance(path, unicode):
             return path
-        return path.decode(get_filesystem_encoding()).encode("utf-8")
+        return path.decode(get_filesystem_encoding())
 
     def hashlib_encode_data(data):
         if is_bytes(data):

platformio/downloader.py

@@ -23,7 +23,7 @@ from time import mktime
 import click
 import requests
 
-from platformio import util
+from platformio import app, util
 from platformio.exception import (
     FDSHASumMismatch,
     FDSizeMismatch,
@@ -38,7 +38,7 @@ class FileDownloader(object):
         self._request = requests.get(
             url,
             stream=True,
-            headers=util.get_request_defheaders(),
+            headers={"User-Agent": app.get_user_agent()},
             verify=sys.version_info >= (2, 7, 9),
         )
         if self._request.status_code != 200:

platformio/exception.py

@@ -92,7 +92,7 @@ class PlatformIOPackageException(PlatformioException):
     pass
 
 
-class UnknownPackage(PlatformIOPackageException):
+class UnknownPackage(UserSideException):
 
     MESSAGE = "Detected unknown package '{0}'"
 
@@ -177,7 +177,7 @@ class NotGlobalLibDir(UserSideException):
     )
 
 
-class InvalidLibConfURL(PlatformioException):
+class InvalidLibConfURL(UserSideException):
 
     MESSAGE = "Invalid library config URL '{0}'"
 
@@ -232,8 +232,8 @@ class InternetIsOffline(UserSideException):
 
     MESSAGE = (
         "You are not connected to the Internet.\n"
-        "If you build a project first time, we need Internet connection "
-        "to install all dependencies and toolchains."
+        "PlatformIO needs the Internet connection to"
+        " download dependent packages or to work with PIO Account."
     )
 
 
@@ -242,12 +242,12 @@ class BuildScriptNotFound(PlatformioException):
     MESSAGE = "Invalid path '{0}' to build script"
 
 
-class InvalidSettingName(PlatformioException):
+class InvalidSettingName(UserSideException):
 
     MESSAGE = "Invalid setting with the name '{0}'"
 
 
-class InvalidSettingValue(PlatformioException):
+class InvalidSettingValue(UserSideException):
 
     MESSAGE = "Invalid value '{0}' for the setting '{1}'"
 
@@ -257,7 +257,7 @@ class InvalidJSONFile(PlatformioException):
     MESSAGE = "Could not load broken JSON: {0}"
 
 
-class CIBuildEnvsEmpty(PlatformioException):
+class CIBuildEnvsEmpty(UserSideException):
 
     MESSAGE = (
         "Can't find PlatformIO build environments.\n"
@@ -295,7 +295,7 @@ class CygwinEnvDetected(PlatformioException):
     )
 
 
-class TestDirNotExists(PlatformioException):
+class TestDirNotExists(UserSideException):
 
     MESSAGE = (
         "A test folder '{0}' does not exist.\nPlease create 'test' "

platformio/fs.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import io
 import json
 import os
 import re
@@ -49,30 +48,6 @@ def get_source_dir():
     return os.path.dirname(curpath)
 
 
-def get_file_contents(path, encoding=None):
-    try:
-        with io.open(path, encoding=encoding) as fp:
-            return fp.read()
-    except UnicodeDecodeError:
-        click.secho(
-            "Unicode decode error has occurred, please remove invalid "
-            "(non-ASCII or non-UTF8) characters from %s file" % path,
-            fg="yellow",
-            err=True,
-        )
-        with io.open(path, encoding="latin-1") as fp:
-            return fp.read()
-
-
-def write_file_contents(path, contents, errors=None):
-    try:
-        with open(path, "w") as fp:
-            return fp.write(contents)
-    except UnicodeEncodeError:
-        with io.open(path, "w", encoding="latin-1", errors=errors) as fp:
-            return fp.write(contents)
-
-
 def load_json(file_path):
     try:
         with open(file_path, "r") as f:
@@ -102,11 +77,14 @@ def ensure_udev_rules():
     from platformio.util import get_systype  # pylint: disable=import-outside-toplevel
 
     def _rules_to_set(rules_path):
-        return set(
-            l.strip()
-            for l in get_file_contents(rules_path).split("\n")
-            if l.strip() and not l.startswith("#")
-        )
+        result = set()
+        with open(rules_path) as fp:
+            for line in fp.readlines():
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                result.add(line)
+        return result
 
     if "linux" not in get_systype():
         return None

platformio/ide/projectgenerator.py

@@ -57,6 +57,20 @@ class ProjectGenerator(object):
 
         return envname
 
+    @staticmethod
+    def filter_includes(includes_map, ignore_scopes=None, to_unix_path=True):
+        ignore_scopes = ignore_scopes or []
+        result = []
+        for scope, includes in includes_map.items():
+            if scope in ignore_scopes:
+                continue
+            for include in includes:
+                if to_unix_path:
+                    include = fs.to_unix_path(include)
+                if include not in result:
+                    result.append(include)
+        return result
+
     def _load_tplvars(self):
         tpl_vars = {
             "config": self.config,
@@ -92,12 +106,13 @@ class ProjectGenerator(object):
         for key, value in tpl_vars.items():
             if key.endswith(("_path", "_dir")):
                 tpl_vars[key] = fs.to_unix_path(value)
-        for key in ("includes", "src_files", "libsource_dirs"):
+        for key in ("src_files", "libsource_dirs"):
             if key not in tpl_vars:
                 continue
             tpl_vars[key] = [fs.to_unix_path(inc) for inc in tpl_vars[key]]
 
         tpl_vars["to_unix_path"] = fs.to_unix_path
+        tpl_vars["filter_includes"] = self.filter_includes
         return tpl_vars
 
     def get_src_files(self):
@@ -136,7 +151,7 @@ class ProjectGenerator(object):
     @staticmethod
     def _render_tpl(tpl_path, tpl_vars):
         with codecs.open(tpl_path, "r", encoding="utf8") as fp:
-            return bottle.SimpleTemplate(fp.read()).render(**tpl_vars)
+            return bottle.template(fp.read(), **tpl_vars)
 
     @staticmethod
     def _merge_contents(dst_path, contents):

platformio/ide/tpls/atom/.clang_complete.tpl

@@ -1,4 +1,4 @@
-% for include in includes:
+% for include in filter_includes(includes):
 -I{{include}}
 % end
 % for define in defines:

platformio/ide/tpls/atom/.gcc-flags.json.tpl

@@ -4,6 +4,6 @@
   "gccDefaultCFlags": "-fsyntax-only {{! to_unix_path(cc_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
   "gccDefaultCppFlags": "-fsyntax-only {{! to_unix_path(cxx_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
   "gccErrorLimit": 15,
-  "gccIncludePaths": "{{ ','.join(includes) }}",
+  "gccIncludePaths": "{{ ','.join(filter_includes(includes)) }}",
   "gccSuppressWarnings": false
 }

platformio/ide/tpls/clion/.gitignore.tpl

@@ -1,2 +1,3 @@
 .pio
 CMakeListsPrivate.txt
+cmake-build-*/ 

platformio/ide/tpls/clion/.idea/clion.iml.tpl

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="CPP_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

platformio/ide/tpls/clion/.idea/misc.xml.tpl

@@ -1,16 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="CMakeWorkspace" PROJECT_DIR="$PROJECT_DIR$" />
-  <component name="CidrRootsConfiguration">
-    <sourceRoots>
-      <file path="$PROJECT_DIR$/src" />
-    </sourceRoots>
-    <libraryRoots>
-      <file path="$PROJECT_DIR$/lib" />
-      <file path="$PROJECT_DIR$/.pio/libdeps" />
-    </libraryRoots>
-    <excludeRoots>
-      <file path="$PROJECT_DIR$/.pio" />
-    </excludeRoots>
-  </component>
-</project>

platformio/ide/tpls/clion/.idea/modules.xml.tpl

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/clion.iml" filepath="$PROJECT_DIR$/.idea/clion.iml" />
-      <module fileurl="file://$PROJECT_DIR$/.idea/platformio.iml" filepath="$PROJECT_DIR$/.idea/platformio.iml" />
-    </modules>
-  </component>
-</project>