Merge branch 'release/v4.3.0'

* Replace installer script with a new one. Resolve #3420

* temp file name fix

* get-platformio.py script update.

* small fix

.appveyor.yml

@@ -1,19 +1,23 @@
 build: off
 
 platform:
-  - x86
   - x64
 
 environment:
   matrix:
     - TOXENV: "py27"
+      PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P2_{build}
+      PYTHON_DIRS: C:\Python27-x64;C:\Python27-x64\Scripts
+
+    - TOXENV: "py36"
+      PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P3_{build}
+      PYTHON_DIRS: C:\Python36-x64;C:\Python36-x64\Scripts
 
 install:
   - cmd: git submodule update --init --recursive
-  - cmd: SET PATH=C:\MinGW\bin;%PATH%
-  - if %PLATFORM% == x64 SET PATH=C:\Python27-x64;C:\Python27-x64\Scripts;%PATH%
-  - if %PLATFORM% == x86 SET PATH=C:\Python27;C:\Python27\Scripts;%PATH%
-  - cmd: pip install tox
+  - cmd: SET PATH=%PYTHON_DIRS%;C:\MinGW\bin;%PATH%
+  - cmd: SET PLATFORMIO_CORE_DIR=C:\.pio
+  - cmd: pip install --force-reinstall tox
 
 test_script:
   - cmd: tox

.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: https://platformio.org/donate

.isort.cfg

@@ -1,3 +1,3 @@
 [settings]
-line_length=79
-known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial
+line_length=88
+known_third_party=SCons, twisted, autobahn, jsonrpc

.pylintrc

@@ -1,23 +1,15 @@
 [MESSAGES CONTROL]
-
-# Only show warnings with the listed confidence levels. Leave empty to show
-# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
-confidence=
-
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time. See also the "--disable" option for examples.
-#enable=
-
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once).You can also use "--disable=all" to
-# disable everything first and then reenable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use"--disable=all --enable=classes
-# --disable=W"
-# disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
-
-disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens
+disable=
+    bad-continuation,
+    bad-whitespace,
+    missing-docstring,
+    ungrouped-imports,
+    invalid-name,
+    cyclic-import,
+    duplicate-code,
+    superfluous-parens,
+    too-few-public-methods,
+    useless-object-inheritance,
+    useless-import-alias,
+    fixme,
+    bad-option-value

.readthedocs.yml

@@ -0,0 +1,12 @@
+# See https://docs.readthedocs.io/en/stable/config-file/index.html
+
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+formats:
+  - pdf
+
+submodules:
+  include: all

.style.yapf

@@ -1,3 +0,0 @@
-[style]
-blank_line_before_nested_class_or_def = true
-allow_multiline_lambdas = true

.travis.yml

@@ -6,14 +6,14 @@ matrix:
       sudo: false
       python: 2.7
       env: TOX_ENV=docs
-    - os: linux
-      sudo: false
-      python: 2.7
-      env: TOX_ENV=lint
     - os: linux
       sudo: required
       python: 2.7
-      env: TOX_ENV=py27
+      env: TOX_ENV=py27 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d)
+    - os: linux
+      sudo: required
+      python: 3.6
+      env: TOX_ENV=py36 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d)
     - os: osx
       language: generic
       env: TOX_ENV=skipexamples
@@ -21,10 +21,10 @@ matrix:
 install:
   - git submodule update --init --recursive
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl -fsSL https://bootstrap.pypa.io/get-pip.py | sudo python; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox==3.0.0"; else pip install -U tox; fi
+  - pip install -U tox
 
   # ChipKIT issue: install 32-bit support for GCC PIC32
-  - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi
 
 script:
   - tox -e $TOX_ENV

.vscode/settings.json

@@ -1,15 +0,0 @@
-{
-  "python.pythonPath": "${workspaceRoot}/.tox/develop/bin/python",
-  "python.formatting.provider": "yapf",
-  "files.exclude": {
-    "**/*.pyc": true,
-    "*.egg-info": true,
-    ".cache": true,
-    "build": true,
-    "dist": true
-  },
-  "editor.rulers": [79],
-  "restructuredtext.builtDocumentationPath": "${workspaceRoot}/docs/_build/html",
-  "restructuredtext.confPath": "${workspaceRoot}/docs",
-  "restructuredtext.linter.executablePath": "${workspaceRoot}/.tox/docs/bin/restructuredtext-lint"
-}

CONTRIBUTING.md

@@ -4,18 +4,18 @@ Contributing
 To get started, <a href="https://www.clahub.com/agreements/platformio/platformio-core">sign the Contributor License Agreement</a>.
 
 1. Fork the repository on GitHub.
-2. Make a branch off of ``develop``
-3. Run ``pip install tox``
-4. Go to the root of project where is located ``tox.ini`` and run ``tox -e develop``
+2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git`
+3. Run `pip install tox`
+4. Go to the root of project where is located `tox.ini` and run `tox -e py27`
 5. Activate current development environment:
 
-   * Windows: ``.tox\develop\Scripts\activate``
-   * Bash/ZSH: ``source .tox/develop/bin/activate``
-   * Fish: ``source .tox/bin/activate.fish``
+   * Windows: `.tox\py27\Scripts\activate`
+   * Bash/ZSH: `source .tox/py27/bin/activate`
+   * Fish: `source .tox/py27/bin/activate.fish`
 
 6. Make changes to code, documentation, etc.
-7. Lint source code ``tox -e lint``
-8. Run the tests ``tox -e py27``
-9. Build documentation ``tox -e docs`` (creates a directory _build under docs where you can find the html)
+7. Lint source code `make lint`
+8. Run the tests `make test`
+9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html)
 10. Commit changes to your forked repository
 11. Submit a Pull Request on GitHub.

HISTORY.rst

@@ -1,17 +1,312 @@
 Release Notes
 =============
 
-PlatformIO 3.0
---------------
+.. _release_notes_4:
+
+PlatformIO Core 4
+-----------------
+
+4.3.0 (2020-03-19)
+~~~~~~~~~~~~~~~~~~
+
+* Initial support for an official `PlatformIO for CLion IDE <https://docs.platformio.org/page/integration/ide/clion.html>`__ plugin:
+
+  - Smart C and C++ editor
+  - Code refactoring
+  - On-the-fly code analysis
+  - "New PlatformIO Project" wizard
+  - Building, Uploading, Testing
+  - Integrated debugger (inline variable view, conditional breakpoints, expressions, watchpoints, peripheral registers, multi-thread support, etc.)
+
+* `Device Monitor 2.0 <https://docs.platformio.org/page/core/userguide/device/cmd_monitor.html>`__
+
+  - Added **PlatformIO Device Monitor Filter API** (dev-platforms can extend base device monitor with a custom functionality, such as exception decoding) (`pull #3383 <https://github.com/platformio/platformio-core/pull/3383>`_)
+  - Configure project device monitor with `monitor_filters <https://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-filters>`__ option
+  - `Capture device monitor output to a file <https://docs.platformio.org/page/core/userguide/device/cmd_monitor.html#capture-output-to-a-file>`__ with ``log2file`` filter (`issue #670 <https://github.com/platformio/platformio-core/issues/670>`_)
+  - Show a timestamp for each new line with ``time`` filter (`issue #981 <https://github.com/platformio/platformio-core/issues/981>`_)
+  - Send a text to device on ENTER with ``send_on_enter`` filter (`issue #926 <https://github.com/platformio/platformio-core/issues/926>`_)
+  - Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter
+
+* New standalone (1-script) `PlatformIO Core Installer <https://github.com/platformio/platformio-core-installer>`_
+* Initial support for `Renode <https://docs.platformio.org/page/plus/debug-tools/qemu.html>`__ simulation framework (`issue #3401 <https://github.com/platformio/platformio-core/issues/3401>`_)
+* Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 <https://github.com/platformio/platformio-core/issues/3400>`_)
+* Improved support for Arduino "library.properties" ``depends`` field
+* Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server
+* Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 <https://github.com/platformio/platformio-core/issues/3396>`_)
+* Fixed an issue when Python 2 does not keep encoding when converting ".ino" (`issue #3393 <https://github.com/platformio/platformio-core/issues/3393>`_)
+* Fixed an issue when ``"libArchive": false`` in "library.json" does not work (`issue #3403 <https://github.com/platformio/platformio-core/issues/3403>`_)
+* Fixed an issue when not all commands in `compilation database "compile_commands.json" <https://docs.platformio.org/page/integration/compile_commands.html>`__ use absolute paths (`pull #3415 <https://github.com/platformio/platformio-core/pull/3415>`_)
+* Fixed an issue when unknown transport is used for `PIO Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html>`__ engine (`issue #3422 <https://github.com/platformio/platformio-core/issues/3422>`_)
+
+4.2.1 (2020-02-17)
+~~~~~~~~~~~~~~~~~~
+
+* Improved VSCode template with special ``forceInclude`` field for direct includes via ``-include`` flag (`issue #3379 <https://github.com/platformio/platformio-core/issues/3379>`_)
+* Improved support of PIO Home on card-sized PC (Raspberry Pi, etc.) (`issue #3313 <https://github.com/platformio/platformio-core/issues/3313>`_)
+* Froze "marshmallow" dependency to 2.X for Python 2 (`issue #3380 <https://github.com/platformio/platformio-core/issues/3380>`_)
+* Fixed "TypeError: unsupported operand type(s)" when system environment variable is used by project configuration parser (`issue #3377 <https://github.com/platformio/platformio-core/issues/3377>`_)
+* Fixed an issue when Library Dependency Finder (LDF) ignores custom "libLDFMode" and "libCompatMode" options in `library.json <http://docs.platformio.org/page/librarymanager/config.html>`__
+* Fixed an issue when generating of compilation database "compile_commands.json" does not work with Python 2.7 (`issue #3378 <https://github.com/platformio/platformio-core/issues/3378>`_)
+
+
+4.2.0 (2020-02-12)
+~~~~~~~~~~~~~~~~~~
+
+* `PlatformIO Home 3.1 <http://docs.platformio.org/page/home/index.html>`__:
+
+  - Project Manager
+  - Project Configuration UI for `"platformio.ini" <https://docs.platformio.org/page/projectconf.html>`__
+
+* `PIO Check <http://docs.platformio.org/page/plus/pio-check.html>`__ – automated code analysis without hassle:
+
+  - Added support for `PVS-Studio <https://docs.platformio.org/page/plus/check-tools/pvs-studio.html>`__ static code analyzer
+
+* Initial support for `Project Manager <https://docs.platformio.org/page/userguide/project/index.html>`_ CLI:
+
+  - Show computed project configuration with a new `platformio project config <https://docs.platformio.org/page/userguide/project/cmd_config.html>`_ command or dump to JSON with ``platformio project config --json-output`` (`issue #3335 <https://github.com/platformio/platformio-core/issues/3335>`_)
+  - Moved ``platformio init`` command to `platformio project init <https://docs.platformio.org/page/userguide/project/cmd_init.html>`_
+
+* Generate `compilation database "compile_commands.json" <https://docs.platformio.org/page/integration/compile_commands.html>`__ (`issue #2990 <https://github.com/platformio/platformio-core/issues/2990>`_)
+* Control debug flags and optimization level with a new `debug_build_flags <https://docs.platformio.org/page/projectconf/section_env_debug.html#debug-build-flags>`__ option
+* Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install <https://docs.platformio.org/page/userguide/platforms/cmd_install.html>`__ command  (`issue #3345 <https://github.com/platformio/platformio-core/issues/3345>`_)
+* Added support for "pythonPackages" in `platform.json <https://docs.platformio.org/page/platforms/creating_platform.html#manifest-file-platform-json>`__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed)
+* Handle project configuration (monitor, test, and upload options) for PIO Remote commands (`issue #2591 <https://github.com/platformio/platformio-core/issues/2591>`_)
+* Added support for Arduino's library.properties ``depends`` field (`issue #2781 <https://github.com/platformio/platformio-core/issues/2781>`_)
+* Autodetect monitor port for boards with specified HWIDs (`issue #3349 <https://github.com/platformio/platformio-core/issues/3349>`_)
+* Updated SCons tool to 3.1.2
+* Updated Unity tool to 2.5.0
+* Made package ManifestSchema compatible with marshmallow >= 3 (`issue #3296 <https://github.com/platformio/platformio-core/issues/3296>`_)
+* Warn about broken library manifest when scanning dependencies (`issue #3268 <https://github.com/platformio/platformio-core/issues/3268>`_)
+* Do not overwrite custom items in VSCode's "extensions.json" (`issue #3374 <https://github.com/platformio/platformio-core/issues/3374>`_)
+* Fixed an issue when ``env.BoardConfig()`` does not work for custom boards in extra scripts of libraries (`issue #3264 <https://github.com/platformio/platformio-core/issues/3264>`_)
+* Fixed an issue with "start-group/end-group" linker flags on Native development platform (`issue #3282 <https://github.com/platformio/platformio-core/issues/3282>`_)
+* Fixed default PIO Unified Debugger configuration for `J-Link probe <http://docs.platformio.org/page/plus/debug-tools/jlink.html>`__
+* Fixed an issue with LDF when header files not found if "libdeps_dir" is within a subdirectory of "lib_extra_dirs" (`issue #3311 <https://github.com/platformio/platformio-core/issues/3311>`_)
+* Fixed an issue "Import of non-existent variable 'projenv''" when development platform does not call "env.BuildProgram()" (`issue #3315 <https://github.com/platformio/platformio-core/issues/3315>`_)
+* Fixed an issue when invalid CLI command does not return non-zero exit code
+* Fixed an issue when Project Inspector crashes when flash use > 100% (`issue #3368 <https://github.com/platformio/platformio-core/issues/3368>`_)
+* Fixed a "UnicodeDecodeError" when listing built-in libraries on macOS with Python 2.7 (`issue #3370 <https://github.com/platformio/platformio-core/issues/3370>`_)
+* Fixed an issue with improperly handled compiler flags with space symbols in VSCode template (`issue #3364 <https://github.com/platformio/platformio-core/issues/3364>`_)
+* Fixed an issue when no error is raised if referred parameter (interpolation) is missing in a project configuration file (`issue #3279 <https://github.com/platformio/platformio-core/issues/3279>`_)
+
+
+4.1.0 (2019-11-07)
+~~~~~~~~~~~~~~~~~~
+
+* `PIO Check <http://docs.platformio.org/page/plus/pio-check.html>`__ – automated code analysis without hassle:
+
+  - Potential NULL pointer dereferences
+  - Possible indexing beyond array bounds
+  - Suspicious assignments
+  - Reads of potentially uninitialized objects
+  - Unused variables or functions
+  - Out of scope memory usage.
+
+* `PlatformIO Home 3.0 <http://docs.platformio.org/page/home/index.html>`__:
+
+  - Project Inspection
+  - Static Code Analysis
+  - Firmware File Explorer
+  - Firmware Memory Inspection
+  - Firmware Sections & Symbols Viewer.
+
+* Added support for `Build Middlewares <http://docs.platformio.org/page/projectconf/advanced_scripting.html#build-middlewares>`__: configure custom build flags per specific file, skip any build nodes from a framework, replace build file with another on-the-fly, etc.
+* Extend project environment configuration in "platformio.ini" with other sections using a new `extends <http://docs.platformio.org/page/projectconf/section_env_advanced.html#extends>`__ option (`issue #2953 <https://github.com/platformio/platformio-core/issues/2953>`_)
+* Generate ``.ccls`` LSP file for `Emacs <https://docs.platformio.org/page/ide/emacs.html>`__ cross references, hierarchies, completion and semantic highlighting
+* Added ``--no-ansi`` flag for `PIO Core <http://docs.platformio.org/page/userguide/index.html>`__ to disable ANSI control characters
+* Added ``--shutdown-timeout`` option to `PIO Home Server <http://docs.platformio.org/page/userguide/cmd_home.html>`__
+* Fixed an issue with project generator for `CLion IDE <http://docs.platformio.org/page/ide/clion.html>`__ when 2 environments were used (`issue #2824 <https://github.com/platformio/platformio-core/issues/2824>`_)
+* Fixed default PIO Unified Debugger configuration for `J-Link probe <http://docs.platformio.org/page/plus/debug-tools/jlink.html>`__
+* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 <https://github.com/platformio/platformio-core/issues/3034>`_)
+* Fixed an issue when installing a package using custom Git tag and submodules were not updated correctly (`issue #3060 <https://github.com/platformio/platformio-core/issues/3060>`_)
+* Fixed an issue with linking process when ``$LDSCRIPT`` contains a space in path
+* Fixed security issue when extracting items from TAR archive (`issue #2995 <https://github.com/platformio/platformio-core/issues/2995>`_)
+* Fixed an issue with project generator when ``src_build_flags`` were not respected (`issue #3137 <https://github.com/platformio/platformio-core/issues/3137>`_)
+* Fixed an issue when booleans in "platformio.ini" are not parsed properly (`issue #3022 <https://github.com/platformio/platformio-core/issues/3022>`_)
+* Fixed an issue with invalid encoding when generating project for Visual Studio (`issue #3183 <https://github.com/platformio/platformio-core/issues/3183>`_)
+* Fixed an issue when Project Config Parser does not remove in-line comments when Python 3 is used (`issue #3213 <https://github.com/platformio/platformio-core/issues/3213>`_)
+* Fixed an issue with a GCC Linter for PlatformIO IDE for Atom (`issue #3218 <https://github.com/platformio/platformio-core/issues/3218>`_)
+
+4.0.3 (2019-08-30)
+~~~~~~~~~~~~~~~~~~
+
+* Added support for multi-environment PlatformIO project for `CLion IDE <http://docs.platformio.org/page/ide/clion.html>`__ (`issue #2824 <https://github.com/platformio/platformio-core/issues/2824>`_)
+* Generate ``.ccls`` LSP file for `Vim <http://docs.platformio.org/en/page/vim.html>`__ cross references, hierarchies, completion and semantic highlighting (`issue #2952 <https://github.com/platformio/platformio-core/issues/2952>`_)
+* Added support for `PLATFORMIO_DISABLE_COLOR <http://docs.platformio.org/page/envvars.html#envvar-PLATFORMIO_DISABLE_COLOR>`__ system environment variable which disables color ANSI-codes in a terminal output (`issue #2956 <https://github.com/platformio/platformio-core/issues/2956>`_)
+* Updated SCons tool to 3.1.1
+* Remove ProjectConfig cache when "platformio.ini" was modified outside
+* Fixed an issue with PIO Unified Debugger on Windows OS when debug server is piped
+* Fixed an issue when `--upload-port <http://docs.platformio.org/page/userguide/cmd_run.html#cmdoption-platformio-run-upload-port>`__ CLI flag does not override declared  `upload_port <http://docs.platformio.org/page/projectconf/section_env_upload.html#upload-port>`__ option in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+
+4.0.2 (2019-08-23)
+~~~~~~~~~~~~~~~~~~
+
+* Fixed an issue with a broken `LDF <http://docs.platformio.org/page/librarymanager/ldf.html>`__ when checking for framework compatibility (`issue #2940 <https://github.com/platformio/platformio-core/issues/2940>`_)
+
+4.0.1 (2019-08-22)
+~~~~~~~~~~~~~~~~~~
+
+* Print `debug tool <http://docs.platformio.org/page/plus/debugging.html#tools-debug-probes>`__ name for the active debugging session
+* Do not shutdown PIO Home Server for "upgrade" operations (`issue #2784 <https://github.com/platformio/platformio-core/issues/2784>`_)
+* Improved computing of project check sum (structure, configuration) and avoid unnecessary rebuilding
+* Improved printing of tabulated results
+* Automatically normalize file system paths to UNIX-style for Project Generator (`issue #2857 <https://github.com/platformio/platformio-core/issues/2857>`_)
+* Ability to set "databaseFilename" for VSCode and C/C++ extension (`issue #2825 <https://github.com/platformio/platformio-core/issues/2825>`_)
+* Renamed "enable_ssl" setting to `strict_ssl <http://docs.platformio.org/page/userguide/cmd_settings.html#strict-ssl>`__
+* Fixed an issue with incorrect escaping of Windows slashes when using `PIO Unified Debugger <http://docs.platformio.org/page/plus/debugging.html>`__ and "piped" openOCD
+* Fixed an issue when "debug", "home", "run", and "test" commands were not shown in "platformio --help" CLI
+* Fixed an issue with PIO Home's "No JSON object could be decoded" (`issue #2823 <https://github.com/platformio/platformio-core/issues/2823>`_)
+* Fixed an issue when `library.json <http://docs.platformio.org/page/librarymanager/config.html>`__ had priority over project configuration for `LDF <http://docs.platformio.org/page/librarymanager/ldf.html>`__ (`issue #2867 <https://github.com/platformio/platformio-core/issues/2867>`_)
+
+4.0.0 (2019-07-10)
+~~~~~~~~~~~~~~~~~~
+
+`Migration Guide from 3.0 to 4.0 <http://docs.platformio.org/page/migration.html>`__.
+
+* `PlatformIO Plus Goes Open Source <https://community.platformio.org/t/platformio-plus-goes-open-source-improving-embedded-development-community-worldwide/8240/4>`__
+
+  - Built-in `PIO Unified Debugger <http://docs.platformio.org/page/plus/debugging.html>`__
+  - Built-in `PIO Unit Testing <http://docs.platformio.org/page/plus/unit-testing.html>`__
+
+* **Project Configuration**
+
+  - New project configuration parser with a strict options typing (`API <https://github.com/platformio/platformio-core/blob/develop/platformio/project/options.py>`__)
+  - Unified workspace storage (`workspace_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#workspace-dir>`__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 <https://github.com/platformio/platformio-core/issues/1778>`_)
+  - Share common (global) options between project environments using `[env] <http://docs.platformio.org/page/projectconf/section_env.html#global-scope-env>`__ section (`issue #1643 <https://github.com/platformio/platformio-core/issues/1643>`_)
+  - Include external configuration files with `extra_configs <http://docs.platformio.org/page/projectconf/section_platformio.html#extra-configs>`__ option (`issue #1590 <https://github.com/platformio/platformio-core/issues/1590>`_)
+  - Custom project ``***_dir`` options declared in `platformio <http://docs.platformio.org/page/projectconf/section_platformio.html>`__ section have higher priority than `Environment variables <http://docs.platformio.org/page/envvars.html>`__
+  - Added support for Unix shell-style wildcards for `monitor_port <http://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-port>`__ option (`issue #2541 <https://github.com/platformio/platformio-core/issues/2541>`_)
+  - Added new `monitor_flags <http://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-flags>`__ option which allows passing extra flags and options to `platformio device monitor <http://docs.platformio.org/page/userguide/cmd_device.html#cmd-device-monitor>`__ command (`issue #2165 <https://github.com/platformio/platformio-core/issues/2165>`_)
+  - Added support for `PLATFORMIO_DEFAULT_ENVS <http://docs.platformio.org/page/envvars.html#envvar-PLATFORMIO_DEFAULT_ENVS>`__ system environment variable (`issue #1967 <https://github.com/platformio/platformio-core/issues/1967>`_)
+  - Added support for `shared_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#shared-dir>`__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a `PIO Remote <http://docs.platformio.org/page/plus/pio-remote.html>`__ machine
+
+* **Library Management**
+
+  - Switched to workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps``
+  - Save libraries passed to `platformio lib install <http://docs.platformio.org/page/userguide/lib/cmd_install.html>`__ command into the project dependency list (`lib_deps <http://docs.platformio.org/page/projectconf/section_env_library.html#lib-deps>`__) with a new ``--save`` flag (`issue #1028 <https://github.com/platformio/platformio-core/issues/1028>`_)
+  - Install all project dependencies declared via `lib_deps <http://docs.platformio.org/page/projectconf/section_env_library.html#lib-deps>`__ option using a simple `platformio lib install <http://docs.platformio.org/page/userguide/lib/cmd_install.html>`__ command (`issue #2147 <https://github.com/platformio/platformio-core/issues/2147>`_)
+  - Use isolated library dependency storage per project build environment (`issue #1696 <https://github.com/platformio/platformio-core/issues/1696>`_)
+  - Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry (`issue #1654 <https://github.com/platformio/platformio-core/issues/1654>`_)
+  - Override default source and include directories for a library via `library.json <http://docs.platformio.org/page/librarymanager/config.html>`__ manifest using ``includeDir`` and ``srcDir`` fields
+  - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 <https://github.com/platformio/platformio-core/issues/1252>`_)
+  - Fixed an issue when `lib_compat_mode = strict <http://docs.platformio.org/page/librarymanager/ldf.html#ldf-compat-mode>`__ does not ignore libraries incompatible with a project framework
+
+* **Build System**
+
+  - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs``
+  - Switch between `Build Configurations <http://docs.platformio.org/page/projectconf/build_configurations.html>`__ (``release`` and ``debug``) with a new project configuration option `build_type <http://docs.platformio.org/page/projectconf/section_env_build.html#build-type>`__
+  - Custom `platform_packages <http://docs.platformio.org/page/projectconf/section_env_general.html#platform>`__ per a build environment with an option to override default (`issue #1367 <https://github.com/platformio/platformio-core/issues/1367>`_)
+  - Print platform package details, such as version, VSC source and commit (`issue #2155 <https://github.com/platformio/platformio-core/issues/2155>`_)
+  - Control a number of parallel build jobs with a new `-j, --jobs <http://docs.platformio.org/page/userguide/cmd_run.html#cmdoption-platformio-run-j>`__ option
+  - Override default `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ with a custom using ``-c, --project-conf`` option for `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__, `platformio debug <http://docs.platformio.org/page/userguide/cmd_debug.html>`__, or `platformio test <http://docs.platformio.org/page/userguide/cmd_test.html>`__ commands (`issue #1913 <https://github.com/platformio/platformio-core/issues/1913>`_)
+  - Override default development platform upload command with a custom `upload_command <http://docs.platformio.org/page/projectconf/section_env_upload.html#upload-command>`__ (`issue #2599 <https://github.com/platformio/platformio-core/issues/2599>`_)
+  - Configure a shared folder for the derived files (objects, firmwares, ELFs) from a build system using `build_cache_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#build-cache-dir>`__ option (`issue #2674 <https://github.com/platformio/platformio-core/issues/2674>`_)
+  - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 <https://github.com/platformio/platformio-core/issues/2508>`_)
+
+* **Infrastructure**
+
+  - Python 3 support (`issue #895 <https://github.com/platformio/platformio-core/issues/895>`_)
+  - Significantly speedup back-end for PIO Home. It works super fast now!
+  - Added support for the latest Python "Click" package (CLI) (`issue #349 <https://github.com/platformio/platformio-core/issues/349>`_)
+  - Added options to override default locations used by PlatformIO Core (`core_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#core-dir>`__, `globallib_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#globallib-dir>`__, `platforms_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#platforms-dir>`__, `packages_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#packages-dir>`__, `cache_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#cache-dir>`__) (`issue #1615 <https://github.com/platformio/platformio-core/issues/1615>`_)
+  - Removed line-buffering from `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__ command which was leading to omitting progress bar from upload tools (`issue #856 <https://github.com/platformio/platformio-core/issues/856>`_)
+  - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #143 <https://github.com/platformio/platformio-core/issues/143>`_, `issue #1342 <https://github.com/platformio/platformio-core/issues/1342>`_, `issue #1959 <https://github.com/platformio/platformio-core/issues/1959>`_, `issue #2100 <https://github.com/platformio/platformio-core/issues/2100>`_)
+
+* **Integration**
+
+  - Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file
+  - Fixed an issue with hardcoded C standard version when generating project for CLion IDE (`issue #2527 <https://github.com/platformio/platformio-core/issues/2527>`_)
+  - Fixed an issue with Project Generator when an include path search order is inconsistent to what passed to the compiler (`issue #2509 <https://github.com/platformio/platformio-core/issues/2509>`_)
+  - Fixed an issue when generating invalid "Eclipse CDT Cross GCC Built-in Compiler Settings" if a custom `PLATFORMIO_CORE_DIR <http://docs.platformio.org/page/envvars.html#envvar-PLATFORMIO_CORE_DIR>`__ is used (`issue #806 <https://github.com/platformio/platformio-core/issues/806>`_)
+
+* **Miscellaneous**
+
+  - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead
+  - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules <http://docs.platformio.org/page/faq.html#platformio-udev-rules>`__ (`issue #2442 <https://github.com/platformio/platformio-core/issues/2442>`_)
+  - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 <https://github.com/platformio/platformio-core/issues/2559>`_)
+
+PlatformIO Core 3
+-----------------
+
+3.6.7 (2019-04-23)
+~~~~~~~~~~~~~~~~~~
+
+* `PIO Unified Debugger <https://docs.platformio.org/page/plus/debugging.html>`__: improved debugging in ``debug_load_mode = modified`` and fixed an issue with useless project rebuilding
+* Project Generator: fixed a VSCode C/C++'s "Cannot find" warning when CPPPATH folder does not exist
+* Fixed an "IndexError: list index out of range" for Arduino sketch preprocessor
+  (`issue #2268 <https://github.com/platformio/platformio-core/issues/2268>`_)
+* Fixed an issue when invalid "env_default" in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ results into unhandled errors
+  (`issue #2265 <https://github.com/platformio/platformio-core/issues/2265>`_)
+
+3.6.6 (2019-03-29)
+~~~~~~~~~~~~~~~~~~
+
+* Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode
+  (`issue #2243 <https://github.com/platformio/platformio-core/issues/2243>`_)
+* Fixed an issue when PlatformIO Build System does not pick up "mbed_lib.json" files from libraries
+  (`issue #2164 <https://github.com/platformio/platformio-core/issues/2164>`_)
+* Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor)
+* Fixed "FileExistsError" when `platformio ci <https://docs.platformio.org/page/userguide/cmd_ci.html>`__ command is used in pair with ``--keep-build-dir`` option
+* Fixed an issue with incorrect order of project "include" and "src" paths in ``CPPPATH``
+  (`issue #1914 <https://github.com/platformio/platformio-core/issues/1914>`_)
+
+3.6.5 (2019-03-07)
+~~~~~~~~~~~~~~~~~~
+
+* Project Generator: added new targets for CLion IDE "BUILD_VERBOSE" and "MONITOR" (serial port monitor)
+  (`issue #359 <https://github.com/platformio/platformio-core/issues/359>`_)
+* Fixed an issue with slow updating of PlatformIO Core packages on Windows
+* Fixed an issue when `platformio ci <https://docs.platformio.org/page/userguide/cmd_ci.html>`__ recompiles project if ``--keep-build-dir`` option is passed
+  (`issue #2109 <https://github.com/platformio/platformio-core/issues/2109>`_)
+* Fixed an issue when ``$PROJECT_HASH`` template was not expanded for the other directory ``***_dir`` options in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+  (`issue #2170 <https://github.com/platformio/platformio-core/issues/2170>`_)
+
+3.6.4 (2019-01-23)
+~~~~~~~~~~~~~~~~~~
+
+* Improved Project Generator for IDEs:
+
+  - Use full path to PlatformIO CLI when generating a project
+    (`issue #1674 <https://github.com/platformio/platformio-core/issues/1674>`_)
+  - CLion: Improved project portability using "${CMAKE_CURRENT_LIST_DIR}" instead of full path
+  - Eclipse: Provide language standard to a project C/C++ indexer
+    (`issue #1010 <https://github.com/platformio/platformio-core/issues/1010>`_)
+
+* Fixed an issue with incorrect detecting of compatibility (LDF) between generic library and Arduino or ARM mbed frameworks
+* Fixed "Runtime Error: Dictionary size changed during iteration"
+  (`issue #2003 <https://github.com/platformio/platformio-core/issues/2003>`_)
+* Fixed an error "Could not extract item..." when extracting TAR archive with symbolic items on Windows platform
+  (`issue #2015 <https://github.com/platformio/platformio-core/issues/2015>`_)
+
+3.6.3 (2018-12-12)
+~~~~~~~~~~~~~~~~~~
+
+* Ignore ``*.asm`` and ``*.ASM`` files when building Arduino-based library (compatibility with Arduino builder)
+* Fixed spurious project's "Problems" for `PlatformIO IDE for VSCode <http://docs.platformio.org/page/ide/vscode.html>`__ when ARM mbed framework is used
+* Fixed an issue with a broken headers list when generating ".clang_complete" for `Emacs <http://docs.platformio.org/page/ide/emacs.html>`__
+  (`issue #1960 <https://github.com/platformio/platformio-core/issues/1960>`_)
+
+3.6.2 (2018-11-29)
+~~~~~~~~~~~~~~~~~~
+
+* Improved IntelliSense for `PlatformIO IDE for VSCode <http://docs.platformio.org/page/ide/vscode.html>`__ via passing extra compiler information for C/C++ Code Parser (resolves issues with spurious project's "Problems")
+* Fixed an issue with VSCode IntelliSense warning about the missed headers located in `include <http://docs.platformio.org/page/projectconf/section_platformio.html#include-dir>`__ folder
+* Fixed incorrect wording when initializing/updating project
+* Fixed an issue with incorrect order for library dependencies ``CPPPATH``
+  (`issue #1914 <https://github.com/platformio/platformio-core/issues/1914>`_)
+* Fixed an issue when Library Dependency Finder (LDF) does not handle project `src_filter <http://docs.platformio.org/page/projectconf/section_env_build.html#src-filter>`__
+  (`issue #1905 <https://github.com/platformio/platformio-core/issues/1905>`_)
+* Fixed an issue when Library Dependency Finder (LDF) finds spurious dependencies in ``chain+`` and ``deep+`` modes
+  (`issue #1930 <https://github.com/platformio/platformio-core/issues/1930>`_)
 
 3.6.1 (2018-10-29)
 ~~~~~~~~~~~~~~~~~~
 
 * Generate an `include <http://docs.platformio.org/page/projectconf/section_platformio.html#include-dir>`__ and `test <http://docs.platformio.org/page/projectconf/section_platformio.html#test-dir>`__ directories with a README file when initializing a new project
-* Support in-line comments for multi-line value (``lib_deps``, ``build_flags``, etc) in `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
-* Added ``$PROJECT_HASH`` template variable for `build_dir <http://docs.platformio.org/en/latest/projectconf/section_platformio.html#projectconf-pio-build-dir>`__. One of the use cases is setting a global storage for project artifacts using `PLATFORMIO_BUILD_DIR <http://docs.platformio.org/en/latest/envvars.html#envvar-PLATFORMIO_BUILD_DIR>`__ system environment variable. For example, ``/tmp/pio-build/$PROJECT_HASH`` (Unix) or ``%TEMP%/pio-build/$PROJECT_HASH`` (Windows)
+* Support in-line comments for multi-line value (``lib_deps``, ``build_flags``, etc) in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+* Added ``$PROJECT_HASH`` template variable for `build_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#projectconf-pio-build-dir>`__. One of the use cases is setting a global storage for project artifacts using `PLATFORMIO_BUILD_DIR <http://docs.platformio.org/page/envvars.html#envvar-PLATFORMIO_BUILD_DIR>`__ system environment variable. For example, ``/tmp/pio-build/$PROJECT_HASH`` (Unix) or ``$[sysenv.TEMP}/pio-build/$PROJECT_HASH`` (Windows)
 * Improved a loading speed of PIO Home "Recent News"
-* Improved `PIO Unified Debugger <https://docs.platformio.org/en/page/plus/debugging.html>`__ for "mbed" framework and fixed issue with missed local variables
+* Improved `PIO Unified Debugger <https://docs.platformio.org/page/plus/debugging.html>`__ for "mbed" framework and fixed issue with missed local variables
 * Introduced `"Release" and "Debug" Build Configurations <http://docs.platformio.org/page/projectconf/build_configurations.html>`__
 * Build project in "Debug Mode" including debugging information with a new ``debug`` target using `platformio run <https://docs.platformio.org/page/userguide/cmd_run.html>`__ command or `targets <http://docs.platformio.org/page/projectconf/section_env_general.html#targets>`__ option in ``platformio.ini``. The last option allows avoiding project rebuilding between "Run/Debug" modes.
   (`issue #1833 <https://github.com/platformio/platformio-core/issues/1833>`_)
@@ -29,7 +324,6 @@ PlatformIO 3.0
   (`issue #1873 <https://github.com/platformio/platformio-core/issues/1873>`_)
 * Fixed an issue with incorrect handling of a custom package name when using `platformio lib install <http://docs.platformio.org/page/userguide/lib/cmd_install.html>`__ or `platformio platform install <http://docs.platformio.org/page/userguide/platforms/cmd_install.html>`__ commands
 
-
 3.6.0 (2018-08-06)
 ~~~~~~~~~~~~~~~~~~
 
@@ -70,7 +364,7 @@ PlatformIO 3.0
   build environment
   (`issue #1665 <https://github.com/platformio/platformio-core/issues/1665>`_)
 * Handle "architectures" data from "library.properties" manifest in
-  `lib_compat_mode = strict <https://docs.platformio.org/en/page/librarymanager/ldf.html#compatibility-mode>`__
+  `lib_compat_mode = strict <https://docs.platformio.org/page/librarymanager/ldf.html#compatibility-mode>`__
 * Added workaround for Python SemVer package's `issue #61 <https://github.com/rbarrois/python-semanticversion/issues/61>`_ with caret range and pre-releases
 * Replaced conflicted "env" pattern by "sysenv" for `"platformio.ini" Dynamic Variables" <https://docs.platformio.org/page/projectconf/dynamic_variables.html>`__
   (`issue #1705 <https://github.com/platformio/platformio-core/issues/1705>`_)
@@ -94,11 +388,11 @@ PlatformIO 3.0
 
 * Simplify configuration for `PIO Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html>`__: separate main program from a test build process, drop
   requirement for ``#ifdef UNIT_TEST`` guard
-* Override any option from board manifest in `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf/section_env_board.html#more-options>`__
+* Override any option from board manifest in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf/section_env_board.html#more-options>`__
   (`issue #1612 <https://github.com/platformio/platformio-core/issues/1612>`_)
 * Configure a custom path to SVD file using `debug_svd_path <https://docs.platformio.org/page/projectconf/section_env_debug.html#debug-svd-path>`__
   option
-* Custom project `description <https://docs.platformio.org/en/page/projectconf/section_platformio.html#description>`_
+* Custom project `description <https://docs.platformio.org/page/projectconf/section_platformio.html#description>`_
   which will be used by `PlatformIO Home <https://docs.platformio.org/page/home/index.html>`_
 * Updated Unity tool to 2.4.3
 * Improved support for Black Magic Probe in "uploader" mode
@@ -124,9 +418,9 @@ PlatformIO 3.0
   - Multiple themes (Dark & Light)
   - Ability to specify a name for new project
 
-* Control `PIO Unified Debugger <https://docs.platformio.org/en/page/plus/debugging.html>`__
+* Control `PIO Unified Debugger <https://docs.platformio.org/page/plus/debugging.html>`__
   and its firmware loading mode using
-  `debug_load_mode <https://docs.platformio.org/en/page/projectconf/section_env_debug.html#debug-load-mode>`__ option
+  `debug_load_mode <https://docs.platformio.org/page/projectconf/section_env_debug.html#debug-load-mode>`__ option
 * Added aliases (off, light, strict) for
   `LDF Compatibility Mode <https://docs.platformio.org/page/librarymanager/ldf.html>`__
 * Search for a library using PIO Library Registry ID ``id:X`` (e.g. ``pio lib search id:13``)
@@ -157,7 +451,7 @@ PlatformIO 3.0
   (`issue #1274 <https://github.com/platformio/platformio-core/issues/1274>`_)
 * Configure a custom firmware/program name in build directory (`example <https://docs.platformio.org/page/projectconf/advanced_scripting.html#custom-firmware-program-name>`__)
 * Renamed ``envs_dir`` option to ``build_dir``
-  in `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf/section_platformio.html#build-dir>`__
+  in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf/section_platformio.html#build-dir>`__
 * Refactored code without "arrow" dependency (resolve issue with "ImportError:
   No module named backports.functools_lru_cache")
 * Improved support of PIO Unified Debugger for Eclipse Oxygen
@@ -192,7 +486,7 @@ PlatformIO 3.0
   folder for project's header files
   (`issue #1107 <https://github.com/platformio/platformio-core/issues/1107>`_)
 * Depend on development platform using VCS URL (Git, Mercurial and Subversion)
-  instead of a name in `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf/section_env_general.html#platform>`__.
+  instead of a name in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf/section_env_general.html#platform>`__.
   Drop support for ``*_stage`` dev/platform names (use VCS URL instead).
 * Reinstall/redownload package with a new ``-f, --force`` option for
   `platformio lib install <https://docs.platformio.org/page/userguide/lib/cmd_install.html>`__
@@ -213,7 +507,7 @@ PlatformIO 3.0
   - Parse library source file in pair with a header when they have the same name
     (`issue #1175 <https://github.com/platformio/platformio-core/issues/1175>`_)
   - Handle library dependencies defined as VCS or SemVer in
-    `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf/section_env_general.html#platform>`__
+    `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf/section_env_general.html#platform>`__
     (`issue #1155 <https://github.com/platformio/platformio-core/issues/1155>`_)
   - Added option to configure library `Compatible Mode <https://docs.platformio.org/page/librarymanager/ldf.html#compatibility-mode>`__
     using `library.json <https://docs.platformio.org/page/librarymanager/config.html>`__
@@ -249,7 +543,7 @@ PlatformIO 3.0
 * Use a root of library when filtering source code using
   `library.json <https://docs.platformio.org/page/librarymanager/config.html>`__
   and ``srcFilter`` field
-* Added ``monitor_*`` options to white-list for `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+* Added ``monitor_*`` options to white-list for `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
   (`issue #982 <https://github.com/platformio/platformio-core/issues/982>`_)
 * Do not ask for board ID when initialize project for desktop platform
 * Handle broken PIO Core state and create new one
@@ -270,18 +564,18 @@ PlatformIO 3.0
   - Integration with `Eclipse <https://docs.platformio.org/page/ide/eclipse.html>`__ and `Sublime Text <https://docs.platformio.org/page/ide/sublimetext.html>`__
 
 * Filter `PIO Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html>`__
-  tests using a new ``test_filter`` option in `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+  tests using a new ``test_filter`` option in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
   or `platformio test --filter <https://docs.platformio.org/page/userguide/cmd_test.html#cmdoption-platformio-test-f>`__ command
   (`issue #934 <https://github.com/platformio/platformio-core/issues/934>`_)
 * Custom ``test_transport`` for `PIO Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html>`__ Engine
-* Configure Serial Port Monitor in `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+* Configure Serial Port Monitor in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
   (`issue #787 <https://github.com/platformio/platformio-core/issues/787>`_)
 * New `monitor <https://docs.platformio.org/page/userguide/cmd_run.html#cmdoption-platformio-run-t>`__
   target which allows to launch Serial Monitor automatically after successful
   "build" or "upload" operations
   (`issue #788 <https://github.com/platformio/platformio-core/issues/788>`_)
 * Project generator for `VIM <https://docs.platformio.org/page/ide/vim.html>`__
-* Multi-line support for the different options in `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__,
+* Multi-line support for the different options in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__,
   such as: ``build_flags``, ``build_unflags``, etc.
   (`issue #889 <https://github.com/platformio/platformio-core/issues/889>`_)
 * Handle dynamic ``SRC_FILTER`` environment variable from
@@ -293,7 +587,7 @@ PlatformIO 3.0
   that were installed from repository
 * Add support for ``.*cc`` extension
   (`issue #939 <https://github.com/platformio/platformio-core/issues/939>`_)
-* Handle ``env_default`` in `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+* Handle ``env_default`` in `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
   when re-initializing a project
   (`issue #950 <https://github.com/platformio/platformio-core/issues/950>`_)
 * Use root directory for PIO Home when path contains non-ascii characters
@@ -391,10 +685,10 @@ PlatformIO 3.0
   (`issue #808 <https://github.com/platformio/platformio-core/issues/808>`_,
   `issue #467 <https://github.com/platformio/platformio-core/issues/467>`_)
 * Inject system environment variables to configuration settings in
-  `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+  `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
   (`issue #792 <https://github.com/platformio/platformio-core/issues/792>`_)
 * Custom boards per project with ``boards_dir`` option in
-  `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+  `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
   (`issue #515 <https://github.com/platformio/platformio-core/issues/515>`_)
 * Unix shell-style wildcards for `upload_port <https://docs.platformio.org/page/projectconf/section_env_upload.html#upload-port>`_
   (`issue #839 <https://github.com/platformio/platformio-core/issues/839>`_)
@@ -405,7 +699,7 @@ PlatformIO 3.0
 * Added new `LDF Modes <https://docs.platformio.org/page/librarymanager/ldf.html#ldf-mode>`__:
   ``chain+`` and ``deep+`` and set ``chain+`` as default
 * Added global ``lib_extra_dirs`` option to ``[platformio]`` section for
-  `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+  `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
   (`issue #842 <https://github.com/platformio/platformio-core/issues/842>`_)
 * Enabled caching by default for API requests and Library Manager (see `enable_cache <https://docs.platformio.org/page/userguide/cmd_settings.html#enable-cache>`__ setting)
 * Native integration with VIM/Neovim using `neomake-platformio <https://github.com/coddingtonbear/neomake-platformio>`__ plugin
@@ -428,7 +722,7 @@ PlatformIO 3.0
 3.1.0 (2016-09-19)
 ~~~~~~~~~~~~~~~~~~
 
-* New! Dynamic variables/templates for `“platformio.ini” (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
+* New! Dynamic variables/templates for `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__
   (`issue #705 <https://github.com/platformio/platformio-core/issues/705>`_)
 * Summary about processed environments
   (`issue #777 <https://github.com/platformio/platformio-core/issues/777>`_)
@@ -532,8 +826,8 @@ PlatformIO 3.0
   (`issue #742 <https://github.com/platformio/platformio-core/issues/742>`_)
 * Stopped supporting Python 2.6
 
-PlatformIO 2.0
---------------
+PlatformIO Core 2
+-----------------
 
 2.11.2 (2016-08-02)
 ~~~~~~~~~~~~~~~~~~~
@@ -1318,8 +1612,8 @@ PlatformIO 2.0
 * Fixed bug with creating copies of source files
   (`issue #177 <https://github.com/platformio/platformio-core/issues/177>`_)
 
-PlatformIO 1.0
---------------
+PlatformIO Core 1
+-----------------
 
 1.5.0 (2015-05-15)
 ~~~~~~~~~~~~~~~~~~
@@ -1509,8 +1803,8 @@ PlatformIO 1.0
   error (`issue #81 <https://github.com/platformio/platformio-core/issues/81>`_)
 * Several bug fixes, increased stability and performance improvements
 
-PlatformIO 0.0
---------------
+PlatformIO Core Preview
+-----------------------
 
 0.10.2 (2015-01-06)
 ~~~~~~~~~~~~~~~~~~~

MANIFEST.in

@@ -0,0 +1 @@
+include LICENSE

Makefile

@@ -1,4 +1,3 @@
-
 lint:
 	pylint --rcfile=./.pylintrc ./platformio
 
@@ -6,13 +5,14 @@ isort:
 	isort -rc ./platformio
 	isort -rc ./tests
 
-yapf:
-	yapf --recursive --in-place platformio/
+format:
+	black --target-version py27 ./platformio
+	black --target-version py27 ./tests
 
 test:
-	py.test -v -s -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py
+	py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
 
-before-commit: isort yapf lint test
+before-commit: isort format lint test
 
 clean-docs:
 	rm -rf docs/_build
@@ -23,4 +23,12 @@ clean: clean-docs
 	rm -rf .cache
 	rm -rf build
 	rm -rf htmlcov
-	rm -f .coverage
+	rm -f .coverage
+
+profile:
+	# Usage $ > make PIOARGS="boards" profile
+	python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS}
+	snakeviz .tox/.tmp/cprofile.prof
+
+publish:
+	python setup.py sdist upload

README.rst

@@ -10,20 +10,16 @@ PlatformIO
 .. image:: https://img.shields.io/pypi/v/platformio.svg
     :target: https://pypi.python.org/pypi/platformio/
     :alt: Latest Version
-.. image:: https://img.shields.io/pypi/l/platformio.svg
+.. image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
     :target: https://pypi.python.org/pypi/platformio/
     :alt:  License
-.. image:: https://img.shields.io/PlatformIO/Community.png
+.. image:: https://img.shields.io/badge/PlatformIO-Community-orange.svg
    :alt: Community Forums
    :target: https://community.platformio.org?utm_source=github&utm_medium=core
-.. image:: https://img.shields.io/PIO/Plus.png?color=orange
-   :alt: PIO Plus: Professional solutions for an awesome open source PlatformIO ecosystem
-   :target: https://platformio.org/pricing?utm_source=github&utm_medium=core
 
 **Quick Links:** `Web <https://platformio.org?utm_source=github&utm_medium=core>`_ |
-`PIO Plus <https://platformio.org/pricing?utm_source=github&utm_medium=core>`_ |
 `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_ |
-`Project Examples <https://github.com/platformio/platformio-examples/>`_ |
+`Project Examples <https://github.com/platformio/platformio-examples/>`__ |
 `Docs <https://docs.platformio.org?utm_source=github&utm_medium=core>`_ |
 `Donate <https://platformio.org/donate?utm_source=github&utm_medium=core>`_ |
 `Contact Us <https://platformio.org/contact?utm_source=github&utm_medium=core>`_
@@ -38,34 +34,37 @@ PlatformIO
 .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png
     :target: https://platformio.org?utm_source=github&utm_medium=core
 
-`PlatformIO <https://platformio.org?utm_source=github&utm_medium=core>`_ is an open source ecosystem for IoT
-development. Cross-platform IDE and unified debugger. Remote unit testing and
-firmware updates.
+`PlatformIO <https://platformio.org?utm_source=github&utm_medium=core>`_ a new generation ecosystem for embedded development
+
+* Open source, maximum permissive Apache 2.0 license
+* Cross-platform IDE and Unified Debugger
+* Static Code Analyzer and Remote Unit Testing
+* Multi-platform and Multi-architecture Build System
+* Firmware File Explorer and Memory Inspection.
 
 Get Started
 -----------
 
 * `What is PlatformIO? <https://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
 
-Open Source
+Instruments
 -----------
 
 * `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_
 * `PlatformIO Core (CLI) <https://docs.platformio.org/en/latest/core.html?utm_source=github&utm_medium=core>`_
 * `Library Management <https://docs.platformio.org/page/librarymanager/index.html?utm_source=github&utm_medium=core>`_
-* `Project Examples <https://github.com/platformio/platformio-examples?utm_source=github&utm_medium=core>`_
+* `Project Examples <https://github.com/platformio/platformio-examples?utm_source=github&utm_medium=core>`__
 * `Desktop IDEs Integration <https://docs.platformio.org/page/ide.html?utm_source=github&utm_medium=core>`_
 * `Continuous Integration <https://docs.platformio.org/page/ci/index.html?utm_source=github&utm_medium=core>`_
 * `Advanced Scripting API <https://docs.platformio.org/page/projectconf/advanced_scripting.html?utm_source=github&utm_medium=core>`_
 
-PIO Plus
---------
+Professional
+------------
 
+* `PIO Check <https://docs.platformio.org/page/plus/pio-check.html?utm_source=github&utm_medium=core>`_
 * `PIO Remote <https://docs.platformio.org/page/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
 * `PIO Unified Debugger <https://docs.platformio.org/page/plus/debugging.html?utm_source=github&utm_medium=core>`_
 * `PIO Unit Testing <https://docs.platformio.org/en/latest/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
-* `Cloud IDEs Integration <https://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core#solution-pio-delivery>`_
-* `Integration Services <https://platformio.org/pricing?utm_source=github&utm_medium=core#enterprise-features>`_
 
 Registry
 --------
@@ -78,6 +77,7 @@ Registry
 Development Platforms
 ---------------------
 
+* `Aceinna IMU <https://platformio.org/platforms/aceinna_imu?utm_source=github&utm_medium=core>`_
 * `Atmel AVR <https://platformio.org/platforms/atmelavr?utm_source=github&utm_medium=core>`_
 * `Atmel SAM <https://platformio.org/platforms/atmelsam?utm_source=github&utm_medium=core>`_
 * `Espressif 32 <https://platformio.org/platforms/espressif32?utm_source=github&utm_medium=core>`_
@@ -86,16 +86,20 @@ Development Platforms
 * `Infineon XMC <https://platformio.org/platforms/infineonxmc?utm_source=github&utm_medium=core>`_
 * `Intel ARC32 <https://platformio.org/platforms/intel_arc32?utm_source=github&utm_medium=core>`_
 * `Intel MCS-51 (8051) <https://platformio.org/platforms/intel_mcs51?utm_source=github&utm_medium=core>`_
+* `Kendryte K210 <https://platformio.org/platforms/kendryte210?utm_source=github&utm_medium=core>`_
 * `Lattice iCE40 <https://platformio.org/platforms/lattice_ice40?utm_source=github&utm_medium=core>`_
 * `Maxim 32 <https://platformio.org/platforms/maxim32?utm_source=github&utm_medium=core>`_
 * `Microchip PIC32 <https://platformio.org/platforms/microchippic32?utm_source=github&utm_medium=core>`_
 * `Nordic nRF51 <https://platformio.org/platforms/nordicnrf51?utm_source=github&utm_medium=core>`_
 * `Nordic nRF52 <https://platformio.org/platforms/nordicnrf52?utm_source=github&utm_medium=core>`_
+* `Nuclei <https://platformio.org/platforms/nuclei?utm_source=github&utm_medium=core>`_
 * `NXP LPC <https://platformio.org/platforms/nxplpc?utm_source=github&utm_medium=core>`_
 * `RISC-V <https://platformio.org/platforms/riscv?utm_source=github&utm_medium=core>`_
-* `Samsung ARTIK <https://platformio.org/platforms/samsung_artik?utm_source=github&utm_medium=core>`_
+* `RISC-V GAP <https://platformio.org/platforms/riscv_gap?utm_source=github&utm_medium=core>`_
+* `Shakti <https://platformio.org/platforms/shakti?utm_source=github&utm_medium=core>`_
 * `Silicon Labs EFM32 <https://platformio.org/platforms/siliconlabsefm32?utm_source=github&utm_medium=core>`_
 * `ST STM32 <https://platformio.org/platforms/ststm32?utm_source=github&utm_medium=core>`_
+* `ST STM8 <https://platformio.org/platforms/ststm8?utm_source=github&utm_medium=core>`_
 * `Teensy <https://platformio.org/platforms/teensy?utm_source=github&utm_medium=core>`_
 * `TI MSP430 <https://platformio.org/platforms/timsp430?utm_source=github&utm_medium=core>`_
 * `TI Tiva <https://platformio.org/platforms/titiva?utm_source=github&utm_medium=core>`_
@@ -105,26 +109,40 @@ Frameworks
 ----------
 
 * `Arduino <https://platformio.org/frameworks/arduino?utm_source=github&utm_medium=core>`_
-* `ARTIK SDK <https://platformio.org/frameworks/artik-sdk?utm_source=github&utm_medium=core>`_
 * `CMSIS <https://platformio.org/frameworks/cmsis?utm_source=github&utm_medium=core>`_
-* `Energia <https://platformio.org/frameworks/energia?utm_source=github&utm_medium=core>`_
 * `ESP-IDF <https://platformio.org/frameworks/espidf?utm_source=github&utm_medium=core>`_
 * `ESP8266 Non-OS SDK <https://platformio.org/frameworks/esp8266-nonos-sdk?utm_source=github&utm_medium=core>`_
 * `ESP8266 RTOS SDK <https://platformio.org/frameworks/esp8266-rtos-sdk?utm_source=github&utm_medium=core>`_
+* `Freedom E SDK <https://platformio.org/frameworks/freedom-e-sdk?utm_source=github&utm_medium=core>`_
+* `GigaDevice GD32V SDK <https://platformio.org/frameworks/gd32vf103-sdk?utm_source=github&utm_medium=core>`_
+* `Kendryte Standalone SDK <https://platformio.org/frameworks/kendryte-standalone-sdk?utm_source=github&utm_medium=core>`_
+* `Kendryte FreeRTOS SDK <https://platformio.org/frameworks/kendryte-freertos-sdk?utm_source=github&utm_medium=core>`_
 * `libOpenCM3 <https://platformio.org/frameworks/libopencm3?utm_source=github&utm_medium=core>`_
-* `mbed <https://platformio.org/frameworks/mbed?utm_source=github&utm_medium=core>`_
+* `Mbed <https://platformio.org/frameworks/mbed?utm_source=github&utm_medium=core>`_
+* `Nuclei SDK <https://platformio.org/frameworks/nuclei-sdk?utm_source=github&utm_medium=core>`_
+* `PULP OS <https://platformio.org/frameworks/pulp-os?utm_source=github&utm_medium=core>`_
 * `Pumbaa <https://platformio.org/frameworks/pumbaa?utm_source=github&utm_medium=core>`_
+* `Shakti SDK <https://platformio.org/frameworks/shakti-sdk?utm_source=github&utm_medium=core>`_
 * `Simba <https://platformio.org/frameworks/simba?utm_source=github&utm_medium=core>`_
 * `SPL <https://platformio.org/frameworks/spl?utm_source=github&utm_medium=core>`_
 * `STM32Cube <https://platformio.org/frameworks/stm32cube?utm_source=github&utm_medium=core>`_
-* `Tizen RT <https://platformio.org/frameworks/tizenrt?utm_source=github&utm_medium=core>`_
 * `WiringPi <https://platformio.org/frameworks/wiringpi?utm_source=github&utm_medium=core>`_
+* `Zephyr <https://platformio.org/frameworks/zephyr?utm_source=github&utm_medium=core>`_
 
 Contributing
 ------------
 
 See `contributing guidelines <https://github.com/platformio/platformio/blob/develop/CONTRIBUTING.md>`_.
 
+Telemetry / Privacy Policy
+--------------------------
+
+Share minimal diagnostics and usage information to help us make PlatformIO better.
+It is enabled by default. For more information see:
+
+* `Telemetry Setting <https://docs.platformio.org/en/latest/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
+* `SSL Setting <https://docs.platformio.org/en/latest/userguide/cmd_settings.html?utm_source=github&utm_medium=core#strict-ssl>`_
+
 License
 -------
 

platformio/__init__.py

@@ -12,18 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
-VERSION = (3, 6, 1)
+VERSION = (4, 3, 0)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
 __description__ = (
-    "An open source ecosystem for IoT development. "
-    "Cross-platform IDE and unified debugger. "
-    "Remote unit testing and firmware updates. "
+    "A new generation ecosystem for embedded development. "
+    "Cross-platform IDE and Unified Debugger. "
+    "Static Code Analyzer and Remote Unit Testing. "
+    "Multi-platform and Multi-architecture Build System. "
+    "Firmware File Explorer and Memory Inspection. "
     "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, "
-    "FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3")
+    "RISC-V, FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3"
+)
 __url__ = "https://platformio.org"
 
 __author__ = "PlatformIO"
@@ -33,10 +34,3 @@ __license__ = "Apache Software License"
 __copyright__ = "Copyright 2014-present PlatformIO"
 
 __apiurl__ = "https://api.platformio.org"
-
-if sys.version_info < (2, 7, 0) or sys.version_info >= (3, 0, 0):
-    msg = ("PlatformIO Core v%s does not run under Python version %s.\n"
-           "Minimum supported version is 2.7, please upgrade Python.\n"
-           "Python 3 is not yet supported.\n")
-    sys.stderr.write(msg % (__version__, sys.version))
-    sys.exit(1)

platformio/__main__.py

@@ -14,92 +14,68 @@
 
 import os
 import sys
-from os.path import join
-from platform import system
 from traceback import format_exc
 
 import click
 
-from platformio import __version__, exception, maintenance
-from platformio.util import get_source_dir
-
-
-class PlatformioCLI(click.MultiCommand):  # pylint: disable=R0904
-
-    def list_commands(self, ctx):
-        cmds = []
-        for filename in os.listdir(join(get_source_dir(), "commands")):
-            if filename.startswith("__init__"):
-                continue
-            if filename.endswith(".py"):
-                cmds.append(filename[:-3])
-        cmds.sort()
-        return cmds
-
-    def get_command(self, ctx, cmd_name):
-        mod = None
-        try:
-            mod = __import__("platformio.commands." + cmd_name, None, None,
-                             ["cli"])
-        except ImportError:
-            try:
-                return self._handle_obsolate_command(cmd_name)
-            except AttributeError:
-                raise click.UsageError('No such command "%s"' % cmd_name, ctx)
-        return mod.cli
-
-    @staticmethod
-    def _handle_obsolate_command(name):
-        if name == "platforms":
-            from platformio.commands import platform
-            return platform.cli
-        elif name == "serialports":
-            from platformio.commands import device
-            return device.cli
-        raise AttributeError()
+from platformio import __version__, exception, maintenance, util
+from platformio.commands import PlatformioCLI
+from platformio.compat import CYGWIN
 
 
 @click.command(
-    cls=PlatformioCLI,
-    context_settings=dict(help_option_names=["-h", "--help"]))
+    cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])
+)
 @click.version_option(__version__, prog_name="PlatformIO")
-@click.option(
-    "--force",
-    "-f",
-    is_flag=True,
-    help="Force to accept any confirmation prompts.")
-@click.option("--caller", "-c", help="Caller ID (service).")
+@click.option("--force", "-f", is_flag=True, help="DEPRECATE")
+@click.option("--caller", "-c", help="Caller ID (service)")
+@click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters")
 @click.pass_context
-def cli(ctx, force, caller):
+def cli(ctx, force, caller, no_ansi):
+    try:
+        if (
+            no_ansi
+            or str(
+                os.getenv("PLATFORMIO_NO_ANSI", os.getenv("PLATFORMIO_DISABLE_COLOR"))
+            ).lower()
+            == "true"
+        ):
+            # pylint: disable=protected-access
+            click._compat.isatty = lambda stream: False
+        elif (
+            str(
+                os.getenv("PLATFORMIO_FORCE_ANSI", os.getenv("PLATFORMIO_FORCE_COLOR"))
+            ).lower()
+            == "true"
+        ):
+            # pylint: disable=protected-access
+            click._compat.isatty = lambda stream: True
+    except:  # pylint: disable=bare-except
+        pass
+
     maintenance.on_platformio_start(ctx, force, caller)
 
 
 @cli.resultcallback()
 @click.pass_context
-def process_result(ctx, result, force, caller):  # pylint: disable=W0613
+def process_result(ctx, result, *_, **__):
     maintenance.on_platformio_end(ctx, result)
 
 
+@util.memoized()
 def configure():
-    if "cygwin" in system().lower():
+    if CYGWIN:
         raise exception.CygwinEnvDetected()
 
     # https://urllib3.readthedocs.org
     # /en/latest/security.html#insecureplatformwarning
     try:
-        import urllib3
+        import urllib3  # pylint: disable=import-outside-toplevel
+
         urllib3.disable_warnings()
     except (AttributeError, ImportError):
         pass
 
-    # handle PLATFORMIO_FORCE_COLOR
-    if str(os.getenv("PLATFORMIO_FORCE_COLOR", "")).lower() == "true":
-        try:
-            # pylint: disable=protected-access
-            click._compat.isatty = lambda stream: True
-        except:  # pylint: disable=bare-except
-            pass
-
     # Handle IOError issue with VSCode's Terminal (Windows)
     click_echo_origin = [click.echo, click.secho]
 
@@ -108,16 +84,25 @@ def configure():
             click_echo_origin[origin](*args, **kwargs)
         except IOError:
             (sys.stderr.write if kwargs.get("err") else sys.stdout.write)(
-                "%s\n" % (args[0] if args else ""))
+                "%s\n" % (args[0] if args else "")
+            )
 
     click.echo = lambda *args, **kwargs: _safe_echo(0, *args, **kwargs)
     click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs)
 
 
-def main():
+def main(argv=None):
+    exit_code = 0
+    prev_sys_argv = sys.argv[:]
+    if argv:
+        assert isinstance(argv, list)
+        sys.argv = argv
     try:
         configure()
-        cli(None, None, None)
+        cli()  # pylint: disable=no-value-for-parameter
+    except SystemExit as e:
+        if e.code and str(e.code).isdigit():
+            exit_code = int(e.code)
     except Exception as e:  # pylint: disable=broad-except
         if not isinstance(e, exception.ReturnErrorCode):
             maintenance.on_platformio_exception(e)
@@ -143,13 +128,13 @@ An unexpected error occurred. Further steps:
 ============================================================
 """
             click.secho(error_str, fg="red", err=True)
-        return int(str(e)) if str(e).isdigit() else 1
-    return 0
+        exit_code = int(str(e)) if str(e).isdigit() else 1
+    sys.argv = prev_sys_argv
+    return exit_code
 
 
 def debug_gdb_main():
-    sys.argv = [sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]
-    return main()
+    return main([sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:])
 
 
 if __name__ == "__main__":

platformio/app.py

@@ -14,102 +14,110 @@
 
 import codecs
 import hashlib
-import json
 import os
 import uuid
-from copy import deepcopy
 from os import environ, getenv, listdir, remove
-from os.path import abspath, dirname, expanduser, isdir, isfile, join
+from os.path import dirname, isdir, isfile, join, realpath
 from time import time
 
 import requests
 
-from platformio import exception, lockfile, util
+from platformio import exception, fs, lockfile
+from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data
+from platformio.proc import is_ci
+from platformio.project.helpers import (
+    get_default_projects_dir,
+    get_project_cache_dir,
+    get_project_core_dir,
+)
 
 
 def projects_dir_validate(projects_dir):
     assert isdir(projects_dir)
-    return abspath(projects_dir)
+    return realpath(projects_dir)
 
 
 DEFAULT_SETTINGS = {
     "auto_update_libraries": {
         "description": "Automatically update libraries (Yes/No)",
-        "value": False
+        "value": False,
     },
     "auto_update_platforms": {
         "description": "Automatically update platforms (Yes/No)",
-        "value": False
+        "value": False,
     },
     "check_libraries_interval": {
         "description": "Check for the library updates interval (days)",
-        "value": 7
+        "value": 7,
     },
     "check_platformio_interval": {
         "description": "Check for the new PlatformIO interval (days)",
-        "value": 3
+        "value": 3,
     },
     "check_platforms_interval": {
         "description": "Check for the platform updates interval (days)",
-        "value": 7
+        "value": 7,
     },
     "enable_cache": {
         "description": "Enable caching for API requests and Library Manager",
-        "value": True
-    },
-    "enable_ssl": {
-        "description": "Enable SSL for PlatformIO Services",
-        "value": False
+        "value": True,
     },
+    "strict_ssl": {"description": "Strict SSL for PlatformIO Services", "value": False},
     "enable_telemetry": {
-        "description":
-        ("Telemetry service <https://docs.platformio.org/page/"
-         "userguide/cmd_settings.html?#enable-telemetry> (Yes/No)"),
-        "value":
-        True
+        "description": ("Telemetry service <http://bit.ly/pio-telemetry> (Yes/No)"),
+        "value": True,
     },
     "force_verbose": {
         "description": "Force verbose output when processing environments",
-        "value": False
+        "value": False,
     },
     "projects_dir": {
         "description": "Default location for PlatformIO projects (PIO Home)",
-        "value": join(expanduser("~"), "Documents", "PlatformIO", "Projects"),
-        "validator": projects_dir_validate
+        "value": get_default_projects_dir(),
+        "validator": projects_dir_validate,
     },
 }
 
-SESSION_VARS = {"command_ctx": None, "force_option": False, "caller_id": None}
+SESSION_VARS = {
+    "command_ctx": None,
+    "force_option": False,
+    "caller_id": None,
+    "custom_project_conf": None,
+}
 
 
 class State(object):
-
     def __init__(self, path=None, lock=False):
         self.path = path
         self.lock = lock
         if not self.path:
-            self.path = join(util.get_home_dir(), "appstate.json")
-        self._state = {}
-        self._prev_state = {}
+            self.path = join(get_project_core_dir(), "appstate.json")
+        self._storage = {}
         self._lockfile = None
+        self.modified = False
 
     def __enter__(self):
         try:
             self._lock_state_file()
             if isfile(self.path):
-                self._state = util.load_json(self.path)
-        except exception.PlatformioException:
-            self._state = {}
-        self._prev_state = deepcopy(self._state)
-        return self._state
+                self._storage = fs.load_json(self.path)
+            assert isinstance(self._storage, dict)
+        except (
+            AssertionError,
+            ValueError,
+            UnicodeDecodeError,
+            exception.InvalidJSONFile,
+        ):
+            self._storage = {}
+        return self
 
     def __exit__(self, type_, value, traceback):
-        if self._prev_state != self._state:
+        if self.modified:
             try:
-                with codecs.open(self.path, "w", encoding="utf8") as fp:
-                    json.dump(self._state, fp)
+                with open(self.path, "w") as fp:
+                    fp.write(dump_json_to_unicode(self._storage))
             except IOError:
-                raise exception.HomeDirPermissionsError(util.get_home_dir())
+                raise exception.HomeDirPermissionsError(get_project_core_dir())
         self._unlock_state_file()
 
     def _lock_state_file(self):
@@ -122,21 +130,49 @@ class State(object):
             raise exception.HomeDirPermissionsError(dirname(self.path))
 
     def _unlock_state_file(self):
-        if self._lockfile:
+        if hasattr(self, "_lockfile") and self._lockfile:
             self._lockfile.release()
 
     def __del__(self):
         self._unlock_state_file()
 
+    # Dictionary Proxy
+
+    def as_dict(self):
+        return self._storage
+
+    def get(self, key, default=True):
+        return self._storage.get(key, default)
+
+    def update(self, *args, **kwargs):
+        self.modified = True
+        return self._storage.update(*args, **kwargs)
+
+    def clear(self):
+        return self._storage.clear()
+
+    def __getitem__(self, key):
+        return self._storage[key]
+
+    def __setitem__(self, key, value):
+        self.modified = True
+        self._storage[key] = value
+
+    def __delitem__(self, key):
+        self.modified = True
+        del self._storage[key]
+
+    def __contains__(self, item):
+        return item in self._storage
+
 
 class ContentCache(object):
-
     def __init__(self, cache_dir=None):
         self.cache_dir = None
         self._db_path = None
         self._lockfile = None
 
-        self.cache_dir = cache_dir or util.get_cache_dir()
+        self.cache_dir = cache_dir or get_project_cache_dir()
         self._db_path = join(self.cache_dir, "db.data")
 
     def __enter__(self):
@@ -163,14 +199,17 @@ class ContentCache(object):
         return True
 
     def get_cache_path(self, key):
+        assert "/" not in key and "\\" not in key
+        key = str(key)
         assert len(key) > 3
         return join(self.cache_dir, key[-2:], key)
 
     @staticmethod
     def key_from_args(*args):
         h = hashlib.md5()
-        for data in args:
-            h.update(str(data))
+        for arg in args:
+            if arg:
+                h.update(hashlib_encode_data(arg))
         return h.hexdigest()
 
     def get(self, key):
@@ -191,7 +230,7 @@ class ContentCache(object):
         if not isdir(self.cache_dir):
             os.makedirs(self.cache_dir)
         tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
-        assert valid.endswith(tuple(tdmap.keys()))
+        assert valid.endswith(tuple(tdmap))
         expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1]))
 
         if not self._lock_dbindex():
@@ -226,20 +265,26 @@ class ContentCache(object):
         newlines = []
         with open(self._db_path) as fp:
             for line in fp.readlines():
+                line = line.strip()
                 if "=" not in line:
                     continue
-                line = line.strip()
                 expire, path = line.split("=")
-                if time() < int(expire) and isfile(path) and \
-                        path not in paths_for_delete:
-                    newlines.append(line)
-                    continue
+                try:
+                    if (
+                        time() < int(expire)
+                        and isfile(path)
+                        and path not in paths_for_delete
+                    ):
+                        newlines.append(line)
+                        continue
+                except ValueError:
+                    pass
                 found = True
                 if isfile(path):
                     try:
                         remove(path)
                         if not listdir(dirname(path)):
-                            util.rmtree_(dirname(path))
+                            fs.rmtree(dirname(path))
                     except OSError:
                         pass
 
@@ -253,7 +298,7 @@ class ContentCache(object):
     def clean(self):
         if not self.cache_dir or not isdir(self.cache_dir):
             return
-        util.rmtree_(self.cache_dir)
+        fs.rmtree(self.cache_dir)
 
 
 def clean_cache():
@@ -268,11 +313,11 @@ def sanitize_setting(name, value):
     defdata = DEFAULT_SETTINGS[name]
     try:
         if "validator" in defdata:
-            value = defdata['validator'](value)
-        elif isinstance(defdata['value'], bool):
+            value = defdata["validator"](value)
+        elif isinstance(defdata["value"], bool):
             if not isinstance(value, bool):
                 value = str(value).lower() in ("true", "yes", "y", "1")
-        elif isinstance(defdata['value'], int):
+        elif isinstance(defdata["value"], int):
             value = int(value)
     except Exception:
         raise exception.InvalidSettingValue(value, name)
@@ -280,19 +325,20 @@ def sanitize_setting(name, value):
 
 
 def get_state_item(name, default=None):
-    with State() as data:
-        return data.get(name, default)
+    with State() as state:
+        return state.get(name, default)
 
 
 def set_state_item(name, value):
-    with State(lock=True) as data:
-        data[name] = value
+    with State(lock=True) as state:
+        state[name] = value
+        state.modified = True
 
 
 def delete_state_item(name):
-    with State(lock=True) as data:
-        if name in data:
-            del data[name]
+    with State(lock=True) as state:
+        if name in state:
+            del state[name]
 
 
 def get_setting(name):
@@ -300,24 +346,25 @@ def get_setting(name):
     if _env_name in environ:
         return sanitize_setting(name, getenv(_env_name))
 
-    with State() as data:
-        if "settings" in data and name in data['settings']:
-            return data['settings'][name]
+    with State() as state:
+        if "settings" in state and name in state["settings"]:
+            return state["settings"][name]
 
-    return DEFAULT_SETTINGS[name]['value']
+    return DEFAULT_SETTINGS[name]["value"]
 
 
 def set_setting(name, value):
-    with State(lock=True) as data:
-        if "settings" not in data:
-            data['settings'] = {}
-        data['settings'][name] = sanitize_setting(name, value)
+    with State(lock=True) as state:
+        if "settings" not in state:
+            state["settings"] = {}
+        state["settings"][name] = sanitize_setting(name, value)
+        state.modified = True
 
 
 def reset_settings():
-    with State(lock=True) as data:
-        if "settings" in data:
-            del data['settings']
+    with State(lock=True) as state:
+        if "settings" in state:
+            del state["settings"]
 
 
 def get_session_var(name, default=None):
@@ -330,30 +377,40 @@ def set_session_var(name, value):
 
 
 def is_disabled_progressbar():
-    return any([
-        get_session_var("force_option"),
-        util.is_ci(),
-        getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true"
-    ])
+    return any(
+        [
+            get_session_var("force_option"),
+            is_ci(),
+            getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true",
+        ]
+    )
 
 
 def get_cid():
     cid = get_state_item("cid")
-    if not cid:
-        _uid = None
-        if getenv("C9_UID"):
-            _uid = getenv("C9_UID")
-        elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")):
-            try:
-                _uid = requests.get("{api}/user?token={token}".format(
-                    api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
-                    token=getenv("USER_TOKEN"))).json().get("id")
-            except:  # pylint: disable=bare-except
-                pass
-        cid = str(
-            uuid.UUID(
-                bytes=hashlib.md5(str(_uid if _uid else uuid.getnode())).
-                digest()))
-        if "windows" in util.get_systype() or os.getuid() > 0:
-            set_state_item("cid", cid)
+    if cid:
+        return cid
+    uid = None
+    if getenv("C9_UID"):
+        uid = getenv("C9_UID")
+    elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")):
+        try:
+            uid = (
+                requests.get(
+                    "{api}/user?token={token}".format(
+                        api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
+                        token=getenv("USER_TOKEN"),
+                    )
+                )
+                .json()
+                .get("id")
+            )
+        except:  # pylint: disable=bare-except
+            pass
+    if not uid:
+        uid = uuid.getnode()
+    cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest())
+    cid = str(cid)
+    if WINDOWS or os.getuid() > 0:  # pylint: disable=no-member
+        set_state_item("cid", cid)
     return cid

platformio/builder/main.py

@@ -12,156 +12,143 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import base64
-import json
 import sys
-from os import environ
-from os.path import expanduser, join
+from os import environ, makedirs
+from os.path import isdir, join
 from time import time
 
-from SCons.Script import (ARGUMENTS, COMMAND_LINE_TARGETS, DEFAULT_TARGETS,
-                          AllowSubstExceptions, AlwaysBuild, Default,
-                          DefaultEnvironment, Variables)
+import click
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
+from SCons.Script import DEFAULT_TARGETS  # pylint: disable=import-error
+from SCons.Script import AllowSubstExceptions  # pylint: disable=import-error
+from SCons.Script import AlwaysBuild  # pylint: disable=import-error
+from SCons.Script import Default  # pylint: disable=import-error
+from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
+from SCons.Script import Import  # pylint: disable=import-error
+from SCons.Script import Variables  # pylint: disable=import-error
 
-from platformio import util
+from platformio import fs
+from platformio.compat import dump_json_to_unicode
+from platformio.managers.platform import PlatformBase
+from platformio.proc import get_pythonexe_path
+from platformio.project.helpers import get_project_dir
 
 AllowSubstExceptions(NameError)
 
-# allow common variables from INI file
-commonvars = Variables(None)
-commonvars.AddVariables(
+# append CLI arguments to build environment
+clivars = Variables(None)
+clivars.AddVariables(
     ("PLATFORM_MANIFEST",),
     ("BUILD_SCRIPT",),
-    ("EXTRA_SCRIPTS",),
+    ("PROJECT_CONFIG",),
     ("PIOENV",),
-    ("PIOTEST",),
-    ("PIOPLATFORM",),
-    ("PIOFRAMEWORK",),
-
-    # build options
-    ("BUILD_FLAGS",),
-    ("SRC_BUILD_FLAGS",),
-    ("BUILD_UNFLAGS",),
-    ("SRC_FILTER",),
-
-    # library options
-    ("LIB_LDF_MODE",),
-    ("LIB_COMPAT_MODE",),
-    ("LIB_DEPS",),
-    ("LIB_IGNORE",),
-    ("LIB_EXTRA_DIRS",),
-    ("LIB_ARCHIVE",),
-
-    # board options
-    ("BOARD",),
-    # deprecated options, use board_{object.path} instead
-    ("BOARD_MCU",),
-    ("BOARD_F_CPU",),
-    ("BOARD_F_FLASH",),
-    ("BOARD_FLASH_MODE",),
-    # end of deprecated options
-
-    # upload options
+    ("PIOTEST_RUNNING_NAME",),
     ("UPLOAD_PORT",),
-    ("UPLOAD_PROTOCOL",),
-    ("UPLOAD_SPEED",),
-    ("UPLOAD_FLAGS",),
-    ("UPLOAD_RESETMETHOD",),
-
-    # test options
-    ("TEST_BUILD_PROJECT_SRC",),
-
-    # debug options
-    ("DEBUG_TOOL",),
-    ("DEBUG_SVD_PATH",),
-
-)  # yapf: disable
-
-MULTILINE_VARS = [
-    "EXTRA_SCRIPTS", "PIOFRAMEWORK", "BUILD_FLAGS", "SRC_BUILD_FLAGS",
-    "BUILD_UNFLAGS", "UPLOAD_FLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE",
-    "LIB_EXTRA_DIRS"
-]
+)
 
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
-        "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform",
-        "piowinhooks", "piolib", "pioupload", "piomisc", "pioide"
-    ],  # yapf: disable
-    toolpath=[join(util.get_source_dir(), "builder", "tools")],
-    variables=commonvars,
-
+        "ar",
+        "as",
+        "cc",
+        "c++",
+        "link",
+        "platformio",
+        "pioplatform",
+        "pioproject",
+        "piomaxlen",
+        "piolib",
+        "pioupload",
+        "piomisc",
+        "pioide",
+        "piosize",
+    ],
+    toolpath=[join(fs.get_source_dir(), "builder", "tools")],
+    variables=clivars,
     # Propagating External Environment
-    PIOVARIABLES=commonvars.keys(),
     ENV=environ,
     UNIX_TIME=int(time()),
-    PIOHOME_DIR=util.get_home_dir(),
-    PROJECT_DIR=util.get_project_dir(),
-    PROJECTINCLUDE_DIR=util.get_projectinclude_dir(),
-    PROJECTSRC_DIR=util.get_projectsrc_dir(),
-    PROJECTTEST_DIR=util.get_projecttest_dir(),
-    PROJECTDATA_DIR=util.get_projectdata_dir(),
-    PROJECTBUILD_DIR=util.get_projectbuild_dir(),
-    BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"),
-    BUILDSRC_DIR=join("$BUILD_DIR", "src"),
-    BUILDTEST_DIR=join("$BUILD_DIR", "test"),
+    BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"),
+    BUILD_SRC_DIR=join("$BUILD_DIR", "src"),
+    BUILD_TEST_DIR=join("$BUILD_DIR", "test"),
+    COMPILATIONDB_PATH=join("$BUILD_DIR", "compile_commands.json"),
     LIBPATH=["$BUILD_DIR"],
-    LIBSOURCE_DIRS=[
-        util.get_projectlib_dir(),
-        util.get_projectlibdeps_dir(),
-        join("$PIOHOME_DIR", "lib")
-    ],
     PROGNAME="program",
     PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
-    PYTHONEXE=util.get_pythonexe_path())
+    PYTHONEXE=get_pythonexe_path(),
+)
 
 if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
-    DEFAULT_ENV_OPTIONS['ARCOMSTR'] = "Archiving $TARGET"
-    DEFAULT_ENV_OPTIONS['LINKCOMSTR'] = "Linking $TARGET"
-    DEFAULT_ENV_OPTIONS['RANLIBCOMSTR'] = "Indexing $TARGET"
+    DEFAULT_ENV_OPTIONS["ARCOMSTR"] = "Archiving $TARGET"
+    DEFAULT_ENV_OPTIONS["LINKCOMSTR"] = "Linking $TARGET"
+    DEFAULT_ENV_OPTIONS["RANLIBCOMSTR"] = "Indexing $TARGET"
     for k in ("ASCOMSTR", "ASPPCOMSTR", "CCCOMSTR", "CXXCOMSTR"):
         DEFAULT_ENV_OPTIONS[k] = "Compiling $TARGET"
 
 env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
 
-# decode common variables
-for k in commonvars.keys():
-    if k in env:
-        env[k] = base64.b64decode(env[k])
-        if k in MULTILINE_VARS:
-            env[k] = util.parse_conf_multi_values(env[k])
+# Load variables from CLI
+env.Replace(
+    **{
+        key: PlatformBase.decode_scons_arg(env[key])
+        for key in list(clivars.keys())
+        if key in env
+    }
+)
 
-if env.GetOption('clean'):
+# Setup project optional directories
+config = env.GetProjectConfig()
+env.Replace(
+    PROJECT_DIR=get_project_dir(),
+    PROJECT_CORE_DIR=config.get_optional_dir("core"),
+    PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"),
+    PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"),
+    PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"),
+    PROJECT_INCLUDE_DIR=config.get_optional_dir("include"),
+    PROJECT_SRC_DIR=config.get_optional_dir("src"),
+    PROJECTSRC_DIR=config.get_optional_dir("src"),  # legacy for dev/platform
+    PROJECT_TEST_DIR=config.get_optional_dir("test"),
+    PROJECT_DATA_DIR=config.get_optional_dir("data"),
+    PROJECTDATA_DIR=config.get_optional_dir("data"),  # legacy for dev/platform
+    PROJECT_BUILD_DIR=config.get_optional_dir("build"),
+    BUILD_CACHE_DIR=config.get_optional_dir("build_cache"),
+    LIBSOURCE_DIRS=[
+        config.get_optional_dir("lib"),
+        join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
+        config.get_optional_dir("globallib"),
+    ],
+)
+
+if env.subst("$BUILD_CACHE_DIR"):
+    if not isdir(env.subst("$BUILD_CACHE_DIR")):
+        makedirs(env.subst("$BUILD_CACHE_DIR"))
+    env.CacheDir("$BUILD_CACHE_DIR")
+
+if int(ARGUMENTS.get("ISATTY", 0)):
+    # pylint: disable=protected-access
+    click._compat.isatty = lambda stream: True
+
+if env.GetOption("clean"):
     env.PioClean(env.subst("$BUILD_DIR"))
     env.Exit(0)
 elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
-    print("Verbose mode can be enabled via `-v, --verbose` option")
+    click.echo("Verbose mode can be enabled via `-v, --verbose` option")
 
-# Handle custom variables from system environment
-for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPTS",
-            "UPLOAD_PORT", "UPLOAD_FLAGS", "LIB_EXTRA_DIRS"):
-    k = "PLATFORMIO_%s" % var
-    if k not in environ:
-        continue
-    if var in ("UPLOAD_PORT", ):
-        env[var] = environ.get(k)
-        continue
-    env.Append(**{var: util.parse_conf_multi_values(environ.get(k))})
+# Dynamically load dependent tools
+if "compiledb" in COMMAND_LINE_TARGETS:
+    env.Tool("compilation_db")
 
-# Configure extra library source directories for LDF
-if util.get_project_optional_dir("lib_extra_dirs"):
-    env.Prepend(
-        LIBSOURCE_DIRS=util.parse_conf_multi_values(
-            util.get_project_optional_dir("lib_extra_dirs")))
-env.Prepend(LIBSOURCE_DIRS=env.get("LIB_EXTRA_DIRS", []))
-env['LIBSOURCE_DIRS'] = [
-    expanduser(d) if d.startswith("~") else d for d in env['LIBSOURCE_DIRS']
-]
+if not isdir(env.subst("$BUILD_DIR")):
+    makedirs(env.subst("$BUILD_DIR"))
 
-env.LoadPioPlatform(commonvars)
+env.LoadProjectOptions()
+env.LoadPioPlatform()
 
 env.SConscriptChdir(0)
-env.SConsignFile(join("$PROJECTBUILD_DIR", ".sconsign.dblite"))
+env.SConsignFile(
+    join("$BUILD_DIR", ".sconsign.py%d%d" % (sys.version_info[0], sys.version_info[1]))
+)
 
 for item in env.GetExtraScripts("pre"):
     env.SConscript(item, exports="env")
@@ -170,6 +157,8 @@ env.SConscript("$BUILD_SCRIPT")
 
 if "UPLOAD_FLAGS" in env:
     env.Prepend(UPLOADERFLAGS=["$UPLOAD_FLAGS"])
+if env.GetProjectOption("upload_command"):
+    env.Replace(UPLOADCMD=env.GetProjectOption("upload_command"))
 
 for item in env.GetExtraScripts("post"):
     env.SConscript(item, exports="env")
@@ -177,7 +166,9 @@ for item in env.GetExtraScripts("post"):
 ##############################################################################
 
 # Checking program size
-if env.get("SIZETOOL") and "nobuild" not in COMMAND_LINE_TARGETS:
+if env.get("SIZETOOL") and not (
+    set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS)
+):
     env.Depends(["upload", "program"], "checkprogsize")
     # Replace platform's "size" target with our
     _new_targets = [t for t in DEFAULT_TARGETS if str(t) != "size"]
@@ -185,31 +176,47 @@ if env.get("SIZETOOL") and "nobuild" not in COMMAND_LINE_TARGETS:
     Default(_new_targets)
     Default("checkprogsize")
 
+if "compiledb" in COMMAND_LINE_TARGETS:
+    env.Alias("compiledb", env.CompilationDatabase("$COMPILATIONDB_PATH"))
+
 # Print configured protocols
 env.AddPreAction(
     ["upload", "program"],
-    env.VerboseAction(lambda source, target, env: env.PrintUploadInfo(),
-                      "Configuring upload protocol..."))
+    env.VerboseAction(
+        lambda source, target, env: env.PrintUploadInfo(),
+        "Configuring upload protocol...",
+    ),
+)
 
 AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS))
-AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS))
 AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS))
 
 ##############################################################################
 
 if "envdump" in COMMAND_LINE_TARGETS:
-    print(env.Dump())
+    click.echo(env.Dump())
     env.Exit(0)
 
 if "idedata" in COMMAND_LINE_TARGETS:
     try:
-        print("\n%s\n" % util.path_to_unicode(
-            json.dumps(env.DumpIDEData(), ensure_ascii=False)))
-        env.Exit(0)
-    except UnicodeDecodeError:
-        sys.stderr.write(
-            "\nUnicodeDecodeError: Non-ASCII characters found in build "
-            "environment\n"
-            "See explanation in FAQ > Troubleshooting > Building\n"
-            "https://docs.platformio.org/page/faq.html\n\n")
-        env.Exit(1)
+        Import("projenv")
+    except:  # pylint: disable=bare-except
+        projenv = env
+    click.echo(
+        "\n%s\n"
+        % dump_json_to_unicode(
+            projenv.DumpIDEData()  # pylint: disable=undefined-variable
+        )
+    )
+    env.Exit(0)
+
+if "sizedata" in COMMAND_LINE_TARGETS:
+    AlwaysBuild(
+        env.Alias(
+            "sizedata",
+            DEFAULT_TARGETS,
+            env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."),
+        )
+    )
+
+    Default("sizedata")

platformio/builder/tools/compilation_db.py

@@ -0,0 +1,219 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+# Copyright 2015 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=unused-argument, protected-access, unused-variable, import-error
+# Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py
+
+from __future__ import absolute_import
+
+import itertools
+import json
+import os
+
+import SCons
+
+from platformio.builder.tools.platformio import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT
+from platformio.proc import where_is_program
+
+# Implements the ability for SCons to emit a compilation database for the MongoDB project. See
+# http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation
+# database is, and why you might want one. The only user visible entry point here is
+# 'env.CompilationDatabase'. This method takes an optional 'target' to name the file that
+# should hold the compilation database, otherwise, the file defaults to compile_commands.json,
+# which is the name that most clang tools search for by default.
+
+# TODO: Is there a better way to do this than this global? Right now this exists so that the
+# emitter we add can record all of the things it emits, so that the scanner for the top level
+# compilation database can access the complete list, and also so that the writer has easy
+# access to write all of the files. But it seems clunky. How can the emitter and the scanner
+# communicate more gracefully?
+__COMPILATION_DB_ENTRIES = []
+
+
+# We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even
+# integrate with the cache, but there doesn't seem to be much call for it.
+class __CompilationDbNode(SCons.Node.Python.Value):
+    def __init__(self, value):
+        SCons.Node.Python.Value.__init__(self, value)
+        self.Decider(changed_since_last_build_node)
+
+
+def changed_since_last_build_node(*args, **kwargs):
+    """ Dummy decider to force always building"""
+    return True
+
+
+def makeEmitCompilationDbEntry(comstr):
+    """
+    Effectively this creates a lambda function to capture:
+    * command line
+    * source
+    * target
+    :param comstr: unevaluated command line
+    :return: an emitter which has captured the above
+    """
+    user_action = SCons.Action.Action(comstr)
+
+    def EmitCompilationDbEntry(target, source, env):
+        """
+        This emitter will be added to each c/c++ object build to capture the info needed
+        for clang tools
+        :param target: target node(s)
+        :param source: source node(s)
+        :param env: Environment for use building this node
+        :return: target(s), source(s)
+        """
+
+        # Resolve absolute path of toolchain
+        for cmd in ("CC", "CXX", "AS"):
+            if cmd not in env:
+                continue
+            if os.path.isabs(env[cmd]):
+                continue
+            env[cmd] = where_is_program(
+                env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")
+            )
+
+        dbtarget = __CompilationDbNode(source)
+
+        entry = env.__COMPILATIONDB_Entry(
+            target=dbtarget,
+            source=[],
+            __COMPILATIONDB_UTARGET=target,
+            __COMPILATIONDB_USOURCE=source,
+            __COMPILATIONDB_UACTION=user_action,
+            __COMPILATIONDB_ENV=env,
+        )
+
+        # TODO: Technically, these next two lines should not be required: it should be fine to
+        # cache the entries. However, they don't seem to update properly. Since they are quick
+        # to re-generate disable caching and sidestep this problem.
+        env.AlwaysBuild(entry)
+        env.NoCache(entry)
+
+        __COMPILATION_DB_ENTRIES.append(dbtarget)
+
+        return target, source
+
+    return EmitCompilationDbEntry
+
+
+def CompilationDbEntryAction(target, source, env, **kw):
+    """
+    Create a dictionary with evaluated command line, target, source
+    and store that info as an attribute on the target
+    (Which has been stored in __COMPILATION_DB_ENTRIES array
+    :param target: target node(s)
+    :param source: source node(s)
+    :param env: Environment for use building this node
+    :param kw:
+    :return: None
+    """
+
+    command = env["__COMPILATIONDB_UACTION"].strfunction(
+        target=env["__COMPILATIONDB_UTARGET"],
+        source=env["__COMPILATIONDB_USOURCE"],
+        env=env["__COMPILATIONDB_ENV"],
+    )
+
+    entry = {
+        "directory": env.Dir("#").abspath,
+        "command": command,
+        "file": str(env["__COMPILATIONDB_USOURCE"][0]),
+    }
+
+    target[0].write(entry)
+
+
+def WriteCompilationDb(target, source, env):
+    entries = []
+
+    for s in __COMPILATION_DB_ENTRIES:
+        item = s.read()
+        item["file"] = os.path.abspath(item["file"])
+        entries.append(item)
+
+    with open(str(target[0]), "w") as target_file:
+        json.dump(
+            entries, target_file, sort_keys=True, indent=4, separators=(",", ": ")
+        )
+
+
+def ScanCompilationDb(node, env, path):
+    return __COMPILATION_DB_ENTRIES
+
+
+def generate(env, **kwargs):
+    static_obj, shared_obj = SCons.Tool.createObjBuilders(env)
+
+    env["COMPILATIONDB_COMSTR"] = kwargs.get(
+        "COMPILATIONDB_COMSTR", "Building compilation database $TARGET"
+    )
+
+    components_by_suffix = itertools.chain(
+        itertools.product(
+            [".%s" % ext for ext in SRC_C_EXT],
+            [
+                (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"),
+                (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"),
+            ],
+        ),
+        itertools.product(
+            [".%s" % ext for ext in SRC_CXX_EXT],
+            [
+                (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"),
+                (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"),
+            ],
+        ),
+        itertools.product(
+            [".%s" % ext for ext in SRC_ASM_EXT],
+            [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASCOM")],
+        ),
+    )
+
+    for entry in components_by_suffix:
+        suffix = entry[0]
+        builder, base_emitter, command = entry[1]
+
+        # Assumes a dictionary emitter
+        emitter = builder.emitter[suffix]
+        builder.emitter[suffix] = SCons.Builder.ListEmitter(
+            [emitter, makeEmitCompilationDbEntry(command)]
+        )
+
+    env["BUILDERS"]["__COMPILATIONDB_Entry"] = SCons.Builder.Builder(
+        action=SCons.Action.Action(CompilationDbEntryAction, None),
+    )
+
+    env["BUILDERS"]["__COMPILATIONDB_Database"] = SCons.Builder.Builder(
+        action=SCons.Action.Action(WriteCompilationDb, "$COMPILATIONDB_COMSTR"),
+        target_scanner=SCons.Scanner.Scanner(
+            function=ScanCompilationDb, node_class=None
+        ),
+    )
+
+    def CompilationDatabase(env, target):
+        result = env.__COMPILATIONDB_Database(target=target, source=[])
+
+        env.AlwaysBuild(result)
+        env.NoCache(result)
+
+        return result
+
+    env.AddMethod(CompilationDatabase, "CompilationDatabase")
+
+
+def exists(env):
+    return True

platformio/builder/tools/pioide.py

@@ -14,18 +14,18 @@
 
 from __future__ import absolute_import
 
+import os
 from glob import glob
-from os import environ
-from os.path import abspath, isfile, join
 
-from SCons.Defaults import processDefines
+from SCons.Defaults import processDefines  # pylint: disable=import-error
 
-from platformio import util
+from platformio.compat import glob_escape
 from platformio.managers.core import get_core_package_dir
+from platformio.proc import exec_command, where_is_program
 
 
 def _dump_includes(env):
-    includes = [env.subst("$PROJECTINCLUDE_DIR"), env.subst("$PROJECTSRC_DIR")]
+    includes = []
 
     for item in env.get("CPPPATH", []):
         includes.append(env.subst(item))
@@ -39,12 +39,12 @@ def _dump_includes(env):
     for name in p.get_installed_packages():
         if p.get_package_type(name) != "toolchain":
             continue
-        toolchain_dir = util.glob_escape(p.get_package_dir(name))
+        toolchain_dir = glob_escape(p.get_package_dir(name))
         toolchain_incglobs = [
-            join(toolchain_dir, "*", "include*"),
-            join(toolchain_dir, "*", "include", "c++", "*"),
-            join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"),
-            join(toolchain_dir, "lib", "gcc", "*", "*", "include*")
+            os.path.join(toolchain_dir, "*", "include*"),
+            os.path.join(toolchain_dir, "*", "include", "c++", "*"),
+            os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"),
+            os.path.join(toolchain_dir, "lib", "gcc", "*", "*", "include*"),
         ]
         for g in toolchain_incglobs:
             includes.extend(glob(g))
@@ -53,11 +53,14 @@ def _dump_includes(env):
     if unity_dir:
         includes.append(unity_dir)
 
+    includes.extend([env.subst("$PROJECT_INCLUDE_DIR"), env.subst("$PROJECT_SRC_DIR")])
+
     # remove duplicates
     result = []
     for item in includes:
+        item = os.path.realpath(item)
         if item not in result:
-            result.append(abspath(item))
+            result.append(item)
 
     return result
 
@@ -65,15 +68,16 @@ def _dump_includes(env):
 def _get_gcc_defines(env):
     items = []
     try:
-        sysenv = environ.copy()
-        sysenv['PATH'] = str(env['ENV']['PATH'])
-        result = util.exec_command(
-            "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True)
+        sysenv = os.environ.copy()
+        sysenv["PATH"] = str(env["ENV"]["PATH"])
+        result = exec_command(
+            "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True
+        )
     except OSError:
         return items
-    if result['returncode'] != 0:
+    if result["returncode"] != 0:
         return items
-    for line in result['out'].split("\n"):
+    for line in result["out"].split("\n"):
         tokens = line.strip().split(" ", 2)
         if not tokens or tokens[0] != "#define":
             continue
@@ -88,17 +92,22 @@ def _dump_defines(env):
     defines = []
     # global symbols
     for item in processDefines(env.get("CPPDEFINES", [])):
-        defines.append(env.subst(item).replace('\\', ''))
+        defines.append(env.subst(item).replace("\\", ""))
 
     # special symbol for Atmel AVR MCU
-    if env['PIOPLATFORM'] == "atmelavr":
+    if env["PIOPLATFORM"] == "atmelavr":
         board_mcu = env.get("BOARD_MCU")
         if not board_mcu and "BOARD" in env:
             board_mcu = env.BoardConfig().get("build.mcu")
         if board_mcu:
             defines.append(
-                str("__AVR_%s__" % board_mcu.upper().replace(
-                    "ATMEGA", "ATmega").replace("ATTINY", "ATtiny")))
+                str(
+                    "__AVR_%s__"
+                    % board_mcu.upper()
+                    .replace("ATMEGA", "ATmega")
+                    .replace("ATTINY", "ATtiny")
+                )
+            )
 
     # built-in GCC marcos
     # if env.GetCompilerType() == "gcc":
@@ -108,9 +117,9 @@ def _dump_defines(env):
 
 
 def _get_svd_path(env):
-    svd_path = env.subst("$DEBUG_SVD_PATH")
+    svd_path = env.GetProjectOption("debug_svd_path")
     if svd_path:
-        return abspath(svd_path)
+        return os.path.realpath(svd_path)
 
     if "BOARD" not in env:
         return None
@@ -120,46 +129,47 @@ def _get_svd_path(env):
     except (AssertionError, KeyError):
         return None
     # custom path to SVD file
-    if isfile(svd_path):
+    if os.path.isfile(svd_path):
         return svd_path
     # default file from ./platform/misc/svd folder
     p = env.PioPlatform()
-    if isfile(join(p.get_dir(), "misc", "svd", svd_path)):
-        return abspath(join(p.get_dir(), "misc", "svd", svd_path))
+    if os.path.isfile(os.path.join(p.get_dir(), "misc", "svd", svd_path)):
+        return os.path.realpath(os.path.join(p.get_dir(), "misc", "svd", svd_path))
     return None
 
 
+def _escape_build_flag(flags):
+    return [flag if " " not in flag else '"%s"' % flag for flag in flags]
+
+
 def DumpIDEData(env):
-    LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS"
-    LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS"
+
+    env["__escape_build_flag"] = _escape_build_flag
+
+    LINTCCOM = (
+        "${__escape_build_flag(CFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS"
+    )
+    LINTCXXCOM = (
+        "${__escape_build_flag(CXXFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS"
+    )
 
     data = {
-        "libsource_dirs":
-        [env.subst(l) for l in env.get("LIBSOURCE_DIRS", [])],
-        "defines":
-        _dump_defines(env),
-        "includes":
-        _dump_includes(env),
-        "cc_flags":
-        env.subst(LINTCCOM),
-        "cxx_flags":
-        env.subst(LINTCXXCOM),
-        "cc_path":
-        util.where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
-        "cxx_path":
-        util.where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
-        "gdb_path":
-        util.where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
-        "prog_path":
-        env.subst("$PROG_PATH"),
-        "flash_extra_images": [{
-            "offset": item[0],
-            "path": env.subst(item[1])
-        } for item in env.get("FLASH_EXTRA_IMAGES", [])],
-        "svd_path":
-        _get_svd_path(env),
-        "compiler_type":
-        env.GetCompilerType()
+        "env_name": env["PIOENV"],
+        "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()],
+        "defines": _dump_defines(env),
+        "includes": _dump_includes(env),
+        "cc_flags": env.subst(LINTCCOM),
+        "cxx_flags": env.subst(LINTCXXCOM),
+        "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
+        "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
+        "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
+        "prog_path": env.subst("$PROG_PATH"),
+        "flash_extra_images": [
+            {"offset": item[0], "path": env.subst(item[1])}
+            for item in env.get("FLASH_EXTRA_IMAGES", [])
+        ],
+        "svd_path": _get_svd_path(env),
+        "compiler_type": env.GetCompilerType(),
     }
 
     env_ = env.Clone()
@@ -173,10 +183,7 @@ def DumpIDEData(env):
             _new_defines.append(item)
     env_.Replace(CPPDEFINES=_new_defines)
 
-    data.update({
-        "cc_flags": env_.subst(LINTCCOM),
-        "cxx_flags": env_.subst(LINTCXXCOM)
-    })
+    data.update({"cc_flags": env_.subst(LINTCCOM), "cxx_flags": env_.subst(LINTCXXCOM)})
 
     return data
 

platformio/builder/tools/piomaxlen.py

@@ -12,19 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import
+
 from hashlib import md5
 from os import makedirs
 from os.path import isdir, isfile, join
-from platform import system
+
+from platformio.compat import WINDOWS, hashlib_encode_data
 
 # Windows CLI has limit with command length to 8192
 # Leave 2000 chars for flags and other options
-MAX_SOURCES_LENGTH = 6000
+MAX_LINE_LENGTH = 6000 if WINDOWS else 128072
 
 
 def long_sources_hook(env, sources):
     _sources = str(sources).replace("\\", "/")
-    if len(str(_sources)) < MAX_SOURCES_LENGTH:
+    if len(str(_sources)) < MAX_LINE_LENGTH:
         return sources
 
     # fix space in paths
@@ -40,7 +43,7 @@ def long_sources_hook(env, sources):
 
 def long_incflags_hook(env, incflags):
     _incflags = env.subst(incflags).replace("\\", "/")
-    if len(_incflags) < MAX_SOURCES_LENGTH:
+    if len(_incflags) < MAX_LINE_LENGTH:
         return incflags
 
     # fix space in paths
@@ -58,7 +61,9 @@ def _file_long_data(env, data):
     build_dir = env.subst("$BUILD_DIR")
     if not isdir(build_dir):
         makedirs(build_dir)
-    tmp_file = join(build_dir, "longcmd-%s" % md5(data).hexdigest())
+    tmp_file = join(
+        build_dir, "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest()
+    )
     if isfile(tmp_file):
         return tmp_file
     with open(tmp_file, "w") as fp:
@@ -71,18 +76,17 @@ def exists(_):
 
 
 def generate(env):
-    if system() != "Windows":
-        return None
-
     env.Replace(_long_sources_hook=long_sources_hook)
     env.Replace(_long_incflags_hook=long_incflags_hook)
     coms = {}
     for key in ("ARCOM", "LINKCOM"):
         coms[key] = env.get(key, "").replace(
-            "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}")
+            "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}"
+        )
     for key in ("_CCCOMCOM", "ASPPCOM"):
         coms[key] = env.get(key, "").replace(
-            "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}")
+            "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}"
+        )
     env.Replace(**coms)
 
     return env

platformio/builder/tools/piomisc.py

@@ -15,17 +15,21 @@
 from __future__ import absolute_import
 
 import atexit
+import io
 import re
 import sys
 from os import environ, remove, walk
 from os.path import basename, isdir, isfile, join, realpath, relpath, sep
 from tempfile import mkstemp
 
-from SCons.Action import Action
-from SCons.Script import ARGUMENTS
+import click
+from SCons.Action import Action  # pylint: disable=import-error
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 
-from platformio import util
+from platformio import fs, util
+from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape
 from platformio.managers.core import get_core_package_dir
+from platformio.proc import exec_command
 
 
 class InoToCPPConverter(object):
@@ -33,17 +37,52 @@ class InoToCPPConverter(object):
     PROTOTYPE_RE = re.compile(
         r"""^(
         (?:template\<.*\>\s*)?      # template
-        ([a-z_\d\&]+\*?\s+){1,2}      # return type
+        ([a-z_\d\&]+\*?\s+){1,2}    # return type
         ([a-z_\d]+\s*)              # name of prototype
         \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
-        )\s*\{                      # must end with {
-        """, re.X | re.M | re.I)
+        )\s*(\{|;)                  # must end with `{` or `;`
+        """,
+        re.X | re.M | re.I,
+    )
     DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
     PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
 
     def __init__(self, env):
         self.env = env
         self._main_ino = None
+        self._safe_encoding = None
+
+    def read_safe_contents(self, path):
+        error_reported = False
+        for encoding in (
+            "utf-8",
+            None,
+            get_filesystem_encoding(),
+            get_locale_encoding(),
+            "latin-1",
+        ):
+            try:
+                with io.open(path, encoding=encoding) as fp:
+                    contents = fp.read()
+                    self._safe_encoding = encoding
+                    return contents
+            except UnicodeDecodeError:
+                if not error_reported:
+                    error_reported = True
+                    click.secho(
+                        "Unicode decode error has occurred, please remove invalid "
+                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
+                        % path,
+                        fg="yellow",
+                        err=True,
+                    )
+        return ""
+
+    def write_safe_contents(self, path, contents):
+        with io.open(
+            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
+        ) as fp:
+            return fp.write(contents)
 
     def is_main_node(self, contents):
         return self.DETECTMAIN_RE.search(contents)
@@ -58,10 +97,8 @@ class InoToCPPConverter(object):
         assert nodes
         lines = []
         for node in nodes:
-            contents = node.get_text_contents()
-            _lines = [
-                '# 1 "%s"' % node.get_path().replace("\\", "/"), contents
-            ]
+            contents = self.read_safe_contents(node.get_path())
+            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
             if self.is_main_node(contents):
                 lines = _lines + lines
                 self._main_ino = node.get_path()
@@ -76,22 +113,22 @@ class InoToCPPConverter(object):
     def process(self, contents):
         out_file = self._main_ino + ".cpp"
         assert self._gcc_preprocess(contents, out_file)
-        with open(out_file) as fp:
-            contents = fp.read()
+        contents = self.read_safe_contents(out_file)
         contents = self._join_multiline_strings(contents)
-        with open(out_file, "w") as fp:
-            fp.write(self.append_prototypes(contents))
+        self.write_safe_contents(out_file, self.append_prototypes(contents))
         return out_file
 
     def _gcc_preprocess(self, contents, out_file):
         tmp_path = mkstemp()[1]
-        with open(tmp_path, "w") as fp:
-            fp.write(contents)
+        self.write_safe_contents(tmp_path, contents)
         self.env.Execute(
             self.env.VerboseAction(
                 '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
-                    out_file, tmp_path),
-                "Converting " + basename(out_file[:-4])))
+                    out_file, tmp_path
+                ),
+                "Converting " + basename(out_file[:-4]),
+            )
+        )
         atexit.register(_delete_file, tmp_path)
         return isfile(out_file)
 
@@ -113,14 +150,15 @@ class InoToCPPConverter(object):
                     stropen = True
                     newlines.append(line[:-1])
                     continue
-                elif stropen:
+                if stropen:
                     newlines[len(newlines) - 1] += line[:-1]
                     continue
             elif stropen and line.endswith(('",', '";')):
                 newlines[len(newlines) - 1] += line
                 stropen = False
-                newlines.append('#line %d "%s"' %
-                                (linenum, self._main_ino.replace("\\", "/")))
+                newlines.append(
+                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
+                )
                 continue
 
             newlines.append(line)
@@ -140,8 +178,10 @@ class InoToCPPConverter(object):
         prototypes = []
         reserved_keywords = set(["if", "else", "while"])
         for match in self.PROTOTYPE_RE.finditer(contents):
-            if (set([match.group(2).strip(),
-                     match.group(3).strip()]) & reserved_keywords):
+            if (
+                set([match.group(2).strip(), match.group(3).strip()])
+                & reserved_keywords
+            ):
                 continue
             prototypes.append(match)
         return prototypes
@@ -158,31 +198,42 @@ class InoToCPPConverter(object):
         return total
 
     def append_prototypes(self, contents):
-        prototypes = self._parse_prototypes(contents)
+        prototypes = self._parse_prototypes(contents) or []
+
+        # skip already declared prototypes
+        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
+        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
+
         if not prototypes:
             return contents
 
-        prototype_names = set([m.group(3).strip() for m in prototypes])
+        prototype_names = set(m.group(3).strip() for m in prototypes)
         split_pos = prototypes[0].start()
         match_ptrs = re.search(
             self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
-            contents[:split_pos], re.M)
+            contents[:split_pos],
+            re.M,
+        )
         if match_ptrs:
             split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
 
         result = []
         result.append(contents[:split_pos].strip())
         result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
-        result.append('#line %d "%s"' % (self._get_total_lines(
-            contents[:split_pos]), self._main_ino.replace("\\", "/")))
+        result.append(
+            '#line %d "%s"'
+            % (
+                self._get_total_lines(contents[:split_pos]),
+                self._main_ino.replace("\\", "/"),
+            )
+        )
         result.append(contents[split_pos:].strip())
         return "\n".join(result)
 
 
 def ConvertInoToCpp(env):
-    src_dir = util.glob_escape(env.subst("$PROJECTSRC_DIR"))
-    ino_nodes = (
-        env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde")))
+    src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR"))
+    ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde"))
     if not ino_nodes:
         return
     c = InoToCPPConverter(env)
@@ -201,18 +252,20 @@ def _delete_file(path):
 
 @util.memoized()
 def _get_compiler_type(env):
+    if env.subst("$CC").endswith("-gcc"):
+        return "gcc"
     try:
         sysenv = environ.copy()
-        sysenv['PATH'] = str(env['ENV']['PATH'])
-        result = util.exec_command([env.subst("$CC"), "-v"], env=sysenv)
+        sysenv["PATH"] = str(env["ENV"]["PATH"])
+        result = exec_command([env.subst("$CC"), "-v"], env=sysenv)
     except OSError:
         return None
-    if result['returncode'] != 0:
+    if result["returncode"] != 0:
         return None
-    output = "".join([result['out'], result['err']]).lower()
+    output = "".join([result["out"], result["err"]]).lower()
     if "clang" in output and "LLVM" in output:
         return "clang"
-    elif "gcc" in output:
+    if "gcc" in output:
         return "gcc"
     return None
 
@@ -222,7 +275,6 @@ def GetCompilerType(env):
 
 
 def GetActualLDScript(env):
-
     def _lookup_in_ldpath(script):
         for d in env.get("LIBPATH", []):
             path = join(env.subst(d), script)
@@ -237,7 +289,7 @@ def GetActualLDScript(env):
         if f == "-T":
             script_in_next = True
             continue
-        elif script_in_next:
+        if script_in_next:
             script_in_next = False
             raw_script = f
         elif f.startswith("-Wl,-T"):
@@ -253,12 +305,13 @@ def GetActualLDScript(env):
 
     if script:
         sys.stderr.write(
-            "Error: Could not find '%s' LD script in LDPATH '%s'\n" %
-            (script, env.subst("$LIBPATH")))
+            "Error: Could not find '%s' LD script in LDPATH '%s'\n"
+            % (script, env.subst("$LIBPATH"))
+        )
         env.Exit(1)
 
     if not script and "LDSCRIPT_PATH" in env:
-        path = _lookup_in_ldpath(env['LDSCRIPT_PATH'])
+        path = _lookup_in_ldpath(env["LDSCRIPT_PATH"])
         if path:
             return path
 
@@ -276,52 +329,70 @@ def PioClean(env, clean_dir):
     if not isdir(clean_dir):
         print("Build environment is clean")
         env.Exit(0)
+    clean_rel_path = relpath(clean_dir)
     for root, _, files in walk(clean_dir):
-        for file_ in files:
-            remove(join(root, file_))
-            print("Removed %s" % relpath(join(root, file_)))
+        for f in files:
+            dst = join(root, f)
+            remove(dst)
+            print(
+                "Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst))
+            )
     print("Done cleaning")
-    util.rmtree_(clean_dir)
+    fs.rmtree(clean_dir)
     env.Exit(0)
 
 
-def ProcessDebug(env):
-    if not env.subst("$PIODEBUGFLAGS"):
-        env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"])
-    env.Append(
-        PIODEBUGFLAGS=["-D__PLATFORMIO_DEBUG__"],
-        BUILD_FLAGS=env.get("PIODEBUGFLAGS", []))
-    unflags = ["-Os"]
-    for level in [0, 1, 2]:
-        for flag in ("O", "g", "ggdb"):
-            unflags.append("-%s%d" % (flag, level))
-    env.Append(BUILD_UNFLAGS=unflags)
+def ConfigureDebugFlags(env):
+    def _cleanup_debug_flags(scope):
+        if scope not in env:
+            return
+        unflags = ["-Os", "-g"]
+        for level in [0, 1, 2, 3]:
+            for flag in ("O", "g", "ggdb"):
+                unflags.append("-%s%d" % (flag, level))
+        env[scope] = [f for f in env.get(scope, []) if f not in unflags]
+
+    env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"])
+
+    for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"):
+        _cleanup_debug_flags(scope)
+
+    debug_flags = env.ParseFlags(env.GetProjectOption("debug_build_flags"))
+    env.MergeFlags(debug_flags)
+    optimization_flags = [
+        f for f in debug_flags.get("CCFLAGS", []) if f.startswith(("-O", "-g"))
+    ]
+
+    if optimization_flags:
+        env.AppendUnique(ASFLAGS=optimization_flags, LINKFLAGS=optimization_flags)
 
 
-def ProcessTest(env):
+def ConfigureTestTarget(env):
     env.Append(
         CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
-        CPPPATH=[join("$BUILD_DIR", "UnityTestLib")])
+        CPPPATH=[join("$BUILD_DIR", "UnityTestLib")],
+    )
     unitylib = env.BuildLibrary(
-        join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity"))
+        join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity")
+    )
     env.Prepend(LIBS=[unitylib])
 
     src_filter = ["+<*.cpp>", "+<*.c>"]
-    if "PIOTEST" in env:
-        src_filter.append("+<%s%s>" % (env['PIOTEST'], sep))
+    if "PIOTEST_RUNNING_NAME" in env:
+        src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], sep))
     env.Replace(PIOTEST_SRC_FILTER=src_filter)
 
 
 def GetExtraScripts(env, scope):
     items = []
-    for item in env.get("EXTRA_SCRIPTS", []):
+    for item in env.GetProjectOption("extra_scripts", []):
         if scope == "post" and ":" not in item:
             items.append(item)
         elif item.startswith("%s:" % scope):
-            items.append(item[len(scope) + 1:])
+            items.append(item[len(scope) + 1 :])
     if not items:
         return items
-    with util.cd(env.subst("$PROJECT_DIR")):
+    with fs.cd(env.subst("$PROJECT_DIR")):
         return [realpath(item) for item in items]
 
 
@@ -335,7 +406,7 @@ def generate(env):
     env.AddMethod(GetActualLDScript)
     env.AddMethod(VerboseAction)
     env.AddMethod(PioClean)
-    env.AddMethod(ProcessDebug)
-    env.AddMethod(ProcessTest)
+    env.AddMethod(ConfigureDebugFlags)
+    env.AddMethod(ConfigureTestTarget)
     env.AddMethod(GetExtraScripts)
     return env

platformio/builder/tools/pioplatform.py

@@ -14,60 +14,58 @@
 
 from __future__ import absolute_import
 
-import base64
 import sys
 from os.path import isdir, isfile, join
 
-from SCons.Script import COMMAND_LINE_TARGETS
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 
-from platformio import exception, util
+from platformio import exception, fs, util
+from platformio.compat import WINDOWS
 from platformio.managers.platform import PlatformFactory
+from platformio.project.config import ProjectOptions
 
 # pylint: disable=too-many-branches, too-many-locals
 
 
 @util.memoized()
-def initPioPlatform(name):
-    return PlatformFactory.newPlatform(name)
-
-
 def PioPlatform(env):
-    variables = {}
-    for name in env['PIOVARIABLES']:
-        if name in env:
-            variables[name.lower()] = env[name]
-    p = initPioPlatform(env['PLATFORM_MANIFEST'])
+    variables = env.GetProjectOptions(as_dict=True)
+    if "framework" in variables:
+        # support PIO Core 3.0 dev/platforms
+        variables["pioframework"] = variables["framework"]
+    p = PlatformFactory.newPlatform(env["PLATFORM_MANIFEST"])
     p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
     return p
 
 
 def BoardConfig(env, board=None):
-    p = initPioPlatform(env['PLATFORM_MANIFEST'])
-    try:
-        board = board or env.get("BOARD")
-        assert board, "BoardConfig: Board is not defined"
-        config = p.board_config(board)
-    except (AssertionError, exception.UnknownBoard) as e:
-        sys.stderr.write("Error: %s\n" % str(e))
-        env.Exit(1)
-    return config
+    with fs.cd(env.subst("$PROJECT_DIR")):
+        try:
+            p = env.PioPlatform()
+            board = board or env.get("BOARD")
+            assert board, "BoardConfig: Board is not defined"
+            return p.board_config(board)
+        except (AssertionError, exception.UnknownBoard) as e:
+            sys.stderr.write("Error: %s\n" % str(e))
+            env.Exit(1)
 
 
 def GetFrameworkScript(env, framework):
     p = env.PioPlatform()
     assert p.frameworks and framework in p.frameworks
-    script_path = env.subst(p.frameworks[framework]['script'])
+    script_path = env.subst(p.frameworks[framework]["script"])
     if not isfile(script_path):
         script_path = join(p.get_dir(), script_path)
     return script_path
 
 
-def LoadPioPlatform(env, variables):
+def LoadPioPlatform(env):
     p = env.PioPlatform()
     installed_packages = p.get_installed_packages()
 
     # Ensure real platform name
-    env['PIOPLATFORM'] = p.name
+    env["PIOPLATFORM"] = p.name
 
     # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH
     systype = util.get_systype()
@@ -77,103 +75,158 @@ def LoadPioPlatform(env, variables):
             continue
         pkg_dir = p.get_package_dir(name)
         env.PrependENVPath(
-            "PATH",
-            join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir)
-        if ("windows" not in systype and isdir(join(pkg_dir, "lib"))
-                and type_ != "toolchain"):
+            "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir
+        )
+        if not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain":
             env.PrependENVPath(
-                "DYLD_LIBRARY_PATH"
-                if "darwin" in systype else "LD_LIBRARY_PATH",
-                join(pkg_dir, "lib"))
+                "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH",
+                join(pkg_dir, "lib"),
+            )
 
     # Platform specific LD Scripts
     if isdir(join(p.get_dir(), "ldscripts")):
         env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")])
 
     if "BOARD" not in env:
-        # handle _MCU and _F_CPU variables for AVR native
-        for key, value in variables.UnknownVariables().items():
-            if not key.startswith("BOARD_"):
-                continue
-            env.Replace(
-                **{key.upper().replace("BUILD.", ""): base64.b64decode(value)})
         return
 
-    # update board manifest with a custom data
+    # update board manifest with overridden data from INI config
     board_config = env.BoardConfig()
-    for key, value in variables.UnknownVariables().items():
-        if not key.startswith("BOARD_"):
+    for option, value in env.GetProjectOptions():
+        if not option.startswith("board_"):
             continue
-        board_config.update(key.lower()[6:], base64.b64decode(value))
+        option = option.lower()[6:]
+        try:
+            if isinstance(board_config.get(option), bool):
+                value = str(value).lower() in ("1", "yes", "true")
+            elif isinstance(board_config.get(option), int):
+                value = int(value)
+        except KeyError:
+            pass
+        board_config.update(option, value)
 
-    # update default environment variables
-    for key in variables.keys():
-        if key in env or \
-                not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]):
+    # load default variables from board config
+    for option_meta in ProjectOptions.values():
+        if not option_meta.buildenvvar or option_meta.buildenvvar in env:
             continue
-        _opt, _val = key.lower().split("_", 1)
-        if _opt == "board":
-            _opt = "build"
-        if _val in board_config.get(_opt):
-            env.Replace(**{key: board_config.get("%s.%s" % (_opt, _val))})
+        data_path = (
+            option_meta.name[6:]
+            if option_meta.name.startswith("board_")
+            else option_meta.name.replace("_", ".")
+        )
+        try:
+            env[option_meta.buildenvvar] = board_config.get(data_path)
+        except KeyError:
+            pass
 
     if "build.ldscript" in board_config:
         env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript"))
 
 
-def PrintConfiguration(env):
+def PrintConfiguration(env):  # pylint: disable=too-many-statements
     platform = env.PioPlatform()
-    platform_data = ["PLATFORM: %s >" % platform.title]
-    system_data = ["SYSTEM:"]
-    configuration_data = ["CONFIGURATION:"]
-    mcu = env.subst("$BOARD_MCU")
-    f_cpu = env.subst("$BOARD_F_CPU")
-    if mcu:
-        system_data.append(mcu.upper())
-    if f_cpu:
-        f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()]))
-        system_data.append("%dMHz" % (f_cpu / 1000000))
+    board_config = env.BoardConfig() if "BOARD" in env else None
 
-    debug_tools = None
-    if "BOARD" in env:
-        board_config = env.BoardConfig()
-        platform_data.append(board_config.get("name"))
+    def _get_configuration_data():
+        return (
+            None
+            if not board_config
+            else [
+                "CONFIGURATION:",
+                "https://docs.platformio.org/page/boards/%s/%s.html"
+                % (platform.name, board_config.id),
+            ]
+        )
 
-        debug_tools = board_config.get("debug", {}).get("tools")
+    def _get_plaform_data():
+        data = ["PLATFORM: %s %s" % (platform.title, platform.version)]
+        src_manifest_path = platform.pm.get_src_manifest_path(platform.get_dir())
+        if src_manifest_path:
+            src_manifest = fs.load_json(src_manifest_path)
+            if "version" in src_manifest:
+                data.append("#" + src_manifest["version"])
+            if int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                data.append("(%s)" % src_manifest["url"])
+        if board_config:
+            data.extend([">", board_config.get("name")])
+        return data
+
+    def _get_hardware_data():
+        data = ["HARDWARE:"]
+        mcu = env.subst("$BOARD_MCU")
+        f_cpu = env.subst("$BOARD_F_CPU")
+        if mcu:
+            data.append(mcu.upper())
+        if f_cpu:
+            f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()]))
+            data.append("%dMHz," % (f_cpu / 1000000))
+        if not board_config:
+            return data
         ram = board_config.get("upload", {}).get("maximum_ram_size")
         flash = board_config.get("upload", {}).get("maximum_size")
-        system_data.append("%s RAM (%s Flash)" % (util.format_filesize(ram),
-                                                  util.format_filesize(flash)))
-        configuration_data.append(
-            "https://docs.platformio.org/page/boards/%s/%s.html" %
-            (platform.name, board_config.id))
+        data.append(
+            "%s RAM, %s Flash" % (fs.format_filesize(ram), fs.format_filesize(flash))
+        )
+        return data
 
-    for data in (configuration_data, platform_data, system_data):
-        if len(data) > 1:
+    def _get_debug_data():
+        debug_tools = (
+            board_config.get("debug", {}).get("tools") if board_config else None
+        )
+        if not debug_tools:
+            return None
+        data = [
+            "DEBUG:",
+            "Current",
+            "(%s)"
+            % board_config.get_debug_tool_name(env.GetProjectOption("debug_tool")),
+        ]
+        onboard = []
+        external = []
+        for key, value in debug_tools.items():
+            if value.get("onboard"):
+                onboard.append(key)
+            else:
+                external.append(key)
+        if onboard:
+            data.extend(["On-board", "(%s)" % ", ".join(sorted(onboard))])
+        if external:
+            data.extend(["External", "(%s)" % ", ".join(sorted(external))])
+        return data
+
+    def _get_packages_data():
+        data = []
+        for name, options in platform.packages.items():
+            if options.get("optional"):
+                continue
+            pkg_dir = platform.get_package_dir(name)
+            if not pkg_dir:
+                continue
+            manifest = platform.pm.load_manifest(pkg_dir)
+            original_version = util.get_original_version(manifest["version"])
+            info = "%s %s" % (manifest["name"], manifest["version"])
+            extra = []
+            if original_version:
+                extra.append(original_version)
+            if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                extra.append(manifest["__src_url"])
+            if extra:
+                info += " (%s)" % ", ".join(extra)
+            data.append(info)
+        if not data:
+            return None
+        return ["PACKAGES:"] + ["\n - %s" % d for d in sorted(data)]
+
+    for data in (
+        _get_configuration_data(),
+        _get_plaform_data(),
+        _get_hardware_data(),
+        _get_debug_data(),
+        _get_packages_data(),
+    ):
+        if data and len(data) > 1:
             print(" ".join(data))
 
-    # Debugging
-    if not debug_tools:
-        return
-
-    data = [
-        "CURRENT(%s)" % board_config.get_debug_tool_name(
-            env.subst("$DEBUG_TOOL"))
-    ]
-    onboard = []
-    external = []
-    for key, value in debug_tools.items():
-        if value.get("onboard"):
-            onboard.append(key)
-        else:
-            external.append(key)
-    if onboard:
-        data.append("ON-BOARD(%s)" % ", ".join(sorted(onboard)))
-    if external:
-        data.append("EXTERNAL(%s)" % ", ".join(sorted(external)))
-
-    print("DEBUG: %s" % " ".join(data))
-
 
 def exists(_):
     return True

platformio/builder/tools/pioproject.py

@@ -0,0 +1,53 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from platformio.project.config import MISSING, ProjectConfig, ProjectOptions
+
+
+def GetProjectConfig(env):
+    return ProjectConfig.get_instance(env["PROJECT_CONFIG"])
+
+
+def GetProjectOptions(env, as_dict=False):
+    return env.GetProjectConfig().items(env=env["PIOENV"], as_dict=as_dict)
+
+
+def GetProjectOption(env, option, default=MISSING):
+    return env.GetProjectConfig().get("env:" + env["PIOENV"], option, default)
+
+
+def LoadProjectOptions(env):
+    for option, value in env.GetProjectOptions():
+        option_meta = ProjectOptions.get("env." + option)
+        if (
+            not option_meta
+            or not option_meta.buildenvvar
+            or option_meta.buildenvvar in env
+        ):
+            continue
+        env[option_meta.buildenvvar] = value
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    env.AddMethod(GetProjectConfig)
+    env.AddMethod(GetProjectOptions)
+    env.AddMethod(GetProjectOption)
+    env.AddMethod(LoadProjectOptions)
+    return env

platformio/builder/tools/piosize.py

@@ -0,0 +1,254 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-locals
+
+from __future__ import absolute_import
+
+import sys
+from os import environ, makedirs, remove
+from os.path import isdir, join, splitdrive
+
+from elftools.elf.descriptions import describe_sh_flags
+from elftools.elf.elffile import ELFFile
+
+from platformio.compat import dump_json_to_unicode
+from platformio.proc import exec_command
+from platformio.util import get_systype
+
+
+def _run_tool(cmd, env, tool_args):
+    sysenv = environ.copy()
+    sysenv["PATH"] = str(env["ENV"]["PATH"])
+
+    build_dir = env.subst("$BUILD_DIR")
+    if not isdir(build_dir):
+        makedirs(build_dir)
+    tmp_file = join(build_dir, "size-data-longcmd.txt")
+
+    with open(tmp_file, "w") as fp:
+        fp.write("\n".join(tool_args))
+
+    cmd.append("@" + tmp_file)
+    result = exec_command(cmd, env=sysenv)
+    remove(tmp_file)
+
+    return result
+
+
+def _get_symbol_locations(env, elf_path, addrs):
+    if not addrs:
+        return {}
+    cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path]
+    result = _run_tool(cmd, env, addrs)
+    locations = [line for line in result["out"].split("\n") if line]
+    assert len(addrs) == len(locations)
+
+    return dict(zip(addrs, [l.strip() for l in locations]))
+
+
+def _get_demangled_names(env, mangled_names):
+    if not mangled_names:
+        return {}
+    result = _run_tool(
+        [env.subst("$CC").replace("-gcc", "-c++filt")], env, mangled_names
+    )
+    demangled_names = [line for line in result["out"].split("\n") if line]
+    assert len(mangled_names) == len(demangled_names)
+
+    return dict(
+        zip(
+            mangled_names,
+            [dn.strip().replace("::__FUNCTION__", "") for dn in demangled_names],
+        )
+    )
+
+
+def _determine_section(sections, symbol_addr):
+    for section, info in sections.items():
+        if not _is_flash_section(info) and not _is_ram_section(info):
+            continue
+        if symbol_addr in range(info["start_addr"], info["start_addr"] + info["size"]):
+            return section
+    return "unknown"
+
+
+def _is_ram_section(section):
+    return (
+        section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS")
+        and section.get("flags", "") == "WA"
+    )
+
+
+def _is_flash_section(section):
+    return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "")
+
+
+def _is_valid_symbol(symbol_name, symbol_type, symbol_address):
+    return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE"
+
+
+def _collect_sections_info(elffile):
+    sections = {}
+    for section in elffile.iter_sections():
+        if section.is_null() or section.name.startswith(".debug"):
+            continue
+
+        section_type = section["sh_type"]
+        section_flags = describe_sh_flags(section["sh_flags"])
+        section_size = section.data_size
+
+        sections[section.name] = {
+            "size": section_size,
+            "start_addr": section["sh_addr"],
+            "type": section_type,
+            "flags": section_flags,
+        }
+
+    return sections
+
+
+def _collect_symbols_info(env, elffile, elf_path, sections):
+    symbols = []
+
+    symbol_section = elffile.get_section_by_name(".symtab")
+    if symbol_section.is_null():
+        sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?")
+        env.Exit(1)
+
+    sysenv = environ.copy()
+    sysenv["PATH"] = str(env["ENV"]["PATH"])
+
+    symbol_addrs = []
+    mangled_names = []
+    for s in symbol_section.iter_symbols():
+        symbol_info = s.entry["st_info"]
+        symbol_addr = s["st_value"]
+        symbol_size = s["st_size"]
+        symbol_type = symbol_info["type"]
+
+        if not _is_valid_symbol(s.name, symbol_type, symbol_addr):
+            continue
+
+        symbol = {
+            "addr": symbol_addr,
+            "bind": symbol_info["bind"],
+            "name": s.name,
+            "type": symbol_type,
+            "size": symbol_size,
+            "section": _determine_section(sections, symbol_addr),
+        }
+
+        if s.name.startswith("_Z"):
+            mangled_names.append(s.name)
+
+        symbol_addrs.append(hex(symbol_addr))
+        symbols.append(symbol)
+
+    symbol_locations = _get_symbol_locations(env, elf_path, symbol_addrs)
+    demangled_names = _get_demangled_names(env, mangled_names)
+    for symbol in symbols:
+        if symbol["name"].startswith("_Z"):
+            symbol["demangled_name"] = demangled_names.get(symbol["name"])
+        location = symbol_locations.get(hex(symbol["addr"]))
+        if not location or "?" in location:
+            continue
+        if "windows" in get_systype():
+            drive, tail = splitdrive(location)
+            location = join(drive.upper(), tail)
+        symbol["file"] = location
+        symbol["line"] = 0
+        if ":" in location:
+            file_, line = location.rsplit(":", 1)
+            if line.isdigit():
+                symbol["file"] = file_
+                symbol["line"] = int(line)
+    return symbols
+
+
+def _calculate_firmware_size(sections):
+    flash_size = ram_size = 0
+    for section_info in sections.values():
+        if _is_flash_section(section_info):
+            flash_size += section_info.get("size", 0)
+        if _is_ram_section(section_info):
+            ram_size += section_info.get("size", 0)
+
+    return ram_size, flash_size
+
+
+def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
+    data = {"device": {}, "memory": {}, "version": 1}
+
+    board = env.BoardConfig()
+    if board:
+        data["device"] = {
+            "mcu": board.get("build.mcu", ""),
+            "cpu": board.get("build.cpu", ""),
+            "frequency": board.get("build.f_cpu"),
+            "flash": int(board.get("upload.maximum_size", 0)),
+            "ram": int(board.get("upload.maximum_ram_size", 0)),
+        }
+        if data["device"]["frequency"] and data["device"]["frequency"].endswith("L"):
+            data["device"]["frequency"] = int(data["device"]["frequency"][0:-1])
+
+    elf_path = env.subst("$PIOMAINPROG")
+
+    with open(elf_path, "rb") as fp:
+        elffile = ELFFile(fp)
+
+        if not elffile.has_dwarf_info():
+            sys.stderr.write("Elf file doesn't contain DWARF information")
+            env.Exit(1)
+
+        sections = _collect_sections_info(elffile)
+        firmware_ram, firmware_flash = _calculate_firmware_size(sections)
+        data["memory"]["total"] = {
+            "ram_size": firmware_ram,
+            "flash_size": firmware_flash,
+            "sections": sections,
+        }
+
+        files = dict()
+        for symbol in _collect_symbols_info(env, elffile, elf_path, sections):
+            file_path = symbol.get("file") or "unknown"
+            if not files.get(file_path, {}):
+                files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0}
+
+            symbol_size = symbol.get("size", 0)
+            section = sections.get(symbol.get("section", ""), {})
+            if _is_ram_section(section):
+                files[file_path]["ram_size"] += symbol_size
+            if _is_flash_section(section):
+                files[file_path]["flash_size"] += symbol_size
+
+            files[file_path]["symbols"].append(symbol)
+
+        data["memory"]["files"] = list()
+        for k, v in files.items():
+            file_data = {"path": k}
+            file_data.update(v)
+            data["memory"]["files"].append(file_data)
+
+    with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp:
+        fp.write(dump_json_to_unicode(data))
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    env.AddMethod(DumpSizeData)
+    return env

platformio/builder/tools/pioupload.py

@@ -22,10 +22,12 @@ from os.path import isfile, join
 from shutil import copyfile
 from time import sleep
 
-from SCons.Script import ARGUMENTS
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from serial import Serial, SerialException
 
-from platformio import exception, util
+from platformio import exception, fs, util
+from platformio.compat import WINDOWS
+from platformio.proc import exec_command
 
 # pylint: disable=unused-argument
 
@@ -58,9 +60,9 @@ def WaitForNewSerialPort(env, before):
     prev_port = env.subst("$UPLOAD_PORT")
     new_port = None
     elapsed = 0
-    before = [p['port'] for p in before]
+    before = [p["port"] for p in before]
     while elapsed < 5 and new_port is None:
-        now = [p['port'] for p in util.get_serial_ports()]
+        now = [p["port"] for p in util.get_serial_ports()]
         for p in now:
             if p not in before:
                 new_port = p
@@ -82,10 +84,12 @@ def WaitForNewSerialPort(env, before):
         sleep(1)
 
     if not new_port:
-        sys.stderr.write("Error: Couldn't find a board on the selected port. "
-                         "Check that you have the correct port selected. "
-                         "If it is correct, try pressing the board's reset "
-                         "button after initiating the upload.\n")
+        sys.stderr.write(
+            "Error: Couldn't find a board on the selected port. "
+            "Check that you have the correct port selected. "
+            "If it is correct, try pressing the board's reset "
+            "button after initiating the upload.\n"
+        )
         env.Exit(1)
 
     return new_port
@@ -97,8 +101,8 @@ def AutodetectUploadPort(*args, **kwargs):
     def _get_pattern():
         if "UPLOAD_PORT" not in env:
             return None
-        if set(["*", "?", "[", "]"]) & set(env['UPLOAD_PORT']):
-            return env['UPLOAD_PORT']
+        if set(["*", "?", "[", "]"]) & set(env["UPLOAD_PORT"]):
+            return env["UPLOAD_PORT"]
         return None
 
     def _is_match_pattern(port):
@@ -110,17 +114,13 @@ def AutodetectUploadPort(*args, **kwargs):
     def _look_for_mbed_disk():
         msdlabels = ("mbed", "nucleo", "frdm", "microbit")
         for item in util.get_logical_devices():
-            if item['path'].startswith("/net") or not _is_match_pattern(
-                    item['path']):
+            if item["path"].startswith("/net") or not _is_match_pattern(item["path"]):
                 continue
-            mbed_pages = [
-                join(item['path'], n) for n in ("mbed.htm", "mbed.html")
-            ]
+            mbed_pages = [join(item["path"], n) for n in ("mbed.htm", "mbed.html")]
             if any(isfile(p) for p in mbed_pages):
-                return item['path']
-            if item['name'] \
-                    and any(l in item['name'].lower() for l in msdlabels):
-                return item['path']
+                return item["path"]
+            if item["name"] and any(l in item["name"].lower() for l in msdlabels):
+                return item["path"]
         return None
 
     def _look_for_serial_port():
@@ -130,18 +130,17 @@ def AutodetectUploadPort(*args, **kwargs):
         if "BOARD" in env and "build.hwids" in env.BoardConfig():
             board_hwids = env.BoardConfig().get("build.hwids")
         for item in util.get_serial_ports(filter_hwid=True):
-            if not _is_match_pattern(item['port']):
+            if not _is_match_pattern(item["port"]):
                 continue
-            port = item['port']
+            port = item["port"]
             if upload_protocol.startswith("blackmagic"):
-                if "windows" in util.get_systype() and \
-                        port.startswith("COM") and len(port) > 4:
+                if WINDOWS and port.startswith("COM") and len(port) > 4:
                     port = "\\\\.\\%s" % port
-                if "GDB" in item['description']:
+                if "GDB" in item["description"]:
                     return port
             for hwid in board_hwids:
                 hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                if hwid_str in item['hwid']:
+                if hwid_str in item["hwid"]:
                     return port
         return port
 
@@ -149,13 +148,13 @@ def AutodetectUploadPort(*args, **kwargs):
         print(env.subst("Use manually specified: $UPLOAD_PORT"))
         return
 
-    if (env.subst("$UPLOAD_PROTOCOL") == "mbed"
-            or ("mbed" in env.subst("$PIOFRAMEWORK")
-                and not env.subst("$UPLOAD_PROTOCOL"))):
+    if env.subst("$UPLOAD_PROTOCOL") == "mbed" or (
+        "mbed" in env.subst("$PIOFRAMEWORK") and not env.subst("$UPLOAD_PROTOCOL")
+    ):
         env.Replace(UPLOAD_PORT=_look_for_mbed_disk())
     else:
         try:
-            util.ensure_udev_rules()
+            fs.ensure_udev_rules()
         except exception.InvalidUdevRules as e:
             sys.stderr.write("\n%s\n\n" % e)
         env.Replace(UPLOAD_PORT=_look_for_serial_port())
@@ -167,7 +166,8 @@ def AutodetectUploadPort(*args, **kwargs):
             "Error: Please specify `upload_port` for environment or use "
             "global `--upload-port` option.\n"
             "For some development platforms it can be a USB flash "
-            "drive (i.e. /media/<user>/<device name>)\n")
+            "drive (i.e. /media/<user>/<device name>)\n"
+        )
         env.Exit(1)
 
 
@@ -178,16 +178,17 @@ def UploadToDisk(_, target, source, env):
         fpath = join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
         if not isfile(fpath):
             continue
-        copyfile(fpath,
-                 join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)))
-    print("Firmware has been successfully uploaded.\n"
-          "(Some boards may require manual hard reset)")
+        copyfile(fpath, join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)))
+    print(
+        "Firmware has been successfully uploaded.\n"
+        "(Some boards may require manual hard reset)"
+    )
 
 
 def CheckUploadSize(_, target, source, env):
     check_conditions = [
         env.get("BOARD"),
-        env.get("SIZETOOL") or env.get("SIZECHECKCMD")
+        env.get("SIZETOOL") or env.get("SIZECHECKCMD"),
     ]
     if not all(check_conditions):
         return
@@ -200,7 +201,8 @@ def CheckUploadSize(_, target, source, env):
         env.Replace(
             SIZECHECKCMD="$SIZETOOL -B -d $SOURCES",
             SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s",
-            SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+")
+            SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+",
+        )
 
     def _get_size_output():
         cmd = env.get("SIZECHECKCMD")
@@ -210,11 +212,11 @@ def CheckUploadSize(_, target, source, env):
             cmd = cmd.split()
         cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
         sysenv = environ.copy()
-        sysenv['PATH'] = str(env['ENV']['PATH'])
-        result = util.exec_command(env.subst(cmd), env=sysenv)
-        if result['returncode'] != 0:
+        sysenv["PATH"] = str(env["ENV"]["PATH"])
+        result = exec_command(env.subst(cmd), env=sysenv)
+        if result["returncode"] != 0:
             return None
-        return result['out'].strip()
+        return result["out"].strip()
 
     def _calculate_size(output, pattern):
         if not output or not pattern:
@@ -238,7 +240,8 @@ def CheckUploadSize(_, target, source, env):
         if used_blocks > blocks_per_progress:
             used_blocks = blocks_per_progress
         return "[{:{}}] {: 6.1%} (used {:d} bytes from {:d} bytes)".format(
-            "=" * used_blocks, blocks_per_progress, percent_raw, value, total)
+            "=" * used_blocks, blocks_per_progress, percent_raw, value, total
+        )
 
     if not env.get("SIZECHECKCMD") and not env.get("SIZEPROGREGEXP"):
         _configure_defaults()
@@ -246,12 +249,11 @@ def CheckUploadSize(_, target, source, env):
     program_size = _calculate_size(output, env.get("SIZEPROGREGEXP"))
     data_size = _calculate_size(output, env.get("SIZEDATAREGEXP"))
 
-    print("Memory Usage -> http://bit.ly/pio-memory-usage")
+    print('Advanced Memory Usage is available via "PlatformIO Home > Project Inspect"')
     if data_max_size and data_size > -1:
-        print("DATA:    %s" % _format_availale_bytes(data_size, data_max_size))
+        print("RAM:   %s" % _format_availale_bytes(data_size, data_max_size))
     if program_size > -1:
-        print("PROGRAM: %s" % _format_availale_bytes(program_size,
-                                                     program_max_size))
+        print("Flash: %s" % _format_availale_bytes(program_size, program_max_size))
     if int(ARGUMENTS.get("PIOVERBOSE", 0)):
         print(output)
 
@@ -262,9 +264,10 @@ def CheckUploadSize(_, target, source, env):
     #         "than maximum allowed (%s bytes)\n" % (data_size, data_max_size))
     #     env.Exit(1)
     if program_size > program_max_size:
-        sys.stderr.write("Error: The program size (%d bytes) is greater "
-                         "than maximum allowed (%s bytes)\n" %
-                         (program_size, program_max_size))
+        sys.stderr.write(
+            "Error: The program size (%d bytes) is greater "
+            "than maximum allowed (%s bytes)\n" % (program_size, program_max_size)
+        )
         env.Exit(1)
 
 
@@ -272,8 +275,7 @@ def PrintUploadInfo(env):
     configured = env.subst("$UPLOAD_PROTOCOL")
     available = [configured] if configured else []
     if "BOARD" in env:
-        available.extend(env.BoardConfig().get("upload", {}).get(
-            "protocols", []))
+        available.extend(env.BoardConfig().get("upload", {}).get("protocols", []))
     if available:
         print("AVAILABLE: %s" % ", ".join(sorted(set(available))))
     if configured:

platformio/builder/tools/platformio.py

@@ -14,23 +14,28 @@
 
 from __future__ import absolute_import
 
-import re
+import fnmatch
+import os
 import sys
-from glob import glob
-from os import sep, walk
-from os.path import basename, dirname, isdir, join, realpath
 
-from SCons import Builder, Util
-from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild,
-                          DefaultEnvironment, Export, SConscript)
+from SCons import Builder, Util  # pylint: disable=import-error
+from SCons.Node import FS  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
+from SCons.Script import AlwaysBuild  # pylint: disable=import-error
+from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
+from SCons.Script import Export  # pylint: disable=import-error
+from SCons.Script import SConscript  # pylint: disable=import-error
 
-from platformio.util import glob_escape, pioversion_to_intstr
+from platformio import fs
+from platformio.compat import string_types
+from platformio.util import pioversion_to_intstr
 
 SRC_HEADER_EXT = ["h", "hpp"]
-SRC_C_EXT = ["c", "cc", "cpp"]
-SRC_BUILD_EXT = SRC_C_EXT + ["S", "spp", "SPP", "sx", "s", "asm", "ASM"]
-SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep]
-SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>")
+SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"]
+SRC_C_EXT = ["c"]
+SRC_CXX_EXT = ["cc", "cpp", "cxx", "c++"]
+SRC_BUILD_EXT = SRC_C_EXT + SRC_CXX_EXT + SRC_ASM_EXT
+SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % os.sep, "-<.svn%s>" % os.sep]
 
 
 def scons_patched_match_splitext(path, suffixes=None):
@@ -41,49 +46,58 @@ def scons_patched_match_splitext(path, suffixes=None):
     return tokens
 
 
-def _build_project_deps(env):
-    project_lib_builder = env.ConfigureProjectLibBuilder()
-
-    # append project libs to the beginning of list
-    env.Prepend(LIBS=project_lib_builder.build())
-    # append extra linker related options from libs
-    env.AppendUnique(
-        **{
-            key: project_lib_builder.env.get(key)
-            for key in ("LIBS", "LIBPATH", "LINKFLAGS")
-            if project_lib_builder.env.get(key)
-        })
-
-    projenv = env.Clone()
-
-    # CPPPATH from dependencies
-    projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH"))
-    # extra build flags from `platformio.ini`
-    projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
-
-    is_test = "__test" in COMMAND_LINE_TARGETS
-    if is_test:
-        projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR",
-                             "$PIOTEST_SRC_FILTER")
-    if not is_test or env.get("TEST_BUILD_PROJECT_SRC") == "true":
-        projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR",
-                             env.get("SRC_FILTER"))
-
-    if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
-        sys.stderr.write(
-            "Error: Nothing to build. Please put your source code files "
-            "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR"))
-        env.Exit(1)
-
-    Export("projenv")
+def GetBuildType(env):
+    return (
+        "debug"
+        if (
+            set(["debug", "sizedata"]) & set(COMMAND_LINE_TARGETS)
+            or env.GetProjectOption("build_type") == "debug"
+        )
+        else "release"
+    )
 
 
 def BuildProgram(env):
+    env.ProcessProgramDeps()
+    env.ProcessProjectDeps()
 
+    # append into the beginning a main LD script
+    if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]):
+        env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")])
+
+    # enable "cyclic reference" for linker
+    if env.get("LIBS") and env.GetCompilerType() == "gcc":
+        env.Prepend(_LIBFLAGS="-Wl,--start-group ")
+        env.Append(_LIBFLAGS=" -Wl,--end-group")
+
+    program = env.Program(
+        os.path.join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"]
+    )
+    env.Replace(PIOMAINPROG=program)
+
+    AlwaysBuild(
+        env.Alias(
+            "checkprogsize",
+            program,
+            env.VerboseAction(env.CheckUploadSize, "Checking size $PIOMAINPROG"),
+        )
+    )
+
+    print("Building in %s mode" % env.GetBuildType())
+
+    return program
+
+
+def ProcessProgramDeps(env):
     def _append_pio_macros():
-        env.AppendUnique(CPPDEFINES=[(
-            "PLATFORMIO",
-            int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())))])
+        env.AppendUnique(
+            CPPDEFINES=[
+                (
+                    "PLATFORMIO",
+                    int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())),
+                )
+            ]
+        )
 
     _append_pio_macros()
 
@@ -93,9 +107,6 @@ def BuildProgram(env):
     if not Util.case_sensitive_suffixes(".s", ".S"):
         env.Replace(AS="$CC", ASCOM="$ASPPCOM")
 
-    if set(["__debug", "debug"]) & set(COMMAND_LINE_TARGETS):
-        env.ProcessDebug()
-
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
         env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
@@ -106,39 +117,55 @@ def BuildProgram(env):
     # process framework scripts
     env.BuildFrameworks(env.get("PIOFRAMEWORK"))
 
-    # restore PIO macros if it was deleted by framework
-    _append_pio_macros()
+    if env.GetBuildType() == "debug":
+        env.ConfigureDebugFlags()
 
     # remove specified flags
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
     if "__test" in COMMAND_LINE_TARGETS:
-        env.ProcessTest()
+        env.ConfigureTestTarget()
 
-    # build project with dependencies
-    _build_project_deps(env)
 
-    # append into the beginning a main LD script
-    if (env.get("LDSCRIPT_PATH")
-            and not any("-Wl,-T" in f for f in env['LINKFLAGS'])):
-        env.Prepend(LINKFLAGS=["-T", "$LDSCRIPT_PATH"])
+def ProcessProjectDeps(env):
+    project_lib_builder = env.ConfigureProjectLibBuilder()
 
-    # enable "cyclic reference" for linker
-    if env.get("LIBS") and env.GetCompilerType() == "gcc":
-        env.Prepend(_LIBFLAGS="-Wl,--start-group ")
-        env.Append(_LIBFLAGS=" -Wl,--end-group")
+    # prepend project libs to the beginning of list
+    env.Prepend(LIBS=project_lib_builder.build())
+    # prepend extra linker related options from libs
+    env.PrependUnique(
+        **{
+            key: project_lib_builder.env.get(key)
+            for key in ("LIBS", "LIBPATH", "LINKFLAGS")
+            if project_lib_builder.env.get(key)
+        }
+    )
 
-    program = env.Program(
-        join("$BUILD_DIR", env.subst("$PROGNAME")), env['PIOBUILDFILES'])
-    env.Replace(PIOMAINPROG=program)
+    projenv = env.Clone()
 
-    AlwaysBuild(
-        env.Alias(
-            "checkprogsize", program,
-            env.VerboseAction(env.CheckUploadSize,
-                              "Checking size $PIOMAINPROG")))
+    # CPPPATH from dependencies
+    projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH"))
+    # extra build flags from `platformio.ini`
+    projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
 
-    return program
+    is_test = "__test" in COMMAND_LINE_TARGETS
+    if is_test:
+        projenv.BuildSources(
+            "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
+        )
+    if not is_test or env.GetProjectOption("test_build_project_src"):
+        projenv.BuildSources(
+            "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER")
+        )
+
+    if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
+        sys.stderr.write(
+            "Error: Nothing to build. Please put your source code files "
+            "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
+        )
+        env.Exit(1)
+
+    Export("projenv")
 
 
 def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
@@ -152,30 +179,30 @@ def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
             result[key].extend(value)
 
     cppdefines = []
-    for item in result['CPPDEFINES']:
+    for item in result["CPPDEFINES"]:
         if not Util.is_Sequence(item):
             cppdefines.append(item)
             continue
         name, value = item[:2]
-        if '\"' in value:
-            value = value.replace('\"', '\\\"')
+        if '"' in value:
+            value = value.replace('"', '\\"')
         elif value.isdigit():
             value = int(value)
         elif value.replace(".", "", 1).isdigit():
             value = float(value)
         cppdefines.append((name, value))
-    result['CPPDEFINES'] = cppdefines
+    result["CPPDEFINES"] = cppdefines
 
     # fix relative CPPPATH & LIBPATH
     for k in ("CPPPATH", "LIBPATH"):
         for i, p in enumerate(result.get(k, [])):
-            if isdir(p):
-                result[k][i] = realpath(p)
+            if os.path.isdir(p):
+                result[k][i] = os.path.realpath(p)
 
     # fix relative path for "-include"
     for i, f in enumerate(result.get("CCFLAGS", [])):
         if isinstance(f, tuple) and f[0] == "-include":
-            result['CCFLAGS'][i] = (f[0], env.File(realpath(f[1].get_path())))
+            result["CCFLAGS"][i] = (f[0], env.File(os.path.realpath(f[1].get_path())))
 
     return result
 
@@ -188,12 +215,15 @@ def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
     # Cancel any previous definition of name, either built in or
     # provided with a -U option // Issue #191
     undefines = [
-        u for u in env.get("CCFLAGS", [])
-        if isinstance(u, basestring) and u.startswith("-U")
+        u
+        for u in env.get("CCFLAGS", [])
+        if isinstance(u, string_types) and u.startswith("-U")
     ]
     if undefines:
         for undef in undefines:
-            env['CCFLAGS'].remove(undef)
+            env["CCFLAGS"].remove(undef)
+            if undef[2:] in env["CPPDEFINES"]:
+                env["CPPDEFINES"].remove(undef[2:])
         env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines))
 
 
@@ -216,87 +246,69 @@ def ProcessUnFlags(env, flags):
             for current in env.get(key, []):
                 conditions = [
                     unflag == current,
-                    isinstance(current, (tuple, list))
-                    and unflag[0] == current[0]
+                    isinstance(current, (tuple, list)) and unflag[0] == current[0],
                 ]
                 if any(conditions):
                     env[key].remove(current)
 
 
-def IsFileWithExt(env, file_, ext):  # pylint: disable=W0613
-    if basename(file_).startswith("."):
-        return False
-    for e in ext:
-        if file_.endswith(".%s" % e):
-            return True
-    return False
-
-
 def MatchSourceFiles(env, src_dir, src_filter=None):
-
-    def _append_build_item(items, item, src_dir):
-        if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT):
-            items.add(item.replace(src_dir + sep, ""))
-
-    src_dir = env.subst(src_dir)
     src_filter = env.subst(src_filter) if src_filter else None
     src_filter = src_filter or SRC_FILTER_DEFAULT
-    if isinstance(src_filter, (list, tuple)):
-        src_filter = " ".join(src_filter)
-
-    matches = set()
-    # correct fs directory separator
-    src_filter = src_filter.replace("/", sep).replace("\\", sep)
-    for (action, pattern) in SRC_FILTER_PATTERNS_RE.findall(src_filter):
-        items = set()
-        for item in glob(join(glob_escape(src_dir), pattern)):
-            if isdir(item):
-                for root, _, files in walk(item, followlinks=True):
-                    for f in files:
-                        _append_build_item(items, join(root, f), src_dir)
-            else:
-                _append_build_item(items, item, src_dir)
-        if action == "+":
-            matches |= items
-        else:
-            matches -= items
-    return sorted(list(matches))
+    return fs.match_src_files(
+        env.subst(src_dir), src_filter, SRC_BUILD_EXT + SRC_HEADER_EXT
+    )
 
 
-def CollectBuildFiles(env,
-                      variant_dir,
-                      src_dir,
-                      src_filter=None,
-                      duplicate=False):
+def CollectBuildFiles(
+    env, variant_dir, src_dir, src_filter=None, duplicate=False
+):  # pylint: disable=too-many-locals
     sources = []
     variants = []
 
     src_dir = env.subst(src_dir)
-    if src_dir.endswith(sep):
+    if src_dir.endswith(os.sep):
         src_dir = src_dir[:-1]
 
     for item in env.MatchSourceFiles(src_dir, src_filter):
-        _reldir = dirname(item)
-        _src_dir = join(src_dir, _reldir) if _reldir else src_dir
-        _var_dir = join(variant_dir, _reldir) if _reldir else variant_dir
+        _reldir = os.path.dirname(item)
+        _src_dir = os.path.join(src_dir, _reldir) if _reldir else src_dir
+        _var_dir = os.path.join(variant_dir, _reldir) if _reldir else variant_dir
 
         if _var_dir not in variants:
             variants.append(_var_dir)
             env.VariantDir(_var_dir, _src_dir, duplicate)
 
-        if env.IsFileWithExt(item, SRC_BUILD_EXT):
-            sources.append(env.File(join(_var_dir, basename(item))))
+        if fs.path_endswith_ext(item, SRC_BUILD_EXT):
+            sources.append(env.File(os.path.join(_var_dir, os.path.basename(item))))
+
+    for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []):
+        tmp = []
+        for node in sources:
+            if pattern and not fnmatch.fnmatch(node.get_path(), pattern):
+                tmp.append(node)
+                continue
+            n = callback(node)
+            if n:
+                tmp.append(n)
+        sources = tmp
 
     return sources
 
 
+def AddBuildMiddleware(env, callback, pattern=None):
+    env.Append(__PIO_BUILD_MIDDLEWARES=[(callback, pattern)])
+
+
 def BuildFrameworks(env, frameworks):
     if not frameworks:
         return
 
     if "BOARD" not in env:
-        sys.stderr.write("Please specify `board` in `platformio.ini` to use "
-                         "with '%s' framework\n" % ", ".join(frameworks))
+        sys.stderr.write(
+            "Please specify `board` in `platformio.ini` to use "
+            "with '%s' framework\n" % ", ".join(frameworks)
+        )
         env.Exit(1)
 
     board_frameworks = env.BoardConfig().get("frameworks", [])
@@ -304,12 +316,11 @@ def BuildFrameworks(env, frameworks):
         if board_frameworks:
             frameworks.insert(0, board_frameworks[0])
         else:
-            sys.stderr.write(
-                "Error: Please specify `board` in `platformio.ini`\n")
+            sys.stderr.write("Error: Please specify `board` in `platformio.ini`\n")
             env.Exit(1)
 
     for f in frameworks:
-        if f in ("arduino", "energia"):
+        if f == "arduino":
             # Arduino IDE appends .o the end of filename
             Builder.match_splitext = scons_patched_match_splitext
             if "nobuild" not in COMMAND_LINE_TARGETS:
@@ -318,22 +329,24 @@ def BuildFrameworks(env, frameworks):
         if f in board_frameworks:
             SConscript(env.GetFrameworkScript(f), exports="env")
         else:
-            sys.stderr.write(
-                "Error: This board doesn't support %s framework!\n" % f)
+            sys.stderr.write("Error: This board doesn't support %s framework!\n" % f)
             env.Exit(1)
 
 
 def BuildLibrary(env, variant_dir, src_dir, src_filter=None):
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
     return env.StaticLibrary(
-        env.subst(variant_dir),
-        env.CollectBuildFiles(variant_dir, src_dir, src_filter))
+        env.subst(variant_dir), env.CollectBuildFiles(variant_dir, src_dir, src_filter)
+    )
 
 
 def BuildSources(env, variant_dir, src_dir, src_filter=None):
     nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter)
     DefaultEnvironment().Append(
-        PIOBUILDFILES=[env.Object(node) for node in nodes])
+        PIOBUILDFILES=[
+            env.Object(node) if isinstance(node, FS.File) else node for node in nodes
+        ]
+    )
 
 
 def exists(_):
@@ -341,13 +354,16 @@ def exists(_):
 
 
 def generate(env):
+    env.AddMethod(GetBuildType)
     env.AddMethod(BuildProgram)
+    env.AddMethod(ProcessProgramDeps)
+    env.AddMethod(ProcessProjectDeps)
     env.AddMethod(ParseFlagsExtended)
     env.AddMethod(ProcessFlags)
     env.AddMethod(ProcessUnFlags)
-    env.AddMethod(IsFileWithExt)
     env.AddMethod(MatchSourceFiles)
     env.AddMethod(CollectBuildFiles)
+    env.AddMethod(AddBuildMiddleware)
     env.AddMethod(BuildFrameworks)
     env.AddMethod(BuildLibrary)
     env.AddMethod(BuildSources)

platformio/commands/__init__.py

@@ -11,3 +11,70 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import os
+
+import click
+
+
+class PlatformioCLI(click.MultiCommand):
+
+    leftover_args = []
+
+    def __init__(self, *args, **kwargs):
+        super(PlatformioCLI, self).__init__(*args, **kwargs)
+        self._pio_cmds_dir = os.path.dirname(__file__)
+
+    @staticmethod
+    def in_silence():
+        args = PlatformioCLI.leftover_args
+        return args and any(
+            [
+                args[0] == "debug" and "--interpreter" in " ".join(args),
+                args[0] == "upgrade",
+                "--json-output" in args,
+                "--version" in args,
+            ]
+        )
+
+    def invoke(self, ctx):
+        PlatformioCLI.leftover_args = ctx.args
+        if hasattr(ctx, "protected_args"):
+            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
+        return super(PlatformioCLI, self).invoke(ctx)
+
+    def list_commands(self, ctx):
+        cmds = []
+        for cmd_name in os.listdir(self._pio_cmds_dir):
+            if cmd_name.startswith("__init__"):
+                continue
+            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
+                cmds.append(cmd_name)
+            elif cmd_name.endswith(".py"):
+                cmds.append(cmd_name[:-3])
+        cmds.sort()
+        return cmds
+
+    def get_command(self, ctx, cmd_name):
+        mod = None
+        try:
+            mod_path = "platformio.commands." + cmd_name
+            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
+                mod_path = "platformio.commands.%s.command" % cmd_name
+            mod = __import__(mod_path, None, None, ["cli"])
+        except ImportError:
+            try:
+                return self._handle_obsolate_command(cmd_name)
+            except AttributeError:
+                pass
+            raise click.UsageError('No such command "%s"' % cmd_name, ctx)
+        return mod.cli
+
+    @staticmethod
+    def _handle_obsolate_command(name):
+        # pylint: disable=import-outside-toplevel
+        if name == "init":
+            from platformio.commands.project import project_init
+
+            return project_init
+        raise AttributeError()

platformio/commands/boards.py

@@ -15,8 +15,10 @@
 import json
 
 import click
+from tabulate import tabulate
 
-from platformio import util
+from platformio import fs
+from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformManager
 
 
@@ -30,45 +32,42 @@ def cli(query, installed, json_output):  # pylint: disable=R0912
 
     grpboards = {}
     for board in _get_boards(installed):
-        if query and query.lower() not in json.dumps(board).lower():
+        if query and not any(
+            query.lower() in str(board.get(k, "")).lower()
+            for k in ("id", "name", "mcu", "vendor", "platform", "frameworks")
+        ):
             continue
-        if board['platform'] not in grpboards:
-            grpboards[board['platform']] = []
-        grpboards[board['platform']].append(board)
+        if board["platform"] not in grpboards:
+            grpboards[board["platform"]] = []
+        grpboards[board["platform"]].append(board)
 
     terminal_width, _ = click.get_terminal_size()
     for (platform, boards) in sorted(grpboards.items()):
         click.echo("")
         click.echo("Platform: ", nl=False)
         click.secho(platform, bold=True)
-        click.echo("-" * terminal_width)
+        click.echo("=" * terminal_width)
         print_boards(boards)
     return True
 
 
 def print_boards(boards):
-    terminal_width, _ = click.get_terminal_size()
-    BOARDLIST_TPL = ("{type:<30} {mcu:<14} {frequency:<8} "
-                     " {flash:<7} {ram:<6} {name}")
     click.echo(
-        BOARDLIST_TPL.format(
-            type=click.style("ID", fg="cyan"),
-            mcu="MCU",
-            frequency="Frequency",
-            flash="Flash",
-            ram="RAM",
-            name="Name"))
-    click.echo("-" * terminal_width)
-
-    for board in boards:
-        click.echo(
-            BOARDLIST_TPL.format(
-                type=click.style(board['id'], fg="cyan"),
-                mcu=board['mcu'],
-                frequency="%dMHz" % (board['fcpu'] / 1000000),
-                flash=util.format_filesize(board['rom']),
-                ram=util.format_filesize(board['ram']),
-                name=board['name']))
+        tabulate(
+            [
+                (
+                    click.style(b["id"], fg="cyan"),
+                    b["mcu"],
+                    "%dMHz" % (b["fcpu"] / 1000000),
+                    fs.format_filesize(b["rom"]),
+                    fs.format_filesize(b["ram"]),
+                    b["name"],
+                )
+                for b in boards
+            ],
+            headers=["ID", "MCU", "Frequency", "Flash", "RAM", "Name"],
+        )
+    )
 
 
 def _get_boards(installed=False):
@@ -80,8 +79,8 @@ def _print_boards_json(query, installed=False):
     result = []
     for board in _get_boards(installed):
         if query:
-            search_data = "%s %s" % (board['id'], json.dumps(board).lower())
+            search_data = "%s %s" % (board["id"], json.dumps(board).lower())
             if query.lower() not in search_data.lower():
                 continue
         result.append(board)
-    click.echo(json.dumps(result))
+    click.echo(dump_json_to_unicode(result))

platformio/commands/check/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/check/command.py

@@ -0,0 +1,316 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
+# pylint: disable=redefined-builtin,too-many-statements
+
+import os
+from collections import Counter
+from os.path import dirname, isfile
+from time import time
+
+import click
+from tabulate import tabulate
+
+from platformio import app, exception, fs, util
+from platformio.commands.check.defect import DefectItem
+from platformio.commands.check.tools import CheckToolFactory
+from platformio.compat import dump_json_to_unicode
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import find_project_dir_above, get_project_dir
+
+
+@click.command("check", short_help="Run a static analysis tool on code")
+@click.option("-e", "--environment", multiple=True)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
+@click.option("--pattern", multiple=True)
+@click.option("--flags", multiple=True)
+@click.option(
+    "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values())
+)
+@click.option("-s", "--silent", is_flag=True)
+@click.option("-v", "--verbose", is_flag=True)
+@click.option("--json-output", is_flag=True)
+@click.option(
+    "--fail-on-defect",
+    multiple=True,
+    type=click.Choice(DefectItem.SEVERITY_LABELS.values()),
+)
+def cli(
+    environment,
+    project_dir,
+    project_conf,
+    pattern,
+    flags,
+    severity,
+    silent,
+    verbose,
+    json_output,
+    fail_on_defect,
+):
+    app.set_session_var("custom_project_conf", project_conf)
+
+    # find project directory on upper level
+    if isfile(project_dir):
+        project_dir = find_project_dir_above(project_dir)
+
+    results = []
+    with fs.cd(project_dir):
+        config = ProjectConfig.get_instance(project_conf)
+        config.validate(environment)
+
+        default_envs = config.default_envs()
+        for envname in config.envs():
+            skipenv = any(
+                [
+                    environment and envname not in environment,
+                    not environment and default_envs and envname not in default_envs,
+                ]
+            )
+
+            env_options = config.items(env=envname, as_dict=True)
+            env_dump = []
+            for k, v in env_options.items():
+                if k not in ("platform", "framework", "board"):
+                    continue
+                env_dump.append(
+                    "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)
+                )
+
+            default_patterns = [
+                config.get_optional_dir("src"),
+                config.get_optional_dir("include"),
+            ]
+            tool_options = dict(
+                verbose=verbose,
+                silent=silent,
+                patterns=pattern or env_options.get("check_patterns", default_patterns),
+                flags=flags or env_options.get("check_flags"),
+                severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]]
+                if silent
+                else severity or config.get("env:" + envname, "check_severity"),
+            )
+
+            for tool in config.get("env:" + envname, "check_tool"):
+                if skipenv:
+                    results.append({"env": envname, "tool": tool})
+                    continue
+                if not silent and not json_output:
+                    print_processing_header(tool, envname, env_dump)
+
+                ct = CheckToolFactory.new(
+                    tool, project_dir, config, envname, tool_options
+                )
+
+                result = {"env": envname, "tool": tool, "duration": time()}
+                rc = ct.check(
+                    on_defect_callback=None
+                    if (json_output or verbose)
+                    else lambda defect: click.echo(repr(defect))
+                )
+
+                result["defects"] = ct.get_defects()
+                result["duration"] = time() - result["duration"]
+
+                result["succeeded"] = rc == 0
+                if fail_on_defect:
+                    result["succeeded"] = rc == 0 and not any(
+                        DefectItem.SEVERITY_LABELS[d.severity] in fail_on_defect
+                        for d in result["defects"]
+                    )
+                result["stats"] = collect_component_stats(result)
+                results.append(result)
+
+                if verbose:
+                    click.echo("\n".join(repr(d) for d in result["defects"]))
+
+                if not json_output and not silent:
+                    if rc != 0:
+                        click.echo(
+                            "Error: %s failed to perform check! Please "
+                            "examine tool output in verbose mode." % tool
+                        )
+                    elif not result["defects"]:
+                        click.echo("No defects found")
+                    print_processing_footer(result)
+
+        if json_output:
+            click.echo(dump_json_to_unicode(results_to_json(results)))
+        elif not silent:
+            print_check_summary(results)
+
+    command_failed = any(r.get("succeeded") is False for r in results)
+    if command_failed:
+        raise exception.ReturnErrorCode(1)
+
+
+def results_to_json(raw):
+    results = []
+    for item in raw:
+        if item.get("succeeded") is None:
+            continue
+        item.update(
+            {
+                "succeeded": bool(item.get("succeeded")),
+                "defects": [d.as_dict() for d in item.get("defects", [])],
+            }
+        )
+        results.append(item)
+
+    return results
+
+
+def print_processing_header(tool, envname, envdump):
+    click.echo(
+        "Checking %s > %s (%s)"
+        % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump))
+    )
+    terminal_width, _ = click.get_terminal_size()
+    click.secho("-" * terminal_width, bold=True)
+
+
+def print_processing_footer(result):
+    is_failed = not result.get("succeeded")
+    util.print_labeled_bar(
+        "[%s] Took %.2f seconds"
+        % (
+            (
+                click.style("FAILED", fg="red", bold=True)
+                if is_failed
+                else click.style("PASSED", fg="green", bold=True)
+            ),
+            result["duration"],
+        ),
+        is_error=is_failed,
+    )
+
+
+def collect_component_stats(result):
+    components = dict()
+
+    def _append_defect(component, defect):
+        if not components.get(component):
+            components[component] = Counter()
+        components[component].update({DefectItem.SEVERITY_LABELS[defect.severity]: 1})
+
+    for defect in result.get("defects", []):
+        component = dirname(defect.file) or defect.file
+        _append_defect(component, defect)
+
+        if component.startswith(get_project_dir()):
+            while os.sep in component:
+                component = dirname(component)
+                _append_defect(component, defect)
+
+    return components
+
+
+def print_defects_stats(results):
+    if not results:
+        return
+
+    component_stats = {}
+    for r in results:
+        for k, v in r.get("stats", {}).items():
+            if not component_stats.get(k):
+                component_stats[k] = Counter()
+            component_stats[k].update(r["stats"][k])
+
+    if not component_stats:
+        return
+
+    severity_labels = list(DefectItem.SEVERITY_LABELS.values())
+    severity_labels.reverse()
+    tabular_data = list()
+    for k, v in component_stats.items():
+        tool_defect = [v.get(s, 0) for s in severity_labels]
+        tabular_data.append([k] + tool_defect)
+
+    total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]]
+    tabular_data.sort()
+    tabular_data.append([])  # Empty line as delimiter
+    tabular_data.append(total)
+
+    headers = ["Component"]
+    headers.extend([l.upper() for l in severity_labels])
+    headers = [click.style(h, bold=True) for h in headers]
+    click.echo(tabulate(tabular_data, headers=headers, numalign="center"))
+    click.echo()
+
+
+def print_check_summary(results):
+    click.echo()
+
+    tabular_data = []
+    succeeded_nums = 0
+    failed_nums = 0
+    duration = 0
+
+    print_defects_stats(results)
+
+    for result in results:
+        duration += result.get("duration", 0)
+        if result.get("succeeded") is False:
+            failed_nums += 1
+            status_str = click.style("FAILED", fg="red")
+        elif result.get("succeeded") is None:
+            status_str = "IGNORED"
+        else:
+            succeeded_nums += 1
+            status_str = click.style("PASSED", fg="green")
+
+        tabular_data.append(
+            (
+                click.style(result["env"], fg="cyan"),
+                result["tool"],
+                status_str,
+                util.humanize_duration_time(result.get("duration")),
+            )
+        )
+
+    click.echo(
+        tabulate(
+            tabular_data,
+            headers=[
+                click.style(s, bold=True)
+                for s in ("Environment", "Tool", "Status", "Duration")
+            ],
+        ),
+        err=failed_nums,
+    )
+
+    util.print_labeled_bar(
+        "%s%d succeeded in %s"
+        % (
+            "%d failed, " % failed_nums if failed_nums else "",
+            succeeded_nums,
+            util.humanize_duration_time(duration),
+        ),
+        is_error=failed_nums,
+        fg="red" if failed_nums else "green",
+    )

platformio/commands/check/defect.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import click
+
+from platformio.project.helpers import get_project_dir
+
+# pylint: disable=too-many-instance-attributes, redefined-builtin
+# pylint: disable=too-many-arguments
+
+
+class DefectItem(object):
+
+    SEVERITY_HIGH = 1
+    SEVERITY_MEDIUM = 2
+    SEVERITY_LOW = 4
+    SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"}
+
+    def __init__(
+        self,
+        severity,
+        category,
+        message,
+        file="unknown",
+        line=0,
+        column=0,
+        id=None,
+        callstack=None,
+        cwe=None,
+    ):
+        assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, self.SEVERITY_LOW)
+        self.severity = severity
+        self.category = category
+        self.message = message
+        self.line = int(line)
+        self.column = int(column)
+        self.callstack = callstack
+        self.cwe = cwe
+        self.id = id
+        self.file = file
+        if file.startswith(get_project_dir()):
+            self.file = os.path.relpath(file, get_project_dir())
+
+    def __repr__(self):
+        defect_color = None
+        if self.severity == self.SEVERITY_HIGH:
+            defect_color = "red"
+        elif self.severity == self.SEVERITY_MEDIUM:
+            defect_color = "yellow"
+
+        format_str = "{file}:{line}: [{severity}:{category}] {message} {id}"
+        return format_str.format(
+            severity=click.style(self.SEVERITY_LABELS[self.severity], fg=defect_color),
+            category=click.style(self.category.lower(), fg=defect_color),
+            file=click.style(self.file, bold=True),
+            message=self.message,
+            line=self.line,
+            id="%s" % "[%s]" % self.id if self.id else "",
+        )
+
+    def __or__(self, defect):
+        return self.severity | defect.severity
+
+    @staticmethod
+    def severity_to_int(label):
+        for key, value in DefectItem.SEVERITY_LABELS.items():
+            if label == value:
+                return key
+        raise Exception("Unknown severity label -> %s" % label)
+
+    def as_dict(self):
+        return {
+            "severity": self.SEVERITY_LABELS[self.severity],
+            "category": self.category,
+            "message": self.message,
+            "file": os.path.realpath(self.file),
+            "line": self.line,
+            "column": self.column,
+            "callstack": self.callstack,
+            "id": self.id,
+            "cwe": self.cwe,
+        }

platformio/commands/check/tools/__init__.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio import exception
+from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool
+from platformio.commands.check.tools.cppcheck import CppcheckCheckTool
+from platformio.commands.check.tools.pvsstudio import PvsStudioCheckTool
+
+
+class CheckToolFactory(object):
+    @staticmethod
+    def new(tool, project_dir, config, envname, options):
+        cls = None
+        if tool == "cppcheck":
+            cls = CppcheckCheckTool
+        elif tool == "clangtidy":
+            cls = ClangtidyCheckTool
+        elif tool == "pvs-studio":
+            cls = PvsStudioCheckTool
+        else:
+            raise exception.PlatformioException("Unknown check tool `%s`" % tool)
+        return cls(project_dir, config, envname, options)

platformio/commands/check/tools/base.py

@@ -0,0 +1,177 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import os
+
+import click
+
+from platformio import fs, proc
+from platformio.commands.check.defect import DefectItem
+from platformio.project.helpers import get_project_dir, load_project_ide_data
+
+
+class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
+    def __init__(self, project_dir, config, envname, options):
+        self.config = config
+        self.envname = envname
+        self.options = options
+        self.cc_flags = []
+        self.cxx_flags = []
+        self.cpp_includes = []
+        self.cpp_defines = []
+        self.toolchain_defines = []
+        self.cc_path = None
+        self.cxx_path = None
+        self._defects = []
+        self._on_defect_callback = None
+        self._bad_input = False
+        self._load_cpp_data(project_dir, envname)
+
+        # detect all defects by default
+        if not self.options.get("severity"):
+            self.options["severity"] = [
+                DefectItem.SEVERITY_LOW,
+                DefectItem.SEVERITY_MEDIUM,
+                DefectItem.SEVERITY_HIGH,
+            ]
+        # cast to severity by ids
+        self.options["severity"] = [
+            s if isinstance(s, int) else DefectItem.severity_to_int(s)
+            for s in self.options["severity"]
+        ]
+
+    def _load_cpp_data(self, project_dir, envname):
+        data = load_project_ide_data(project_dir, envname)
+        if not data:
+            return
+        self.cc_flags = data.get("cc_flags", "").split(" ")
+        self.cxx_flags = data.get("cxx_flags", "").split(" ")
+        self.cpp_includes = data.get("includes", [])
+        self.cpp_defines = data.get("defines", [])
+        self.cc_path = data.get("cc_path")
+        self.cxx_path = data.get("cxx_path")
+        self.toolchain_defines = self._get_toolchain_defines(self.cc_path)
+
+    def get_flags(self, tool):
+        result = []
+        flags = self.options.get("flags") or []
+        for flag in flags:
+            if ":" not in flag or flag.startswith("-"):
+                result.extend([f for f in flag.split(" ") if f])
+            elif flag.startswith("%s:" % tool):
+                result.extend([f for f in flag.split(":", 1)[1].split(" ") if f])
+
+        return result
+
+    @staticmethod
+    def _get_toolchain_defines(cc_path):
+        defines = []
+        result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path, shell=True)
+
+        for line in result["out"].split("\n"):
+            tokens = line.strip().split(" ", 2)
+            if not tokens or tokens[0] != "#define":
+                continue
+            if len(tokens) > 2:
+                defines.append("%s=%s" % (tokens[1], tokens[2]))
+            else:
+                defines.append(tokens[1])
+
+        return defines
+
+    @staticmethod
+    def is_flag_set(flag, flags):
+        return any(flag in f for f in flags)
+
+    def get_defects(self):
+        return self._defects
+
+    def configure_command(self):
+        raise NotImplementedError
+
+    def on_tool_output(self, line):
+        line = self.tool_output_filter(line)
+        if not line:
+            return
+
+        defect = self.parse_defect(line)
+
+        if not isinstance(defect, DefectItem):
+            if self.options.get("verbose"):
+                click.echo(line)
+            return
+
+        if defect.severity not in self.options["severity"]:
+            return
+
+        self._defects.append(defect)
+        if self._on_defect_callback:
+            self._on_defect_callback(defect)
+
+    @staticmethod
+    def tool_output_filter(line):
+        return line
+
+    @staticmethod
+    def parse_defect(raw_line):
+        return raw_line
+
+    def clean_up(self):
+        pass
+
+    def get_project_target_files(self):
+        allowed_extensions = (".h", ".hpp", ".c", ".cc", ".cpp", ".ino")
+        result = []
+
+        def _add_file(path):
+            if not path.endswith(allowed_extensions):
+                return
+            result.append(os.path.realpath(path))
+
+        for pattern in self.options["patterns"]:
+            for item in glob.glob(pattern):
+                if not os.path.isdir(item):
+                    _add_file(item)
+                for root, _, files in os.walk(item, followlinks=True):
+                    for f in files:
+                        _add_file(os.path.join(root, f))
+
+        return result
+
+    def get_source_language(self):
+        with fs.cd(get_project_dir()):
+            for _, __, files in os.walk(self.config.get_optional_dir("src")):
+                for name in files:
+                    if "." not in name:
+                        continue
+                    if os.path.splitext(name)[1].lower() in (".cpp", ".cxx", ".ino"):
+                        return "c++"
+            return "c"
+
+    def check(self, on_defect_callback=None):
+        self._on_defect_callback = on_defect_callback
+        cmd = self.configure_command()
+        if self.options.get("verbose"):
+            click.echo(" ".join(cmd))
+
+        proc.exec_command(
+            cmd,
+            stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
+            stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
+        )
+
+        self.clean_up()
+
+        return self._bad_input

platformio/commands/check/tools/clangtidy.py

@@ -0,0 +1,67 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from os.path import join
+
+from platformio.commands.check.defect import DefectItem
+from platformio.commands.check.tools.base import CheckToolBase
+from platformio.managers.core import get_core_package_dir
+
+
+class ClangtidyCheckTool(CheckToolBase):
+    def tool_output_filter(self, line):
+        if not self.options.get("verbose") and "[clang-diagnostic-error]" in line:
+            return ""
+
+        if "[CommonOptionsParser]" in line:
+            self._bad_input = True
+            return line
+
+        if any(d in line for d in ("note: ", "error: ", "warning: ")):
+            return line
+
+        return ""
+
+    def parse_defect(self, raw_line):
+        match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line)
+        if not match:
+            return raw_line
+
+        file_, line, column, category, message, defect_id = match.groups()
+
+        severity = DefectItem.SEVERITY_LOW
+        if category == "error":
+            severity = DefectItem.SEVERITY_HIGH
+        elif category == "warning":
+            severity = DefectItem.SEVERITY_MEDIUM
+
+        return DefectItem(severity, category, message, file_, line, column, defect_id)
+
+    def configure_command(self):
+        tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy")
+
+        cmd = [tool_path, "--quiet"]
+        flags = self.get_flags("clangtidy")
+        if not self.is_flag_set("--checks", flags):
+            cmd.append("--checks=*")
+
+        cmd.extend(flags)
+        cmd.extend(self.get_project_target_files())
+        cmd.append("--")
+
+        cmd.extend(["-D%s" % d for d in self.cpp_defines + self.toolchain_defines])
+        cmd.extend(["-I%s" % inc for inc in self.cpp_includes])
+
+        return cmd

platformio/commands/check/tools/cppcheck.py

@@ -0,0 +1,158 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os import remove
+from os.path import isfile, join
+from tempfile import NamedTemporaryFile
+
+from platformio.commands.check.defect import DefectItem
+from platformio.commands.check.tools.base import CheckToolBase
+from platformio.managers.core import get_core_package_dir
+
+
+class CppcheckCheckTool(CheckToolBase):
+    def __init__(self, *args, **kwargs):
+        self._tmp_files = []
+        self.defect_fields = [
+            "severity",
+            "message",
+            "file",
+            "line",
+            "column",
+            "callstack",
+            "cwe",
+            "id",
+        ]
+        super(CppcheckCheckTool, self).__init__(*args, **kwargs)
+
+    def tool_output_filter(self, line):
+        if (
+            not self.options.get("verbose")
+            and "--suppress=unmatchedSuppression:" in line
+        ):
+            return ""
+
+        if any(
+            msg in line
+            for msg in (
+                "No C or C++ source files found",
+                "unrecognized command line option",
+            )
+        ):
+            self._bad_input = True
+
+        return line
+
+    def parse_defect(self, raw_line):
+        if "<&PIO&>" not in raw_line or any(
+            f not in raw_line for f in self.defect_fields
+        ):
+            return None
+
+        args = dict()
+        for field in raw_line.split("<&PIO&>"):
+            field = field.strip().replace('"', "")
+            name, value = field.split("=", 1)
+            args[name] = value
+
+        args["category"] = args["severity"]
+        if args["severity"] == "error":
+            args["severity"] = DefectItem.SEVERITY_HIGH
+        elif args["severity"] == "warning":
+            args["severity"] = DefectItem.SEVERITY_MEDIUM
+        else:
+            args["severity"] = DefectItem.SEVERITY_LOW
+
+        return DefectItem(**args)
+
+    def configure_command(self):
+        tool_path = join(get_core_package_dir("tool-cppcheck"), "cppcheck")
+
+        cmd = [
+            tool_path,
+            "--error-exitcode=1",
+            "--verbose" if self.options.get("verbose") else "--quiet",
+        ]
+
+        cmd.append(
+            '--template="%s"'
+            % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields])
+        )
+
+        flags = self.get_flags("cppcheck")
+        if not flags:
+            # by default user can suppress reporting individual defects
+            # directly in code // cppcheck-suppress warningID
+            cmd.append("--inline-suppr")
+        if not self.is_flag_set("--platform", flags):
+            cmd.append("--platform=unspecified")
+        if not self.is_flag_set("--enable", flags):
+            enabled_checks = [
+                "warning",
+                "style",
+                "performance",
+                "portability",
+                "unusedFunction",
+            ]
+            cmd.append("--enable=%s" % ",".join(enabled_checks))
+
+        if not self.is_flag_set("--language", flags):
+            if self.get_source_language() == "c++":
+                cmd.append("--language=c++")
+
+                if not self.is_flag_set("--std", flags):
+                    for f in self.cxx_flags + self.cc_flags:
+                        if "-std" in f:
+                            # Standards with GNU extensions are not allowed
+                            cmd.append("-" + f.replace("gnu", "c"))
+
+        cmd.extend(["-D%s" % d for d in self.cpp_defines + self.toolchain_defines])
+        cmd.extend(flags)
+
+        cmd.append("--file-list=%s" % self._generate_src_file())
+        cmd.append("--includes-file=%s" % self._generate_inc_file())
+
+        core_dir = self.config.get_optional_dir("packages")
+        cmd.append("--suppress=*:%s*" % core_dir)
+        cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir)
+
+        return cmd
+
+    def _create_tmp_file(self, data):
+        with NamedTemporaryFile("w", delete=False) as fp:
+            fp.write(data)
+            self._tmp_files.append(fp.name)
+            return fp.name
+
+    def _generate_src_file(self):
+        src_files = [
+            f for f in self.get_project_target_files() if not f.endswith((".h", ".hpp"))
+        ]
+        return self._create_tmp_file("\n".join(src_files))
+
+    def _generate_inc_file(self):
+        return self._create_tmp_file("\n".join(self.cpp_includes))
+
+    def clean_up(self):
+        for f in self._tmp_files:
+            if isfile(f):
+                remove(f)
+
+        # delete temporary dump files generated by addons
+        if not self.is_flag_set("--addon", self.get_flags("cppcheck")):
+            return
+        for f in self.get_project_target_files():
+            dump_file = f + ".dump"
+            if isfile(dump_file):
+                remove(dump_file)

platformio/commands/check/tools/pvsstudio.py

@@ -0,0 +1,226 @@
+# Copyright (c) 2020-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tempfile
+from xml.etree.ElementTree import fromstring
+
+import click
+
+from platformio import proc, util
+from platformio.commands.check.defect import DefectItem
+from platformio.commands.check.tools.base import CheckToolBase
+from platformio.managers.core import get_core_package_dir
+
+
+class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-attributes
+    def __init__(self, *args, **kwargs):
+        self._tmp_dir = tempfile.mkdtemp(prefix="piocheck")
+        self._tmp_preprocessed_file = self._generate_tmp_file_path() + ".i"
+        self._tmp_output_file = self._generate_tmp_file_path() + ".pvs"
+        self._tmp_cfg_file = self._generate_tmp_file_path() + ".cfg"
+        self._tmp_cmd_file = self._generate_tmp_file_path() + ".cmd"
+        self.tool_path = os.path.join(
+            get_core_package_dir("tool-pvs-studio"),
+            "x64" if "windows" in util.get_systype() else "bin",
+            "pvs-studio",
+        )
+        super(PvsStudioCheckTool, self).__init__(*args, **kwargs)
+
+        with open(self._tmp_cfg_file, "w") as fp:
+            fp.write(
+                "exclude-path = "
+                + self.config.get_optional_dir("packages").replace("\\", "/")
+            )
+
+        with open(self._tmp_cmd_file, "w") as fp:
+            fp.write(
+                " ".join(
+                    ['-I"%s"' % inc.replace("\\", "/") for inc in self.cpp_includes]
+                )
+            )
+
+    def _process_defects(self, defects):
+        for defect in defects:
+            if not isinstance(defect, DefectItem):
+                return
+            if defect.severity not in self.options["severity"]:
+                return
+            self._defects.append(defect)
+            if self._on_defect_callback:
+                self._on_defect_callback(defect)
+
+    def _demangle_report(self, output_file):
+        converter_tool = os.path.join(
+            get_core_package_dir("tool-pvs-studio"),
+            "HtmlGenerator"
+            if "windows" in util.get_systype()
+            else os.path.join("bin", "plog-converter"),
+        )
+
+        cmd = (
+            converter_tool,
+            "-t",
+            "xml",
+            output_file,
+            "-m",
+            "cwe",
+            "-m",
+            "misra",
+            "-a",
+            # Enable all possible analyzers and defect levels
+            "GA:1,2,3;64:1,2,3;OP:1,2,3;CS:1,2,3;MISRA:1,2,3",
+            "--cerr",
+        )
+
+        result = proc.exec_command(cmd)
+        if result["returncode"] != 0:
+            click.echo(result["err"])
+            self._bad_input = True
+
+        return result["err"]
+
+    def parse_defects(self, output_file):
+        defects = []
+
+        report = self._demangle_report(output_file)
+        if not report:
+            self._bad_input = True
+            return []
+
+        try:
+            defects_data = fromstring(report)
+        except:  # pylint: disable=bare-except
+            click.echo("Error: Couldn't decode generated report!")
+            self._bad_input = True
+            return []
+
+        for table in defects_data.iter("PVS-Studio_Analysis_Log"):
+            message = table.find("Message").text
+            category = table.find("ErrorType").text
+            line = table.find("Line").text
+            file_ = table.find("File").text
+            defect_id = table.find("ErrorCode").text
+            cwe = table.find("CWECode")
+            cwe_id = None
+            if cwe is not None:
+                cwe_id = cwe.text.lower().replace("cwe-", "")
+            misra = table.find("MISRA")
+            if misra is not None:
+                message += " [%s]" % misra.text
+
+            severity = DefectItem.SEVERITY_LOW
+            if category == "error":
+                severity = DefectItem.SEVERITY_HIGH
+            elif category == "warning":
+                severity = DefectItem.SEVERITY_MEDIUM
+
+            defects.append(
+                DefectItem(
+                    severity, category, message, file_, line, id=defect_id, cwe=cwe_id
+                )
+            )
+
+        return defects
+
+    def configure_command(self, src_file):  # pylint: disable=arguments-differ
+        if os.path.isfile(self._tmp_output_file):
+            os.remove(self._tmp_output_file)
+
+        if not os.path.isfile(self._tmp_preprocessed_file):
+            click.echo(
+                "Error: Missing preprocessed file '%s'" % (self._tmp_preprocessed_file)
+            )
+            return ""
+
+        cmd = [
+            self.tool_path,
+            "--skip-cl-exe",
+            "yes",
+            "--language",
+            "C" if src_file.endswith(".c") else "C++",
+            "--preprocessor",
+            "gcc",
+            "--cfg",
+            self._tmp_cfg_file,
+            "--source-file",
+            src_file,
+            "--i-file",
+            self._tmp_preprocessed_file,
+            "--output-file",
+            self._tmp_output_file,
+        ]
+
+        flags = self.get_flags("pvs-studio")
+        if not self.is_flag_set("--platform", flags):
+            cmd.append("--platform=arm")
+        cmd.extend(flags)
+
+        return cmd
+
+    def _generate_tmp_file_path(self):
+        # pylint: disable=protected-access
+        return os.path.join(self._tmp_dir, next(tempfile._get_candidate_names()))
+
+    def _prepare_preprocessed_file(self, src_file):
+        flags = self.cxx_flags
+        compiler = self.cxx_path
+        if src_file.endswith(".c"):
+            flags = self.cc_flags
+            compiler = self.cc_path
+
+        cmd = [compiler, src_file, "-E", "-o", self._tmp_preprocessed_file]
+        cmd.extend([f for f in flags if f])
+        cmd.extend(["-D%s" % d for d in self.cpp_defines])
+        cmd.append('@"%s"' % self._tmp_cmd_file)
+
+        result = proc.exec_command(" ".join(cmd), shell=True)
+        if result["returncode"] != 0:
+            if self.options.get("verbose"):
+                click.echo(" ".join(cmd))
+            click.echo(result["err"])
+            self._bad_input = True
+
+    def clean_up(self):
+        if os.path.isdir(self._tmp_dir):
+            shutil.rmtree(self._tmp_dir)
+
+    def check(self, on_defect_callback=None):
+        self._on_defect_callback = on_defect_callback
+        src_files = [
+            f for f in self.get_project_target_files() if not f.endswith((".h", ".hpp"))
+        ]
+
+        for src_file in src_files:
+            self._prepare_preprocessed_file(src_file)
+            cmd = self.configure_command(src_file)
+            if self.options.get("verbose"):
+                click.echo(" ".join(cmd))
+            if not cmd:
+                self._bad_input = True
+                continue
+
+            result = proc.exec_command(cmd)
+            # pylint: disable=unsupported-membership-test
+            if result["returncode"] != 0 or "License was not entered" in result["err"]:
+                self._bad_input = True
+                click.echo(result["err"])
+                continue
+
+            self._process_defects(self.parse_defects(self._tmp_output_file))
+
+        self.clean_up()
+
+        return self._bad_input

platformio/commands/ci.py

@@ -14,17 +14,19 @@
 
 from glob import glob
 from os import getenv, makedirs, remove
-from os.path import abspath, basename, expanduser, isdir, isfile, join
+from os.path import basename, isdir, isfile, join, realpath
 from shutil import copyfile, copytree
 from tempfile import mkdtemp
 
 import click
 
-from platformio import app, util
-from platformio.commands.init import cli as cmd_init
-from platformio.commands.init import validate_boards
-from platformio.commands.run import cli as cmd_run
+from platformio import app, fs
+from platformio.commands.project import project_init as cmd_project_init
+from platformio.commands.project import validate_boards
+from platformio.commands.run.command import cli as cmd_run
+from platformio.compat import glob_escape
 from platformio.exception import CIBuildEnvsEmpty
+from platformio.project.config import ProjectConfig
 
 
 def validate_path(ctx, param, value):  # pylint: disable=unused-argument
@@ -32,8 +34,8 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
     value = list(value)
     for i, p in enumerate(value):
         if p.startswith("~"):
-            value[i] = expanduser(p)
-        value[i] = abspath(value[i])
+            value[i] = fs.expanduser(p)
+        value[i] = realpath(value[i])
         if not glob(value[i]):
             invalid_path = p
             break
@@ -46,36 +48,37 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
 
 @click.command("ci", short_help="Continuous Integration")
 @click.argument("src", nargs=-1, callback=validate_path)
-@click.option(
-    "-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY")
+@click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY")
 @click.option("--exclude", multiple=True)
-@click.option(
-    "-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
+@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
 @click.option(
     "--build-dir",
     default=mkdtemp,
-    type=click.Path(
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True))
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
+)
 @click.option("--keep-build-dir", is_flag=True)
 @click.option(
-    "-C",
+    "-c",
     "--project-conf",
     type=click.Path(
-        exists=True,
-        file_okay=True,
-        dir_okay=False,
-        readable=True,
-        resolve_path=True))
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
 @click.option("-O", "--project-option", multiple=True)
 @click.option("-v", "--verbose", is_flag=True)
 @click.pass_context
-def cli(  # pylint: disable=too-many-arguments
-        ctx, src, lib, exclude, board, build_dir, keep_build_dir, project_conf,
-        project_option, verbose):
+def cli(  # pylint: disable=too-many-arguments, too-many-branches
+    ctx,
+    src,
+    lib,
+    exclude,
+    board,
+    build_dir,
+    keep_build_dir,
+    project_conf,
+    project_option,
+    verbose,
+):
 
     if not src and getenv("PLATFORMIO_CI_SRC"):
         src = validate_path(ctx, None, getenv("PLATFORMIO_CI_SRC").split(":"))
@@ -84,7 +87,11 @@ def cli(  # pylint: disable=too-many-arguments
 
     try:
         app.set_session_var("force_option", True)
-        _clean_dir(build_dir)
+
+        if not keep_build_dir and isdir(build_dir):
+            fs.rmtree(build_dir)
+        if not isdir(build_dir):
+            makedirs(build_dir)
 
         for dir_name, patterns in dict(lib=lib, src=src).items():
             if not patterns:
@@ -104,21 +111,17 @@ def cli(  # pylint: disable=too-many-arguments
 
         # initialise project
         ctx.invoke(
-            cmd_init,
+            cmd_project_init,
             project_dir=build_dir,
             board=board,
-            project_option=project_option)
+            project_option=project_option,
+        )
 
         # process project
         ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose)
     finally:
         if not keep_build_dir:
-            util.rmtree_(build_dir)
-
-
-def _clean_dir(dirpath):
-    util.rmtree_(dirpath)
-    makedirs(dirpath)
+            fs.rmtree(build_dir)
 
 
 def _copy_contents(dst_dir, contents):
@@ -126,44 +129,47 @@ def _copy_contents(dst_dir, contents):
 
     for path in contents:
         if isdir(path):
-            items['dirs'].add(path)
+            items["dirs"].add(path)
         elif isfile(path):
-            items['files'].add(path)
+            items["files"].add(path)
 
     dst_dir_name = basename(dst_dir)
 
-    if dst_dir_name == "src" and len(items['dirs']) == 1:
-        copytree(list(items['dirs']).pop(), dst_dir, symlinks=True)
+    if dst_dir_name == "src" and len(items["dirs"]) == 1:
+        copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
     else:
-        makedirs(dst_dir)
-        for d in items['dirs']:
+        if not isdir(dst_dir):
+            makedirs(dst_dir)
+        for d in items["dirs"]:
             copytree(d, join(dst_dir, basename(d)), symlinks=True)
 
-    if not items['files']:
+    if not items["files"]:
         return
 
     if dst_dir_name == "lib":
         dst_dir = join(dst_dir, mkdtemp(dir=dst_dir))
 
-    for f in items['files']:
-        copyfile(f, join(dst_dir, basename(f)))
+    for f in items["files"]:
+        dst_file = join(dst_dir, basename(f))
+        if f == dst_file:
+            continue
+        copyfile(f, dst_file)
 
 
 def _exclude_contents(dst_dir, patterns):
     contents = []
     for p in patterns:
-        contents += glob(join(util.glob_escape(dst_dir), p))
+        contents += glob(join(glob_escape(dst_dir), p))
     for path in contents:
-        path = abspath(path)
+        path = realpath(path)
         if isdir(path):
-            util.rmtree_(path)
+            fs.rmtree(path)
         elif isfile(path):
             remove(path)
 
 
 def _copy_project_conf(build_dir, project_conf):
-    config = util.load_project_config(project_conf)
+    config = ProjectConfig(project_conf, parse_extra=False)
     if config.has_section("platformio"):
         config.remove_section("platformio")
-    with open(join(build_dir, "platformio.ini"), "w") as fp:
-        config.write(fp)
+    config.save(join(build_dir, "platformio.ini"))

platformio/commands/debug.py

@@ -1,42 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-from os import getcwd
-
-import click
-
-from platformio.managers.core import pioplus_call
-
-
-@click.command(
-    "debug",
-    context_settings=dict(ignore_unknown_options=True),
-    short_help="PIO Unified Debugger")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=getcwd,
-    type=click.Path(
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True))
-@click.option("--environment", "-e", metavar="<environment>")
-@click.option("--verbose", "-v", is_flag=True)
-@click.option("--interface", type=click.Choice(["gdb"]))
-@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
-def cli(*args, **kwargs):  # pylint: disable=unused-argument
-    pioplus_call(sys.argv[1:])

platformio/commands/debug/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/debug/client.py

@@ -0,0 +1,276 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import signal
+import time
+from hashlib import sha1
+from os.path import basename, dirname, isdir, join, realpath, splitext
+from tempfile import mkdtemp
+
+from twisted.internet import defer  # pylint: disable=import-error
+from twisted.internet import protocol  # pylint: disable=import-error
+from twisted.internet import reactor  # pylint: disable=import-error
+from twisted.internet import stdio  # pylint: disable=import-error
+from twisted.internet import task  # pylint: disable=import-error
+
+from platformio import app, fs, proc, telemetry, util
+from platformio.commands.debug import helpers
+from platformio.commands.debug.exception import DebugInvalidOptionsError
+from platformio.commands.debug.initcfgs import get_gdb_init_config
+from platformio.commands.debug.process import BaseProcess
+from platformio.commands.debug.server import DebugServer
+from platformio.compat import hashlib_encode_data, is_bytes
+from platformio.project.helpers import get_project_cache_dir
+
+
+class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
+
+    PIO_SRC_NAME = ".pioinit"
+    INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed"
+
+    def __init__(self, project_dir, args, debug_options, env_options):
+        super(GDBClient, self).__init__()
+        self.project_dir = project_dir
+        self.args = list(args)
+        self.debug_options = debug_options
+        self.env_options = env_options
+
+        self._debug_server = DebugServer(debug_options, env_options)
+        self._session_id = None
+
+        if not isdir(get_project_cache_dir()):
+            os.makedirs(get_project_cache_dir())
+        self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), prefix=".piodebug-")
+
+        self._target_is_run = False
+        self._auto_continue_timer = None
+        self._errors_buffer = b""
+
+    @defer.inlineCallbacks
+    def spawn(self, gdb_path, prog_path):
+        session_hash = gdb_path + prog_path
+        self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest()
+        self._kill_previous_session()
+
+        patterns = {
+            "PROJECT_DIR": self.project_dir,
+            "PROG_PATH": prog_path,
+            "PROG_DIR": dirname(prog_path),
+            "PROG_NAME": basename(splitext(prog_path)[0]),
+            "DEBUG_PORT": self.debug_options["port"],
+            "UPLOAD_PROTOCOL": self.debug_options["upload_protocol"],
+            "INIT_BREAK": self.debug_options["init_break"] or "",
+            "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []),
+        }
+
+        yield self._debug_server.spawn(patterns)
+        if not patterns["DEBUG_PORT"]:
+            patterns["DEBUG_PORT"] = self._debug_server.get_debug_port()
+
+        self.generate_pioinit(self._gdbsrc_dir, patterns)
+
+        # start GDB client
+        args = [
+            "piogdb",
+            "-q",
+            "--directory",
+            self._gdbsrc_dir,
+            "--directory",
+            self.project_dir,
+            "-l",
+            "10",
+        ]
+        args.extend(self.args)
+        if not gdb_path:
+            raise DebugInvalidOptionsError("GDB client is not configured")
+        gdb_data_dir = self._get_data_dir(gdb_path)
+        if gdb_data_dir:
+            args.extend(["--data-directory", gdb_data_dir])
+        args.append(patterns["PROG_PATH"])
+
+        transport = reactor.spawnProcess(
+            self, gdb_path, args, path=self.project_dir, env=os.environ
+        )
+        defer.returnValue(transport)
+
+    @staticmethod
+    def _get_data_dir(gdb_path):
+        if "msp430" in gdb_path:
+            return None
+        gdb_data_dir = realpath(join(dirname(gdb_path), "..", "share", "gdb"))
+        return gdb_data_dir if isdir(gdb_data_dir) else None
+
+    def generate_pioinit(self, dst_dir, patterns):
+        # default GDB init commands depending on debug tool
+        commands = get_gdb_init_config(self.debug_options).split("\n")
+
+        if self.debug_options["init_cmds"]:
+            commands = self.debug_options["init_cmds"]
+        commands.extend(self.debug_options["extra_cmds"])
+
+        if not any("define pio_reset_run_target" in cmd for cmd in commands):
+            commands = [
+                "define pio_reset_run_target",
+                "   echo Warning! Undefined pio_reset_run_target command\\n",
+                "   monitor reset",
+                "end",
+            ] + commands
+        if not any("define pio_reset_halt_target" in cmd for cmd in commands):
+            commands = [
+                "define pio_reset_halt_target",
+                "   echo Warning! Undefined pio_reset_halt_target command\\n",
+                "   monitor reset halt",
+                "end",
+            ] + commands
+        if not any("define pio_restart_target" in cmd for cmd in commands):
+            commands += [
+                "define pio_restart_target",
+                "   pio_reset_halt_target",
+                "   $INIT_BREAK",
+                "   %s" % ("continue" if patterns["INIT_BREAK"] else "next"),
+                "end",
+            ]
+
+        banner = [
+            "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n",
+            "echo PlatformIO: debug_tool = %s\\n" % self.debug_options["tool"],
+            "echo PlatformIO: Initializing remote target...\\n",
+        ]
+        footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER]
+        commands = banner + commands + footer
+
+        with open(join(dst_dir, self.PIO_SRC_NAME), "w") as fp:
+            fp.write("\n".join(self.apply_patterns(commands, patterns)))
+
+    def connectionMade(self):
+        self._lock_session(self.transport.pid)
+
+        p = protocol.Protocol()
+        p.dataReceived = self.onStdInData
+        stdio.StandardIO(p)
+
+    def onStdInData(self, data):
+        super(GDBClient, self).onStdInData(data)
+        if b"-exec-run" in data:
+            if self._target_is_run:
+                token, _ = data.split(b"-", 1)
+                self.outReceived(token + b"^running\n")
+                return
+            data = data.replace(b"-exec-run", b"-exec-continue")
+
+        if b"-exec-continue" in data:
+            self._target_is_run = True
+        if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"):
+            # Allow terminating via SIGINT/CTRL+C
+            signal.signal(signal.SIGINT, signal.default_int_handler)
+            self.transport.write(b"pio_reset_run_target\n")
+        self.transport.write(data)
+
+    def processEnded(self, reason):  # pylint: disable=unused-argument
+        self._unlock_session()
+        if self._gdbsrc_dir and isdir(self._gdbsrc_dir):
+            fs.rmtree(self._gdbsrc_dir)
+        if self._debug_server:
+            self._debug_server.terminate()
+
+        reactor.stop()
+
+    def outReceived(self, data):
+        super(GDBClient, self).outReceived(data)
+        self._handle_error(data)
+        # go to init break automatically
+        if self.INIT_COMPLETED_BANNER.encode() in data:
+            telemetry.send_event(
+                "Debug", "Started", telemetry.encode_run_environment(self.env_options)
+            )
+            self._auto_continue_timer = task.LoopingCall(self._auto_exec_continue)
+            self._auto_continue_timer.start(0.1)
+
+    def errReceived(self, data):
+        super(GDBClient, self).errReceived(data)
+        self._handle_error(data)
+
+    def console_log(self, msg):
+        if helpers.is_gdbmi_mode():
+            msg = helpers.escape_gdbmi_stream("~", msg)
+        self.outReceived(msg if is_bytes(msg) else msg.encode())
+
+    def _auto_exec_continue(self):
+        auto_exec_delay = 0.5  # in seconds
+        if self._last_activity > (time.time() - auto_exec_delay):
+            return
+        if self._auto_continue_timer:
+            self._auto_continue_timer.stop()
+        self._auto_continue_timer = None
+
+        if not self.debug_options["init_break"] or self._target_is_run:
+            return
+        self.console_log(
+            "PlatformIO: Resume the execution to `debug_init_break = %s`\n"
+            % self.debug_options["init_break"]
+        )
+        self.console_log(
+            "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n"
+        )
+        self.transport.write(
+            b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n"
+        )
+        self._target_is_run = True
+
+    def _handle_error(self, data):
+        self._errors_buffer += data
+        if self.PIO_SRC_NAME.encode() not in data or b"Error in sourced" not in data:
+            return
+
+        last_erros = self._errors_buffer.decode()
+        last_erros = " ".join(reversed(last_erros.split("\n")))
+        last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M)
+
+        err = "%s -> %s" % (
+            telemetry.encode_run_environment(self.env_options),
+            last_erros,
+        )
+        telemetry.send_exception("DebugInitError: %s" % err)
+        self.transport.loseConnection()
+
+    def _kill_previous_session(self):
+        assert self._session_id
+        pid = None
+        with app.ContentCache() as cc:
+            pid = cc.get(self._session_id)
+            cc.delete(self._session_id)
+        if not pid:
+            return
+        if "windows" in util.get_systype():
+            kill = ["Taskkill", "/PID", pid, "/F"]
+        else:
+            kill = ["kill", pid]
+        try:
+            proc.exec_command(kill)
+        except:  # pylint: disable=bare-except
+            pass
+
+    def _lock_session(self, pid):
+        if not self._session_id:
+            return
+        with app.ContentCache() as cc:
+            cc.set(self._session_id, str(pid), "1h")
+
+    def _unlock_session(self):
+        if not self._session_id:
+            return
+        with app.ContentCache() as cc:
+            cc.delete(self._session_id)

platformio/commands/debug/command.py

@@ -0,0 +1,159 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-arguments, too-many-statements
+# pylint: disable=too-many-locals, too-many-branches
+
+import os
+import signal
+from os.path import isfile
+
+import click
+
+from platformio import app, exception, fs, proc, util
+from platformio.commands.debug import helpers
+from platformio.commands.debug.exception import DebugInvalidOptionsError
+from platformio.managers.core import inject_contrib_pysite
+from platformio.project.config import ProjectConfig
+from platformio.project.exception import ProjectEnvsNotAvailableError
+from platformio.project.helpers import is_platformio_project, load_project_ide_data
+
+
+@click.command(
+    "debug",
+    context_settings=dict(ignore_unknown_options=True),
+    short_help="PIO Unified Debugger",
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
+@click.option("--environment", "-e", metavar="<environment>")
+@click.option("--verbose", "-v", is_flag=True)
+@click.option("--interface", type=click.Choice(["gdb"]))
+@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
+@click.pass_context
+def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unprocessed):
+    app.set_session_var("custom_project_conf", project_conf)
+
+    # use env variables from Eclipse or CLion
+    for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"):
+        if is_platformio_project(project_dir):
+            break
+        if os.getenv(sysenv):
+            project_dir = os.getenv(sysenv)
+
+    with fs.cd(project_dir):
+        config = ProjectConfig.get_instance(project_conf)
+        config.validate(envs=[environment] if environment else None)
+
+        env_name = environment or helpers.get_default_debug_env(config)
+        env_options = config.items(env=env_name, as_dict=True)
+        if not set(env_options.keys()) >= set(["platform", "board"]):
+            raise ProjectEnvsNotAvailableError()
+        debug_options = helpers.validate_debug_options(ctx, env_options)
+        assert debug_options
+
+    if not interface:
+        return helpers.predebug_project(ctx, project_dir, env_name, False, verbose)
+
+    configuration = load_project_ide_data(project_dir, env_name)
+    if not configuration:
+        raise DebugInvalidOptionsError("Could not load debug configuration")
+
+    if "--version" in __unprocessed:
+        result = proc.exec_command([configuration["gdb_path"], "--version"])
+        if result["returncode"] == 0:
+            return click.echo(result["out"])
+        raise exception.PlatformioException("\n".join([result["out"], result["err"]]))
+
+    try:
+        fs.ensure_udev_rules()
+    except exception.InvalidUdevRules as e:
+        click.echo(
+            helpers.escape_gdbmi_stream("~", str(e) + "\n")
+            if helpers.is_gdbmi_mode()
+            else str(e) + "\n",
+            nl=False,
+        )
+
+    debug_options["load_cmds"] = helpers.configure_esp32_load_cmds(
+        debug_options, configuration
+    )
+
+    rebuild_prog = False
+    preload = debug_options["load_cmds"] == ["preload"]
+    load_mode = debug_options["load_mode"]
+    if load_mode == "always":
+        rebuild_prog = preload or not helpers.has_debug_symbols(
+            configuration["prog_path"]
+        )
+    elif load_mode == "modified":
+        rebuild_prog = helpers.is_prog_obsolete(
+            configuration["prog_path"]
+        ) or not helpers.has_debug_symbols(configuration["prog_path"])
+    else:
+        rebuild_prog = not isfile(configuration["prog_path"])
+
+    if preload or (not rebuild_prog and load_mode != "always"):
+        # don't load firmware through debug server
+        debug_options["load_cmds"] = []
+
+    if rebuild_prog:
+        if helpers.is_gdbmi_mode():
+            click.echo(
+                helpers.escape_gdbmi_stream(
+                    "~", "Preparing firmware for debugging...\n"
+                ),
+                nl=False,
+            )
+            stream = helpers.GDBMIConsoleStream()
+            with util.capture_std_streams(stream):
+                helpers.predebug_project(ctx, project_dir, env_name, preload, verbose)
+            stream.close()
+        else:
+            click.echo("Preparing firmware for debugging...")
+            helpers.predebug_project(ctx, project_dir, env_name, preload, verbose)
+
+        # save SHA sum of newly created prog
+        if load_mode == "modified":
+            helpers.is_prog_obsolete(configuration["prog_path"])
+
+    if not isfile(configuration["prog_path"]):
+        raise DebugInvalidOptionsError("Program/firmware is missed")
+
+    # run debugging client
+    inject_contrib_pysite()
+
+    # pylint: disable=import-outside-toplevel
+    from platformio.commands.debug.client import GDBClient, reactor
+
+    client = GDBClient(project_dir, __unprocessed, debug_options, env_options)
+    client.spawn(configuration["gdb_path"], configuration["prog_path"])
+
+    signal.signal(signal.SIGINT, lambda *args, **kwargs: None)
+    reactor.run()
+
+    return True

platformio/commands/debug/exception.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.exception import PlatformioException, UserSideException
+
+
+class DebugError(PlatformioException):
+    pass
+
+
+class DebugSupportError(DebugError, UserSideException):
+
+    MESSAGE = (
+        "Currently, PlatformIO does not support debugging for `{0}`.\n"
+        "Please request support at https://github.com/platformio/"
+        "platformio-core/issues \nor visit -> https://docs.platformio.org"
+        "/page/plus/debugging.html"
+    )
+
+
+class DebugInvalidOptionsError(DebugError, UserSideException):
+    pass

platformio/commands/debug/helpers.py

@@ -0,0 +1,311 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import sys
+import time
+from fnmatch import fnmatch
+from hashlib import sha1
+from io import BytesIO
+from os.path import isfile
+
+from platformio import exception, fs, util
+from platformio.commands import PlatformioCLI
+from platformio.commands.debug.exception import DebugInvalidOptionsError
+from platformio.commands.platform import platform_install as cmd_platform_install
+from platformio.commands.run.command import cli as cmd_run
+from platformio.compat import is_bytes
+from platformio.managers.platform import PlatformFactory
+from platformio.project.config import ProjectConfig
+from platformio.project.options import ProjectOptions
+
+
+class GDBMIConsoleStream(BytesIO):  # pylint: disable=too-few-public-methods
+
+    STDOUT = sys.stdout
+
+    def write(self, text):
+        self.STDOUT.write(escape_gdbmi_stream("~", text))
+        self.STDOUT.flush()
+
+
+def is_gdbmi_mode():
+    return "--interpreter" in " ".join(PlatformioCLI.leftover_args)
+
+
+def escape_gdbmi_stream(prefix, stream):
+    bytes_stream = False
+    if is_bytes(stream):
+        bytes_stream = True
+        stream = stream.decode()
+
+    if not stream:
+        return b"" if bytes_stream else ""
+
+    ends_nl = stream.endswith("\n")
+    stream = re.sub(r"\\+", "\\\\\\\\", stream)
+    stream = stream.replace('"', '\\"')
+    stream = stream.replace("\n", "\\n")
+    stream = '%s"%s"' % (prefix, stream)
+    if ends_nl:
+        stream += "\n"
+
+    return stream.encode() if bytes_stream else stream
+
+
+def get_default_debug_env(config):
+    default_envs = config.default_envs()
+    all_envs = config.envs()
+    for env in default_envs:
+        if config.get("env:" + env, "build_type") == "debug":
+            return env
+    for env in all_envs:
+        if config.get("env:" + env, "build_type") == "debug":
+            return env
+    return default_envs[0] if default_envs else all_envs[0]
+
+
+def predebug_project(ctx, project_dir, env_name, preload, verbose):
+    ctx.invoke(
+        cmd_run,
+        project_dir=project_dir,
+        environment=[env_name],
+        target=["debug"] + (["upload"] if preload else []),
+        verbose=verbose,
+    )
+    if preload:
+        time.sleep(5)
+
+
+def validate_debug_options(cmd_ctx, env_options):
+    def _cleanup_cmds(items):
+        items = ProjectConfig.parse_multi_values(items)
+        return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items]
+
+    try:
+        platform = PlatformFactory.newPlatform(env_options["platform"])
+    except exception.UnknownPlatform:
+        cmd_ctx.invoke(
+            cmd_platform_install,
+            platforms=[env_options["platform"]],
+            skip_default_package=True,
+        )
+        platform = PlatformFactory.newPlatform(env_options["platform"])
+
+    board_config = platform.board_config(env_options["board"])
+    tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool"))
+    tool_settings = board_config.get("debug", {}).get("tools", {}).get(tool_name, {})
+    server_options = None
+
+    # specific server per a system
+    if isinstance(tool_settings.get("server", {}), list):
+        for item in tool_settings["server"][:]:
+            tool_settings["server"] = item
+            if util.get_systype() in item.get("system", []):
+                break
+
+    # user overwrites debug server
+    if env_options.get("debug_server"):
+        server_options = {
+            "cwd": None,
+            "executable": None,
+            "arguments": env_options.get("debug_server"),
+        }
+        server_options["executable"] = server_options["arguments"][0]
+        server_options["arguments"] = server_options["arguments"][1:]
+    elif "server" in tool_settings:
+        server_options = tool_settings["server"]
+        server_package = server_options.get("package")
+        server_package_dir = (
+            platform.get_package_dir(server_package) if server_package else None
+        )
+        if server_package and not server_package_dir:
+            platform.install_packages(
+                with_packages=[server_package], skip_default_package=True, silent=True
+            )
+            server_package_dir = platform.get_package_dir(server_package)
+        server_options.update(
+            dict(
+                cwd=server_package_dir if server_package else None,
+                executable=server_options.get("executable"),
+                arguments=[
+                    a.replace("$PACKAGE_DIR", server_package_dir)
+                    if server_package_dir
+                    else a
+                    for a in server_options.get("arguments", [])
+                ],
+            )
+        )
+
+    extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds"))
+    extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds")))
+    result = dict(
+        tool=tool_name,
+        upload_protocol=env_options.get(
+            "upload_protocol", board_config.get("upload", {}).get("protocol")
+        ),
+        load_cmds=_cleanup_cmds(
+            env_options.get(
+                "debug_load_cmds",
+                tool_settings.get(
+                    "load_cmds",
+                    tool_settings.get(
+                        "load_cmd", ProjectOptions["env.debug_load_cmds"].default
+                    ),
+                ),
+            )
+        ),
+        load_mode=env_options.get(
+            "debug_load_mode",
+            tool_settings.get(
+                "load_mode", ProjectOptions["env.debug_load_mode"].default
+            ),
+        ),
+        init_break=env_options.get(
+            "debug_init_break",
+            tool_settings.get(
+                "init_break", ProjectOptions["env.debug_init_break"].default
+            ),
+        ),
+        init_cmds=_cleanup_cmds(
+            env_options.get("debug_init_cmds", tool_settings.get("init_cmds"))
+        ),
+        extra_cmds=extra_cmds,
+        require_debug_port=tool_settings.get("require_debug_port", False),
+        port=reveal_debug_port(
+            env_options.get("debug_port", tool_settings.get("port")),
+            tool_name,
+            tool_settings,
+        ),
+        server=server_options,
+    )
+    return result
+
+
+def configure_esp32_load_cmds(debug_options, configuration):
+    ignore_conds = [
+        debug_options["load_cmds"] != ["load"],
+        "xtensa-esp32" not in configuration.get("cc_path", ""),
+        not configuration.get("flash_extra_images"),
+        not all(
+            [isfile(item["path"]) for item in configuration.get("flash_extra_images")]
+        ),
+    ]
+    if any(ignore_conds):
+        return debug_options["load_cmds"]
+
+    mon_cmds = [
+        'monitor program_esp32 "{{{path}}}" {offset} verify'.format(
+            path=fs.to_unix_path(item["path"]), offset=item["offset"]
+        )
+        for item in configuration.get("flash_extra_images")
+    ]
+    mon_cmds.append(
+        'monitor program_esp32 "{%s.bin}" 0x10000 verify'
+        % fs.to_unix_path(configuration["prog_path"][:-4])
+    )
+    return mon_cmds
+
+
+def has_debug_symbols(prog_path):
+    if not isfile(prog_path):
+        return False
+    matched = {
+        b".debug_info": False,
+        b".debug_abbrev": False,
+        b" -Og": False,
+        b" -g": False,
+        b"__PLATFORMIO_BUILD_DEBUG__": False,
+    }
+    with open(prog_path, "rb") as fp:
+        last_data = b""
+        while True:
+            data = fp.read(1024)
+            if not data:
+                break
+            for pattern, found in matched.items():
+                if found:
+                    continue
+                if pattern in last_data + data:
+                    matched[pattern] = True
+            last_data = data
+    return all(matched.values())
+
+
+def is_prog_obsolete(prog_path):
+    prog_hash_path = prog_path + ".sha1"
+    if not isfile(prog_path):
+        return True
+    shasum = sha1()
+    with open(prog_path, "rb") as fp:
+        while True:
+            data = fp.read(1024)
+            if not data:
+                break
+            shasum.update(data)
+    new_digest = shasum.hexdigest()
+    old_digest = None
+    if isfile(prog_hash_path):
+        with open(prog_hash_path) as fp:
+            old_digest = fp.read()
+    if new_digest == old_digest:
+        return False
+    with open(prog_hash_path, "w") as fp:
+        fp.write(new_digest)
+    return True
+
+
+def reveal_debug_port(env_debug_port, tool_name, tool_settings):
+    def _get_pattern():
+        if not env_debug_port:
+            return None
+        if set(["*", "?", "[", "]"]) & set(env_debug_port):
+            return env_debug_port
+        return None
+
+    def _is_match_pattern(port):
+        pattern = _get_pattern()
+        if not pattern:
+            return True
+        return fnmatch(port, pattern)
+
+    def _look_for_serial_port(hwids):
+        for item in util.get_serialports(filter_hwid=True):
+            if not _is_match_pattern(item["port"]):
+                continue
+            port = item["port"]
+            if tool_name.startswith("blackmagic"):
+                if (
+                    "windows" in util.get_systype()
+                    and port.startswith("COM")
+                    and len(port) > 4
+                ):
+                    port = "\\\\.\\%s" % port
+                if "GDB" in item["description"]:
+                    return port
+            for hwid in hwids:
+                hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
+                if hwid_str in item["hwid"]:
+                    return port
+        return None
+
+    if env_debug_port and not _get_pattern():
+        return env_debug_port
+    if not tool_settings.get("require_debug_port"):
+        return None
+
+    debug_port = _look_for_serial_port(tool_settings.get("hwids", []))
+    if not debug_port:
+        raise DebugInvalidOptionsError("Please specify `debug_port` for environment")
+    return debug_port

platformio/commands/debug/initcfgs.py

@@ -0,0 +1,161 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GDB_DEFAULT_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor reset halt
+end
+
+define pio_reset_run_target
+    monitor reset
+end
+
+target extended-remote $DEBUG_PORT
+monitor init
+$LOAD_CMDS
+pio_reset_halt_target
+$INIT_BREAK
+"""
+
+GDB_STUTIL_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor reset
+    monitor halt
+end
+
+define pio_reset_run_target
+    monitor reset
+end
+
+target extended-remote $DEBUG_PORT
+$LOAD_CMDS
+pio_reset_halt_target
+$INIT_BREAK
+"""
+
+GDB_JLINK_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor reset
+    monitor halt
+end
+
+define pio_reset_run_target
+    monitor clrbp
+    monitor reset
+    monitor go
+end
+
+target extended-remote $DEBUG_PORT
+monitor clrbp
+monitor speed auto
+pio_reset_halt_target
+$LOAD_CMDS
+$INIT_BREAK
+"""
+
+GDB_BLACKMAGIC_INIT_CONFIG = """
+define pio_reset_halt_target
+    set language c
+    set *0xE000ED0C = 0x05FA0004
+    set $busy = (*0xE000ED0C & 0x4)
+    while ($busy)
+        set $busy = (*0xE000ED0C & 0x4)
+    end
+    set language auto
+end
+
+define pio_reset_run_target
+    pio_reset_halt_target
+end
+
+target extended-remote $DEBUG_PORT
+monitor swdp_scan
+attach 1
+set mem inaccessible-by-default off
+$LOAD_CMDS
+$INIT_BREAK
+
+set language c
+set *0xE000ED0C = 0x05FA0004
+set $busy = (*0xE000ED0C & 0x4)
+while ($busy)
+    set $busy = (*0xE000ED0C & 0x4)
+end
+set language auto
+"""
+
+GDB_MSPDEBUG_INIT_CONFIG = """
+define pio_reset_halt_target
+end
+
+define pio_reset_run_target
+end
+
+target extended-remote $DEBUG_PORT
+monitor erase
+$LOAD_CMDS
+pio_reset_halt_target
+$INIT_BREAK
+"""
+
+GDB_QEMU_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor system_reset
+end
+
+define pio_reset_run_target
+    monitor system_reset
+end
+
+target extended-remote $DEBUG_PORT
+$LOAD_CMDS
+pio_reset_halt_target
+$INIT_BREAK
+"""
+
+GDB_RENODE_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor machine Reset
+    $LOAD_CMDS
+    monitor start
+end
+
+define pio_reset_run_target
+    pio_reset_halt_target
+end
+
+target extended-remote $DEBUG_PORT
+$LOAD_CMDS
+$INIT_BREAK
+monitor start
+"""
+
+
+TOOL_TO_CONFIG = {
+    "jlink": GDB_JLINK_INIT_CONFIG,
+    "mspdebug": GDB_MSPDEBUG_INIT_CONFIG,
+    "qemu": GDB_QEMU_INIT_CONFIG,
+    "blackmagic": GDB_BLACKMAGIC_INIT_CONFIG,
+    "renode": GDB_RENODE_INIT_CONFIG,
+}
+
+
+def get_gdb_init_config(debug_options):
+    tool = debug_options.get("tool")
+    if tool and tool in TOOL_TO_CONFIG:
+        return TOOL_TO_CONFIG[tool]
+    server_exe = (debug_options.get("server") or {}).get("executable", "").lower()
+    if "st-util" in server_exe:
+        return GDB_STUTIL_INIT_CONFIG
+    return GDB_DEFAULT_INIT_CONFIG

platformio/commands/debug/process.py

@@ -0,0 +1,93 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import signal
+import time
+
+import click
+from twisted.internet import protocol  # pylint: disable=import-error
+
+from platformio import fs
+from platformio.compat import string_types
+from platformio.proc import get_pythonexe_path
+from platformio.project.helpers import get_project_core_dir
+
+
+class BaseProcess(protocol.ProcessProtocol, object):
+
+    STDOUT_CHUNK_SIZE = 2048
+    LOG_FILE = None
+
+    COMMON_PATTERNS = {
+        "PLATFORMIO_HOME_DIR": get_project_core_dir(),
+        "PLATFORMIO_CORE_DIR": get_project_core_dir(),
+        "PYTHONEXE": get_pythonexe_path(),
+    }
+
+    def __init__(self):
+        self._last_activity = 0
+
+    def apply_patterns(self, source, patterns=None):
+        _patterns = self.COMMON_PATTERNS.copy()
+        _patterns.update(patterns or {})
+
+        for key, value in _patterns.items():
+            if key.endswith(("_DIR", "_PATH")):
+                _patterns[key] = fs.to_unix_path(value)
+
+        def _replace(text):
+            for key, value in _patterns.items():
+                pattern = "$%s" % key
+                text = text.replace(pattern, value or "")
+            return text
+
+        if isinstance(source, string_types):
+            source = _replace(source)
+        elif isinstance(source, (list, dict)):
+            items = enumerate(source) if isinstance(source, list) else source.items()
+            for key, value in items:
+                if isinstance(value, string_types):
+                    source[key] = _replace(value)
+                elif isinstance(value, (list, dict)):
+                    source[key] = self.apply_patterns(value, patterns)
+
+        return source
+
+    def onStdInData(self, data):
+        self._last_activity = time.time()
+        if self.LOG_FILE:
+            with open(self.LOG_FILE, "ab") as fp:
+                fp.write(data)
+
+    def outReceived(self, data):
+        self._last_activity = time.time()
+        if self.LOG_FILE:
+            with open(self.LOG_FILE, "ab") as fp:
+                fp.write(data)
+        while data:
+            chunk = data[: self.STDOUT_CHUNK_SIZE]
+            click.echo(chunk, nl=False)
+            data = data[self.STDOUT_CHUNK_SIZE :]
+
+    def errReceived(self, data):
+        self._last_activity = time.time()
+        if self.LOG_FILE:
+            with open(self.LOG_FILE, "ab") as fp:
+                fp.write(data)
+        click.echo(data, nl=False, err=True)
+
+    def processEnded(self, _):
+        self._last_activity = time.time()
+        # Allow terminating via SIGINT/CTRL+C
+        signal.signal(signal.SIGINT, signal.default_int_handler)

platformio/commands/debug/server.py

@@ -0,0 +1,166 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+from os.path import isdir, isfile, join
+
+from twisted.internet import defer  # pylint: disable=import-error
+from twisted.internet import reactor  # pylint: disable=import-error
+
+from platformio import fs, util
+from platformio.commands.debug.exception import DebugInvalidOptionsError
+from platformio.commands.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode
+from platformio.commands.debug.process import BaseProcess
+from platformio.proc import where_is_program
+
+
+class DebugServer(BaseProcess):
+    def __init__(self, debug_options, env_options):
+        super(DebugServer, self).__init__()
+        self.debug_options = debug_options
+        self.env_options = env_options
+
+        self._debug_port = ":3333"
+        self._transport = None
+        self._process_ended = False
+        self._ready = False
+
+    @defer.inlineCallbacks
+    def spawn(self, patterns):  # pylint: disable=too-many-branches
+        systype = util.get_systype()
+        server = self.debug_options.get("server")
+        if not server:
+            return None
+        server = self.apply_patterns(server, patterns)
+        server_executable = server["executable"]
+        if not server_executable:
+            return None
+        if server["cwd"]:
+            server_executable = join(server["cwd"], server_executable)
+        if (
+            "windows" in systype
+            and not server_executable.endswith(".exe")
+            and isfile(server_executable + ".exe")
+        ):
+            server_executable = server_executable + ".exe"
+
+        if not isfile(server_executable):
+            server_executable = where_is_program(server_executable)
+        if not isfile(server_executable):
+            raise DebugInvalidOptionsError(
+                "\nCould not launch Debug Server '%s'. Please check that it "
+                "is installed and is included in a system PATH\n\n"
+                "See documentation or contact contact@platformio.org:\n"
+                "https://docs.platformio.org/page/plus/debugging.html\n"
+                % server_executable
+            )
+
+        openocd_pipe_allowed = all(
+            [not self.debug_options["port"], "openocd" in server_executable]
+        )
+        if openocd_pipe_allowed:
+            args = []
+            if server["cwd"]:
+                args.extend(["-s", server["cwd"]])
+            args.extend(
+                ["-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"]
+            )
+            args.extend(server["arguments"])
+            str_args = " ".join(
+                [arg if arg.startswith("-") else '"%s"' % arg for arg in args]
+            )
+            self._debug_port = '| "%s" %s' % (server_executable, str_args)
+            self._debug_port = fs.to_unix_path(self._debug_port)
+            return self._debug_port
+
+        env = os.environ.copy()
+        # prepend server "lib" folder to LD path
+        if (
+            "windows" not in systype
+            and server["cwd"]
+            and isdir(join(server["cwd"], "lib"))
+        ):
+            ld_key = "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH"
+            env[ld_key] = join(server["cwd"], "lib")
+            if os.environ.get(ld_key):
+                env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key))
+        # prepend BIN to PATH
+        if server["cwd"] and isdir(join(server["cwd"], "bin")):
+            env["PATH"] = "%s%s%s" % (
+                join(server["cwd"], "bin"),
+                os.pathsep,
+                os.environ.get("PATH", os.environ.get("Path", "")),
+            )
+
+        self._transport = reactor.spawnProcess(
+            self,
+            server_executable,
+            [server_executable] + server["arguments"],
+            path=server["cwd"],
+            env=env,
+        )
+        if "mspdebug" in server_executable.lower():
+            self._debug_port = ":2000"
+        elif "jlink" in server_executable.lower():
+            self._debug_port = ":2331"
+        elif "qemu" in server_executable.lower():
+            self._debug_port = ":1234"
+
+        yield self._wait_until_ready()
+
+        return self._debug_port
+
+    @defer.inlineCallbacks
+    def _wait_until_ready(self):
+        timeout = 10
+        elapsed = 0
+        delay = 0.5
+        auto_ready_delay = 0.5
+        while not self._ready and not self._process_ended and elapsed < timeout:
+            yield self.async_sleep(delay)
+            if not self.debug_options.get("server", {}).get("ready_pattern"):
+                self._ready = self._last_activity < (time.time() - auto_ready_delay)
+            elapsed += delay
+
+    @staticmethod
+    def async_sleep(secs):
+        d = defer.Deferred()
+        reactor.callLater(secs, d.callback, None)
+        return d
+
+    def get_debug_port(self):
+        return self._debug_port
+
+    def outReceived(self, data):
+        super(DebugServer, self).outReceived(
+            escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data
+        )
+        if self._ready:
+            return
+        ready_pattern = self.debug_options.get("server", {}).get("ready_pattern")
+        if ready_pattern:
+            self._ready = ready_pattern.encode() in data
+
+    def processEnded(self, reason):
+        self._process_ended = True
+        super(DebugServer, self).processEnded(reason)
+
+    def terminate(self):
+        if self._process_ended or not self._transport:
+            return
+        try:
+            self._transport.signalProcess("KILL")
+        except:  # pylint: disable=bare-except
+            pass

platformio/commands/device.py

@@ -1,228 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import sys
-from os import getcwd
-
-import click
-from serial.tools import miniterm
-
-from platformio import exception, util
-
-
-@click.group(short_help="Monitor device or list existing")
-def cli():
-    pass
-
-
-@cli.command("list", short_help="List devices")
-@click.option("--serial", is_flag=True, help="List serial ports, default")
-@click.option("--logical", is_flag=True, help="List logical devices")
-@click.option("--mdns", is_flag=True, help="List multicast DNS services")
-@click.option("--json-output", is_flag=True)
-def device_list(  # pylint: disable=too-many-branches
-        serial, logical, mdns, json_output):
-    if not logical and not mdns:
-        serial = True
-    data = {}
-    if serial:
-        data['serial'] = util.get_serial_ports()
-    if logical:
-        data['logical'] = util.get_logical_devices()
-    if mdns:
-        data['mdns'] = util.get_mdns_services()
-
-    single_key = data.keys()[0] if len(data.keys()) == 1 else None
-
-    if json_output:
-        return click.echo(json.dumps(data[single_key] if single_key else data))
-
-    titles = {
-        "serial": "Serial Ports",
-        "logical": "Logical Devices",
-        "mdns": "Multicast DNS Services"
-    }
-
-    for key, value in data.items():
-        if not single_key:
-            click.secho(titles[key], bold=True)
-            click.echo("=" * len(titles[key]))
-
-        if key == "serial":
-            for item in value:
-                click.secho(item['port'], fg="cyan")
-                click.echo("-" * len(item['port']))
-                click.echo("Hardware ID: %s" % item['hwid'])
-                click.echo("Description: %s" % item['description'])
-                click.echo("")
-
-        if key == "logical":
-            for item in value:
-                click.secho(item['path'], fg="cyan")
-                click.echo("-" * len(item['path']))
-                click.echo("Name: %s" % item['name'])
-                click.echo("")
-
-        if key == "mdns":
-            for item in value:
-                click.secho(item['name'], fg="cyan")
-                click.echo("-" * len(item['name']))
-                click.echo("Type: %s" % item['type'])
-                click.echo("IP: %s" % item['ip'])
-                click.echo("Port: %s" % item['port'])
-                if item['properties']:
-                    click.echo("Properties: %s" % ("; ".join([
-                        "%s=%s" % (k, v)
-                        for k, v in item['properties'].items()
-                    ])))
-                click.echo("")
-
-        if single_key:
-            click.echo("")
-
-    return True
-
-
-@cli.command("monitor", short_help="Monitor device (Serial)")
-@click.option("--port", "-p", help="Port, a number or a device name")
-@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
-@click.option(
-    "--parity",
-    default="N",
-    type=click.Choice(["N", "E", "O", "S", "M"]),
-    help="Set parity, default=N")
-@click.option(
-    "--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
-@click.option(
-    "--xonxoff",
-    is_flag=True,
-    help="Enable software flow control, default=Off")
-@click.option(
-    "--rts",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial RTS line state")
-@click.option(
-    "--dtr",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial DTR line state")
-@click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
-@click.option(
-    "--encoding",
-    default="UTF-8",
-    help="Set the encoding for the serial port (e.g. hexlify, "
-    "Latin1, UTF-8), default: UTF-8")
-@click.option("--filter", "-f", multiple=True, help="Add text transformation")
-@click.option(
-    "--eol",
-    default="CRLF",
-    type=click.Choice(["CR", "LF", "CRLF"]),
-    help="End of line mode, default=CRLF")
-@click.option(
-    "--raw", is_flag=True, help="Do not apply any encodings/transformations")
-@click.option(
-    "--exit-char",
-    type=int,
-    default=3,
-    help="ASCII code of special character that is used to exit "
-    "the application, default=3 (Ctrl+C)")
-@click.option(
-    "--menu-char",
-    type=int,
-    default=20,
-    help="ASCII code of special character that is used to "
-    "control miniterm (menu), default=20 (DEC)")
-@click.option(
-    "--quiet",
-    is_flag=True,
-    help="Diagnostics: suppress non-error messages, default=Off")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=getcwd,
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, resolve_path=True))
-@click.option(
-    "-e",
-    "--environment",
-    help="Load configuration from `platformio.ini` and specified environment")
-def device_monitor(**kwargs):  # pylint: disable=too-many-branches
-    try:
-        project_options = get_project_options(kwargs['project_dir'],
-                                              kwargs['environment'])
-        monitor_options = {k: v for k, v in project_options or []}
-        if monitor_options:
-            for k in ("port", "baud", "speed", "rts", "dtr"):
-                k2 = "monitor_%s" % k
-                if k == "speed":
-                    k = "baud"
-                if kwargs[k] is None and k2 in monitor_options:
-                    kwargs[k] = monitor_options[k2]
-                    if k != "port":
-                        kwargs[k] = int(kwargs[k])
-    except exception.NotPlatformIOProject:
-        pass
-
-    if not kwargs['port']:
-        ports = util.get_serial_ports(filter_hwid=True)
-        if len(ports) == 1:
-            kwargs['port'] = ports[0]['port']
-
-    sys.argv = ["monitor"]
-    for k, v in kwargs.items():
-        if k in ("port", "baud", "rts", "dtr", "environment", "project_dir"):
-            continue
-        k = "--" + k.replace("_", "-")
-        if isinstance(v, bool):
-            if v:
-                sys.argv.append(k)
-        elif isinstance(v, tuple):
-            for i in v:
-                sys.argv.extend([k, i])
-        else:
-            sys.argv.extend([k, str(v)])
-
-    try:
-        miniterm.main(
-            default_port=kwargs['port'],
-            default_baudrate=kwargs['baud'] or 9600,
-            default_rts=kwargs['rts'],
-            default_dtr=kwargs['dtr'])
-    except Exception as e:
-        raise exception.MinitermException(e)
-
-
-def get_project_options(project_dir, environment):
-    config = util.load_project_config(project_dir)
-    if not config.sections():
-        return None
-
-    known_envs = [s[4:] for s in config.sections() if s.startswith("env:")]
-    if environment:
-        if environment in known_envs:
-            return config.items("env:%s" % environment)
-        raise exception.UnknownEnvNames(environment, ", ".join(known_envs))
-
-    if not known_envs:
-        return None
-
-    if config.has_option("platformio", "env_default"):
-        env_default = config.get("platformio",
-                                 "env_default").split(", ")[0].strip()
-        if env_default and env_default in known_envs:
-            return config.items("env:%s" % env_default)
-
-    return config.items("env:%s" % known_envs[0])

platformio/commands/device/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.device.filters.base import DeviceMonitorFilter

platformio/commands/device/command.py

@@ -0,0 +1,243 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from fnmatch import fnmatch
+
+import click
+from serial.tools import miniterm
+
+from platformio import exception, fs, util
+from platformio.commands.device import helpers as device_helpers
+from platformio.compat import dump_json_to_unicode
+from platformio.managers.platform import PlatformFactory
+from platformio.project.exception import NotPlatformIOProjectError
+
+
+@click.group(short_help="Monitor device or list existing")
+def cli():
+    pass
+
+
+@cli.command("list", short_help="List devices")
+@click.option("--serial", is_flag=True, help="List serial ports, default")
+@click.option("--logical", is_flag=True, help="List logical devices")
+@click.option("--mdns", is_flag=True, help="List multicast DNS services")
+@click.option("--json-output", is_flag=True)
+def device_list(  # pylint: disable=too-many-branches
+    serial, logical, mdns, json_output
+):
+    if not logical and not mdns:
+        serial = True
+    data = {}
+    if serial:
+        data["serial"] = util.get_serial_ports()
+    if logical:
+        data["logical"] = util.get_logical_devices()
+    if mdns:
+        data["mdns"] = util.get_mdns_services()
+
+    single_key = list(data)[0] if len(list(data)) == 1 else None
+
+    if json_output:
+        return click.echo(
+            dump_json_to_unicode(data[single_key] if single_key else data)
+        )
+
+    titles = {
+        "serial": "Serial Ports",
+        "logical": "Logical Devices",
+        "mdns": "Multicast DNS Services",
+    }
+
+    for key, value in data.items():
+        if not single_key:
+            click.secho(titles[key], bold=True)
+            click.echo("=" * len(titles[key]))
+
+        if key == "serial":
+            for item in value:
+                click.secho(item["port"], fg="cyan")
+                click.echo("-" * len(item["port"]))
+                click.echo("Hardware ID: %s" % item["hwid"])
+                click.echo("Description: %s" % item["description"])
+                click.echo("")
+
+        if key == "logical":
+            for item in value:
+                click.secho(item["path"], fg="cyan")
+                click.echo("-" * len(item["path"]))
+                click.echo("Name: %s" % item["name"])
+                click.echo("")
+
+        if key == "mdns":
+            for item in value:
+                click.secho(item["name"], fg="cyan")
+                click.echo("-" * len(item["name"]))
+                click.echo("Type: %s" % item["type"])
+                click.echo("IP: %s" % item["ip"])
+                click.echo("Port: %s" % item["port"])
+                if item["properties"]:
+                    click.echo(
+                        "Properties: %s"
+                        % (
+                            "; ".join(
+                                [
+                                    "%s=%s" % (k, v)
+                                    for k, v in item["properties"].items()
+                                ]
+                            )
+                        )
+                    )
+                click.echo("")
+
+        if single_key:
+            click.echo("")
+
+    return True
+
+
+@cli.command("monitor", short_help="Monitor device (Serial)")
+@click.option("--port", "-p", help="Port, a number or a device name")
+@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
+@click.option(
+    "--parity",
+    default="N",
+    type=click.Choice(["N", "E", "O", "S", "M"]),
+    help="Set parity, default=N",
+)
+@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
+@click.option(
+    "--xonxoff", is_flag=True, help="Enable software flow control, default=Off"
+)
+@click.option(
+    "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state"
+)
+@click.option(
+    "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state"
+)
+@click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
+@click.option(
+    "--encoding",
+    default="UTF-8",
+    help="Set the encoding for the serial port (e.g. hexlify, "
+    "Latin1, UTF-8), default: UTF-8",
+)
+@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations")
+@click.option(
+    "--eol",
+    default="CRLF",
+    type=click.Choice(["CR", "LF", "CRLF"]),
+    help="End of line mode, default=CRLF",
+)
+@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations")
+@click.option(
+    "--exit-char",
+    type=int,
+    default=3,
+    help="ASCII code of special character that is used to exit "
+    "the application, default=3 (Ctrl+C)",
+)
+@click.option(
+    "--menu-char",
+    type=int,
+    default=20,
+    help="ASCII code of special character that is used to "
+    "control miniterm (menu), default=20 (DEC)",
+)
+@click.option(
+    "--quiet",
+    is_flag=True,
+    help="Diagnostics: suppress non-error messages, default=Off",
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+)
+@click.option(
+    "-e",
+    "--environment",
+    help="Load configuration from `platformio.ini` and specified environment",
+)
+def device_monitor(**kwargs):  # pylint: disable=too-many-branches
+    # load default monitor filters
+    filters_dir = os.path.join(fs.get_source_dir(), "commands", "device", "filters")
+    for name in os.listdir(filters_dir):
+        if not name.endswith(".py"):
+            continue
+        device_helpers.load_monitor_filter(os.path.join(filters_dir, name))
+
+    project_options = {}
+    try:
+        with fs.cd(kwargs["project_dir"]):
+            project_options = device_helpers.get_project_options(kwargs["environment"])
+        kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options)
+    except NotPlatformIOProjectError:
+        pass
+
+    platform = None
+    if "platform" in project_options:
+        with fs.cd(kwargs["project_dir"]):
+            platform = PlatformFactory.newPlatform(project_options["platform"])
+            device_helpers.register_platform_filters(
+                platform, kwargs["project_dir"], kwargs["environment"]
+            )
+
+    if not kwargs["port"]:
+        ports = util.get_serial_ports(filter_hwid=True)
+        if len(ports) == 1:
+            kwargs["port"] = ports[0]["port"]
+        elif "platform" in project_options and "board" in project_options:
+            board_hwids = device_helpers.get_board_hwids(
+                kwargs["project_dir"], platform, project_options["board"],
+            )
+            for item in ports:
+                for hwid in board_hwids:
+                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
+                    if hwid_str in item["hwid"]:
+                        kwargs["port"] = item["port"]
+                        break
+                if kwargs["port"]:
+                    break
+    elif kwargs["port"] and (set(["*", "?", "[", "]"]) & set(kwargs["port"])):
+        for item in util.get_serial_ports():
+            if fnmatch(item["port"], kwargs["port"]):
+                kwargs["port"] = item["port"]
+                break
+
+    # override system argv with patched options
+    sys.argv = ["monitor"] + device_helpers.options_to_argv(
+        kwargs,
+        project_options,
+        ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"),
+    )
+
+    if not kwargs["quiet"]:
+        click.echo(
+            "--- Available filters and text transformations: %s"
+            % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys()))
+        )
+        click.echo("--- More details at http://bit.ly/pio-monitor-filters")
+    try:
+        miniterm.main(
+            default_port=kwargs["port"],
+            default_baudrate=kwargs["baud"] or 9600,
+            default_rts=kwargs["rts"],
+            default_dtr=kwargs["dtr"],
+        )
+    except Exception as e:
+        raise exception.MinitermException(e)

platformio/commands/device/filters/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/device/filters/base.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from serial.tools import miniterm
+
+from platformio.project.config import ProjectConfig
+
+
+class DeviceMonitorFilter(miniterm.Transform):
+    def __init__(self, project_dir=None, environment=None):
+        """ Called by PlatformIO to pass context """
+        super(DeviceMonitorFilter, self).__init__()
+
+        self.project_dir = project_dir
+        self.environment = environment
+
+        self.config = ProjectConfig.get_instance()
+        if not self.environment:
+            default_envs = self.config.default_envs()
+            if default_envs:
+                self.environment = default_envs[0]
+            elif self.config.envs():
+                self.environment = self.config.envs()[0]
+
+    def __call__(self):
+        """ Called by the miniterm library when the filter is actually used """
+        return self
+
+    @property
+    def NAME(self):
+        raise NotImplementedError("Please declare NAME attribute for the filter class")

platformio/commands/device/filters/hexlify.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import serial
+
+from platformio.commands.device import DeviceMonitorFilter
+
+
+class Hexlify(DeviceMonitorFilter):
+    NAME = "hexlify"
+
+    def __init__(self, *args, **kwargs):
+        super(Hexlify, self).__init__(*args, **kwargs)
+        self._counter = 0
+
+    def rx(self, text):
+        result = ""
+        for b in serial.iterbytes(text):
+            if (self._counter % 16) == 0:
+                result += "\n{:04X} | ".format(self._counter)
+            asciicode = ord(b)
+            if asciicode <= 255:
+                result += "{:02X} ".format(asciicode)
+            else:
+                result += "?? "
+            self._counter += 1
+        return result

platformio/commands/device/filters/log2file.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import os.path
+from datetime import datetime
+
+from platformio.commands.device import DeviceMonitorFilter
+
+
+class LogToFile(DeviceMonitorFilter):
+    NAME = "log2file"
+
+    def __init__(self, *args, **kwargs):
+        super(LogToFile, self).__init__(*args, **kwargs)
+        self._log_fp = None
+
+    def __call__(self):
+        log_file_name = "platformio-device-monitor-%s.log" % datetime.now().strftime(
+            "%y%m%d-%H%M%S"
+        )
+        print("--- Logging an output to %s" % os.path.abspath(log_file_name))
+        self._log_fp = io.open(log_file_name, "w", encoding="utf-8")
+        return self
+
+    def __del__(self):
+        if self._log_fp:
+            self._log_fp.close()
+
+    def rx(self, text):
+        self._log_fp.write(text)
+        self._log_fp.flush()
+        return text

platformio/commands/device/filters/send_on_enter.py

@@ -12,20 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
-import click
-
-from platformio.managers.core import pioplus_call
+from platformio.commands.device import DeviceMonitorFilter
 
 
-@click.command("home", short_help="PIO Home")
-@click.option("--port", type=int, default=8008, help="HTTP port, default=8008")
-@click.option(
-    "--host",
-    default="127.0.0.1",
-    help="HTTP host, default=127.0.0.1. "
-    "You can open PIO Home for inbound connections with --host=0.0.0.0")
-@click.option("--no-open", is_flag=True)
-def cli(*args, **kwargs):  # pylint: disable=unused-argument
-    pioplus_call(sys.argv[1:])
+class SendOnEnter(DeviceMonitorFilter):
+    NAME = "send_on_enter"
+
+    def __init__(self, *args, **kwargs):
+        super(SendOnEnter, self).__init__(*args, **kwargs)
+        self._buffer = ""
+
+    def tx(self, text):
+        self._buffer += text
+        if self._buffer.endswith("\r\n"):
+            text = self._buffer[:-2]
+            self._buffer = ""
+            return text
+        return ""

platformio/commands/device/filters/time.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+
+from platformio.commands.device import DeviceMonitorFilter
+
+
+class Timestamp(DeviceMonitorFilter):
+    NAME = "time"
+
+    def __init__(self, *args, **kwargs):
+        super(Timestamp, self).__init__(*args, **kwargs)
+        self._first_text_received = False
+
+    def rx(self, text):
+        if self._first_text_received and "\n" not in text:
+            return text
+        timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
+        if not self._first_text_received:
+            self._first_text_received = True
+            return "%s > %s" % (timestamp, text)
+        return text.replace("\n", "\n%s > " % timestamp)

platformio/commands/device/helpers.py

@@ -0,0 +1,106 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import os
+
+from serial.tools import miniterm
+
+from platformio import fs
+from platformio.commands.device import DeviceMonitorFilter
+from platformio.compat import get_object_members, load_python_module
+from platformio.project.config import ProjectConfig
+
+
+def apply_project_monitor_options(cli_options, project_options):
+    for k in ("port", "speed", "rts", "dtr"):
+        k2 = "monitor_%s" % k
+        if k == "speed":
+            k = "baud"
+        if cli_options[k] is None and k2 in project_options:
+            cli_options[k] = project_options[k2]
+            if k != "port":
+                cli_options[k] = int(cli_options[k])
+    return cli_options
+
+
+def options_to_argv(cli_options, project_options, ignore=None):
+    confmon_flags = project_options.get("monitor_flags", [])
+    result = confmon_flags[::]
+
+    for f in project_options.get("monitor_filters", []):
+        result.extend(["--filter", f])
+
+    for k, v in cli_options.items():
+        if v is None or (ignore and k in ignore):
+            continue
+        k = "--" + k.replace("_", "-")
+        if k in confmon_flags:
+            continue
+        if isinstance(v, bool):
+            if v:
+                result.append(k)
+        elif isinstance(v, tuple):
+            for i in v:
+                result.extend([k, i])
+        else:
+            result.extend([k, str(v)])
+    return result
+
+
+def get_project_options(environment=None):
+    config = ProjectConfig.get_instance()
+    config.validate(envs=[environment] if environment else None)
+    if not environment:
+        default_envs = config.default_envs()
+        if default_envs:
+            environment = default_envs[0]
+        else:
+            environment = config.envs()[0]
+    return config.items(env=environment, as_dict=True)
+
+
+def get_board_hwids(project_dir, platform, board):
+    with fs.cd(project_dir):
+        return platform.board_config(board).get("build.hwids", [])
+
+
+def load_monitor_filter(path, project_dir=None, environment=None):
+    name = os.path.basename(path)
+    name = name[: name.find(".")]
+    module = load_python_module("platformio.commands.device.filters.%s" % name, path)
+    for cls in get_object_members(module).values():
+        if (
+            not inspect.isclass(cls)
+            or not issubclass(cls, DeviceMonitorFilter)
+            or cls == DeviceMonitorFilter
+        ):
+            continue
+        obj = cls(project_dir, environment)
+        miniterm.TRANSFORMATIONS[obj.NAME] = obj
+    return True
+
+
+def register_platform_filters(platform, project_dir, environment):
+    monitor_dir = os.path.join(platform.get_dir(), "monitor")
+    if not os.path.isdir(monitor_dir):
+        return
+
+    for name in os.listdir(monitor_dir):
+        if not name.startswith("filter_") or not name.endswith(".py"):
+            continue
+        path = os.path.join(monitor_dir, name)
+        if not os.path.isfile(path):
+            continue
+        load_monitor_filter(path, project_dir, environment)

platformio/commands/home/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/home/command.py

@@ -0,0 +1,159 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-locals,too-many-statements
+
+import mimetypes
+import socket
+from os.path import isdir
+
+import click
+
+from platformio import exception
+from platformio.compat import WINDOWS
+from platformio.managers.core import (
+    build_contrib_pysite_deps,
+    get_core_package_dir,
+    inject_contrib_pysite,
+)
+
+
+@click.command("home", short_help="PIO Home")
+@click.option("--port", type=int, default=8008, help="HTTP port, default=8008")
+@click.option(
+    "--host",
+    default="127.0.0.1",
+    help=(
+        "HTTP host, default=127.0.0.1. You can open PIO Home for inbound "
+        "connections with --host=0.0.0.0"
+    ),
+)
+@click.option("--no-open", is_flag=True)
+@click.option(
+    "--shutdown-timeout",
+    default=0,
+    type=int,
+    help=(
+        "Automatically shutdown server on timeout (in seconds) when no clients "
+        "are connected. Default is 0 which means never auto shutdown"
+    ),
+)
+def cli(port, host, no_open, shutdown_timeout):
+    # pylint: disable=import-error, import-outside-toplevel
+
+    # import contrib modules
+    inject_contrib_pysite()
+
+    try:
+        from autobahn.twisted.resource import WebSocketResource
+    except:  # pylint: disable=bare-except
+        build_contrib_pysite_deps(get_core_package_dir("contrib-pysite"))
+        from autobahn.twisted.resource import WebSocketResource
+
+    from twisted.internet import reactor
+    from twisted.web import server
+    from twisted.internet.error import CannotListenError
+
+    from platformio.commands.home.rpc.handlers.app import AppRPC
+    from platformio.commands.home.rpc.handlers.ide import IDERPC
+    from platformio.commands.home.rpc.handlers.misc import MiscRPC
+    from platformio.commands.home.rpc.handlers.os import OSRPC
+    from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
+    from platformio.commands.home.rpc.handlers.project import ProjectRPC
+    from platformio.commands.home.rpc.server import JSONRPCServerFactory
+    from platformio.commands.home.web import WebRoot
+
+    factory = JSONRPCServerFactory(shutdown_timeout)
+    factory.addHandler(AppRPC(), namespace="app")
+    factory.addHandler(IDERPC(), namespace="ide")
+    factory.addHandler(MiscRPC(), namespace="misc")
+    factory.addHandler(OSRPC(), namespace="os")
+    factory.addHandler(PIOCoreRPC(), namespace="core")
+    factory.addHandler(ProjectRPC(), namespace="project")
+
+    contrib_dir = get_core_package_dir("contrib-piohome")
+    if not isdir(contrib_dir):
+        raise exception.PlatformioException("Invalid path to PIO Home Contrib")
+
+    # Ensure PIO Home mimetypes are known
+    mimetypes.add_type("text/html", ".html")
+    mimetypes.add_type("text/css", ".css")
+    mimetypes.add_type("application/javascript", ".js")
+
+    root = WebRoot(contrib_dir)
+    root.putChild(b"wsrpc", WebSocketResource(factory))
+    site = server.Site(root)
+
+    # hook for `platformio-node-helpers`
+    if host == "__do_not_start__":
+        return
+
+    already_started = is_port_used(host, port)
+    home_url = "http://%s:%d" % (host, port)
+    if not no_open:
+        if already_started:
+            click.launch(home_url)
+        else:
+            reactor.callLater(1, lambda: click.launch(home_url))
+
+    click.echo(
+        "\n".join(
+            [
+                "",
+                "  ___I_",
+                " /\\-_--\\   PlatformIO Home",
+                "/  \\_-__\\",
+                "|[]| [] |  %s" % home_url,
+                "|__|____|______________%s" % ("_" * len(host)),
+            ]
+        )
+    )
+    click.echo("")
+    click.echo("Open PlatformIO Home in your browser by this URL => %s" % home_url)
+
+    try:
+        reactor.listenTCP(port, site, interface=host)
+    except CannotListenError as e:
+        click.secho(str(e), fg="red", err=True)
+        already_started = True
+
+    if already_started:
+        click.secho(
+            "PlatformIO Home server is already started in another process.", fg="yellow"
+        )
+        return
+
+    click.echo("PIO Home has been started. Press Ctrl+C to shutdown.")
+
+    reactor.run()
+
+
+def is_port_used(host, port):
+    socket.setdefaulttimeout(1)
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    if WINDOWS:
+        try:
+            s.bind((host, port))
+            s.close()
+            return False
+        except (OSError, socket.error):
+            pass
+    else:
+        try:
+            s.connect((host, port))
+            s.close()
+        except socket.error:
+            return False
+
+    return True

platformio/commands/home/helpers.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=keyword-arg-before-vararg, arguments-differ
+
+import os
+import socket
+
+import requests
+from twisted.internet import defer  # pylint: disable=import-error
+from twisted.internet import reactor  # pylint: disable=import-error
+from twisted.internet import threads  # pylint: disable=import-error
+
+from platformio import util
+from platformio.proc import where_is_program
+
+
+class AsyncSession(requests.Session):
+    def __init__(self, n=None, *args, **kwargs):
+        if n:
+            pool = reactor.getThreadPool()
+            pool.adjustPoolsize(0, n)
+
+        super(AsyncSession, self).__init__(*args, **kwargs)
+
+    def request(self, *args, **kwargs):
+        func = super(AsyncSession, self).request
+        return threads.deferToThread(func, *args, **kwargs)
+
+    def wrap(self, *args, **kwargs):  # pylint: disable=no-self-use
+        return defer.ensureDeferred(*args, **kwargs)
+
+
+@util.memoized(expire="60s")
+def requests_session():
+    return AsyncSession(n=5)
+
+
+@util.memoized(expire="60s")
+def get_core_fullpath():
+    return where_is_program(
+        "platformio" + (".exe" if "windows" in util.get_systype() else "")
+    )
+
+
+@util.memoized(expire="10s")
+def is_twitter_blocked():
+    ip = "104.244.42.1"
+    timeout = 2
+    try:
+        if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")):
+            requests.get("http://%s" % ip, allow_redirects=False, timeout=timeout)
+        else:
+            socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80))
+        return False
+    except:  # pylint: disable=bare-except
+        pass
+    return True

platformio/commands/home/rpc/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/home/rpc/handlers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/home/rpc/handlers/app.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from os.path import join
+
+from platformio import __version__, app, fs, util
+from platformio.project.helpers import get_project_core_dir, is_platformio_project
+
+
+class AppRPC(object):
+
+    APPSTATE_PATH = join(get_project_core_dir(), "homestate.json")
+
+    IGNORE_STORAGE_KEYS = [
+        "cid",
+        "coreVersion",
+        "coreSystype",
+        "coreCaller",
+        "coreSettings",
+        "homeDir",
+        "projectsDir",
+    ]
+
+    @staticmethod
+    def load_state():
+        with app.State(AppRPC.APPSTATE_PATH, lock=True) as state:
+            storage = state.get("storage", {})
+
+            # base data
+            caller_id = app.get_session_var("caller_id")
+            storage["cid"] = app.get_cid()
+            storage["coreVersion"] = __version__
+            storage["coreSystype"] = util.get_systype()
+            storage["coreCaller"] = str(caller_id).lower() if caller_id else None
+            storage["coreSettings"] = {
+                name: {
+                    "description": data["description"],
+                    "default_value": data["value"],
+                    "value": app.get_setting(name),
+                }
+                for name, data in app.DEFAULT_SETTINGS.items()
+            }
+
+            storage["homeDir"] = fs.expanduser("~")
+            storage["projectsDir"] = storage["coreSettings"]["projects_dir"]["value"]
+
+            # skip non-existing recent projects
+            storage["recentProjects"] = [
+                p for p in storage.get("recentProjects", []) if is_platformio_project(p)
+            ]
+
+            state["storage"] = storage
+            state.modified = False  # skip saving extra fields
+            return state.as_dict()
+
+    @staticmethod
+    def get_state():
+        return AppRPC.load_state()
+
+    @staticmethod
+    def save_state(state):
+        with app.State(AppRPC.APPSTATE_PATH, lock=True) as s:
+            s.clear()
+            s.update(state)
+            storage = s.get("storage", {})
+            for k in AppRPC.IGNORE_STORAGE_KEYS:
+                if k in storage:
+                    del storage[k]
+        return True

platformio/commands/home/rpc/handlers/ide.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+import jsonrpc  # pylint: disable=import-error
+from twisted.internet import defer  # pylint: disable=import-error
+
+
+class IDERPC(object):
+    def __init__(self):
+        self._queue = {}
+
+    def send_command(self, sid, command, params):
+        if not self._queue.get(sid):
+            raise jsonrpc.exceptions.JSONRPCDispatchException(
+                code=4005, message="PIO Home IDE agent is not started"
+            )
+        while self._queue[sid]:
+            self._queue[sid].pop().callback(
+                {"id": time.time(), "method": command, "params": params}
+            )
+
+    def listen_commands(self, sid=0):
+        if sid not in self._queue:
+            self._queue[sid] = []
+        self._queue[sid].append(defer.Deferred())
+        return self._queue[sid][-1]
+
+    def open_project(self, sid, project_dir):
+        return self.send_command(sid, "open_project", project_dir)
+
+    def open_text_document(self, sid, path, line=None, column=None):
+        return self.send_command(
+            sid, "open_text_document", dict(path=path, line=line, column=column)
+        )

platformio/commands/home/rpc/handlers/misc.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import time
+
+from twisted.internet import defer, reactor  # pylint: disable=import-error
+
+from platformio import app
+from platformio.commands.home.rpc.handlers.os import OSRPC
+
+
+class MiscRPC(object):
+    def load_latest_tweets(self, data_url):
+        cache_key = app.ContentCache.key_from_args(data_url, "tweets")
+        cache_valid = "7d"
+        with app.ContentCache() as cc:
+            cache_data = cc.get(cache_key)
+            if cache_data:
+                cache_data = json.loads(cache_data)
+                # automatically update cache in background every 12 hours
+                if cache_data["time"] < (time.time() - (3600 * 12)):
+                    reactor.callLater(
+                        5, self._preload_latest_tweets, data_url, cache_key, cache_valid
+                    )
+                return cache_data["result"]
+
+        result = self._preload_latest_tweets(data_url, cache_key, cache_valid)
+        return result
+
+    @staticmethod
+    @defer.inlineCallbacks
+    def _preload_latest_tweets(data_url, cache_key, cache_valid):
+        result = json.loads((yield OSRPC.fetch_content(data_url)))
+        with app.ContentCache() as cc:
+            cc.set(
+                cache_key,
+                json.dumps({"time": int(time.time()), "result": result}),
+                cache_valid,
+            )
+        defer.returnValue(result)

platformio/commands/home/rpc/handlers/os.py

@@ -0,0 +1,156 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import glob
+import io
+import os
+import shutil
+from functools import cmp_to_key
+
+import click
+from twisted.internet import defer  # pylint: disable=import-error
+
+from platformio import app, fs, util
+from platformio.commands.home import helpers
+from platformio.compat import PY2, get_filesystem_encoding
+
+
+class OSRPC(object):
+    @staticmethod
+    @defer.inlineCallbacks
+    def fetch_content(uri, data=None, headers=None, cache_valid=None):
+        if not headers:
+            headers = {
+                "User-Agent": (
+                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) "
+                    "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 "
+                    "Safari/603.3.8"
+                )
+            }
+        cache_key = app.ContentCache.key_from_args(uri, data) if cache_valid else None
+        with app.ContentCache() as cc:
+            if cache_key:
+                result = cc.get(cache_key)
+                if result is not None:
+                    defer.returnValue(result)
+
+        # check internet before and resolve issue with 60 seconds timeout
+        util.internet_on(raise_exception=True)
+
+        session = helpers.requests_session()
+        if data:
+            r = yield session.post(uri, data=data, headers=headers)
+        else:
+            r = yield session.get(uri, headers=headers)
+
+        r.raise_for_status()
+        result = r.text
+        if cache_valid:
+            with app.ContentCache() as cc:
+                cc.set(cache_key, result, cache_valid)
+        defer.returnValue(result)
+
+    def request_content(self, uri, data=None, headers=None, cache_valid=None):
+        if uri.startswith("http"):
+            return self.fetch_content(uri, data, headers, cache_valid)
+        if os.path.isfile(uri):
+            with io.open(uri, encoding="utf-8") as fp:
+                return fp.read()
+        return None
+
+    @staticmethod
+    def open_url(url):
+        return click.launch(url)
+
+    @staticmethod
+    def reveal_file(path):
+        return click.launch(
+            path.encode(get_filesystem_encoding()) if PY2 else path, locate=True
+        )
+
+    @staticmethod
+    def open_file(path):
+        return click.launch(path.encode(get_filesystem_encoding()) if PY2 else path)
+
+    @staticmethod
+    def is_file(path):
+        return os.path.isfile(path)
+
+    @staticmethod
+    def is_dir(path):
+        return os.path.isdir(path)
+
+    @staticmethod
+    def make_dirs(path):
+        return os.makedirs(path)
+
+    @staticmethod
+    def get_file_mtime(path):
+        return os.path.getmtime(path)
+
+    @staticmethod
+    def rename(src, dst):
+        return os.rename(src, dst)
+
+    @staticmethod
+    def copy(src, dst):
+        return shutil.copytree(src, dst)
+
+    @staticmethod
+    def glob(pathnames, root=None):
+        if not isinstance(pathnames, list):
+            pathnames = [pathnames]
+        result = set()
+        for pathname in pathnames:
+            result |= set(glob.glob(os.path.join(root, pathname) if root else pathname))
+        return list(result)
+
+    @staticmethod
+    def list_dir(path):
+        def _cmp(x, y):
+            if x[1] and not y[1]:
+                return -1
+            if not x[1] and y[1]:
+                return 1
+            if x[0].lower() > y[0].lower():
+                return 1
+            if x[0].lower() < y[0].lower():
+                return -1
+            return 0
+
+        items = []
+        if path.startswith("~"):
+            path = fs.expanduser(path)
+        if not os.path.isdir(path):
+            return items
+        for item in os.listdir(path):
+            try:
+                item_is_dir = os.path.isdir(os.path.join(path, item))
+                if item_is_dir:
+                    os.listdir(os.path.join(path, item))
+                items.append((item, item_is_dir))
+            except OSError:
+                pass
+        return sorted(items, key=cmp_to_key(_cmp))
+
+    @staticmethod
+    def get_logical_devices():
+        items = []
+        for item in util.get_logical_devices():
+            if item["name"]:
+                item["name"] = item["name"]
+            items.append(item)
+        return items

platformio/commands/home/rpc/handlers/piocore.py

@@ -0,0 +1,164 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import json
+import os
+import sys
+from io import BytesIO, StringIO
+
+import click
+import jsonrpc  # pylint: disable=import-error
+from twisted.internet import defer  # pylint: disable=import-error
+from twisted.internet import threads  # pylint: disable=import-error
+from twisted.internet import utils  # pylint: disable=import-error
+
+from platformio import __main__, __version__, fs
+from platformio.commands.home import helpers
+from platformio.compat import PY2, get_filesystem_encoding, is_bytes, string_types
+
+try:
+    from thread import get_ident as thread_get_ident
+except ImportError:
+    from threading import get_ident as thread_get_ident
+
+
+class MultiThreadingStdStream(object):
+    def __init__(self, parent_stream):
+        self._buffers = {thread_get_ident(): parent_stream}
+
+    def __getattr__(self, name):
+        thread_id = thread_get_ident()
+        self._ensure_thread_buffer(thread_id)
+        return getattr(self._buffers[thread_id], name)
+
+    def _ensure_thread_buffer(self, thread_id):
+        if thread_id not in self._buffers:
+            self._buffers[thread_id] = BytesIO() if PY2 else StringIO()
+
+    def write(self, value):
+        thread_id = thread_get_ident()
+        self._ensure_thread_buffer(thread_id)
+        return self._buffers[thread_id].write(
+            value.decode() if is_bytes(value) else value
+        )
+
+    def get_value_and_reset(self):
+        result = ""
+        try:
+            result = self.getvalue()
+            self.truncate(0)
+            self.seek(0)
+        except AttributeError:
+            pass
+        return result
+
+
+class PIOCoreRPC(object):
+    @staticmethod
+    def version():
+        return __version__
+
+    @staticmethod
+    def setup_multithreading_std_streams():
+        if isinstance(sys.stdout, MultiThreadingStdStream):
+            return
+        PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout)
+        PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr)
+        sys.stdout = PIOCoreRPC.thread_stdout
+        sys.stderr = PIOCoreRPC.thread_stderr
+
+    @staticmethod
+    def call(args, options=None):
+        return defer.maybeDeferred(PIOCoreRPC._call_generator, args, options)
+
+    @staticmethod
+    @defer.inlineCallbacks
+    def _call_generator(args, options=None):
+        for i, arg in enumerate(args):
+            if isinstance(arg, string_types):
+                args[i] = arg.encode(get_filesystem_encoding()) if PY2 else arg
+            else:
+                args[i] = str(arg)
+
+        to_json = "--json-output" in args
+
+        try:
+            if args and args[0] in ("account", "remote"):
+                result = yield PIOCoreRPC._call_subprocess(args, options)
+                defer.returnValue(PIOCoreRPC._process_result(result, to_json))
+            else:
+                result = yield PIOCoreRPC._call_inline(args, options)
+                try:
+                    defer.returnValue(PIOCoreRPC._process_result(result, to_json))
+                except ValueError:
+                    # fall-back to subprocess method
+                    result = yield PIOCoreRPC._call_subprocess(args, options)
+                    defer.returnValue(PIOCoreRPC._process_result(result, to_json))
+        except Exception as e:  # pylint: disable=bare-except
+            raise jsonrpc.exceptions.JSONRPCDispatchException(
+                code=4003, message="PIO Core Call Error", data=str(e)
+            )
+
+    @staticmethod
+    def _call_inline(args, options):
+        PIOCoreRPC.setup_multithreading_std_streams()
+        cwd = (options or {}).get("cwd") or os.getcwd()
+
+        def _thread_task():
+            with fs.cd(cwd):
+                exit_code = __main__.main(["-c"] + args)
+            return (
+                PIOCoreRPC.thread_stdout.get_value_and_reset(),
+                PIOCoreRPC.thread_stderr.get_value_and_reset(),
+                exit_code,
+            )
+
+        return threads.deferToThread(_thread_task)
+
+    @staticmethod
+    def _call_subprocess(args, options):
+        cwd = (options or {}).get("cwd") or os.getcwd()
+        return utils.getProcessOutputAndValue(
+            helpers.get_core_fullpath(),
+            args,
+            path=cwd,
+            env={k: v for k, v in os.environ.items() if "%" not in k},
+        )
+
+    @staticmethod
+    def _process_result(result, to_json=False):
+        out, err, code = result
+        text = ("%s\n\n%s" % (out, err)).strip()
+        if code != 0:
+            raise Exception(text)
+        if not to_json:
+            return text
+        if is_bytes(out):
+            out = out.decode()
+        try:
+            return json.loads(out)
+        except ValueError as e:
+            click.secho("%s => `%s`" % (e, out), fg="red", err=True)
+            # if PIO Core prints unhandled warnings
+            for line in out.split("\n"):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    return json.loads(line)
+                except ValueError:
+                    pass
+            raise e

platformio/commands/home/rpc/handlers/project.py

@@ -0,0 +1,327 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import os
+import shutil
+import time
+
+import jsonrpc  # pylint: disable=import-error
+
+from platformio import exception, fs
+from platformio.commands.home.rpc.handlers.app import AppRPC
+from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
+from platformio.compat import PY2, get_filesystem_encoding
+from platformio.ide.projectgenerator import ProjectGenerator
+from platformio.managers.platform import PlatformManager
+from platformio.project.config import ProjectConfig
+from platformio.project.exception import ProjectError
+from platformio.project.helpers import get_project_dir, is_platformio_project
+from platformio.project.options import get_config_options_schema
+
+
+class ProjectRPC(object):
+    @staticmethod
+    def config_call(init_kwargs, method, *args):
+        assert isinstance(init_kwargs, dict)
+        assert "path" in init_kwargs
+        project_dir = get_project_dir()
+        if os.path.isfile(init_kwargs["path"]):
+            project_dir = os.path.dirname(init_kwargs["path"])
+        with fs.cd(project_dir):
+            return getattr(ProjectConfig(**init_kwargs), method)(*args)
+
+    @staticmethod
+    def config_load(path):
+        return ProjectConfig(
+            path, parse_extra=False, expand_interpolations=False
+        ).as_tuple()
+
+    @staticmethod
+    def config_dump(path, data):
+        config = ProjectConfig(path, parse_extra=False, expand_interpolations=False)
+        config.update(data, clear=True)
+        return config.save()
+
+    @staticmethod
+    def config_update_description(path, text):
+        config = ProjectConfig(path, parse_extra=False, expand_interpolations=False)
+        if not config.has_section("platformio"):
+            config.add_section("platformio")
+        if text:
+            config.set("platformio", "description", text)
+        else:
+            if config.has_option("platformio", "description"):
+                config.remove_option("platformio", "description")
+            if not config.options("platformio"):
+                config.remove_section("platformio")
+        return config.save()
+
+    @staticmethod
+    def get_config_schema():
+        return get_config_options_schema()
+
+    @staticmethod
+    def get_projects():
+        def _get_project_data():
+            data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []}
+            config = ProjectConfig()
+            data["envs"] = config.envs()
+            data["description"] = config.get("platformio", "description")
+            data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", []))
+
+            libdeps_dir = config.get_optional_dir("libdeps")
+            for section in config.sections():
+                if not section.startswith("env:"):
+                    continue
+                data["envLibdepsDirs"].append(os.path.join(libdeps_dir, section[4:]))
+                if config.has_option(section, "board"):
+                    data["boards"].append(config.get(section, "board"))
+                data["libExtraDirs"].extend(config.get(section, "lib_extra_dirs", []))
+
+            # skip non existing folders and resolve full path
+            for key in ("envLibdepsDirs", "libExtraDirs"):
+                data[key] = [
+                    fs.expanduser(d) if d.startswith("~") else os.path.realpath(d)
+                    for d in data[key]
+                    if os.path.isdir(d)
+                ]
+
+            return data
+
+        def _path_to_name(path):
+            return (os.path.sep).join(path.split(os.path.sep)[-2:])
+
+        result = []
+        pm = PlatformManager()
+        for project_dir in AppRPC.load_state()["storage"]["recentProjects"]:
+            if not os.path.isdir(project_dir):
+                continue
+            data = {}
+            boards = []
+            try:
+                with fs.cd(project_dir):
+                    data = _get_project_data()
+            except ProjectError:
+                continue
+
+            for board_id in data.get("boards", []):
+                name = board_id
+                try:
+                    name = pm.board_config(board_id)["name"]
+                except exception.PlatformioException:
+                    pass
+                boards.append({"id": board_id, "name": name})
+
+            result.append(
+                {
+                    "path": project_dir,
+                    "name": _path_to_name(project_dir),
+                    "modified": int(os.path.getmtime(project_dir)),
+                    "boards": boards,
+                    "description": data.get("description"),
+                    "envs": data.get("envs", []),
+                    "envLibStorages": [
+                        {"name": os.path.basename(d), "path": d}
+                        for d in data.get("envLibdepsDirs", [])
+                    ],
+                    "extraLibStorages": [
+                        {"name": _path_to_name(d), "path": d}
+                        for d in data.get("libExtraDirs", [])
+                    ],
+                }
+            )
+        return result
+
+    @staticmethod
+    def get_project_examples():
+        result = []
+        for manifest in PlatformManager().get_installed():
+            examples_dir = os.path.join(manifest["__pkg_dir"], "examples")
+            if not os.path.isdir(examples_dir):
+                continue
+            items = []
+            for project_dir, _, __ in os.walk(examples_dir):
+                project_description = None
+                try:
+                    config = ProjectConfig(os.path.join(project_dir, "platformio.ini"))
+                    config.validate(silent=True)
+                    project_description = config.get("platformio", "description")
+                except ProjectError:
+                    continue
+
+                path_tokens = project_dir.split(os.path.sep)
+                items.append(
+                    {
+                        "name": "/".join(
+                            path_tokens[path_tokens.index("examples") + 1 :]
+                        ),
+                        "path": project_dir,
+                        "description": project_description,
+                    }
+                )
+            result.append(
+                {
+                    "platform": {
+                        "title": manifest["title"],
+                        "version": manifest["version"],
+                    },
+                    "items": sorted(items, key=lambda item: item["name"]),
+                }
+            )
+        return sorted(result, key=lambda data: data["platform"]["title"])
+
+    def init(self, board, framework, project_dir):
+        assert project_dir
+        state = AppRPC.load_state()
+        if not os.path.isdir(project_dir):
+            os.makedirs(project_dir)
+        args = ["init", "--board", board]
+        if framework:
+            args.extend(["--project-option", "framework = %s" % framework])
+        if (
+            state["storage"]["coreCaller"]
+            and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides()
+        ):
+            args.extend(["--ide", state["storage"]["coreCaller"]])
+        d = PIOCoreRPC.call(args, options={"cwd": project_dir})
+        d.addCallback(self._generate_project_main, project_dir, framework)
+        return d
+
+    @staticmethod
+    def _generate_project_main(_, project_dir, framework):
+        main_content = None
+        if framework == "arduino":
+            main_content = "\n".join(
+                [
+                    "#include <Arduino.h>",
+                    "",
+                    "void setup() {",
+                    "  // put your setup code here, to run once:",
+                    "}",
+                    "",
+                    "void loop() {",
+                    "  // put your main code here, to run repeatedly:",
+                    "}",
+                    "",
+                ]
+            )
+        elif framework == "mbed":
+            main_content = "\n".join(
+                [
+                    "#include <mbed.h>",
+                    "",
+                    "int main() {",
+                    "",
+                    "  // put your setup code here, to run once:",
+                    "",
+                    "  while(1) {",
+                    "    // put your main code here, to run repeatedly:",
+                    "  }",
+                    "}",
+                    "",
+                ]
+            )
+        if not main_content:
+            return project_dir
+        with fs.cd(project_dir):
+            config = ProjectConfig()
+            src_dir = config.get_optional_dir("src")
+            main_path = os.path.join(src_dir, "main.cpp")
+            if os.path.isfile(main_path):
+                return project_dir
+            if not os.path.isdir(src_dir):
+                os.makedirs(src_dir)
+            with open(main_path, "w") as fp:
+                fp.write(main_content.strip())
+        return project_dir
+
+    def import_arduino(self, board, use_arduino_libs, arduino_project_dir):
+        board = str(board)
+        if arduino_project_dir and PY2:
+            arduino_project_dir = arduino_project_dir.encode(get_filesystem_encoding())
+        # don't import PIO Project
+        if is_platformio_project(arduino_project_dir):
+            return arduino_project_dir
+
+        is_arduino_project = any(
+            [
+                os.path.isfile(
+                    os.path.join(
+                        arduino_project_dir,
+                        "%s.%s" % (os.path.basename(arduino_project_dir), ext),
+                    )
+                )
+                for ext in ("ino", "pde")
+            ]
+        )
+        if not is_arduino_project:
+            raise jsonrpc.exceptions.JSONRPCDispatchException(
+                code=4000, message="Not an Arduino project: %s" % arduino_project_dir
+            )
+
+        state = AppRPC.load_state()
+        project_dir = os.path.join(
+            state["storage"]["projectsDir"], time.strftime("%y%m%d-%H%M%S-") + board
+        )
+        if not os.path.isdir(project_dir):
+            os.makedirs(project_dir)
+        args = ["init", "--board", board]
+        args.extend(["--project-option", "framework = arduino"])
+        if use_arduino_libs:
+            args.extend(
+                ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"]
+            )
+        if (
+            state["storage"]["coreCaller"]
+            and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides()
+        ):
+            args.extend(["--ide", state["storage"]["coreCaller"]])
+        d = PIOCoreRPC.call(args, options={"cwd": project_dir})
+        d.addCallback(self._finalize_arduino_import, project_dir, arduino_project_dir)
+        return d
+
+    @staticmethod
+    def _finalize_arduino_import(_, project_dir, arduino_project_dir):
+        with fs.cd(project_dir):
+            config = ProjectConfig()
+            src_dir = config.get_optional_dir("src")
+            if os.path.isdir(src_dir):
+                fs.rmtree(src_dir)
+            shutil.copytree(arduino_project_dir, src_dir)
+        return project_dir
+
+    @staticmethod
+    def import_pio(project_dir):
+        if not project_dir or not is_platformio_project(project_dir):
+            raise jsonrpc.exceptions.JSONRPCDispatchException(
+                code=4001, message="Not an PlatformIO project: %s" % project_dir
+            )
+        new_project_dir = os.path.join(
+            AppRPC.load_state()["storage"]["projectsDir"],
+            time.strftime("%y%m%d-%H%M%S-") + os.path.basename(project_dir),
+        )
+        shutil.copytree(project_dir, new_project_dir)
+
+        state = AppRPC.load_state()
+        args = ["init"]
+        if (
+            state["storage"]["coreCaller"]
+            and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides()
+        ):
+            args.extend(["--ide", state["storage"]["coreCaller"]])
+        d = PIOCoreRPC.call(args, options={"cwd": new_project_dir})
+        d.addCallback(lambda _: new_project_dir)
+        return d

platformio/commands/home/rpc/server.py

@@ -0,0 +1,101 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=import-error
+
+import click
+import jsonrpc
+from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol
+from jsonrpc.exceptions import JSONRPCDispatchException
+from twisted.internet import defer, reactor
+
+from platformio.compat import PY2, dump_json_to_unicode, is_bytes
+
+
+class JSONRPCServerProtocol(WebSocketServerProtocol):
+    def onOpen(self):
+        self.factory.connection_nums += 1
+        if self.factory.shutdown_timer:
+            self.factory.shutdown_timer.cancel()
+            self.factory.shutdown_timer = None
+
+    def onClose(self, wasClean, code, reason):  # pylint: disable=unused-argument
+        self.factory.connection_nums -= 1
+        if self.factory.connection_nums == 0:
+            self.factory.shutdownByTimeout()
+
+    def onMessage(self, payload, isBinary):  # pylint: disable=unused-argument
+        # click.echo("> %s" % payload)
+        response = jsonrpc.JSONRPCResponseManager.handle(
+            payload, self.factory.dispatcher
+        ).data
+        # if error
+        if "result" not in response:
+            self.sendJSONResponse(response)
+            return None
+
+        d = defer.maybeDeferred(lambda: response["result"])
+        d.addCallback(self._callback, response)
+        d.addErrback(self._errback, response)
+
+        return None
+
+    def _callback(self, result, response):
+        response["result"] = result
+        self.sendJSONResponse(response)
+
+    def _errback(self, failure, response):
+        if isinstance(failure.value, JSONRPCDispatchException):
+            e = failure.value
+        else:
+            e = JSONRPCDispatchException(code=4999, message=failure.getErrorMessage())
+        del response["result"]
+        response["error"] = e.error._data  # pylint: disable=protected-access
+        self.sendJSONResponse(response)
+
+    def sendJSONResponse(self, response):
+        # click.echo("< %s" % response)
+        if "error" in response:
+            click.secho("Error: %s" % response["error"], fg="red", err=True)
+        response = dump_json_to_unicode(response)
+        if not PY2 and not is_bytes(response):
+            response = response.encode("utf-8")
+        self.sendMessage(response)
+
+
+class JSONRPCServerFactory(WebSocketServerFactory):
+
+    protocol = JSONRPCServerProtocol
+    connection_nums = 0
+    shutdown_timer = 0
+
+    def __init__(self, shutdown_timeout=0):
+        super(JSONRPCServerFactory, self).__init__()
+        self.shutdown_timeout = shutdown_timeout
+        self.dispatcher = jsonrpc.Dispatcher()
+
+    def shutdownByTimeout(self):
+        if self.shutdown_timeout < 1:
+            return
+
+        def _auto_shutdown_server():
+            click.echo("Automatically shutdown server on timeout")
+            reactor.stop()
+
+        self.shutdown_timer = reactor.callLater(
+            self.shutdown_timeout, _auto_shutdown_server
+        )
+
+    def addHandler(self, handler, namespace):
+        self.dispatcher.build_method_map(handler, prefix="%s." % namespace)

platformio/commands/home/web.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import reactor  # pylint: disable=import-error
+from twisted.web import static  # pylint: disable=import-error
+
+
+class WebRoot(static.File):
+    def render_GET(self, request):
+        if request.args.get(b"__shutdown__", False):
+            reactor.stop()
+            return "Server has been stopped"
+
+        request.setHeader("cache-control", "no-cache, no-store, must-revalidate")
+        request.setHeader("pragma", "no-cache")
+        request.setHeader("expires", "0")
+        return static.File.render_GET(self, request)

platformio/commands/lib.py

@@ -14,101 +14,197 @@
 
 # pylint: disable=too-many-branches, too-many-locals
 
-import json
+import os
 import time
-from os.path import isdir, join
-from urllib import quote
 
 import click
+import semantic_version
+from tabulate import tabulate
 
-from platformio import exception, util
-from platformio.managers.lib import LibraryManager, get_builtin_libs
-from platformio.util import get_api_result
+from platformio import exception, fs, util
+from platformio.commands import PlatformioCLI
+from platformio.compat import dump_json_to_unicode
+from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
+from platformio.package.manifest.parser import ManifestParserFactory
+from platformio.package.manifest.schema import ManifestSchema
+from platformio.proc import is_ci
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import get_project_dir, is_platformio_project
+
+try:
+    from urllib.parse import quote
+except ImportError:
+    from urllib import quote
+
+CTX_META_INPUT_DIRS_KEY = __name__ + ".input_dirs"
+CTX_META_PROJECT_ENVIRONMENTS_KEY = __name__ + ".project_environments"
+CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs"
+CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps"
+
+
+def get_project_global_lib_dir():
+    return ProjectConfig.get_instance().get_optional_dir("globallib")
 
 
 @click.group(short_help="Library Manager")
-@click.option(
-    "-g",
-    "--global",
-    is_flag=True,
-    help="Manage global PlatformIO library storage")
 @click.option(
     "-d",
     "--storage-dir",
+    multiple=True,
     default=None,
     type=click.Path(
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True),
-    help="Manage custom library storage")
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+    help="Manage custom library storage",
+)
+@click.option(
+    "-g", "--global", is_flag=True, help="Manage global PlatformIO library storage"
+)
+@click.option(
+    "-e",
+    "--environment",
+    multiple=True,
+    help=(
+        "Manage libraries for the specific project build environments "
+        "declared in `platformio.ini`"
+    ),
+)
 @click.pass_context
 def cli(ctx, **options):
-    non_storage_cmds = ("search", "show", "register", "stats", "builtin")
+    storage_cmds = ("install", "uninstall", "update", "list")
     # skip commands that don't need storage folder
-    if ctx.invoked_subcommand in non_storage_cmds or \
-            (len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")):
+    if ctx.invoked_subcommand not in storage_cmds or (
+        len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")
+    ):
         return
-    storage_dir = options['storage_dir']
-    if not storage_dir:
-        if options['global']:
-            storage_dir = join(util.get_home_dir(), "lib")
-        elif util.is_platformio_project():
-            storage_dir = util.get_projectlibdeps_dir()
-        elif util.is_ci():
-            storage_dir = join(util.get_home_dir(), "lib")
+    storage_dirs = list(options["storage_dir"])
+    if options["global"]:
+        storage_dirs.append(get_project_global_lib_dir())
+    if not storage_dirs:
+        if is_platformio_project():
+            storage_dirs = [get_project_dir()]
+        elif is_ci():
+            storage_dirs = [get_project_global_lib_dir()]
             click.secho(
                 "Warning! Global library storage is used automatically. "
                 "Please use `platformio lib --global %s` command to remove "
                 "this warning." % ctx.invoked_subcommand,
-                fg="yellow")
-    elif util.is_platformio_project(storage_dir):
-        with util.cd(storage_dir):
-            storage_dir = util.get_projectlibdeps_dir()
+                fg="yellow",
+            )
 
-    if not storage_dir and not util.is_platformio_project():
-        raise exception.NotGlobalLibDir(util.get_project_dir(),
-                                        join(util.get_home_dir(), "lib"),
-                                        ctx.invoked_subcommand)
+    if not storage_dirs:
+        raise exception.NotGlobalLibDir(
+            get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand
+        )
 
-    ctx.obj = LibraryManager(storage_dir)
-    if "--json-output" not in ctx.args:
-        click.echo("Library Storage: " + storage_dir)
+    in_silence = PlatformioCLI.in_silence()
+    ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options["environment"]
+    ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs
+    ctx.meta[CTX_META_STORAGE_DIRS_KEY] = []
+    ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] = {}
+    for storage_dir in storage_dirs:
+        if not is_platformio_project(storage_dir):
+            ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
+            continue
+        with fs.cd(storage_dir):
+            config = ProjectConfig.get_instance(
+                os.path.join(storage_dir, "platformio.ini")
+            )
+            config.validate(options["environment"], silent=in_silence)
+            libdeps_dir = config.get_optional_dir("libdeps")
+            for env in config.envs():
+                if options["environment"] and env not in options["environment"]:
+                    continue
+                storage_dir = os.path.join(libdeps_dir, env)
+                ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
+                ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get(
+                    "env:" + env, "lib_deps", []
+                )
 
 
 @cli.command("install", short_help="Install library")
 @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]")
-# @click.option(
-#     "--save",
-#     is_flag=True,
-#     help="Save installed libraries into the project's platformio.ini "
-#     "library dependencies")
 @click.option(
-    "-s", "--silent", is_flag=True, help="Suppress progress reporting")
-@click.option(
-    "--interactive",
+    "--save",
     is_flag=True,
-    help="Allow to make a choice for all prompts")
+    help="Save installed libraries into the `platformio.ini` dependency list",
+)
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option(
-    "-f",
-    "--force",
-    is_flag=True,
-    help="Reinstall/redownload library if exists")
-@click.pass_obj
-def lib_install(lm, libraries, silent, interactive, force):
-    # @TODO: "save" option
-    for library in libraries:
-        lm.install(
-            library, silent=silent, interactive=interactive, force=force)
+    "--interactive", is_flag=True, help="Allow to make a choice for all prompts"
+)
+@click.option(
+    "-f", "--force", is_flag=True, help="Reinstall/redownload library if exists"
+)
+@click.pass_context
+def lib_install(  # pylint: disable=too-many-arguments
+    ctx, libraries, save, silent, interactive, force
+):
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
+    storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, [])
+
+    installed_manifests = {}
+    for storage_dir in storage_dirs:
+        if not silent and (libraries or storage_dir in storage_libdeps):
+            print_storage_header(storage_dirs, storage_dir)
+        lm = LibraryManager(storage_dir)
+        if libraries:
+            for library in libraries:
+                pkg_dir = lm.install(
+                    library, silent=silent, interactive=interactive, force=force
+                )
+                installed_manifests[library] = lm.load_manifest(pkg_dir)
+        elif storage_dir in storage_libdeps:
+            builtin_lib_storages = None
+            for library in storage_libdeps[storage_dir]:
+                try:
+                    pkg_dir = lm.install(
+                        library, silent=silent, interactive=interactive, force=force
+                    )
+                    installed_manifests[library] = lm.load_manifest(pkg_dir)
+                except exception.LibNotFound as e:
+                    if builtin_lib_storages is None:
+                        builtin_lib_storages = get_builtin_libs()
+                    if not silent or not is_builtin_lib(builtin_lib_storages, library):
+                        click.secho("Warning! %s" % e, fg="yellow")
+
+    if not save or not libraries:
+        return
+
+    input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
+    project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
+    for input_dir in input_dirs:
+        config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini"))
+        config.validate(project_environments)
+        for env in config.envs():
+            if project_environments and env not in project_environments:
+                continue
+            config.expand_interpolations = False
+            lib_deps = config.get("env:" + env, "lib_deps", [])
+            for library in libraries:
+                if library in lib_deps:
+                    continue
+                manifest = installed_manifests[library]
+                try:
+                    assert library.lower() == manifest["name"].lower()
+                    assert semantic_version.Version(manifest["version"])
+                    lib_deps.append("{name}@^{version}".format(**manifest))
+                except (AssertionError, ValueError):
+                    lib_deps.append(library)
+            config.set("env:" + env, "lib_deps", lib_deps)
+            config.save()
 
 
 @cli.command("uninstall", short_help="Uninstall libraries")
 @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]")
-@click.pass_obj
-def lib_uninstall(lm, libraries):
-    for library in libraries:
-        lm.uninstall(library)
+@click.pass_context
+def lib_uninstall(ctx, libraries):
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
+    for storage_dir in storage_dirs:
+        print_storage_header(storage_dirs, storage_dir)
+        lm = LibraryManager(storage_dir)
+        for library in libraries:
+            lm.uninstall(library)
 
 
 @cli.command("update", short_help="Update installed libraries")
@@ -117,69 +213,85 @@ def lib_uninstall(lm, libraries):
     "-c",
     "--only-check",
     is_flag=True,
-    help="Do not update, only check for new version")
+    help="DEPRECATED. Please use `--dry-run` instead",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
+)
 @click.option("--json-output", is_flag=True)
-@click.pass_obj
-def lib_update(lm, libraries, only_check, json_output):
-    if not libraries:
-        libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()]
+@click.pass_context
+def lib_update(ctx, libraries, only_check, dry_run, json_output):
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
+    only_check = dry_run or only_check
+    json_result = {}
+    for storage_dir in storage_dirs:
+        if not json_output:
+            print_storage_header(storage_dirs, storage_dir)
+        lm = LibraryManager(storage_dir)
 
-    if only_check and json_output:
-        result = []
-        for library in libraries:
-            pkg_dir = library if isdir(library) else None
-            requirements = None
-            url = None
-            if not pkg_dir:
-                name, requirements, url = lm.parse_pkg_uri(library)
-                pkg_dir = lm.get_package_dir(name, requirements, url)
-            if not pkg_dir:
-                continue
-            latest = lm.outdated(pkg_dir, requirements)
-            if not latest:
-                continue
-            manifest = lm.load_manifest(pkg_dir)
-            manifest['versionLatest'] = latest
-            result.append(manifest)
-        return click.echo(json.dumps(result))
-    else:
-        for library in libraries:
-            lm.update(library, only_check=only_check)
+        _libraries = libraries
+        if not _libraries:
+            _libraries = [manifest["__pkg_dir"] for manifest in lm.get_installed()]
+
+        if only_check and json_output:
+            result = []
+            for library in _libraries:
+                pkg_dir = library if os.path.isdir(library) else None
+                requirements = None
+                url = None
+                if not pkg_dir:
+                    name, requirements, url = lm.parse_pkg_uri(library)
+                    pkg_dir = lm.get_package_dir(name, requirements, url)
+                if not pkg_dir:
+                    continue
+                latest = lm.outdated(pkg_dir, requirements)
+                if not latest:
+                    continue
+                manifest = lm.load_manifest(pkg_dir)
+                manifest["versionLatest"] = latest
+                result.append(manifest)
+            json_result[storage_dir] = result
+        else:
+            for library in _libraries:
+                lm.update(library, only_check=only_check)
+
+    if json_output:
+        return click.echo(
+            dump_json_to_unicode(
+                json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
+            )
+        )
 
     return True
 
 
-def print_lib_item(item):
-    click.secho(item['name'], fg="cyan")
-    click.echo("=" * len(item['name']))
-    if "id" in item:
-        click.secho("#ID: %d" % item['id'], bold=True)
-    if "description" in item or "url" in item:
-        click.echo(item.get("description", item.get("url", "")))
-    click.echo()
-
-    for key in ("version", "homepage", "license", "keywords"):
-        if key not in item or not item[key]:
-            continue
-        if isinstance(item[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
+@cli.command("list", short_help="List installed libraries")
+@click.option("--json-output", is_flag=True)
+@click.pass_context
+def lib_list(ctx, json_output):
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
+    json_result = {}
+    for storage_dir in storage_dirs:
+        if not json_output:
+            print_storage_header(storage_dirs, storage_dir)
+        lm = LibraryManager(storage_dir)
+        items = lm.get_installed()
+        if json_output:
+            json_result[storage_dir] = items
+        elif items:
+            for item in sorted(items, key=lambda i: i["name"]):
+                print_lib_item(item)
         else:
-            click.echo("%s: %s" % (key.title(), item[key]))
+            click.echo("No items found")
 
-    for key in ("frameworks", "platforms"):
-        if key not in item:
-            continue
-        click.echo("Compatible %s: %s" % (key, ", ".join(
-            [i['title'] if isinstance(i, dict) else i for i in item[key]])))
+    if json_output:
+        return click.echo(
+            dump_json_to_unicode(
+                json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
+            )
+        )
 
-    if "authors" in item or "authornames" in item:
-        click.echo("Authors: %s" % ", ".join(
-            item.get("authornames",
-                     [a.get("name", "") for a in item.get("authors", [])])))
-
-    if "__src_url" in item:
-        click.secho("Source: %s" % item['__src_url'])
-    click.echo()
+    return True
 
 
 @cli.command("search", short_help="Search for a library")
@@ -196,7 +308,8 @@ def print_lib_item(item):
 @click.option(
     "--noninteractive",
     is_flag=True,
-    help="Do not prompt, automatically paginate with delay")
+    help="Do not prompt, automatically paginate with delay",
+)
 def lib_search(query, json_output, page, noninteractive, **filters):
     if not query:
         query = []
@@ -207,77 +320,61 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         for value in values:
             query.append('%s:"%s"' % (key, value))
 
-    result = get_api_result(
-        "/v2/lib/search",
-        dict(query=" ".join(query), page=page),
-        cache_valid="1d")
+    result = util.get_api_result(
+        "/v2/lib/search", dict(query=" ".join(query), page=page), cache_valid="1d"
+    )
 
     if json_output:
-        click.echo(json.dumps(result))
+        click.echo(dump_json_to_unicode(result))
         return
 
-    if result['total'] == 0:
+    if result["total"] == 0:
         click.secho(
             "Nothing has been found by your request\n"
             "Try a less-specific search or use truncation (or wildcard) "
             "operator",
             fg="yellow",
-            nl=False)
+            nl=False,
+        )
         click.secho(" *", fg="green")
         click.secho("For example: DS*, PCA*, DHT* and etc.\n", fg="yellow")
-        click.echo("For more examples and advanced search syntax, "
-                   "please use documentation:")
+        click.echo(
+            "For more examples and advanced search syntax, please use documentation:"
+        )
         click.secho(
             "https://docs.platformio.org/page/userguide/lib/cmd_search.html\n",
-            fg="cyan")
+            fg="cyan",
+        )
         return
 
     click.secho(
-        "Found %d libraries:\n" % result['total'],
-        fg="green" if result['total'] else "yellow")
+        "Found %d libraries:\n" % result["total"],
+        fg="green" if result["total"] else "yellow",
+    )
 
     while True:
-        for item in result['items']:
+        for item in result["items"]:
             print_lib_item(item)
 
-        if (int(result['page']) * int(result['perpage']) >= int(
-                result['total'])):
+        if int(result["page"]) * int(result["perpage"]) >= int(result["total"]):
             break
 
         if noninteractive:
             click.echo()
             click.secho(
-                "Loading next %d libraries... Press Ctrl+C to stop!" %
-                result['perpage'],
-                fg="yellow")
+                "Loading next %d libraries... Press Ctrl+C to stop!"
+                % result["perpage"],
+                fg="yellow",
+            )
             click.echo()
             time.sleep(5)
         elif not click.confirm("Show next libraries?"):
             break
-        result = get_api_result(
-            "/v2/lib/search", {
-                "query": " ".join(query),
-                "page": int(result['page']) + 1
-            },
-            cache_valid="1d")
-
-
-@cli.command("list", short_help="List installed libraries")
-@click.option("--json-output", is_flag=True)
-@click.pass_obj
-def lib_list(lm, json_output):
-    items = lm.get_installed()
-
-    if json_output:
-        return click.echo(json.dumps(items))
-
-    if not items:
-        return None
-
-    for item in sorted(items, key=lambda i: i['name']):
-        print_lib_item(item)
-
-    return True
+        result = util.get_api_result(
+            "/v2/lib/search",
+            {"query": " ".join(query), "page": int(result["page"]) + 1},
+            cache_valid="1d",
+        )
 
 
 @cli.command("builtin", short_help="List built-in libraries")
@@ -286,16 +383,16 @@ def lib_list(lm, json_output):
 def lib_builtin(storage, json_output):
     items = get_builtin_libs(storage)
     if json_output:
-        return click.echo(json.dumps(items))
+        return click.echo(dump_json_to_unicode(items))
 
     for storage_ in items:
-        if not storage_['items']:
+        if not storage_["items"]:
             continue
-        click.secho(storage_['name'], fg="green")
-        click.echo("*" * len(storage_['name']))
+        click.secho(storage_["name"], fg="green")
+        click.echo("*" * len(storage_["name"]))
         click.echo()
 
-        for item in sorted(storage_['items'], key=lambda i: i['name']):
+        for item in sorted(storage_["items"], key=lambda i: i["name"]):
             print_lib_item(item)
 
     return True
@@ -307,27 +404,29 @@ def lib_builtin(storage, json_output):
 def lib_show(library, json_output):
     lm = LibraryManager()
     name, requirements, _ = lm.parse_pkg_uri(library)
-    lib_id = lm.search_lib_id({
-        "name": name,
-        "requirements": requirements
-    },
-                              silent=json_output,
-                              interactive=not json_output)
-    lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
+    lib_id = lm.search_lib_id(
+        {"name": name, "requirements": requirements},
+        silent=json_output,
+        interactive=not json_output,
+    )
+    lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
     if json_output:
-        return click.echo(json.dumps(lib))
+        return click.echo(dump_json_to_unicode(lib))
 
-    click.secho(lib['name'], fg="cyan")
-    click.echo("=" * len(lib['name']))
-    click.secho("#ID: %d" % lib['id'], bold=True)
-    click.echo(lib['description'])
+    click.secho(lib["name"], fg="cyan")
+    click.echo("=" * len(lib["name"]))
+    click.secho("#ID: %d" % lib["id"], bold=True)
+    click.echo(lib["description"])
     click.echo()
 
     click.echo(
-        "Version: %s, released %s" %
-        (lib['version']['name'],
-         time.strftime("%c", util.parse_date(lib['version']['released']))))
-    click.echo("Manifest: %s" % lib['confurl'])
+        "Version: %s, released %s"
+        % (
+            lib["version"]["name"],
+            time.strftime("%c", util.parse_date(lib["version"]["released"])),
+        )
+    )
+    click.echo("Manifest: %s" % lib["confurl"])
     for key in ("homepage", "repository", "license"):
         if key not in lib or not lib[key]:
             continue
@@ -354,23 +453,33 @@ def lib_show(library, json_output):
     if _authors:
         blocks.append(("Authors", _authors))
 
-    blocks.append(("Keywords", lib['keywords']))
+    blocks.append(("Keywords", lib["keywords"]))
     for key in ("frameworks", "platforms"):
         if key not in lib or not lib[key]:
             continue
-        blocks.append(("Compatible %s" % key, [i['title'] for i in lib[key]]))
-    blocks.append(("Headers", lib['headers']))
-    blocks.append(("Examples", lib['examples']))
-    blocks.append(("Versions", [
-        "%s, released %s" %
-        (v['name'], time.strftime("%c", util.parse_date(v['released'])))
-        for v in lib['versions']
-    ]))
-    blocks.append(("Unique Downloads", [
-        "Today: %s" % lib['dlstats']['day'],
-        "Week: %s" % lib['dlstats']['week'],
-        "Month: %s" % lib['dlstats']['month']
-    ]))
+        blocks.append(("Compatible %s" % key, [i["title"] for i in lib[key]]))
+    blocks.append(("Headers", lib["headers"]))
+    blocks.append(("Examples", lib["examples"]))
+    blocks.append(
+        (
+            "Versions",
+            [
+                "%s, released %s"
+                % (v["name"], time.strftime("%c", util.parse_date(v["released"])))
+                for v in lib["versions"]
+            ],
+        )
+    )
+    blocks.append(
+        (
+            "Unique Downloads",
+            [
+                "Today: %s" % lib["dlstats"]["day"],
+                "Week: %s" % lib["dlstats"]["week"],
+                "Month: %s" % lib["dlstats"]["month"],
+            ],
+        )
+    )
 
     for (title, rows) in blocks:
         click.echo()
@@ -385,88 +494,136 @@ def lib_show(library, json_output):
 @cli.command("register", short_help="Register a new library")
 @click.argument("config_url")
 def lib_register(config_url):
-    if (not config_url.startswith("http://")
-            and not config_url.startswith("https://")):
+    if not config_url.startswith("http://") and not config_url.startswith("https://"):
         raise exception.InvalidLibConfURL(config_url)
 
-    result = get_api_result("/lib/register", data=dict(config_url=config_url))
-    if "message" in result and result['message']:
+    # Validate manifest
+    ManifestSchema().load_manifest(
+        ManifestParserFactory.new_from_url(config_url).as_dict()
+    )
+
+    result = util.get_api_result("/lib/register", data=dict(config_url=config_url))
+    if "message" in result and result["message"]:
         click.secho(
-            result['message'],
-            fg="green"
-            if "successed" in result and result['successed'] else "red")
+            result["message"],
+            fg="green" if "successed" in result and result["successed"] else "red",
+        )
 
 
 @cli.command("stats", short_help="Library Registry Statistics")
 @click.option("--json-output", is_flag=True)
 def lib_stats(json_output):
-    result = get_api_result("/lib/stats", cache_valid="1h")
+    result = util.get_api_result("/lib/stats", cache_valid="1h")
 
     if json_output:
-        return click.echo(json.dumps(result))
-
-    printitem_tpl = "{name:<33} {url}"
-    printitemdate_tpl = "{name:<33} {date:23} {url}"
-
-    def _print_title(title):
-        click.secho(title.upper(), bold=True)
-        click.echo("*" * len(title))
-
-    def _print_header(with_date=False):
-        click.echo((printitemdate_tpl if with_date else printitem_tpl).format(
-            name=click.style("Name", fg="cyan"),
-            date="Date",
-            url=click.style("Url", fg="blue")))
-
-        terminal_width, _ = click.get_terminal_size()
-        click.echo("-" * terminal_width)
-
-    def _print_lib_item(item):
-        date = str(
-            time.strftime("%c", util.parse_date(item['date'])) if "date" in
-            item else "")
-        url = click.style(
-            "https://platformio.org/lib/show/%s/%s" % (item['id'],
-                                                       quote(item['name'])),
-            fg="blue")
-        click.echo(
-            (printitemdate_tpl if "date" in item else printitem_tpl).format(
-                name=click.style(item['name'], fg="cyan"), date=date, url=url))
-
-    def _print_tag_item(name):
-        click.echo(
-            printitem_tpl.format(
-                name=click.style(name, fg="cyan"),
-                url=click.style(
-                    "https://platformio.org/lib/search?query=" + quote(
-                        "keyword:%s" % name),
-                    fg="blue")))
+        return click.echo(dump_json_to_unicode(result))
 
     for key in ("updated", "added"):
-        _print_title("Recently " + key)
-        _print_header(with_date=True)
-        for item in result.get(key, []):
-            _print_lib_item(item)
+        tabular_data = [
+            (
+                click.style(item["name"], fg="cyan"),
+                time.strftime("%c", util.parse_date(item["date"])),
+                "https://platformio.org/lib/show/%s/%s"
+                % (item["id"], quote(item["name"])),
+            )
+            for item in result.get(key, [])
+        ]
+        table = tabulate(
+            tabular_data,
+            headers=[click.style("RECENTLY " + key.upper(), bold=True), "Date", "URL"],
+        )
+        click.echo(table)
         click.echo()
 
-    _print_title("Recent keywords")
-    _print_header(with_date=False)
-    for item in result.get("lastkeywords"):
-        _print_tag_item(item)
-    click.echo()
+    for key in ("lastkeywords", "topkeywords"):
+        tabular_data = [
+            (
+                click.style(name, fg="cyan"),
+                "https://platformio.org/lib/search?query=" + quote("keyword:%s" % name),
+            )
+            for name in result.get(key, [])
+        ]
+        table = tabulate(
+            tabular_data,
+            headers=[
+                click.style(
+                    ("RECENT" if key == "lastkeywords" else "POPULAR") + " KEYWORDS",
+                    bold=True,
+                ),
+                "URL",
+            ],
+        )
+        click.echo(table)
+        click.echo()
 
-    _print_title("Popular keywords")
-    _print_header(with_date=False)
-    for item in result.get("topkeywords"):
-        _print_tag_item(item)
-    click.echo()
-
-    for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth",
-                                                                "Month")):
-        _print_title("Featured: " + title)
-        _print_header(with_date=False)
-        for item in result.get(key, []):
-            _print_lib_item(item)
+    for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth", "Month")):
+        tabular_data = [
+            (
+                click.style(item["name"], fg="cyan"),
+                "https://platformio.org/lib/show/%s/%s"
+                % (item["id"], quote(item["name"])),
+            )
+            for item in result.get(key, [])
+        ]
+        table = tabulate(
+            tabular_data,
+            headers=[click.style("FEATURED: " + title.upper(), bold=True), "URL"],
+        )
+        click.echo(table)
         click.echo()
 
     return True
+
+
+def print_storage_header(storage_dirs, storage_dir):
+    if storage_dirs and storage_dirs[0] != storage_dir:
+        click.echo("")
+    click.echo(
+        click.style("Library Storage: ", bold=True)
+        + click.style(storage_dir, fg="blue")
+    )
+
+
+def print_lib_item(item):
+    click.secho(item["name"], fg="cyan")
+    click.echo("=" * len(item["name"]))
+    if "id" in item:
+        click.secho("#ID: %d" % item["id"], bold=True)
+    if "description" in item or "url" in item:
+        click.echo(item.get("description", item.get("url", "")))
+    click.echo()
+
+    for key in ("version", "homepage", "license", "keywords"):
+        if key not in item or not item[key]:
+            continue
+        if isinstance(item[key], list):
+            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
+        else:
+            click.echo("%s: %s" % (key.title(), item[key]))
+
+    for key in ("frameworks", "platforms"):
+        if key not in item:
+            continue
+        click.echo(
+            "Compatible %s: %s"
+            % (
+                key,
+                ", ".join(
+                    [i["title"] if isinstance(i, dict) else i for i in item[key]]
+                ),
+            )
+        )
+
+    if "authors" in item or "authornames" in item:
+        click.echo(
+            "Authors: %s"
+            % ", ".join(
+                item.get(
+                    "authornames", [a.get("name", "") for a in item.get("authors", [])]
+                )
+            )
+        )
+
+    if "__src_url" in item:
+        click.secho("Source: %s" % item["__src_url"])
+    click.echo()

platformio/commands/platform.py

@@ -12,14 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 from os.path import dirname, isdir
 
 import click
 
 from platformio import app, exception, util
 from platformio.commands.boards import print_boards
+from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.package.pack import PackagePacker
 
 
 @click.group(short_help="Platform Manager")
@@ -29,20 +30,27 @@ def cli():
 
 def _print_platforms(platforms):
     for platform in platforms:
-        click.echo("{name} ~ {title}".format(
-            name=click.style(platform['name'], fg="cyan"),
-            title=platform['title']))
-        click.echo("=" * (3 + len(platform['name'] + platform['title'])))
-        click.echo(platform['description'])
+        click.echo(
+            "{name} ~ {title}".format(
+                name=click.style(platform["name"], fg="cyan"), title=platform["title"]
+            )
+        )
+        click.echo("=" * (3 + len(platform["name"] + platform["title"])))
+        click.echo(platform["description"])
         click.echo()
         if "homepage" in platform:
-            click.echo("Home: %s" % platform['homepage'])
-        if "frameworks" in platform and platform['frameworks']:
-            click.echo("Frameworks: %s" % ", ".join(platform['frameworks']))
+            click.echo("Home: %s" % platform["homepage"])
+        if "frameworks" in platform and platform["frameworks"]:
+            click.echo("Frameworks: %s" % ", ".join(platform["frameworks"]))
         if "packages" in platform:
-            click.echo("Packages: %s" % ", ".join(platform['packages']))
+            click.echo("Packages: %s" % ", ".join(platform["packages"]))
         if "version" in platform:
-            click.echo("Version: " + platform['version'])
+            if "__src_url" in platform:
+                click.echo(
+                    "Version: #%s (%s)" % (platform["version"], platform["__src_url"])
+                )
+            else:
+                click.echo("Version: " + platform["version"])
         click.echo()
 
 
@@ -50,22 +58,10 @@ def _get_registry_platforms():
     platforms = util.get_api_result("/platforms", cache_valid="7d")
     pm = PlatformManager()
     for platform in platforms or []:
-        platform['versions'] = pm.get_all_repo_versions(platform['name'])
+        platform["versions"] = pm.get_all_repo_versions(platform["name"])
     return platforms
 
 
-def _original_version(version):
-    if version.count(".") != 2:
-        return None
-    _, y = version.split(".")[:2]
-    if int(y) < 100:
-        return None
-    if len(y) % 2 != 0:
-        y = "0" + y
-    parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(len(y) / 2)]
-    return ".".join(parts)
-
-
 def _get_platform_data(*args, **kwargs):
     try:
         return _get_installed_platform_data(*args, **kwargs)
@@ -73,9 +69,7 @@ def _get_platform_data(*args, **kwargs):
         return _get_registry_platform_data(*args, **kwargs)
 
 
-def _get_installed_platform_data(platform,
-                                 with_boards=True,
-                                 expose_packages=True):
+def _get_installed_platform_data(platform, with_boards=True, expose_packages=True):
     p = PlatformFactory.newPlatform(platform)
     data = dict(
         name=p.name,
@@ -88,8 +82,9 @@ def _get_installed_platform_data(platform,
         docs=p.docs_url,
         license=p.license,
         forDesktop=not p.is_embedded(),
-        frameworks=sorted(p.frameworks.keys() if p.frameworks else []),
-        packages=p.packages.keys() if p.packages else [])
+        frameworks=sorted(list(p.frameworks) if p.frameworks else []),
+        packages=list(p.packages) if p.packages else [],
+    )
 
     # if dump to API
     # del data['version']
@@ -103,38 +98,38 @@ def _get_installed_platform_data(platform,
             data[key] = manifest[key]
 
     if with_boards:
-        data['boards'] = [c.get_brief_data() for c in p.get_boards().values()]
+        data["boards"] = [c.get_brief_data() for c in p.get_boards().values()]
 
-    if not data['packages'] or not expose_packages:
+    if not data["packages"] or not expose_packages:
         return data
 
-    data['packages'] = []
+    data["packages"] = []
     installed_pkgs = p.get_installed_packages()
     for name, opts in p.packages.items():
         item = dict(
             name=name,
             type=p.get_package_type(name),
             requirements=opts.get("version"),
-            optional=opts.get("optional") is True)
+            optional=opts.get("optional") is True,
+        )
         if name in installed_pkgs:
             for key, value in installed_pkgs[name].items():
                 if key not in ("url", "version", "description"):
                     continue
                 item[key] = value
                 if key == "version":
-                    item["originalVersion"] = _original_version(value)
-        data['packages'].append(item)
+                    item["originalVersion"] = util.get_original_version(value)
+        data["packages"].append(item)
 
     return data
 
 
 def _get_registry_platform_data(  # pylint: disable=unused-argument
-        platform,
-        with_boards=True,
-        expose_packages=True):
+    platform, with_boards=True, expose_packages=True
+):
     _data = None
     for p in _get_registry_platforms():
-        if p['name'] == platform:
+        if p["name"] == platform:
             _data = p
             break
 
@@ -142,22 +137,24 @@ def _get_registry_platform_data(  # pylint: disable=unused-argument
         return None
 
     data = dict(
-        name=_data['name'],
-        title=_data['title'],
-        description=_data['description'],
-        homepage=_data['homepage'],
-        repository=_data['repository'],
-        url=_data['url'],
-        license=_data['license'],
-        forDesktop=_data['forDesktop'],
-        frameworks=_data['frameworks'],
-        packages=_data['packages'],
-        versions=_data['versions'])
+        name=_data["name"],
+        title=_data["title"],
+        description=_data["description"],
+        homepage=_data["homepage"],
+        repository=_data["repository"],
+        url=_data["url"],
+        license=_data["license"],
+        forDesktop=_data["forDesktop"],
+        frameworks=_data["frameworks"],
+        packages=_data["packages"],
+        versions=_data["versions"],
+    )
 
     if with_boards:
-        data['boards'] = [
-            board for board in PlatformManager().get_registered_boards()
-            if board['platform'] == _data['name']
+        data["boards"] = [
+            board
+            for board in PlatformManager().get_registered_boards()
+            if board["platform"] == _data["name"]
         ]
 
     return data
@@ -171,15 +168,17 @@ def platform_search(query, json_output):
     for platform in _get_registry_platforms():
         if query == "all":
             query = ""
-        search_data = json.dumps(platform)
+        search_data = dump_json_to_unicode(platform)
         if query and query.lower() not in search_data.lower():
             continue
         platforms.append(
             _get_registry_platform_data(
-                platform['name'], with_boards=False, expose_packages=False))
+                platform["name"], with_boards=False, expose_packages=False
+            )
+        )
 
     if json_output:
-        click.echo(json.dumps(platforms))
+        click.echo(dump_json_to_unicode(platforms))
     else:
         _print_platforms(platforms)
 
@@ -192,20 +191,20 @@ def platform_frameworks(query, json_output):
     for framework in util.get_api_result("/frameworks", cache_valid="7d"):
         if query == "all":
             query = ""
-        search_data = json.dumps(framework)
+        search_data = dump_json_to_unicode(framework)
         if query and query.lower() not in search_data.lower():
             continue
-        framework['homepage'] = (
-            "https://platformio.org/frameworks/" + framework['name'])
-        framework['platforms'] = [
-            platform['name'] for platform in _get_registry_platforms()
-            if framework['name'] in platform['frameworks']
+        framework["homepage"] = "https://platformio.org/frameworks/" + framework["name"]
+        framework["platforms"] = [
+            platform["name"]
+            for platform in _get_registry_platforms()
+            if framework["name"] in platform["frameworks"]
         ]
         frameworks.append(framework)
 
-    frameworks = sorted(frameworks, key=lambda manifest: manifest['name'])
+    frameworks = sorted(frameworks, key=lambda manifest: manifest["name"])
     if json_output:
-        click.echo(json.dumps(frameworks))
+        click.echo(dump_json_to_unicode(frameworks))
     else:
         _print_platforms(frameworks)
 
@@ -218,13 +217,13 @@ def platform_list(json_output):
     for manifest in pm.get_installed():
         platforms.append(
             _get_installed_platform_data(
-                manifest['__pkg_dir'],
-                with_boards=False,
-                expose_packages=False))
+                manifest["__pkg_dir"], with_boards=False, expose_packages=False
+            )
+        )
 
-    platforms = sorted(platforms, key=lambda manifest: manifest['name'])
+    platforms = sorted(platforms, key=lambda manifest: manifest["name"])
     if json_output:
-        click.echo(json.dumps(platforms))
+        click.echo(dump_json_to_unicode(platforms))
     else:
         _print_platforms(platforms)
 
@@ -237,56 +236,60 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     if not data:
         raise exception.UnknownPlatform(platform)
     if json_output:
-        return click.echo(json.dumps(data))
+        return click.echo(dump_json_to_unicode(data))
 
-    click.echo("{name} ~ {title}".format(
-        name=click.style(data['name'], fg="cyan"), title=data['title']))
-    click.echo("=" * (3 + len(data['name'] + data['title'])))
-    click.echo(data['description'])
+    click.echo(
+        "{name} ~ {title}".format(
+            name=click.style(data["name"], fg="cyan"), title=data["title"]
+        )
+    )
+    click.echo("=" * (3 + len(data["name"] + data["title"])))
+    click.echo(data["description"])
     click.echo()
     if "version" in data:
-        click.echo("Version: %s" % data['version'])
-    if data['homepage']:
-        click.echo("Home: %s" % data['homepage'])
-    if data['repository']:
-        click.echo("Repository: %s" % data['repository'])
-    if data['url']:
-        click.echo("Vendor: %s" % data['url'])
-    if data['license']:
-        click.echo("License: %s" % data['license'])
-    if data['frameworks']:
-        click.echo("Frameworks: %s" % ", ".join(data['frameworks']))
+        click.echo("Version: %s" % data["version"])
+    if data["homepage"]:
+        click.echo("Home: %s" % data["homepage"])
+    if data["repository"]:
+        click.echo("Repository: %s" % data["repository"])
+    if data["url"]:
+        click.echo("Vendor: %s" % data["url"])
+    if data["license"]:
+        click.echo("License: %s" % data["license"])
+    if data["frameworks"]:
+        click.echo("Frameworks: %s" % ", ".join(data["frameworks"]))
 
-    if not data['packages']:
+    if not data["packages"]:
         return None
 
-    if not isinstance(data['packages'][0], dict):
-        click.echo("Packages: %s" % ", ".join(data['packages']))
+    if not isinstance(data["packages"][0], dict):
+        click.echo("Packages: %s" % ", ".join(data["packages"]))
     else:
         click.echo()
         click.secho("Packages", bold=True)
         click.echo("--------")
-        for item in data['packages']:
+        for item in data["packages"]:
             click.echo()
-            click.echo("Package %s" % click.style(item['name'], fg="yellow"))
-            click.echo("-" * (8 + len(item['name'])))
-            if item['type']:
-                click.echo("Type: %s" % item['type'])
-            click.echo("Requirements: %s" % item['requirements'])
-            click.echo("Installed: %s" %
-                       ("Yes" if item.get("version") else "No (optional)"))
+            click.echo("Package %s" % click.style(item["name"], fg="yellow"))
+            click.echo("-" * (8 + len(item["name"])))
+            if item["type"]:
+                click.echo("Type: %s" % item["type"])
+            click.echo("Requirements: %s" % item["requirements"])
+            click.echo(
+                "Installed: %s" % ("Yes" if item.get("version") else "No (optional)")
+            )
             if "version" in item:
-                click.echo("Version: %s" % item['version'])
+                click.echo("Version: %s" % item["version"])
             if "originalVersion" in item:
-                click.echo("Original version: %s" % item['originalVersion'])
+                click.echo("Original version: %s" % item["originalVersion"])
             if "description" in item:
-                click.echo("Description: %s" % item['description'])
+                click.echo("Description: %s" % item["description"])
 
-    if data['boards']:
+    if data["boards"]:
         click.echo()
         click.secho("Boards", bold=True)
         click.echo("------")
-        print_boards(data['boards'])
+        print_boards(data["boards"])
 
     return True
 
@@ -296,26 +299,37 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
 @click.option("--with-package", multiple=True)
 @click.option("--without-package", multiple=True)
 @click.option("--skip-default-package", is_flag=True)
+@click.option("--with-all-packages", is_flag=True)
 @click.option(
     "-f",
     "--force",
     is_flag=True,
-    help="Reinstall/redownload dev/platform and its packages if exist")
-def platform_install(platforms, with_package, without_package,
-                     skip_default_package, force):
+    help="Reinstall/redownload dev/platform and its packages if exist",
+)
+def platform_install(  # pylint: disable=too-many-arguments
+    platforms,
+    with_package,
+    without_package,
+    skip_default_package,
+    with_all_packages,
+    force,
+):
     pm = PlatformManager()
     for platform in platforms:
         if pm.install(
-                name=platform,
-                with_packages=with_package,
-                without_packages=without_package,
-                skip_default_package=skip_default_package,
-                force=force):
+            name=platform,
+            with_packages=with_package,
+            without_packages=without_package,
+            skip_default_package=skip_default_package,
+            with_all_packages=with_all_packages,
+            force=force,
+        ):
             click.secho(
                 "The platform '%s' has been successfully installed!\n"
                 "The rest of packages will be installed automatically "
                 "depending on your build environment." % platform,
-                fg="green")
+                fg="green",
+            )
 
 
 @cli.command("uninstall", short_help="Uninstall development platform")
@@ -325,33 +339,40 @@ def platform_uninstall(platforms):
     for platform in platforms:
         if pm.uninstall(platform):
             click.secho(
-                "The platform '%s' has been successfully "
-                "uninstalled!" % platform,
-                fg="green")
+                "The platform '%s' has been successfully uninstalled!" % platform,
+                fg="green",
+            )
 
 
 @cli.command("update", short_help="Update installed development platforms")
 @click.argument("platforms", nargs=-1, required=False, metavar="[PLATFORM...]")
 @click.option(
-    "-p",
-    "--only-packages",
-    is_flag=True,
-    help="Update only the platform packages")
+    "-p", "--only-packages", is_flag=True, help="Update only the platform packages"
+)
 @click.option(
     "-c",
     "--only-check",
     is_flag=True,
-    help="Do not update, only check for a new version")
+    help="DEPRECATED. Please use `--dry-run` instead",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
+)
 @click.option("--json-output", is_flag=True)
-def platform_update(platforms, only_packages, only_check, json_output):
+def platform_update(  # pylint: disable=too-many-locals
+    platforms, only_packages, only_check, dry_run, json_output
+):
     pm = PlatformManager()
     pkg_dir_to_name = {}
     if not platforms:
         platforms = []
         for manifest in pm.get_installed():
-            platforms.append(manifest['__pkg_dir'])
-            pkg_dir_to_name[manifest['__pkg_dir']] = manifest.get(
-                "title", manifest['name'])
+            platforms.append(manifest["__pkg_dir"])
+            pkg_dir_to_name[manifest["__pkg_dir"]] = manifest.get(
+                "title", manifest["name"]
+            )
+
+    only_check = dry_run or only_check
 
     if only_check and json_output:
         result = []
@@ -365,24 +386,38 @@ def platform_update(platforms, only_packages, only_check, json_output):
             if not pkg_dir:
                 continue
             latest = pm.outdated(pkg_dir, requirements)
-            if (not latest and not PlatformFactory.newPlatform(pkg_dir).
-                    are_outdated_packages()):
+            if (
+                not latest
+                and not PlatformFactory.newPlatform(pkg_dir).are_outdated_packages()
+            ):
                 continue
             data = _get_installed_platform_data(
-                pkg_dir, with_boards=False, expose_packages=False)
+                pkg_dir, with_boards=False, expose_packages=False
+            )
             if latest:
-                data['versionLatest'] = latest
+                data["versionLatest"] = latest
             result.append(data)
-        return click.echo(json.dumps(result))
-    else:
-        # cleanup cached board and platform lists
-        app.clean_cache()
-        for platform in platforms:
-            click.echo("Platform %s" % click.style(
-                pkg_dir_to_name.get(platform, platform), fg="cyan"))
-            click.echo("--------")
-            pm.update(
-                platform, only_packages=only_packages, only_check=only_check)
-            click.echo()
+        return click.echo(dump_json_to_unicode(result))
+
+    # cleanup cached board and platform lists
+    app.clean_cache()
+    for platform in platforms:
+        click.echo(
+            "Platform %s"
+            % click.style(pkg_dir_to_name.get(platform, platform), fg="cyan")
+        )
+        click.echo("--------")
+        pm.update(platform, only_packages=only_packages, only_check=only_check)
+        click.echo()
 
     return True
+
+
+@cli.command(
+    "pack", short_help="Create a tarball from development platform/tool package"
+)
+@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]")
+def platform_pack(package):
+    p = PackagePacker(package)
+    tarball_path = p.pack()
+    click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green")

platformio/commands/project.py

@@ -14,17 +14,59 @@
 
 # pylint: disable=too-many-arguments,too-many-locals, too-many-branches
 
-from os import getcwd, makedirs
-from os.path import isdir, isfile, join
-from shutil import copyfile
+import os
 
 import click
+from tabulate import tabulate
 
-from platformio import exception, util
-from platformio.commands.platform import \
-    platform_install as cli_platform_install
+from platformio import exception, fs
+from platformio.commands.platform import platform_install as cli_platform_install
 from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.managers.platform import PlatformManager
+from platformio.project.config import ProjectConfig
+from platformio.project.exception import NotPlatformIOProjectError
+from platformio.project.helpers import is_platformio_project
+
+
+@click.group(short_help="Project Manager")
+def cli():
+    pass
+
+
+@cli.command("config", short_help="Show computed configuration")
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option("--json-output", is_flag=True)
+def project_config(project_dir, json_output):
+    if not is_platformio_project(project_dir):
+        raise NotPlatformIOProjectError(project_dir)
+    with fs.cd(project_dir):
+        config = ProjectConfig.get_instance()
+    if json_output:
+        return click.echo(config.to_json())
+    click.echo(
+        "Computed project configuration for %s" % click.style(project_dir, fg="cyan")
+    )
+    for section, options in config.as_tuple():
+        click.echo()
+        click.secho(section, fg="cyan")
+        click.echo("-" * len(section))
+        click.echo(
+            tabulate(
+                [
+                    (name, "=", "\n".join(value) if isinstance(value, list) else value)
+                    for name, value in options
+                ],
+                tablefmt="plain",
+            )
+        )
+    return None
 
 
 def validate_boards(ctx, param, value):  # pylint: disable=W0613
@@ -35,70 +77,69 @@ def validate_boards(ctx, param, value):  # pylint: disable=W0613
         except exception.UnknownBoard:
             raise click.BadParameter(
                 "`%s`. Please search for board ID using `platformio boards` "
-                "command" % id_)
+                "command" % id_
+            )
     return value
 
 
-@click.command(
-    "init", short_help="Initialize PlatformIO project or update existing")
+@cli.command("init", short_help="Initialize a project or update existing")
 @click.option(
     "--project-dir",
     "-d",
-    default=getcwd,
+    default=os.getcwd,
     type=click.Path(
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True))
-@click.option(
-    "-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
-@click.option(
-    "--ide", type=click.Choice(ProjectGenerator.get_supported_ides()))
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
+@click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides()))
 @click.option("-O", "--project-option", multiple=True)
 @click.option("--env-prefix", default="")
 @click.option("-s", "--silent", is_flag=True)
 @click.pass_context
-def cli(
-        ctx,  # pylint: disable=R0913
-        project_dir,
-        board,
-        ide,
-        project_option,
-        env_prefix,
-        silent):
-
+def project_init(
+    ctx,  # pylint: disable=R0913
+    project_dir,
+    board,
+    ide,
+    project_option,
+    env_prefix,
+    silent,
+):
     if not silent:
-        if project_dir == getcwd():
-            click.secho(
-                "\nThe current working directory", fg="yellow", nl=False)
+        if project_dir == os.getcwd():
+            click.secho("\nThe current working directory", fg="yellow", nl=False)
             click.secho(" %s " % project_dir, fg="cyan", nl=False)
             click.secho("will be used for the project.", fg="yellow")
             click.echo("")
 
-        click.echo("The next files/directories have been created in %s" %
-                   click.style(project_dir, fg="cyan"))
-        click.echo("%s - Put project header files here" % click.style(
-            "include", fg="cyan"))
-        click.echo("%s - Put here project specific (private) libraries" %
-                   click.style("lib", fg="cyan"))
-        click.echo("%s - Put project source files here" % click.style(
-            "src", fg="cyan"))
-        click.echo("%s - Project Configuration File" % click.style(
-            "platformio.ini", fg="cyan"))
+        click.echo(
+            "The next files/directories have been created in %s"
+            % click.style(project_dir, fg="cyan")
+        )
+        click.echo(
+            "%s - Put project header files here" % click.style("include", fg="cyan")
+        )
+        click.echo(
+            "%s - Put here project specific (private) libraries"
+            % click.style("lib", fg="cyan")
+        )
+        click.echo("%s - Put project source files here" % click.style("src", fg="cyan"))
+        click.echo(
+            "%s - Project Configuration File" % click.style("platformio.ini", fg="cyan")
+        )
 
-    is_new_project = util.is_platformio_project(project_dir)
-    init_base_project(project_dir)
+    is_new_project = not is_platformio_project(project_dir)
+    if is_new_project:
+        init_base_project(project_dir)
 
     if board:
-        fill_project_envs(ctx, project_dir, board, project_option, env_prefix,
-                          ide is not None)
+        fill_project_envs(
+            ctx, project_dir, board, project_option, env_prefix, ide is not None
+        )
 
     if ide:
-        env_name = get_best_envname(project_dir, board)
-        if not env_name:
-            raise exception.BoardNotDefined()
-        pg = ProjectGenerator(project_dir, ide, env_name)
+        pg = ProjectGenerator(project_dir, ide, board)
         pg.generate()
 
     if is_new_project:
@@ -111,9 +152,9 @@ def cli(
     if ide:
         click.secho(
             "\nProject has been successfully %s including configuration files "
-            "for `%s` IDE." % ("initialized" if is_new_project else "updated",
-                               ide),
-            fg="green")
+            "for `%s` IDE." % ("initialized" if is_new_project else "updated", ide),
+            fg="green",
+        )
     else:
         click.secho(
             "\nProject has been successfully %s! Useful commands:\n"
@@ -121,55 +162,34 @@ def cli(
             "`pio run --target upload` or `pio run -t upload` "
             "- upload firmware to a target\n"
             "`pio run --target clean` - clean project (remove compiled files)"
-            "\n`pio run --help` - additional information" %
-            ("initialized" if is_new_project else "updated"),
-            fg="green")
-
-
-def get_best_envname(project_dir, boards=None):
-    config = util.load_project_config(project_dir)
-    env_default = None
-    if config.has_option("platformio", "env_default"):
-        env_default = config.get("platformio",
-                                 "env_default").split(", ")[0].strip()
-    if env_default:
-        return env_default
-    section = None
-    for section in config.sections():
-        if not section.startswith("env:"):
-            continue
-        elif config.has_option(section, "board") and (not boards or config.get(
-                section, "board") in boards):
-            break
-    return section[4:] if section else None
+            "\n`pio run --help` - additional information"
+            % ("initialized" if is_new_project else "updated"),
+            fg="green",
+        )
 
 
 def init_base_project(project_dir):
-    if util.is_platformio_project(project_dir):
-        return
-
-    copyfile(
-        join(util.get_source_dir(), "projectconftpl.ini"),
-        join(project_dir, "platformio.ini"))
-
-    with util.cd(project_dir):
+    with fs.cd(project_dir):
+        config = ProjectConfig()
+        config.save()
         dir_to_readme = [
-            (util.get_projectsrc_dir(), None),
-            (util.get_projectinclude_dir(), init_include_readme),
-            (util.get_projectlib_dir(), init_lib_readme),
-            (util.get_projecttest_dir(), init_test_readme),
+            (config.get_optional_dir("src"), None),
+            (config.get_optional_dir("include"), init_include_readme),
+            (config.get_optional_dir("lib"), init_lib_readme),
+            (config.get_optional_dir("test"), init_test_readme),
         ]
         for (path, cb) in dir_to_readme:
-            if isdir(path):
+            if os.path.isdir(path):
                 continue
-            makedirs(path)
+            os.makedirs(path)
             if cb:
                 cb(path)
 
 
 def init_include_readme(include_dir):
-    with open(join(include_dir, "README"), "w") as f:
-        f.write("""
+    with open(os.path.join(include_dir, "README"), "w") as fp:
+        fp.write(
+            """
 This directory is intended for project header files.
 
 A header file is a file containing C declarations and macro definitions
@@ -208,12 +228,15 @@ Read more about using header files in official GCC documentation:
 * Computed Includes
 
 https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html
-""")
+""",
+        )
 
 
 def init_lib_readme(lib_dir):
-    with open(join(lib_dir, "README"), "w") as f:
-        f.write("""
+    # pylint: disable=line-too-long
+    with open(os.path.join(lib_dir, "README"), "w") as fp:
+        fp.write(
+            """
 This directory is intended for project specific (private) libraries.
 PlatformIO will compile them to static libraries and link into executable file.
 
@@ -259,12 +282,14 @@ libraries scanning project source files.
 
 More information about PlatformIO Library Dependency Finder
 - https://docs.platformio.org/page/librarymanager/ldf.html
-""")
+""",
+        )
 
 
 def init_test_readme(test_dir):
-    with open(join(test_dir, "README"), "w") as f:
-        f.write("""
+    with open(os.path.join(test_dir, "README"), "w") as fp:
+        fp.write(
+            """
 This directory is intended for PIO Unit Testing and project tests.
 
 Unit Testing is a software testing method by which individual units of
@@ -275,15 +300,17 @@ in the development cycle.
 
 More information about PIO Unit Testing:
 - https://docs.platformio.org/page/plus/unit-testing.html
-""")
+""",
+        )
 
 
 def init_ci_conf(project_dir):
-    conf_path = join(project_dir, ".travis.yml")
-    if isfile(conf_path):
+    conf_path = os.path.join(project_dir, ".travis.yml")
+    if os.path.isfile(conf_path):
         return
-    with open(conf_path, "w") as f:
-        f.write("""# Continuous Integration (CI) is the practice, in software
+    with open(conf_path, "w") as fp:
+        fp.write(
+            """# Continuous Integration (CI) is the practice, in software
 # engineering, of merging all developer working copies with a shared mainline
 # several times a day < https://docs.platformio.org/page/ci/index.html >
 #
@@ -350,45 +377,46 @@ def init_ci_conf(project_dir):
 #
 # script:
 #     - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N
-""")
+""",
+        )
 
 
 def init_cvs_ignore(project_dir):
-    conf_path = join(project_dir, ".gitignore")
-    if isfile(conf_path):
+    conf_path = os.path.join(project_dir, ".gitignore")
+    if os.path.isfile(conf_path):
         return
     with open(conf_path, "w") as fp:
-        fp.writelines([".pio\n", ".pioenvs\n", ".piolibdeps\n"])
+        fp.write(".pio\n")
 
 
-def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix,
-                      force_download):
-    content = []
+def fill_project_envs(
+    ctx, project_dir, board_ids, project_option, env_prefix, force_download
+):
+    config = ProjectConfig(
+        os.path.join(project_dir, "platformio.ini"), parse_extra=False
+    )
     used_boards = []
-    used_platforms = []
-
-    config = util.load_project_config(project_dir)
     for section in config.sections():
-        cond = [
-            section.startswith("env:"),
-            config.has_option(section, "board")
-        ]
+        cond = [section.startswith("env:"), config.has_option(section, "board")]
         if all(cond):
             used_boards.append(config.get(section, "board"))
 
     pm = PlatformManager()
+    used_platforms = []
+    modified = False
     for id_ in board_ids:
         board_config = pm.board_config(id_)
-        used_platforms.append(board_config['platform'])
+        used_platforms.append(board_config["platform"])
         if id_ in used_boards:
             continue
         used_boards.append(id_)
+        modified = True
 
-        envopts = {"platform": board_config['platform'], "board": id_}
+        envopts = {"platform": board_config["platform"], "board": id_}
         # find default framework for board
         frameworks = board_config.get("frameworks")
         if frameworks:
-            envopts['framework'] = frameworks[0]
+            envopts["framework"] = frameworks[0]
 
         for item in project_option:
             if "=" not in item:
@@ -396,28 +424,23 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix,
             _name, _value = item.split("=", 1)
             envopts[_name.strip()] = _value.strip()
 
-        content.append("")
-        content.append("[env:%s%s]" % (env_prefix, id_))
-        for name, value in envopts.items():
-            content.append("%s = %s" % (name, value))
+        section = "env:%s%s" % (env_prefix, id_)
+        config.add_section(section)
+
+        for option, value in envopts.items():
+            config.set(section, option, value)
 
     if force_download and used_platforms:
         _install_dependent_platforms(ctx, used_platforms)
 
-    if not content:
-        return
-
-    with open(join(project_dir, "platformio.ini"), "a") as f:
-        content.append("")
-        f.write("\n".join(content))
+    if modified:
+        config.save()
 
 
 def _install_dependent_platforms(ctx, platforms):
-    installed_platforms = [
-        p['name'] for p in PlatformManager().get_installed()
-    ]
+    installed_platforms = [p["name"] for p in PlatformManager().get_installed()]
     if set(platforms) <= set(installed_platforms):
         return
     ctx.invoke(
-        cli_platform_install,
-        platforms=list(set(platforms) - set(installed_platforms)))
+        cli_platform_install, platforms=list(set(platforms) - set(installed_platforms))
+    )

platformio/commands/remote.py

@@ -12,18 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import sys
 import threading
-from os import getcwd
-from os.path import isfile, join
 from tempfile import mkdtemp
 from time import sleep
 
 import click
 
-from platformio import exception, util
-from platformio.commands.device import device_monitor as cmd_device_monitor
+from platformio import exception, fs
+from platformio.commands.device import helpers as device_helpers
+from platformio.commands.device.command import device_monitor as cmd_device_monitor
 from platformio.managers.core import pioplus_call
+from platformio.project.exception import NotPlatformIOProjectError
 
 # pylint: disable=unused-argument
 
@@ -46,8 +47,8 @@ def remote_agent():
     "-d",
     "--working-dir",
     envvar="PLATFORMIO_REMOTE_AGENT_DIR",
-    type=click.Path(
-        file_okay=False, dir_okay=True, writable=True, resolve_path=True))
+    type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
+)
 def remote_agent_start(**kwargs):
     pioplus_call(sys.argv[1:])
 
@@ -62,14 +63,17 @@ def remote_agent_list():
     pioplus_call(sys.argv[1:])
 
 
-@cli.command(
-    "update", short_help="Update installed Platforms, Packages and Libraries")
+@cli.command("update", short_help="Update installed Platforms, Packages and Libraries")
 @click.option(
     "-c",
     "--only-check",
     is_flag=True,
-    help="Do not update, only check for new version")
-def remote_update(only_check):
+    help="DEPRECATED. Please use `--dry-run` instead",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
+)
+def remote_update(only_check, dry_run):
     pioplus_call(sys.argv[1:])
 
 
@@ -80,13 +84,11 @@ def remote_update(only_check):
 @click.option(
     "-d",
     "--project-dir",
-    default=getcwd,
+    default=os.getcwd,
     type=click.Path(
-        exists=True,
-        file_okay=True,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True))
+        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
 @click.option("--disable-auto-clean", is_flag=True)
 @click.option("-r", "--force-remote", is_flag=True)
 @click.option("-s", "--silent", is_flag=True)
@@ -103,13 +105,11 @@ def remote_run(**kwargs):
 @click.option(
     "-d",
     "--project-dir",
-    default=getcwd,
+    default=os.getcwd,
     type=click.Path(
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True))
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
 @click.option("-r", "--force-remote", is_flag=True)
 @click.option("--without-building", is_flag=True)
 @click.option("--without-uploading", is_flag=True)
@@ -131,79 +131,101 @@ def device_list(json_output):
 
 @remote_device.command("monitor", short_help="Monitor remote device")
 @click.option("--port", "-p", help="Port, a number or a device name")
-@click.option(
-    "--baud", "-b", type=int, default=9600, help="Set baud rate, default=9600")
+@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
 @click.option(
     "--parity",
     default="N",
     type=click.Choice(["N", "E", "O", "S", "M"]),
-    help="Set parity, default=N")
+    help="Set parity, default=N",
+)
+@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
 @click.option(
-    "--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
+    "--xonxoff", is_flag=True, help="Enable software flow control, default=Off"
+)
 @click.option(
-    "--xonxoff",
-    is_flag=True,
-    help="Enable software flow control, default=Off")
+    "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state"
+)
 @click.option(
-    "--rts",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial RTS line state")
-@click.option(
-    "--dtr",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial DTR line state")
+    "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state"
+)
 @click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
 @click.option(
     "--encoding",
     default="UTF-8",
     help="Set the encoding for the serial port (e.g. hexlify, "
-    "Latin1, UTF-8), default: UTF-8")
+    "Latin1, UTF-8), default: UTF-8",
+)
 @click.option("--filter", "-f", multiple=True, help="Add text transformation")
 @click.option(
     "--eol",
     default="CRLF",
     type=click.Choice(["CR", "LF", "CRLF"]),
-    help="End of line mode, default=CRLF")
-@click.option(
-    "--raw", is_flag=True, help="Do not apply any encodings/transformations")
+    help="End of line mode, default=CRLF",
+)
+@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations")
 @click.option(
     "--exit-char",
     type=int,
     default=3,
     help="ASCII code of special character that is used to exit "
-    "the application, default=3 (Ctrl+C)")
+    "the application, default=3 (Ctrl+C)",
+)
 @click.option(
     "--menu-char",
     type=int,
     default=20,
     help="ASCII code of special character that is used to "
-    "control miniterm (menu), default=20 (DEC)")
+    "control miniterm (menu), default=20 (DEC)",
+)
 @click.option(
     "--quiet",
     is_flag=True,
-    help="Diagnostics: suppress non-error messages, default=Off")
+    help="Diagnostics: suppress non-error messages, default=Off",
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+)
+@click.option(
+    "-e",
+    "--environment",
+    help="Load configuration from `platformio.ini` and specified environment",
+)
 @click.pass_context
 def device_monitor(ctx, **kwargs):
+    project_options = {}
+    try:
+        with fs.cd(kwargs["project_dir"]):
+            project_options = device_helpers.get_project_options(kwargs["environment"])
+        kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options)
+    except NotPlatformIOProjectError:
+        pass
+
+    kwargs["baud"] = kwargs["baud"] or 9600
 
     def _tx_target(sock_dir):
+        pioplus_argv = ["remote", "device", "monitor"]
+        pioplus_argv.extend(device_helpers.options_to_argv(kwargs, project_options))
+        pioplus_argv.extend(["--sock", sock_dir])
         try:
-            pioplus_call(sys.argv[1:] + ["--sock", sock_dir])
+            pioplus_call(pioplus_argv)
         except exception.ReturnErrorCode:
             pass
 
     sock_dir = mkdtemp(suffix="pioplus")
-    sock_file = join(sock_dir, "sock")
+    sock_file = os.path.join(sock_dir, "sock")
     try:
-        t = threading.Thread(target=_tx_target, args=(sock_dir, ))
+        t = threading.Thread(target=_tx_target, args=(sock_dir,))
         t.start()
-        while t.is_alive() and not isfile(sock_file):
+        while t.is_alive() and not os.path.isfile(sock_file):
             sleep(0.1)
         if not t.is_alive():
             return
-        kwargs['port'] = open(sock_file).read()
+        with open(sock_file) as fp:
+            kwargs["port"] = fp.read()
         ctx.invoke(cmd_device_monitor, **kwargs)
         t.join(2)
     finally:
-        util.rmtree_(sock_dir)
+        fs.rmtree(sock_dir)

platformio/commands/run.py

@@ -1,435 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from hashlib import sha1
-from os import getcwd, makedirs, walk
-from os.path import getmtime, isdir, isfile, join
-from time import time
-
-import click
-
-from platformio import __version__, exception, telemetry, util
-from platformio.commands.device import device_monitor as cmd_device_monitor
-from platformio.commands.lib import lib_install as cmd_lib_install
-from platformio.commands.platform import \
-    platform_install as cmd_platform_install
-from platformio.managers.lib import LibraryManager, is_builtin_lib
-from platformio.managers.platform import PlatformFactory
-
-# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
-
-
-@click.command("run", short_help="Process project environments")
-@click.option("-e", "--environment", multiple=True)
-@click.option("-t", "--target", multiple=True)
-@click.option("--upload-port")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=getcwd,
-    type=click.Path(
-        exists=True,
-        file_okay=True,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True))
-@click.option("-s", "--silent", is_flag=True)
-@click.option("-v", "--verbose", is_flag=True)
-@click.option("--disable-auto-clean", is_flag=True)
-@click.pass_context
-def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
-        disable_auto_clean):
-    # find project directory on upper level
-    if isfile(project_dir):
-        project_dir = util.find_project_dir_above(project_dir)
-
-    if not util.is_platformio_project(project_dir):
-        raise exception.NotPlatformIOProject(project_dir)
-
-    with util.cd(project_dir):
-        # clean obsolete build dir
-        if not disable_auto_clean:
-            try:
-                _clean_build_dir(util.get_projectbuild_dir())
-            except:  # pylint: disable=bare-except
-                click.secho(
-                    "Can not remove temporary directory `%s`. Please remove "
-                    "it manually to avoid build issues" %
-                    util.get_projectbuild_dir(force=True),
-                    fg="yellow")
-
-        config = util.load_project_config()
-        env_default = None
-        if config.has_option("platformio", "env_default"):
-            env_default = util.parse_conf_multi_values(
-                config.get("platformio", "env_default"))
-
-        check_project_defopts(config)
-        check_project_envs(config, environment or env_default)
-
-        results = []
-        start_time = time()
-        for section in config.sections():
-            if not section.startswith("env:"):
-                continue
-
-            envname = section[4:]
-            skipenv = any([
-                environment and envname not in environment, not environment
-                and env_default and envname not in env_default
-            ])
-            if skipenv:
-                results.append((envname, None))
-                continue
-
-            if not silent and results:
-                click.echo()
-
-            options = {}
-            for k, v in config.items(section):
-                options[k] = v
-            if "piotest" not in options and "piotest" in ctx.meta:
-                options['piotest'] = ctx.meta['piotest']
-
-            ep = EnvironmentProcessor(ctx, envname, options, target,
-                                      upload_port, silent, verbose)
-            result = (envname, ep.process())
-            results.append(result)
-            if result[1] and "monitor" in ep.get_build_targets() and \
-                    "nobuild" not in ep.get_build_targets():
-                ctx.invoke(
-                    cmd_device_monitor,
-                    environment=environment[0] if environment else None)
-
-        found_error = any(status is False for (_, status) in results)
-
-        if (found_error or not silent) and len(results) > 1:
-            click.echo()
-            print_summary(results, start_time)
-
-        if found_error:
-            raise exception.ReturnErrorCode(1)
-        return True
-
-
-class EnvironmentProcessor(object):
-
-    DEFAULT_DUMP_OPTIONS = ("platform", "framework", "board")
-
-    KNOWN_PLATFORMIO_OPTIONS = [
-        "description", "env_default", "home_dir", "lib_dir", "libdeps_dir",
-        "include_dir", "src_dir", "build_dir", "data_dir", "test_dir",
-        "boards_dir", "lib_extra_dirs"
-    ]
-
-    KNOWN_ENV_OPTIONS = [
-        "platform", "framework", "board", "build_flags", "src_build_flags",
-        "build_unflags", "src_filter", "extra_scripts", "targets",
-        "upload_port", "upload_protocol", "upload_speed", "upload_flags",
-        "upload_resetmethod", "lib_deps", "lib_ignore", "lib_extra_dirs",
-        "lib_ldf_mode", "lib_compat_mode", "lib_archive", "piotest",
-        "test_transport", "test_filter", "test_ignore", "test_port",
-        "test_speed", "test_build_project_src", "debug_tool", "debug_port",
-        "debug_init_cmds", "debug_extra_cmds", "debug_server",
-        "debug_init_break", "debug_load_cmd", "debug_load_mode",
-        "debug_svd_path", "monitor_port", "monitor_speed", "monitor_rts",
-        "monitor_dtr"
-    ]
-
-    IGNORE_BUILD_OPTIONS = [
-        "test_transport", "test_filter", "test_ignore", "test_port",
-        "test_speed", "debug_port", "debug_init_cmds", "debug_extra_cmds",
-        "debug_server", "debug_init_break", "debug_load_cmd",
-        "debug_load_mode", "monitor_port", "monitor_speed", "monitor_rts",
-        "monitor_dtr"
-    ]
-
-    REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"}
-
-    RENAMED_OPTIONS = {
-        "lib_use": "lib_deps",
-        "lib_force": "lib_deps",
-        "extra_script": "extra_scripts",
-        "monitor_baud": "monitor_speed",
-        "board_mcu": "board_build.mcu",
-        "board_f_cpu": "board_build.f_cpu",
-        "board_f_flash": "board_build.f_flash",
-        "board_flash_mode": "board_build.flash_mode"
-    }
-
-    RENAMED_PLATFORMS = {"espressif": "espressif8266"}
-
-    def __init__(
-            self,  # pylint: disable=R0913
-            cmd_ctx,
-            name,
-            options,
-            targets,
-            upload_port,
-            silent,
-            verbose):
-        self.cmd_ctx = cmd_ctx
-        self.name = name
-        self.options = options
-        self.targets = targets
-        self.upload_port = upload_port
-        self.silent = silent
-        self.verbose = verbose
-
-    def process(self):
-        terminal_width, _ = click.get_terminal_size()
-        start_time = time()
-        env_dump = []
-
-        for k, v in self.options.items():
-            self.options[k] = self.options[k].strip()
-            if self.verbose or k in self.DEFAULT_DUMP_OPTIONS:
-                env_dump.append(
-                    "%s: %s" % (k, ", ".join(util.parse_conf_multi_values(v))))
-
-        if not self.silent:
-            click.echo("Processing %s (%s)" % (click.style(
-                self.name, fg="cyan", bold=True), "; ".join(env_dump)))
-            click.secho("-" * terminal_width, bold=True)
-
-        self.options = self._validate_options(self.options)
-        result = self._run()
-        is_error = result['returncode'] != 0
-
-        if self.silent and not is_error:
-            return True
-
-        if is_error or "piotest_processor" not in self.cmd_ctx.meta:
-            print_header(
-                "[%s] Took %.2f seconds" % (
-                    (click.style("ERROR", fg="red", bold=True) if is_error else
-                     click.style("SUCCESS", fg="green", bold=True)),
-                    time() - start_time),
-                is_error=is_error)
-
-        return not is_error
-
-    def _validate_options(self, options):
-        result = {}
-        for k, v in options.items():
-            # process obsolete options
-            if k in self.RENAMED_OPTIONS:
-                click.secho(
-                    "Warning! `%s` option is deprecated and will be "
-                    "removed in the next release! Please use "
-                    "`%s` instead." % (k, self.RENAMED_OPTIONS[k]),
-                    fg="yellow")
-                k = self.RENAMED_OPTIONS[k]
-            # process renamed platforms
-            if k == "platform" and v in self.RENAMED_PLATFORMS:
-                click.secho(
-                    "Warning! Platform `%s` is deprecated and will be "
-                    "removed in the next release! Please use "
-                    "`%s` instead." % (v, self.RENAMED_PLATFORMS[v]),
-                    fg="yellow")
-                v = self.RENAMED_PLATFORMS[v]
-
-            # warn about unknown options
-            unknown_conditions = [
-                k not in self.KNOWN_ENV_OPTIONS, not k.startswith("custom_"),
-                not k.startswith("board_")
-            ]
-            if all(unknown_conditions):
-                click.secho(
-                    "Detected non-PlatformIO `%s` option in `[env:%s]` section"
-                    % (k, self.name),
-                    fg="yellow")
-            result[k] = v
-        return result
-
-    def get_build_variables(self):
-        variables = {"pioenv": self.name}
-        if self.upload_port:
-            variables['upload_port'] = self.upload_port
-        for k, v in self.options.items():
-            if k in self.REMAPED_OPTIONS:
-                k = self.REMAPED_OPTIONS[k]
-            if k in self.IGNORE_BUILD_OPTIONS:
-                continue
-            if k == "targets" or (k == "upload_port" and self.upload_port):
-                continue
-            variables[k] = v
-        return variables
-
-    def get_build_targets(self):
-        targets = []
-        if self.targets:
-            targets = [t for t in self.targets]
-        elif "targets" in self.options:
-            targets = self.options['targets'].split(", ")
-        return targets
-
-    def _run(self):
-        if "platform" not in self.options:
-            raise exception.UndefinedEnvPlatform(self.name)
-
-        build_vars = self.get_build_variables()
-        build_targets = self.get_build_targets()
-
-        telemetry.on_run_environment(self.options, build_targets)
-
-        # skip monitor target, we call it above
-        if "monitor" in build_targets:
-            build_targets.remove("monitor")
-        if "nobuild" not in build_targets:
-            # install dependent libraries
-            if "lib_install" in self.options:
-                _autoinstall_libdeps(self.cmd_ctx, [
-                    int(d.strip())
-                    for d in self.options['lib_install'].split(",")
-                    if d.strip()
-                ], self.verbose)
-            if "lib_deps" in self.options:
-                _autoinstall_libdeps(
-                    self.cmd_ctx,
-                    util.parse_conf_multi_values(self.options['lib_deps']),
-                    self.verbose)
-
-        try:
-            p = PlatformFactory.newPlatform(self.options['platform'])
-        except exception.UnknownPlatform:
-            self.cmd_ctx.invoke(
-                cmd_platform_install,
-                platforms=[self.options['platform']],
-                skip_default_package=True)
-            p = PlatformFactory.newPlatform(self.options['platform'])
-
-        return p.run(build_vars, build_targets, self.silent, self.verbose)
-
-
-def _autoinstall_libdeps(ctx, libraries, verbose=False):
-    if not libraries:
-        return
-    storage_dir = util.get_projectlibdeps_dir()
-    ctx.obj = LibraryManager(storage_dir)
-    if verbose:
-        click.echo("Library Storage: " + storage_dir)
-    for lib in libraries:
-        try:
-            ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose)
-        except exception.LibNotFound as e:
-            if verbose or not is_builtin_lib(lib):
-                click.secho("Warning! %s" % e, fg="yellow")
-        except exception.InternetIsOffline as e:
-            click.secho(str(e), fg="yellow")
-
-
-def _clean_build_dir(build_dir):
-    structhash_file = join(build_dir, "structure.hash")
-    proj_hash = calculate_project_hash()
-
-    # if project's config is modified
-    if (isdir(build_dir)
-            and getmtime(join(util.get_project_dir(),
-                              "platformio.ini")) > getmtime(build_dir)):
-        util.rmtree_(build_dir)
-
-    # check project structure
-    if isdir(build_dir) and isfile(structhash_file):
-        with open(structhash_file) as f:
-            if f.read() == proj_hash:
-                return
-        util.rmtree_(build_dir)
-
-    if not isdir(build_dir):
-        makedirs(build_dir)
-
-    with open(structhash_file, "w") as f:
-        f.write(proj_hash)
-
-
-def print_header(label, is_error=False):
-    terminal_width, _ = click.get_terminal_size()
-    width = len(click.unstyle(label))
-    half_line = "=" * ((terminal_width - width - 2) / 2)
-    click.echo("%s %s %s" % (half_line, label, half_line), err=is_error)
-
-
-def print_summary(results, start_time):
-    print_header("[%s]" % click.style("SUMMARY"))
-
-    envname_max_len = 0
-    for (envname, _) in results:
-        if len(envname) > envname_max_len:
-            envname_max_len = len(envname)
-
-    successed = True
-    for (envname, status) in results:
-        status_str = click.style("SUCCESS", fg="green")
-        if status is False:
-            successed = False
-            status_str = click.style("ERROR", fg="red")
-        elif status is None:
-            status_str = click.style("SKIP", fg="yellow")
-
-        format_str = (
-            "Environment {0:<" + str(envname_max_len + 9) + "}\t[{1}]")
-        click.echo(
-            format_str.format(click.style(envname, fg="cyan"), status_str),
-            err=status is False)
-
-    print_header(
-        "[%s] Took %.2f seconds" % (
-            (click.style("SUCCESS", fg="green", bold=True) if successed else
-             click.style("ERROR", fg="red", bold=True)), time() - start_time),
-        is_error=not successed)
-
-
-def check_project_defopts(config):
-    if not config.has_section("platformio"):
-        return True
-    unknown = set([k for k, _ in config.items("platformio")]) - set(
-        EnvironmentProcessor.KNOWN_PLATFORMIO_OPTIONS)
-    if not unknown:
-        return True
-    click.secho(
-        "Warning! Ignore unknown `%s` option in `[platformio]` section" %
-        ", ".join(unknown),
-        fg="yellow")
-    return False
-
-
-def check_project_envs(config, environments=None):
-    if not config.sections():
-        raise exception.ProjectEnvsNotAvailable()
-
-    known = set([s[4:] for s in config.sections() if s.startswith("env:")])
-    unknown = set(environments or []) - known
-    if unknown:
-        raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known))
-    return True
-
-
-def calculate_project_hash():
-    check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
-    chunks = [__version__]
-    for d in (util.get_projectsrc_dir(), util.get_projectlib_dir()):
-        if not isdir(d):
-            continue
-        for root, _, files in walk(d):
-            for f in files:
-                path = join(root, f)
-                if path.endswith(check_suffixes):
-                    chunks.append(path)
-    chunks_to_str = ",".join(sorted(chunks))
-    if "windows" in util.get_systype():
-        # Fix issue with useless project rebuilding for case insensitive FS.
-        # A case of disk drive can differ...
-        chunks_to_str = chunks_to_str.lower()
-    return sha1(chunks_to_str).hexdigest()

platformio/commands/run/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/run/command.py

@@ -0,0 +1,263 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from multiprocessing import cpu_count
+from os import getcwd
+from os.path import isfile
+from time import time
+
+import click
+from tabulate import tabulate
+
+from platformio import app, exception, fs, util
+from platformio.commands.device.command import device_monitor as cmd_device_monitor
+from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libdeps
+from platformio.commands.run.processor import EnvironmentProcessor
+from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import find_project_dir_above
+
+# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
+
+try:
+    DEFAULT_JOB_NUMS = cpu_count()
+except NotImplementedError:
+    DEFAULT_JOB_NUMS = 1
+
+
+@click.command("run", short_help="Process project environments")
+@click.option("-e", "--environment", multiple=True)
+@click.option("-t", "--target", multiple=True)
+@click.option("--upload-port")
+@click.option(
+    "-d",
+    "--project-dir",
+    default=getcwd,
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-j",
+    "--jobs",
+    type=int,
+    default=DEFAULT_JOB_NUMS,
+    help=(
+        "Allow N jobs at once. "
+        "Default is a number of CPUs in a system (N=%d)" % DEFAULT_JOB_NUMS
+    ),
+)
+@click.option("-s", "--silent", is_flag=True)
+@click.option("-v", "--verbose", is_flag=True)
+@click.option("--disable-auto-clean", is_flag=True)
+@click.pass_context
+def cli(
+    ctx,
+    environment,
+    target,
+    upload_port,
+    project_dir,
+    project_conf,
+    jobs,
+    silent,
+    verbose,
+    disable_auto_clean,
+):
+    app.set_session_var("custom_project_conf", project_conf)
+
+    # find project directory on upper level
+    if isfile(project_dir):
+        project_dir = find_project_dir_above(project_dir)
+
+    is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta
+
+    with fs.cd(project_dir):
+        config = ProjectConfig.get_instance(project_conf)
+        config.validate(environment)
+
+        # clean obsolete build dir
+        if not disable_auto_clean:
+            build_dir = config.get_optional_dir("build")
+            try:
+                clean_build_dir(build_dir, config)
+            except:  # pylint: disable=bare-except
+                click.secho(
+                    "Can not remove temporary directory `%s`. Please remove "
+                    "it manually to avoid build issues" % build_dir,
+                    fg="yellow",
+                )
+
+        handle_legacy_libdeps(project_dir, config)
+
+        default_envs = config.default_envs()
+        results = []
+        for env in config.envs():
+            skipenv = any(
+                [
+                    environment and env not in environment,
+                    not environment and default_envs and env not in default_envs,
+                ]
+            )
+            if skipenv:
+                results.append({"env": env})
+                continue
+
+            # print empty line between multi environment project
+            if not silent and any(r.get("succeeded") is not None for r in results):
+                click.echo()
+
+            results.append(
+                process_env(
+                    ctx,
+                    env,
+                    config,
+                    environment,
+                    target,
+                    upload_port,
+                    silent,
+                    verbose,
+                    jobs,
+                    is_test_running,
+                )
+            )
+
+        command_failed = any(r.get("succeeded") is False for r in results)
+
+        if not is_test_running and (command_failed or not silent) and len(results) > 1:
+            print_processing_summary(results)
+
+        if command_failed:
+            raise exception.ReturnErrorCode(1)
+        return True
+
+
+def process_env(
+    ctx,
+    name,
+    config,
+    environments,
+    targets,
+    upload_port,
+    silent,
+    verbose,
+    jobs,
+    is_test_running,
+):
+    if not is_test_running and not silent:
+        print_processing_header(name, config, verbose)
+
+    ep = EnvironmentProcessor(
+        ctx, name, config, targets, upload_port, silent, verbose, jobs
+    )
+    result = {"env": name, "duration": time(), "succeeded": ep.process()}
+    result["duration"] = time() - result["duration"]
+
+    # print footer on error or when is not unit testing
+    if not is_test_running and (not silent or not result["succeeded"]):
+        print_processing_footer(result)
+
+    if (
+        result["succeeded"]
+        and "monitor" in ep.get_build_targets()
+        and "nobuild" not in ep.get_build_targets()
+    ):
+        ctx.invoke(
+            cmd_device_monitor, environment=environments[0] if environments else None
+        )
+
+    return result
+
+
+def print_processing_header(env, config, verbose=False):
+    env_dump = []
+    for k, v in config.items(env=env):
+        if verbose or k in ("platform", "framework", "board"):
+            env_dump.append("%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v))
+    click.echo(
+        "Processing %s (%s)"
+        % (click.style(env, fg="cyan", bold=True), "; ".join(env_dump))
+    )
+    terminal_width, _ = click.get_terminal_size()
+    click.secho("-" * terminal_width, bold=True)
+
+
+def print_processing_footer(result):
+    is_failed = not result.get("succeeded")
+    util.print_labeled_bar(
+        "[%s] Took %.2f seconds"
+        % (
+            (
+                click.style("FAILED", fg="red", bold=True)
+                if is_failed
+                else click.style("SUCCESS", fg="green", bold=True)
+            ),
+            result["duration"],
+        ),
+        is_error=is_failed,
+    )
+
+
+def print_processing_summary(results):
+    tabular_data = []
+    succeeded_nums = 0
+    failed_nums = 0
+    duration = 0
+
+    for result in results:
+        duration += result.get("duration", 0)
+        if result.get("succeeded") is False:
+            failed_nums += 1
+            status_str = click.style("FAILED", fg="red")
+        elif result.get("succeeded") is None:
+            status_str = "IGNORED"
+        else:
+            succeeded_nums += 1
+            status_str = click.style("SUCCESS", fg="green")
+
+        tabular_data.append(
+            (
+                click.style(result["env"], fg="cyan"),
+                status_str,
+                util.humanize_duration_time(result.get("duration")),
+            )
+        )
+
+    click.echo()
+    click.echo(
+        tabulate(
+            tabular_data,
+            headers=[
+                click.style(s, bold=True) for s in ("Environment", "Status", "Duration")
+            ],
+        ),
+        err=failed_nums,
+    )
+
+    util.print_labeled_bar(
+        "%s%d succeeded in %s"
+        % (
+            "%d failed, " % failed_nums if failed_nums else "",
+            succeeded_nums,
+            util.humanize_duration_time(duration),
+        ),
+        is_error=failed_nums,
+        fg="red" if failed_nums else "green",
+    )

platformio/commands/run/helpers.py

@@ -0,0 +1,64 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os import makedirs
+from os.path import isdir, isfile, join
+
+import click
+
+from platformio import fs
+from platformio.project.helpers import compute_project_checksum, get_project_dir
+
+
+def handle_legacy_libdeps(project_dir, config):
+    legacy_libdeps_dir = join(project_dir, ".piolibdeps")
+    if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get_optional_dir(
+        "libdeps"
+    ):
+        return
+    if not config.has_section("env"):
+        config.add_section("env")
+    lib_extra_dirs = config.get("env", "lib_extra_dirs", [])
+    lib_extra_dirs.append(legacy_libdeps_dir)
+    config.set("env", "lib_extra_dirs", lib_extra_dirs)
+    click.secho(
+        "DEPRECATED! A legacy library storage `{0}` has been found in a "
+        "project. \nPlease declare project dependencies in `platformio.ini`"
+        " file using `lib_deps` option and remove `{0}` folder."
+        "\nMore details -> https://docs.platformio.org/page/projectconf/"
+        "section_env_library.html#lib-deps".format(legacy_libdeps_dir),
+        fg="yellow",
+    )
+
+
+def clean_build_dir(build_dir, config):
+    # remove legacy ".pioenvs" folder
+    legacy_build_dir = join(get_project_dir(), ".pioenvs")
+    if isdir(legacy_build_dir) and legacy_build_dir != build_dir:
+        fs.rmtree(legacy_build_dir)
+
+    checksum_file = join(build_dir, "project.checksum")
+    checksum = compute_project_checksum(config)
+
+    if isdir(build_dir):
+        # check project structure
+        if isfile(checksum_file):
+            with open(checksum_file) as fp:
+                if fp.read() == checksum:
+                    return
+        fs.rmtree(build_dir)
+
+    makedirs(build_dir)
+    with open(checksum_file, "w") as fp:
+        fp.write(checksum)

platformio/commands/run/processor.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio import exception, telemetry
+from platformio.commands.platform import platform_install as cmd_platform_install
+from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME
+from platformio.managers.platform import PlatformFactory
+from platformio.project.exception import UndefinedEnvPlatformError
+
+# pylint: disable=too-many-instance-attributes
+
+
+class EnvironmentProcessor(object):
+    def __init__(  # pylint: disable=too-many-arguments
+        self, cmd_ctx, name, config, targets, upload_port, silent, verbose, jobs
+    ):
+        self.cmd_ctx = cmd_ctx
+        self.name = name
+        self.config = config
+        self.targets = [str(t) for t in targets]
+        self.upload_port = upload_port
+        self.silent = silent
+        self.verbose = verbose
+        self.jobs = jobs
+        self.options = config.items(env=name, as_dict=True)
+
+    def get_build_variables(self):
+        variables = {"pioenv": self.name, "project_config": self.config.path}
+
+        if CTX_META_TEST_RUNNING_NAME in self.cmd_ctx.meta:
+            variables["piotest_running_name"] = self.cmd_ctx.meta[
+                CTX_META_TEST_RUNNING_NAME
+            ]
+
+        if self.upload_port:
+            # override upload port with a custom from CLI
+            variables["upload_port"] = self.upload_port
+        return variables
+
+    def get_build_targets(self):
+        return (
+            self.targets
+            if self.targets
+            else self.config.get("env:" + self.name, "targets", [])
+        )
+
+    def process(self):
+        if "platform" not in self.options:
+            raise UndefinedEnvPlatformError(self.name)
+
+        build_vars = self.get_build_variables()
+        build_targets = list(self.get_build_targets())
+
+        telemetry.send_run_environment(self.options, build_targets)
+
+        # skip monitor target, we call it above
+        if "monitor" in build_targets:
+            build_targets.remove("monitor")
+
+        try:
+            p = PlatformFactory.newPlatform(self.options["platform"])
+        except exception.UnknownPlatform:
+            self.cmd_ctx.invoke(
+                cmd_platform_install,
+                platforms=[self.options["platform"]],
+                skip_default_package=True,
+            )
+            p = PlatformFactory.newPlatform(self.options["platform"])
+
+        result = p.run(build_vars, build_targets, self.silent, self.verbose, self.jobs)
+        return result["returncode"] == 0

platformio/commands/settings.py

@@ -13,8 +13,18 @@
 # limitations under the License.
 
 import click
+from tabulate import tabulate
 
 from platformio import app
+from platformio.compat import string_types
+
+
+def format_value(raw):
+    if isinstance(raw, bool):
+        return "Yes" if raw else "No"
+    if isinstance(raw, string_types):
+        return raw
+    return str(raw)
 
 
 @click.group(short_help="Manage PlatformIO settings")
@@ -25,41 +35,31 @@ def cli():
 @cli.command("get", short_help="Get existing setting/-s")
 @click.argument("name", required=False)
 def settings_get(name):
+    tabular_data = []
+    for key, options in sorted(app.DEFAULT_SETTINGS.items()):
+        if name and name != key:
+            continue
+        raw_value = app.get_setting(key)
+        formatted_value = format_value(raw_value)
 
-    list_tpl = "{name:<40} {value:<35} {description}"
-    terminal_width, _ = click.get_terminal_size()
+        if raw_value != options["value"]:
+            default_formatted_value = format_value(options["value"])
+            formatted_value += "%s" % (
+                "\n" if len(default_formatted_value) > 10 else " "
+            )
+            formatted_value += "[%s]" % click.style(
+                default_formatted_value, fg="yellow"
+            )
+
+        tabular_data.append(
+            (click.style(key, fg="cyan"), formatted_value, options["description"])
+        )
 
     click.echo(
-        list_tpl.format(
-            name=click.style("Name", fg="cyan"),
-            value=(click.style("Value", fg="green") + click.style(
-                " [Default]", fg="yellow")),
-            description="Description"))
-    click.echo("-" * terminal_width)
-
-    for _name, _data in sorted(app.DEFAULT_SETTINGS.items()):
-        if name and name != _name:
-            continue
-        _value = app.get_setting(_name)
-
-        _value_str = str(_value)
-        if isinstance(_value, bool):
-            _value_str = "Yes" if _value else "No"
-        _value_str = click.style(_value_str, fg="green")
-
-        if _value != _data['value']:
-            _defvalue_str = str(_data['value'])
-            if isinstance(_data['value'], bool):
-                _defvalue_str = "Yes" if _data['value'] else "No"
-            _value_str += click.style(" [%s]" % _defvalue_str, fg="yellow")
-        else:
-            _value_str += click.style(" ", fg="yellow")
-
-        click.echo(
-            list_tpl.format(
-                name=click.style(_name, fg="cyan"),
-                value=_value_str,
-                description=_data['description']))
+        tabulate(
+            tabular_data, headers=["Name", "Current value [Default]", "Description"]
+        )
+    )
 
 
 @cli.command("set", short_help="Set new value for the setting")

platformio/commands/test.py

@@ -1,67 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-from os import getcwd
-
-import click
-
-from platformio.managers.core import pioplus_call
-
-
-@click.command("test", short_help="Local Unit Testing")
-@click.option("--environment", "-e", multiple=True, metavar="<environment>")
-@click.option(
-    "--filter",
-    "-f",
-    multiple=True,
-    metavar="<pattern>",
-    help="Filter tests by a pattern")
-@click.option(
-    "--ignore",
-    "-i",
-    multiple=True,
-    metavar="<pattern>",
-    help="Ignore tests by a pattern")
-@click.option("--upload-port")
-@click.option("--test-port")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=getcwd,
-    type=click.Path(
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True))
-@click.option("--without-building", is_flag=True)
-@click.option("--without-uploading", is_flag=True)
-@click.option(
-    "--no-reset",
-    is_flag=True,
-    help="Disable software reset via Serial.DTR/RST")
-@click.option(
-    "--monitor-rts",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial RTS line state for Serial Monitor")
-@click.option(
-    "--monitor-dtr",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial DTR line state for Serial Monitor")
-@click.option("--verbose", "-v", is_flag=True)
-def cli(*args, **kwargs):  # pylint: disable=unused-argument
-    pioplus_call(sys.argv[1:])

platformio/commands/test/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/test/command.py

@@ -0,0 +1,273 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
+
+from fnmatch import fnmatch
+from os import getcwd, listdir
+from os.path import isdir, join
+from time import time
+
+import click
+from tabulate import tabulate
+
+from platformio import app, exception, fs, util
+from platformio.commands.test.embedded import EmbeddedTestProcessor
+from platformio.commands.test.native import NativeTestProcessor
+from platformio.project.config import ProjectConfig
+
+
+@click.command("test", short_help="Unit Testing")
+@click.option("--environment", "-e", multiple=True, metavar="<environment>")
+@click.option(
+    "--filter",
+    "-f",
+    multiple=True,
+    metavar="<pattern>",
+    help="Filter tests by a pattern",
+)
+@click.option(
+    "--ignore",
+    "-i",
+    multiple=True,
+    metavar="<pattern>",
+    help="Ignore tests by a pattern",
+)
+@click.option("--upload-port")
+@click.option("--test-port")
+@click.option(
+    "-d",
+    "--project-dir",
+    default=getcwd,
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
+@click.option("--without-building", is_flag=True)
+@click.option("--without-uploading", is_flag=True)
+@click.option("--without-testing", is_flag=True)
+@click.option("--no-reset", is_flag=True)
+@click.option(
+    "--monitor-rts",
+    default=None,
+    type=click.IntRange(0, 1),
+    help="Set initial RTS line state for Serial Monitor",
+)
+@click.option(
+    "--monitor-dtr",
+    default=None,
+    type=click.IntRange(0, 1),
+    help="Set initial DTR line state for Serial Monitor",
+)
+@click.option("--verbose", "-v", is_flag=True)
+@click.pass_context
+def cli(  # pylint: disable=redefined-builtin
+    ctx,
+    environment,
+    ignore,
+    filter,
+    upload_port,
+    test_port,
+    project_dir,
+    project_conf,
+    without_building,
+    without_uploading,
+    without_testing,
+    no_reset,
+    monitor_rts,
+    monitor_dtr,
+    verbose,
+):
+    app.set_session_var("custom_project_conf", project_conf)
+
+    with fs.cd(project_dir):
+        config = ProjectConfig.get_instance(project_conf)
+        config.validate(envs=environment)
+
+        test_dir = config.get_optional_dir("test")
+        if not isdir(test_dir):
+            raise exception.TestDirNotExists(test_dir)
+        test_names = get_test_names(test_dir)
+
+        if not verbose:
+            click.echo("Verbose mode can be enabled via `-v, --verbose` option")
+        click.secho("Collected %d items" % len(test_names), bold=True)
+
+        results = []
+        default_envs = config.default_envs()
+        for testname in test_names:
+
+            for envname in config.envs():
+                section = "env:%s" % envname
+
+                # filter and ignore patterns
+                patterns = dict(filter=list(filter), ignore=list(ignore))
+                for key in patterns:
+                    patterns[key].extend(config.get(section, "test_%s" % key, []))
+
+                skip_conditions = [
+                    environment and envname not in environment,
+                    not environment and default_envs and envname not in default_envs,
+                    testname != "*"
+                    and patterns["filter"]
+                    and not any([fnmatch(testname, p) for p in patterns["filter"]]),
+                    testname != "*"
+                    and any([fnmatch(testname, p) for p in patterns["ignore"]]),
+                ]
+                if any(skip_conditions):
+                    results.append({"env": envname, "test": testname})
+                    continue
+
+                click.echo()
+                print_processing_header(testname, envname)
+
+                cls = (
+                    NativeTestProcessor
+                    if config.get(section, "platform") == "native"
+                    else EmbeddedTestProcessor
+                )
+                tp = cls(
+                    ctx,
+                    testname,
+                    envname,
+                    dict(
+                        project_config=config,
+                        project_dir=project_dir,
+                        upload_port=upload_port,
+                        test_port=test_port,
+                        without_building=without_building,
+                        without_uploading=without_uploading,
+                        without_testing=without_testing,
+                        no_reset=no_reset,
+                        monitor_rts=monitor_rts,
+                        monitor_dtr=monitor_dtr,
+                        verbose=verbose,
+                        silent=not verbose,
+                    ),
+                )
+                result = {
+                    "env": envname,
+                    "test": testname,
+                    "duration": time(),
+                    "succeeded": tp.process(),
+                }
+                result["duration"] = time() - result["duration"]
+                results.append(result)
+
+                print_processing_footer(result)
+
+    if without_testing:
+        return
+
+    print_testing_summary(results)
+
+    command_failed = any(r.get("succeeded") is False for r in results)
+    if command_failed:
+        raise exception.ReturnErrorCode(1)
+
+
+def get_test_names(test_dir):
+    names = []
+    for item in sorted(listdir(test_dir)):
+        if isdir(join(test_dir, item)):
+            names.append(item)
+    if not names:
+        names = ["*"]
+    return names
+
+
+def print_processing_header(test, env):
+    click.echo(
+        "Processing %s in %s environment"
+        % (
+            click.style(test, fg="yellow", bold=True),
+            click.style(env, fg="cyan", bold=True),
+        )
+    )
+    terminal_width, _ = click.get_terminal_size()
+    click.secho("-" * terminal_width, bold=True)
+
+
+def print_processing_footer(result):
+    is_failed = not result.get("succeeded")
+    util.print_labeled_bar(
+        "[%s] Took %.2f seconds"
+        % (
+            (
+                click.style("FAILED", fg="red", bold=True)
+                if is_failed
+                else click.style("PASSED", fg="green", bold=True)
+            ),
+            result["duration"],
+        ),
+        is_error=is_failed,
+    )
+
+
+def print_testing_summary(results):
+    click.echo()
+
+    tabular_data = []
+    succeeded_nums = 0
+    failed_nums = 0
+    duration = 0
+
+    for result in results:
+        duration += result.get("duration", 0)
+        if result.get("succeeded") is False:
+            failed_nums += 1
+            status_str = click.style("FAILED", fg="red")
+        elif result.get("succeeded") is None:
+            status_str = "IGNORED"
+        else:
+            succeeded_nums += 1
+            status_str = click.style("PASSED", fg="green")
+
+        tabular_data.append(
+            (
+                result["test"],
+                click.style(result["env"], fg="cyan"),
+                status_str,
+                util.humanize_duration_time(result.get("duration")),
+            )
+        )
+
+    click.echo(
+        tabulate(
+            tabular_data,
+            headers=[
+                click.style(s, bold=True)
+                for s in ("Test", "Environment", "Status", "Duration")
+            ],
+        ),
+        err=failed_nums,
+    )
+
+    util.print_labeled_bar(
+        "%s%d succeeded in %s"
+        % (
+            "%d failed, " % failed_nums if failed_nums else "",
+            succeeded_nums,
+            util.humanize_duration_time(duration),
+        ),
+        is_error=failed_nums,
+        fg="red" if failed_nums else "green",
+    )

platformio/commands/test/embedded.py

@@ -0,0 +1,138 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from time import sleep
+
+import click
+import serial
+
+from platformio import exception, util
+from platformio.commands.test.processor import TestProcessorBase
+from platformio.managers.platform import PlatformFactory
+
+
+class EmbeddedTestProcessor(TestProcessorBase):
+
+    SERIAL_TIMEOUT = 600
+
+    def process(self):
+        if not self.options["without_building"]:
+            self.print_progress("Building...")
+            target = ["__test"]
+            if self.options["without_uploading"]:
+                target.append("checkprogsize")
+            if not self.build_or_upload(target):
+                return False
+
+        if not self.options["without_uploading"]:
+            self.print_progress("Uploading...")
+            target = ["upload"]
+            if self.options["without_building"]:
+                target.append("nobuild")
+            else:
+                target.append("__test")
+            if not self.build_or_upload(target):
+                return False
+
+        if self.options["without_testing"]:
+            return True
+
+        self.print_progress("Testing...")
+        return self.run()
+
+    def run(self):
+        click.echo(
+            "If you don't see any output for the first 10 secs, "
+            "please reset board (press reset button)"
+        )
+        click.echo()
+
+        try:
+            ser = serial.Serial(
+                baudrate=self.get_baudrate(), timeout=self.SERIAL_TIMEOUT
+            )
+            ser.port = self.get_test_port()
+            ser.rts = self.options["monitor_rts"]
+            ser.dtr = self.options["monitor_dtr"]
+            ser.open()
+        except serial.SerialException as e:
+            click.secho(str(e), fg="red", err=True)
+            return False
+
+        if not self.options["no_reset"]:
+            ser.flushInput()
+            ser.setDTR(False)
+            ser.setRTS(False)
+            sleep(0.1)
+            ser.setDTR(True)
+            ser.setRTS(True)
+            sleep(0.1)
+
+        while True:
+            line = ser.readline().strip()
+
+            # fix non-ascii output from device
+            for i, c in enumerate(line[::-1]):
+                if not isinstance(c, int):
+                    c = ord(c)
+                if c > 127:
+                    line = line[-i:]
+                    break
+
+            if not line:
+                continue
+            if isinstance(line, bytes):
+                line = line.decode("utf8", "ignore")
+            self.on_run_out(line)
+            if all([l in line for l in ("Tests", "Failures", "Ignored")]):
+                break
+        ser.close()
+        return not self._run_failed
+
+    def get_test_port(self):
+        # if test port is specified manually or in config
+        if self.options.get("test_port"):
+            return self.options.get("test_port")
+        if self.env_options.get("test_port"):
+            return self.env_options.get("test_port")
+
+        assert set(["platform", "board"]) & set(self.env_options.keys())
+        p = PlatformFactory.newPlatform(self.env_options["platform"])
+        board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", [])
+        port = None
+        elapsed = 0
+        while elapsed < 5 and not port:
+            for item in util.get_serialports():
+                port = item["port"]
+                for hwid in board_hwids:
+                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
+                    if hwid_str in item["hwid"]:
+                        return port
+
+            # check if port is already configured
+            try:
+                serial.Serial(port, timeout=self.SERIAL_TIMEOUT).close()
+            except serial.SerialException:
+                port = None
+
+            if not port:
+                sleep(0.25)
+                elapsed += 0.25
+
+        if not port:
+            raise exception.PlatformioException(
+                "Please specify `test_port` for environment or use "
+                "global `--test-port` option."
+            )
+        return port

platformio/commands/test/native.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os.path import join
+
+from platformio import proc
+from platformio.commands.test.processor import TestProcessorBase
+from platformio.proc import LineBufferedAsyncPipe
+
+
+class NativeTestProcessor(TestProcessorBase):
+    def process(self):
+        if not self.options["without_building"]:
+            self.print_progress("Building...")
+            if not self.build_or_upload(["__test"]):
+                return False
+        if self.options["without_testing"]:
+            return None
+        self.print_progress("Testing...")
+        return self.run()
+
+    def run(self):
+        build_dir = self.options["project_config"].get_optional_dir("build")
+        result = proc.exec_command(
+            [join(build_dir, self.env_name, "program")],
+            stdout=LineBufferedAsyncPipe(self.on_run_out),
+            stderr=LineBufferedAsyncPipe(self.on_run_out),
+        )
+        assert "returncode" in result
+        return result["returncode"] == 0 and not self._run_failed

platformio/commands/test/processor.py

@@ -0,0 +1,204 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import atexit
+from os import remove
+from os.path import isdir, isfile, join
+from string import Template
+
+import click
+
+from platformio import exception
+
+TRANSPORT_OPTIONS = {
+    "arduino": {
+        "include": "#include <Arduino.h>",
+        "object": "",
+        "putchar": "Serial.write(c)",
+        "flush": "Serial.flush()",
+        "begin": "Serial.begin($baudrate)",
+        "end": "Serial.end()",
+    },
+    "mbed": {
+        "include": "#include <mbed.h>",
+        "object": "Serial pc(USBTX, USBRX);",
+        "putchar": "pc.putc(c)",
+        "flush": "",
+        "begin": "pc.baud($baudrate)",
+        "end": "",
+    },
+    "espidf": {
+        "include": "#include <stdio.h>",
+        "object": "",
+        "putchar": "putchar(c)",
+        "flush": "fflush(stdout)",
+        "begin": "",
+        "end": "",
+    },
+    "native": {
+        "include": "#include <stdio.h>",
+        "object": "",
+        "putchar": "putchar(c)",
+        "flush": "fflush(stdout)",
+        "begin": "",
+        "end": "",
+    },
+    "custom": {
+        "include": '#include "unittest_transport.h"',
+        "object": "",
+        "putchar": "unittest_uart_putchar(c)",
+        "flush": "unittest_uart_flush()",
+        "begin": "unittest_uart_begin()",
+        "end": "unittest_uart_end()",
+    },
+}
+
+CTX_META_TEST_IS_RUNNING = __name__ + ".test_running"
+CTX_META_TEST_RUNNING_NAME = __name__ + ".test_running_name"
+
+
+class TestProcessorBase(object):
+
+    DEFAULT_BAUDRATE = 115200
+
+    def __init__(self, cmd_ctx, testname, envname, options):
+        self.cmd_ctx = cmd_ctx
+        self.cmd_ctx.meta[CTX_META_TEST_IS_RUNNING] = True
+        self.test_name = testname
+        self.options = options
+        self.env_name = envname
+        self.env_options = options["project_config"].items(env=envname, as_dict=True)
+        self._run_failed = False
+        self._outputcpp_generated = False
+
+    def get_transport(self):
+        transport = None
+        if self.env_options.get("platform") == "native":
+            transport = "native"
+        elif "framework" in self.env_options:
+            transport = self.env_options.get("framework")[0]
+        if "test_transport" in self.env_options:
+            transport = self.env_options["test_transport"]
+        if transport not in TRANSPORT_OPTIONS:
+            raise exception.PlatformioException(
+                "Unknown Unit Test transport `%s`. Please check a documentation how "
+                "to create an own 'Test Transport':\n"
+                "- https://docs.platformio.org/page/plus/unit-testing.html" % transport
+            )
+        return transport.lower()
+
+    def get_baudrate(self):
+        return int(self.env_options.get("test_speed", self.DEFAULT_BAUDRATE))
+
+    def print_progress(self, text):
+        click.secho(text, bold=self.options.get("verbose"))
+
+    def build_or_upload(self, target):
+        if not self._outputcpp_generated:
+            self.generate_outputcpp(
+                self.options["project_config"].get_optional_dir("test")
+            )
+            self._outputcpp_generated = True
+
+        if self.test_name != "*":
+            self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name
+
+        try:
+            # pylint: disable=import-outside-toplevel
+            from platformio.commands.run.command import cli as cmd_run
+
+            return self.cmd_ctx.invoke(
+                cmd_run,
+                project_dir=self.options["project_dir"],
+                upload_port=self.options["upload_port"],
+                verbose=self.options["verbose"],
+                silent=self.options["silent"],
+                environment=[self.env_name],
+                disable_auto_clean="nobuild" in target,
+                target=target,
+            )
+        except exception.ReturnErrorCode:
+            return False
+
+    def process(self):
+        raise NotImplementedError
+
+    def run(self):
+        raise NotImplementedError
+
+    def on_run_out(self, line):
+        line = line.strip()
+        if line.endswith(":PASS"):
+            click.echo("%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green")))
+        elif ":FAIL" in line:
+            self._run_failed = True
+            click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red")))
+        else:
+            click.echo(line)
+
+    def generate_outputcpp(self, test_dir):
+        assert isdir(test_dir)
+
+        cpp_tpl = "\n".join(
+            [
+                "$include",
+                "#include <output_export.h>",
+                "",
+                "$object",
+                "",
+                "#ifdef __GNUC__",
+                "void output_start(unsigned int baudrate __attribute__((unused)))",
+                "#else",
+                "void output_start(unsigned int baudrate)",
+                "#endif",
+                "{",
+                "    $begin;",
+                "}",
+                "",
+                "void output_char(int c)",
+                "{",
+                "    $putchar;",
+                "}",
+                "",
+                "void output_flush(void)",
+                "{",
+                "    $flush;",
+                "}",
+                "",
+                "void output_complete(void)",
+                "{",
+                "   $end;",
+                "}",
+            ]
+        )
+
+        def delete_tmptest_file(file_):
+            try:
+                remove(file_)
+            except:  # pylint: disable=bare-except
+                if isfile(file_):
+                    click.secho(
+                        "Warning: Could not remove temporary file '%s'. "
+                        "Please remove it manually." % file_,
+                        fg="yellow",
+                    )
+
+        tpl = Template(cpp_tpl).substitute(TRANSPORT_OPTIONS[self.get_transport()])
+        data = Template(tpl).substitute(baudrate=self.get_baudrate())
+
+        tmp_file = join(test_dir, "output_export.cpp")
+        with open(tmp_file, "w") as fp:
+            fp.write(data)
+
+        atexit.register(delete_tmptest_file, tmp_file)

platformio/commands/update.py

@@ -15,6 +15,7 @@
 import click
 
 from platformio import app
+from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY
 from platformio.commands.lib import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.managers.core import update_core_packages
@@ -22,19 +23,25 @@ from platformio.managers.lib import LibraryManager
 
 
 @click.command(
-    "update", short_help="Update installed platforms, packages and libraries")
-@click.option(
-    "--core-packages", is_flag=True, help="Update only the core packages")
+    "update", short_help="Update installed platforms, packages and libraries"
+)
+@click.option("--core-packages", is_flag=True, help="Update only the core packages")
 @click.option(
     "-c",
     "--only-check",
     is_flag=True,
-    help="Do not update, only check for new version")
+    help="DEPRECATED. Please use `--dry-run` instead",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
+)
 @click.pass_context
-def cli(ctx, core_packages, only_check):
+def cli(ctx, core_packages, only_check, dry_run):
     # cleanup lib search results, cached board and platform lists
     app.clean_cache()
 
+    only_check = dry_run or only_check
+
     update_core_packages(only_check)
 
     if core_packages:
@@ -48,5 +55,5 @@ def cli(ctx, core_packages, only_check):
     click.echo()
     click.echo("Library Manager")
     click.echo("===============")
-    ctx.obj = LibraryManager()
+    ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryManager().package_dir]
     ctx.invoke(cmd_lib_update, only_check=only_check)

platformio/commands/upgrade.py

@@ -19,57 +19,60 @@ from zipfile import ZipFile
 import click
 import requests
 
-from platformio import VERSION, __version__, exception, util
-from platformio.managers.core import shutdown_piohome_servers
+from platformio import VERSION, __version__, app, exception, util
+from platformio.compat import WINDOWS
+from platformio.proc import exec_command, get_pythonexe_path
+from platformio.project.helpers import get_project_cache_dir
 
 
-@click.command(
-    "upgrade", short_help="Upgrade PlatformIO to the latest version")
+@click.command("upgrade", short_help="Upgrade PlatformIO to the latest version")
 @click.option("--dev", is_flag=True, help="Use development branch")
 def cli(dev):
     if not dev and __version__ == get_latest_version():
         return click.secho(
             "You're up-to-date!\nPlatformIO %s is currently the "
             "newest version available." % __version__,
-            fg="green")
+            fg="green",
+        )
 
     click.secho("Please wait while upgrading PlatformIO ...", fg="yellow")
 
-    # kill all PIO Home servers, they block `pioplus` binary
-    shutdown_piohome_servers()
-
     to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".")
-    cmds = (["pip", "install", "--upgrade",
-             get_pip_package(to_develop)], ["platformio", "--version"])
+    cmds = (
+        ["pip", "install", "--upgrade", get_pip_package(to_develop)],
+        ["platformio", "--version"],
+    )
 
     cmd = None
-    r = None
+    r = {}
     try:
         for cmd in cmds:
-            cmd = [util.get_pythonexe_path(), "-m"] + cmd
-            r = None
-            r = util.exec_command(cmd)
+            cmd = [get_pythonexe_path(), "-m"] + cmd
+            r = exec_command(cmd)
 
             # try pip with disabled cache
-            if r['returncode'] != 0 and cmd[2] == "pip":
+            if r["returncode"] != 0 and cmd[2] == "pip":
                 cmd.insert(3, "--no-cache-dir")
-                r = util.exec_command(cmd)
+                r = exec_command(cmd)
 
-            assert r['returncode'] == 0
-        assert "version" in r['out']
-        actual_version = r['out'].strip().split("version", 1)[1].strip()
+            assert r["returncode"] == 0
+        assert "version" in r["out"]
+        actual_version = r["out"].strip().split("version", 1)[1].strip()
         click.secho(
             "PlatformIO has been successfully upgraded to %s" % actual_version,
-            fg="green")
+            fg="green",
+        )
         click.echo("Release notes: ", nl=False)
-        click.secho(
-            "https://docs.platformio.org/en/latest/history.html", fg="cyan")
+        click.secho("https://docs.platformio.org/en/latest/history.html", fg="cyan")
+        if app.get_session_var("caller_id"):
+            click.secho(
+                "Warning! Please restart IDE to affect PIO Home changes", fg="yellow"
+            )
     except Exception as e:  # pylint: disable=broad-except
         if not r:
             raise exception.UpgradeError("\n".join([str(cmd), str(e)]))
         permission_errors = ("permission denied", "not permitted")
-        if (any(m in r['err'].lower() for m in permission_errors)
-                and "windows" not in util.get_systype()):
+        if any(m in r["err"].lower() for m in permission_errors) and not WINDOWS:
             click.secho(
                 """
 -----------------
@@ -82,11 +85,10 @@ You need the `sudo` permission to install Python packages. Try
 WARNING! Don't use `sudo` for the rest PlatformIO commands.
 """,
                 fg="yellow",
-                err=True)
+                err=True,
+            )
             raise exception.ReturnErrorCode(1)
-        else:
-            raise exception.UpgradeError("\n".join(
-                [str(cmd), r['out'], r['err']]))
+        raise exception.UpgradeError("\n".join([str(cmd), r["out"], r["err"]]))
 
     return True
 
@@ -94,18 +96,17 @@ WARNING! Don't use `sudo` for the rest PlatformIO commands.
 def get_pip_package(to_develop):
     if not to_develop:
         return "platformio"
-    dl_url = ("https://github.com/platformio/"
-              "platformio-core/archive/develop.zip")
-    cache_dir = util.get_cache_dir()
+    dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip"
+    cache_dir = get_project_cache_dir()
     if not os.path.isdir(cache_dir):
         os.makedirs(cache_dir)
     pkg_name = os.path.join(cache_dir, "piocoredevelop.zip")
     try:
         with open(pkg_name, "w") as fp:
-            r = util.exec_command(["curl", "-fsSL", dl_url],
-                                  stdout=fp,
-                                  universal_newlines=True)
-            assert r['returncode'] == 0
+            r = exec_command(
+                ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True
+            )
+            assert r["returncode"] == 0
         # check ZIP structure
         with ZipFile(pkg_name) as zp:
             assert zp.testzip() is None
@@ -132,7 +133,8 @@ def get_develop_latest_version():
     r = requests.get(
         "https://raw.githubusercontent.com/platformio/platformio"
         "/develop/platformio/__init__.py",
-        headers=util.get_request_defheaders())
+        headers=util.get_request_defheaders(),
+    )
     r.raise_for_status()
     for line in r.text.split("\n"):
         line = line.strip()
@@ -151,7 +153,7 @@ def get_develop_latest_version():
 
 def get_pypi_latest_version():
     r = requests.get(
-        "https://pypi.org/pypi/platformio/json",
-        headers=util.get_request_defheaders())
+        "https://pypi.org/pypi/platformio/json", headers=util.get_request_defheaders()
+    )
     r.raise_for_status()
-    return r.json()['info']['version']
+    return r.json()["info"]["version"]

platformio/compat.py

@@ -0,0 +1,129 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=unused-import, no-name-in-module, import-error,
+# pylint: disable=no-member, undefined-variable
+
+import inspect
+import json
+import locale
+import os
+import re
+import sys
+
+PY2 = sys.version_info[0] == 2
+CYGWIN = sys.platform.startswith("cygwin")
+WINDOWS = sys.platform.startswith("win")
+
+
+def get_filesystem_encoding():
+    return sys.getfilesystemencoding() or sys.getdefaultencoding()
+
+
+def get_locale_encoding():
+    try:
+        return locale.getdefaultlocale()[1]
+    except ValueError:
+        return None
+
+
+def get_object_members(obj, ignore_private=True):
+    members = inspect.getmembers(obj, lambda a: not inspect.isroutine(a))
+    if not ignore_private:
+        return members
+    return {
+        item[0]: item[1]
+        for item in members
+        if not (item[0].startswith("__") and item[0].endswith("__"))
+    }
+
+
+if PY2:
+    import imp
+
+    string_types = (str, unicode)
+
+    def is_bytes(x):
+        return isinstance(x, (buffer, bytearray))
+
+    def path_to_unicode(path):
+        if isinstance(path, unicode):
+            return path
+        return path.decode(get_filesystem_encoding())
+
+    def hashlib_encode_data(data):
+        if is_bytes(data):
+            return data
+        if isinstance(data, unicode):
+            data = data.encode(get_filesystem_encoding())
+        elif not isinstance(data, string_types):
+            data = str(data)
+        return data
+
+    def dump_json_to_unicode(obj):
+        if isinstance(obj, unicode):
+            return obj
+        return json.dumps(
+            obj, encoding=get_filesystem_encoding(), ensure_ascii=False, sort_keys=True
+        ).encode("utf8")
+
+    _magic_check = re.compile("([*?[])")
+    _magic_check_bytes = re.compile(b"([*?[])")
+
+    def glob_escape(pathname):
+        """Escape all special characters."""
+        # https://github.com/python/cpython/blob/master/Lib/glob.py#L161
+        # Escaping is done by wrapping any of "*?[" between square brackets.
+        # Metacharacters do not work in the drive part and shouldn't be
+        # escaped.
+        drive, pathname = os.path.splitdrive(pathname)
+        if isinstance(pathname, bytes):
+            pathname = _magic_check_bytes.sub(br"[\1]", pathname)
+        else:
+            pathname = _magic_check.sub(r"[\1]", pathname)
+        return drive + pathname
+
+    def load_python_module(name, pathname):
+        return imp.load_source(name, pathname)
+
+
+else:
+    import importlib.util
+    from glob import escape as glob_escape
+
+    string_types = (str,)
+
+    def is_bytes(x):
+        return isinstance(x, (bytes, memoryview, bytearray))
+
+    def path_to_unicode(path):
+        return path
+
+    def hashlib_encode_data(data):
+        if is_bytes(data):
+            return data
+        if not isinstance(data, string_types):
+            data = str(data)
+        return data.encode()
+
+    def dump_json_to_unicode(obj):
+        if isinstance(obj, string_types):
+            return obj
+        return json.dumps(obj, ensure_ascii=False, sort_keys=True)
+
+    def load_python_module(name, pathname):
+        spec = importlib.util.spec_from_file_location(name, pathname)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        return module

platformio/downloader.py

@@ -12,24 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
+import io
+import math
+import sys
 from email.utils import parsedate_tz
-from math import ceil
 from os.path import getsize, join
-from sys import getfilesystemencoding, version_info
 from time import mktime
 
 import click
 import requests
 
 from platformio import util
-from platformio.exception import (FDSHASumMismatch, FDSizeMismatch,
-                                  FDUnrecognizedStatusCode)
+from platformio.exception import (
+    FDSHASumMismatch,
+    FDSizeMismatch,
+    FDUnrecognizedStatusCode,
+)
 
 
 class FileDownloader(object):
-
-    CHUNK_SIZE = 1024
-
     def __init__(self, url, dest_dir=None):
         self._request = None
         # make connection
@@ -37,22 +39,24 @@ class FileDownloader(object):
             url,
             stream=True,
             headers=util.get_request_defheaders(),
-            verify=version_info >= (2, 7, 9))
+            verify=sys.version_info >= (2, 7, 9),
+        )
         if self._request.status_code != 200:
             raise FDUnrecognizedStatusCode(self._request.status_code, url)
 
         disposition = self._request.headers.get("content-disposition")
         if disposition and "filename=" in disposition:
-            self._fname = disposition[disposition.index("filename=") +
-                                      9:].replace('"', "").replace("'", "")
-            self._fname = self._fname.encode("utf8")
+            self._fname = (
+                disposition[disposition.index("filename=") + 9 :]
+                .replace('"', "")
+                .replace("'", "")
+            )
         else:
             self._fname = [p for p in url.split("/") if p][-1]
-
+        self._fname = str(self._fname)
         self._destination = self._fname
         if dest_dir:
-            self.set_destination(
-                join(dest_dir.decode(getfilesystemencoding()), self._fname))
+            self.set_destination(join(dest_dir, self._fname))
 
     def set_destination(self, destination):
         self._destination = destination
@@ -66,20 +70,21 @@ class FileDownloader(object):
     def get_size(self):
         if "content-length" not in self._request.headers:
             return -1
-        return int(self._request.headers['content-length'])
+        return int(self._request.headers["content-length"])
 
-    def start(self, with_progress=True):
+    def start(self, with_progress=True, silent=False):
         label = "Downloading"
-        itercontent = self._request.iter_content(chunk_size=self.CHUNK_SIZE)
+        itercontent = self._request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE)
         f = open(self._destination, "wb")
         try:
             if not with_progress or self.get_size() == -1:
-                click.echo("%s..." % label)
+                if not silent:
+                    click.echo("%s..." % label)
                 for chunk in itercontent:
                     if chunk:
                         f.write(chunk)
             else:
-                chunks = int(ceil(self.get_size() / float(self.CHUNK_SIZE)))
+                chunks = int(math.ceil(self.get_size() / float(io.DEFAULT_BUFFER_SIZE)))
                 with click.progressbar(length=chunks, label=label) as pb:
                     for _ in pb:
                         f.write(next(itercontent))
@@ -96,26 +101,20 @@ class FileDownloader(object):
         _dlsize = getsize(self._destination)
         if self.get_size() != -1 and _dlsize != self.get_size():
             raise FDSizeMismatch(_dlsize, self._fname, self.get_size())
-
         if not sha1:
-            return
+            return None
 
-        dlsha1 = None
-        try:
-            result = util.exec_command(["sha1sum", self._destination])
-            dlsha1 = result['out']
-        except (OSError, ValueError):
-            try:
-                result = util.exec_command(
-                    ["shasum", "-a", "1", self._destination])
-                dlsha1 = result['out']
-            except (OSError, ValueError):
-                pass
+        checksum = hashlib.sha1()
+        with io.open(self._destination, "rb", buffering=0) as fp:
+            while True:
+                chunk = fp.read(io.DEFAULT_BUFFER_SIZE)
+                if not chunk:
+                    break
+                checksum.update(chunk)
 
-        if dlsha1:
-            dlsha1 = dlsha1[1:41] if dlsha1.startswith("\\") else dlsha1[:40]
-            if sha1 != dlsha1:
-                raise FDSHASumMismatch(dlsha1, self._fname, sha1)
+        if sha1.lower() != checksum.hexdigest().lower():
+            raise FDSHASumMismatch(checksum.hexdigest(), self._fname, sha1)
+        return True
 
     def _preserve_filemtime(self, lmdate):
         timedata = parsedate_tz(lmdate)

platformio/exception.py

@@ -19,8 +19,10 @@ class PlatformioException(Exception):
 
     def __str__(self):  # pragma: no cover
         if self.MESSAGE:
+            # pylint: disable=not-an-iterable
             return self.MESSAGE.format(*self.args)
-        return Exception.__str__(self)
+
+        return super(PlatformioException, self).__str__()
 
 
 class ReturnErrorCode(PlatformioException):
@@ -40,11 +42,16 @@ class UserSideException(PlatformioException):
     pass
 
 
-class AbortedByUser(PlatformioException):
+class AbortedByUser(UserSideException):
 
     MESSAGE = "Aborted by user"
 
 
+#
+# Development Platform
+#
+
+
 class UnknownPlatform(PlatformioException):
 
     MESSAGE = "Unknown development platform '{0}'"
@@ -57,15 +64,10 @@ class IncompatiblePlatform(PlatformioException):
 
 class PlatformNotInstalledYet(PlatformioException):
 
-    MESSAGE = ("The platform '{0}' has not been installed yet. "
-               "Use `platformio platform install {0}` command")
-
-
-class BoardNotDefined(PlatformioException):
-
     MESSAGE = (
-        "You need to specify board ID using `-b` or `--board` option. "
-        "Supported boards list is available via `platformio boards` command")
+        "The platform '{0}' has not been installed yet. "
+        "Use `platformio platform install {0}` command"
+    )
 
 
 class UnknownBoard(PlatformioException):
@@ -83,79 +85,132 @@ class UnknownFramework(PlatformioException):
     MESSAGE = "Unknown framework '{0}'"
 
 
-class UnknownPackage(PlatformioException):
+# Package Manager
+
+
+class PlatformIOPackageException(PlatformioException):
+    pass
+
+
+class UnknownPackage(PlatformIOPackageException):
 
     MESSAGE = "Detected unknown package '{0}'"
 
 
-class MissingPackageManifest(PlatformioException):
+class MissingPackageManifest(PlatformIOPackageException):
 
     MESSAGE = "Could not find one of '{0}' manifest files in the package"
 
 
-class UndefinedPackageVersion(PlatformioException):
+class UndefinedPackageVersion(PlatformIOPackageException):
 
-    MESSAGE = ("Could not find a version that satisfies the requirement '{0}'"
-               " for your system '{1}'")
+    MESSAGE = (
+        "Could not find a version that satisfies the requirement '{0}'"
+        " for your system '{1}'"
+    )
 
 
-class PackageInstallError(PlatformioException):
+class PackageInstallError(PlatformIOPackageException):
 
-    MESSAGE = ("Could not install '{0}' with version requirements '{1}' "
-               "for your system '{2}'.\n\n"
-               "Please try this solution -> http://bit.ly/faq-package-manager")
+    MESSAGE = (
+        "Could not install '{0}' with version requirements '{1}' "
+        "for your system '{2}'.\n\n"
+        "Please try this solution -> http://bit.ly/faq-package-manager"
+    )
 
 
-class ExtractArchiveItemError(PlatformioException):
+class ExtractArchiveItemError(PlatformIOPackageException):
 
     MESSAGE = (
         "Could not extract `{0}` to `{1}`. Try to disable antivirus "
-        "tool or check this solution -> http://bit.ly/faq-package-manager")
+        "tool or check this solution -> http://bit.ly/faq-package-manager"
+    )
 
 
-class FDUnrecognizedStatusCode(PlatformioException):
-
-    MESSAGE = "Got an unrecognized status code '{0}' when downloaded {1}"
-
-
-class FDSizeMismatch(PlatformioException):
-
-    MESSAGE = ("The size ({0:d} bytes) of downloaded file '{1}' "
-               "is not equal to remote size ({2:d} bytes)")
-
-
-class FDSHASumMismatch(PlatformioException):
-
-    MESSAGE = ("The 'sha1' sum '{0}' of downloaded file '{1}' "
-               "is not equal to remote '{2}'")
-
-
-class NotPlatformIOProject(PlatformioException):
-
-    MESSAGE = (
-        "Not a PlatformIO project. `platformio.ini` file has not been "
-        "found in current working directory ({0}). To initialize new project "
-        "please use `platformio init` command")
-
-
-class UndefinedEnvPlatform(PlatformioException):
-
-    MESSAGE = "Please specify platform for '{0}' environment"
-
-
-class UnsupportedArchiveType(PlatformioException):
+class UnsupportedArchiveType(PlatformIOPackageException):
 
     MESSAGE = "Can not unpack file '{0}'"
 
 
-class ProjectEnvsNotAvailable(PlatformioException):
+class FDUnrecognizedStatusCode(PlatformIOPackageException):
 
-    MESSAGE = "Please setup environments in `platformio.ini` file"
+    MESSAGE = "Got an unrecognized status code '{0}' when downloaded {1}"
 
 
-class UnknownEnvNames(PlatformioException):
+class FDSizeMismatch(PlatformIOPackageException):
 
-    MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'"
+    MESSAGE = (
+        "The size ({0:d} bytes) of downloaded file '{1}' "
+        "is not equal to remote size ({2:d} bytes)"
+    )
+
+
+class FDSHASumMismatch(PlatformIOPackageException):
+
+    MESSAGE = (
+        "The 'sha1' sum '{0}' of downloaded file '{1}' is not equal to remote '{2}'"
+    )
+
+
+#
+# Library
+#
+
+
+class LibNotFound(PlatformioException):
+
+    MESSAGE = (
+        "Library `{0}` has not been found in PlatformIO Registry.\n"
+        "You can ignore this message, if `{0}` is a built-in library "
+        "(included in framework, SDK). E.g., SPI, Wire, etc."
+    )
+
+
+class NotGlobalLibDir(UserSideException):
+
+    MESSAGE = (
+        "The `{0}` is not a PlatformIO project.\n\n"
+        "To manage libraries in global storage `{1}`,\n"
+        "please use `platformio lib --global {2}` or specify custom storage "
+        "`platformio lib --storage-dir /path/to/storage/ {2}`.\n"
+        "Check `platformio lib --help` for details."
+    )
+
+
+class InvalidLibConfURL(PlatformioException):
+
+    MESSAGE = "Invalid library config URL '{0}'"
+
+
+#
+# UDEV Rules
+#
+
+
+class InvalidUdevRules(PlatformioException):
+    pass
+
+
+class MissedUdevRules(InvalidUdevRules):
+
+    MESSAGE = (
+        "Warning! Please install `99-platformio-udev.rules`. \nMode details: "
+        "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules"
+    )
+
+
+class OutdatedUdevRules(InvalidUdevRules):
+
+    MESSAGE = (
+        "Warning! Your `{0}` are outdated. Please update or reinstall them."
+        "\n Mode details: https://docs.platformio.org"
+        "/en/latest/faq.html#platformio-udev-rules"
+    )
+
+
+#
+# Misc
+#
 
 
 class GetSerialPortsError(PlatformioException):
@@ -173,39 +228,13 @@ class APIRequestError(PlatformioException):
     MESSAGE = "[API] {0}"
 
 
-class InternetIsOffline(PlatformioException):
+class InternetIsOffline(UserSideException):
 
     MESSAGE = (
         "You are not connected to the Internet.\n"
         "If you build a project first time, we need Internet connection "
-        "to install all dependencies and toolchains.")
-
-
-class LibNotFound(PlatformioException):
-
-    MESSAGE = ("Library `{0}` has not been found in PlatformIO Registry.\n"
-               "You can ignore this message, if `{0}` is a built-in library "
-               "(included in framework, SDK). E.g., SPI, Wire, etc.")
-
-
-class NotGlobalLibDir(PlatformioException):
-
-    MESSAGE = (
-        "The `{0}` is not a PlatformIO project.\n\n"
-        "To manage libraries in global storage `{1}`,\n"
-        "please use `platformio lib --global {2}` or specify custom storage "
-        "`platformio lib --storage-dir /path/to/storage/ {2}`.\n"
-        "Check `platformio lib --help` for details.")
-
-
-class InvalidLibConfURL(PlatformioException):
-
-    MESSAGE = "Invalid library config URL '{0}'"
-
-
-class InvalidProjectConf(PlatformioException):
-
-    MESSAGE = "Invalid `platformio.ini`, project configuration file: '{0}'"
+        "to install all dependencies and toolchains."
+    )
 
 
 class BuildScriptNotFound(PlatformioException):
@@ -230,28 +259,11 @@ class InvalidJSONFile(PlatformioException):
 
 class CIBuildEnvsEmpty(PlatformioException):
 
-    MESSAGE = ("Can't find PlatformIO build environments.\n"
-               "Please specify `--board` or path to `platformio.ini` with "
-               "predefined environments using `--project-conf` option")
-
-
-class InvalidUdevRules(PlatformioException):
-    pass
-
-
-class MissedUdevRules(InvalidUdevRules):
-
     MESSAGE = (
-        "Warning! Please install `99-platformio-udev.rules`. \nMode details: "
-        "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules")
-
-
-class OutdatedUdevRules(InvalidUdevRules):
-
-    MESSAGE = (
-        "Warning! Your `{0}` are outdated. Please update or reinstall them."
-        "\n Mode details: https://docs.platformio.org"
-        "/en/latest/faq.html#platformio-udev-rules")
+        "Can't find PlatformIO build environments.\n"
+        "Please specify `--board` or path to `platformio.ini` with "
+        "predefined environments using `--project-conf` option"
+    )
 
 
 class UpgradeError(PlatformioException):
@@ -264,29 +276,31 @@ class UpgradeError(PlatformioException):
 """
 
 
-class HomeDirPermissionsError(PlatformioException):
+class HomeDirPermissionsError(UserSideException):
 
     MESSAGE = (
         "The directory `{0}` or its parent directory is not owned by the "
         "current user and PlatformIO can not store configuration data.\n"
         "Please check the permissions and owner of that directory.\n"
         "Otherwise, please remove manually `{0}` directory and PlatformIO "
-        "will create new from the current user.")
+        "will create new from the current user."
+    )
 
 
 class CygwinEnvDetected(PlatformioException):
 
-    MESSAGE = ("PlatformIO does not work within Cygwin environment. "
-               "Use native Terminal instead.")
+    MESSAGE = (
+        "PlatformIO does not work within Cygwin environment. "
+        "Use native Terminal instead."
+    )
 
 
-class DebugSupportError(PlatformioException):
+class TestDirNotExists(PlatformioException):
 
-    MESSAGE = ("Currently, PlatformIO does not support debugging for `{0}`.\n"
-               "Please contact support@pioplus.com or visit "
-               "< https://docs.platformio.org/page/plus/debugging.html >")
-
-
-class DebugInvalidOptions(PlatformioException):
-
-    pass
+    MESSAGE = (
+        "A test folder '{0}' does not exist.\nPlease create 'test' "
+        "directory in project's root and put a test set.\n"
+        "More details about Unit "
+        "Testing: https://docs.platformio.org/page/plus/"
+        "unit-testing.html"
+    )

platformio/fs.py

@@ -0,0 +1,181 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+import shutil
+import stat
+import sys
+from glob import glob
+
+import click
+
+from platformio import exception
+from platformio.compat import WINDOWS, glob_escape
+
+
+class cd(object):
+    def __init__(self, new_path):
+        self.new_path = new_path
+        self.prev_path = os.getcwd()
+
+    def __enter__(self):
+        os.chdir(self.new_path)
+
+    def __exit__(self, etype, value, traceback):
+        os.chdir(self.prev_path)
+
+
+def get_source_dir():
+    curpath = os.path.realpath(__file__)
+    if not os.path.isfile(curpath):
+        for p in sys.path:
+            if os.path.isfile(os.path.join(p, __file__)):
+                curpath = os.path.join(p, __file__)
+                break
+    return os.path.dirname(curpath)
+
+
+def load_json(file_path):
+    try:
+        with open(file_path, "r") as f:
+            return json.load(f)
+    except ValueError:
+        raise exception.InvalidJSONFile(file_path)
+
+
+def format_filesize(filesize):
+    base = 1024
+    unit = 0
+    suffix = "B"
+    filesize = float(filesize)
+    if filesize < base:
+        return "%d%s" % (filesize, suffix)
+    for i, suffix in enumerate("KMGTPEZY"):
+        unit = base ** (i + 2)
+        if filesize >= unit:
+            continue
+        if filesize % (base ** (i + 1)):
+            return "%.2f%sB" % ((base * filesize / unit), suffix)
+        break
+    return "%d%sB" % ((base * filesize / unit), suffix)
+
+
+def ensure_udev_rules():
+    from platformio.util import get_systype  # pylint: disable=import-outside-toplevel
+
+    def _rules_to_set(rules_path):
+        result = set()
+        with open(rules_path) as fp:
+            for line in fp.readlines():
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                result.add(line)
+        return result
+
+    if "linux" not in get_systype():
+        return None
+    installed_rules = [
+        "/etc/udev/rules.d/99-platformio-udev.rules",
+        "/lib/udev/rules.d/99-platformio-udev.rules",
+    ]
+    if not any(os.path.isfile(p) for p in installed_rules):
+        raise exception.MissedUdevRules
+
+    origin_path = os.path.realpath(
+        os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules")
+    )
+    if not os.path.isfile(origin_path):
+        return None
+
+    origin_rules = _rules_to_set(origin_path)
+    for rules_path in installed_rules:
+        if not os.path.isfile(rules_path):
+            continue
+        current_rules = _rules_to_set(rules_path)
+        if not origin_rules <= current_rules:
+            raise exception.OutdatedUdevRules(rules_path)
+
+    return True
+
+
+def path_endswith_ext(path, extensions):
+    if not isinstance(extensions, (list, tuple)):
+        extensions = [extensions]
+    for ext in extensions:
+        if path.endswith("." + ext):
+            return True
+    return False
+
+
+def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True):
+    def _append_build_item(items, item, src_dir):
+        if not src_exts or path_endswith_ext(item, src_exts):
+            items.add(os.path.relpath(item, src_dir))
+
+    src_filter = src_filter or ""
+    if isinstance(src_filter, (list, tuple)):
+        src_filter = " ".join(src_filter)
+
+    matches = set()
+    # correct fs directory separator
+    src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep)
+    for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter):
+        items = set()
+        for item in glob(os.path.join(glob_escape(src_dir), pattern)):
+            if os.path.isdir(item):
+                for root, _, files in os.walk(item, followlinks=followlinks):
+                    for f in files:
+                        _append_build_item(items, os.path.join(root, f), src_dir)
+            else:
+                _append_build_item(items, item, src_dir)
+        if action == "+":
+            matches |= items
+        else:
+            matches -= items
+    return sorted(list(matches))
+
+
+def to_unix_path(path):
+    if not WINDOWS or not path:
+        return path
+    return re.sub(r"[\\]+", "/", path)
+
+
+def expanduser(path):
+    """
+    Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE
+    """
+    if not WINDOWS or not path.startswith("~") or "USERPROFILE" not in os.environ:
+        return os.path.expanduser(path)
+    return os.environ["USERPROFILE"] + path[1:]
+
+
+def rmtree(path):
+    def _onerror(func, path, __):
+        try:
+            st_mode = os.stat(path).st_mode
+            if st_mode & stat.S_IREAD:
+                os.chmod(path, st_mode | stat.S_IWRITE)
+            func(path)
+        except Exception as e:  # pylint: disable=broad-except
+            click.secho(
+                "%s \nPlease manually remove the file `%s`" % (str(e), path),
+                fg="red",
+                err=True,
+            )
+
+    return shutil.rmtree(path, onerror=_onerror)

platformio/ide/projectgenerator.py

@@ -12,90 +12,105 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
+import codecs
 import os
-import re
-from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath
+import sys
+from os.path import basename, isdir, isfile, join, realpath, relpath
 
 import bottle
-from click.testing import CliRunner
 
-from platformio import exception, util
-from platformio.commands.run import cli as cmd_run
+from platformio import fs, util
+from platformio.proc import where_is_program
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import load_project_ide_data
 
 
 class ProjectGenerator(object):
-
-    def __init__(self, project_dir, ide, env_name):
+    def __init__(self, project_dir, ide, boards):
+        self.config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
+        self.config.validate()
         self.project_dir = project_dir
-        self.ide = ide
-        self.env_name = env_name
-
-        self._tplvars = {}
-        self._gather_tplvars()
+        self.ide = str(ide)
+        self.env_name = str(self.get_best_envname(boards))
 
     @staticmethod
     def get_supported_ides():
-        tpls_dir = join(util.get_source_dir(), "ide", "tpls")
-        return sorted(
-            [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))])
+        tpls_dir = join(fs.get_source_dir(), "ide", "tpls")
+        return sorted([d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))])
 
-    @util.memoized()
-    def get_project_env(self):
-        data = {}
-        config = util.load_project_config(self.project_dir)
-        for section in config.sections():
-            if not section.startswith("env:"):
-                continue
-            if self.env_name != section[4:]:
-                continue
-            data = {"env_name": section[4:]}
-            for k, v in config.items(section):
-                data[k] = v
-        return data
+    def get_best_envname(self, boards=None):
+        envname = None
+        default_envs = self.config.default_envs()
+        if default_envs:
+            envname = default_envs[0]
+            if not boards:
+                return envname
 
-    def get_project_build_data(self):
-        data = {
-            "defines": [],
-            "includes": [],
-            "cxx_path": None,
-            "prog_path": None
+        for env in self.config.envs():
+            if not boards:
+                return env
+            if not envname:
+                envname = env
+            items = self.config.items(env=env, as_dict=True)
+            if "board" in items and items.get("board") in boards:
+                return env
+
+        return envname
+
+    def _load_tplvars(self):
+        tpl_vars = {
+            "config": self.config,
+            "systype": util.get_systype(),
+            "project_name": basename(self.project_dir),
+            "project_dir": self.project_dir,
+            "env_name": self.env_name,
+            "user_home_dir": realpath(fs.expanduser("~")),
+            "platformio_path": sys.argv[0]
+            if isfile(sys.argv[0])
+            else where_is_program("platformio"),
+            "env_path": os.getenv("PATH"),
+            "env_pathsep": os.pathsep,
         }
-        envdata = self.get_project_env()
-        if not envdata:
-            return data
 
-        result = CliRunner().invoke(cmd_run, [
-            "--project-dir", self.project_dir, "--environment",
-            envdata['env_name'], "--target", "idedata"
-        ])
+        # default env configuration
+        tpl_vars.update(self.config.items(env=self.env_name, as_dict=True))
+        # build data
+        tpl_vars.update(load_project_ide_data(self.project_dir, self.env_name) or {})
 
-        if result.exit_code != 0 and not isinstance(result.exception,
-                                                    exception.ReturnErrorCode):
-            raise result.exception
-        if '"includes":' not in result.output:
-            raise exception.PlatformioException(result.output)
+        with fs.cd(self.project_dir):
+            tpl_vars.update(
+                {
+                    "src_files": self.get_src_files(),
+                    "project_src_dir": self.config.get_optional_dir("src"),
+                    "project_lib_dir": self.config.get_optional_dir("lib"),
+                    "project_libdeps_dir": join(
+                        self.config.get_optional_dir("libdeps"), self.env_name
+                    ),
+                }
+            )
 
-        for line in result.output.split("\n"):
-            line = line.strip()
-            if line.startswith('{"') and line.endswith("}"):
-                data = json.loads(line)
-        return data
+        for key, value in tpl_vars.items():
+            if key.endswith(("_path", "_dir")):
+                tpl_vars[key] = fs.to_unix_path(value)
+        for key in ("includes", "src_files", "libsource_dirs"):
+            if key not in tpl_vars:
+                continue
+            tpl_vars[key] = [fs.to_unix_path(inc) for inc in tpl_vars[key]]
 
-    def get_project_name(self):
-        return basename(self.project_dir)
+        tpl_vars["to_unix_path"] = fs.to_unix_path
+        return tpl_vars
 
     def get_src_files(self):
         result = []
-        with util.cd(self.project_dir):
-            for root, _, files in os.walk(util.get_projectsrc_dir()):
+        with fs.cd(self.project_dir):
+            for root, _, files in os.walk(self.config.get_optional_dir("src")):
                 for f in files:
                     result.append(relpath(join(root, f)))
         return result
 
     def get_tpls(self):
         tpls = []
-        tpls_dir = join(util.get_source_dir(), "ide", "tpls", self.ide)
+        tpls_dir = join(fs.get_source_dir(), "ide", "tpls", self.ide)
         for root, _, files in os.walk(tpls_dir):
             for f in files:
                 if not f.endswith(".tpl"):
@@ -107,51 +122,25 @@ class ProjectGenerator(object):
         return tpls
 
     def generate(self):
+        tpl_vars = self._load_tplvars()
         for tpl_relpath, tpl_path in self.get_tpls():
             dst_dir = self.project_dir
             if tpl_relpath:
                 dst_dir = join(self.project_dir, tpl_relpath)
                 if not isdir(dst_dir):
                     os.makedirs(dst_dir)
-
             file_name = basename(tpl_path)[:-4]
-            self._merge_contents(
-                join(dst_dir, file_name),
-                self._render_tpl(tpl_path).encode("utf8"))
+            contents = self._render_tpl(tpl_path, tpl_vars)
+            self._merge_contents(join(dst_dir, file_name), contents)
 
-    def _render_tpl(self, tpl_path):
-        content = ""
-        with open(tpl_path) as f:
-            content = f.read()
-        return bottle.template(content, **self._tplvars)
+    @staticmethod
+    def _render_tpl(tpl_path, tpl_vars):
+        with codecs.open(tpl_path, "r", encoding="utf8") as fp:
+            return bottle.SimpleTemplate(fp.read()).render(**tpl_vars)
 
     @staticmethod
     def _merge_contents(dst_path, contents):
         if basename(dst_path) == ".gitignore" and isfile(dst_path):
             return
-        with open(dst_path, "w") as f:
-            f.write(contents)
-
-    def _gather_tplvars(self):
-        self._tplvars.update(self.get_project_env())
-        self._tplvars.update(self.get_project_build_data())
-        with util.cd(self.project_dir):
-            self._tplvars.update({
-                "project_name": self.get_project_name(),
-                "src_files": self.get_src_files(),
-                "user_home_dir": abspath(expanduser("~")),
-                "project_dir": self.project_dir,
-                "project_src_dir": util.get_projectsrc_dir(),
-                "project_lib_dir": util.get_projectlib_dir(),
-                "project_libdeps_dir": util.get_projectlibdeps_dir(),
-                "systype": util.get_systype(),
-                "platformio_path": self._fix_os_path(
-                    util.where_is_program("platformio")),
-                "env_pathsep": os.pathsep,
-                "env_path": self._fix_os_path(os.getenv("PATH"))
-            })  # yapf: disable
-
-    @staticmethod
-    def _fix_os_path(path):
-        return (re.sub(r"[\\]+", '\\' * 4, path)
-                if "windows" in util.get_systype() else path)
+        with codecs.open(dst_path, "w", encoding="utf8") as fp:
+            fp.write(contents)

platformio/ide/tpls/atom/.gcc-flags.json.tpl

@@ -1,8 +1,9 @@
+% _defines = " ".join(["-D%s" % d.replace(" ", "\\\\ ") for d in defines])
 {
-  "execPath": "{{ cxx_path.replace("\\", "/") }}",
-  "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }}",
-  "gccDefaultCppFlags": "-fsyntax-only {{! cxx_flags.replace(' -MMD ', ' ').replace('"', '\\"') }}",
+  "execPath": "{{ cxx_path }}",
+  "gccDefaultCFlags": "-fsyntax-only {{! to_unix_path(cc_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
+  "gccDefaultCppFlags": "-fsyntax-only {{! to_unix_path(cxx_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
   "gccErrorLimit": 15,
-  "gccIncludePaths": "{{ ','.join(includes).replace("\\", "/") }}",
+  "gccIncludePaths": "{{ ','.join(includes) }}",
   "gccSuppressWarnings": false
 }

platformio/ide/tpls/atom/.gitignore.tpl

@@ -1,5 +1,3 @@
 .pio
-.pioenvs
-.piolibdeps
 .clang_complete
 .gcc-flags.json

platformio/ide/tpls/clion/.gitignore.tpl

@@ -1,4 +1,3 @@
 .pio
-.pioenvs
-.piolibdeps
 CMakeListsPrivate.txt
+cmake-build-*/ 

platformio/ide/tpls/clion/.idea/clion.iml.tpl

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="CPP_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

platformio/ide/tpls/clion/.idea/misc.xml.tpl

@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="CMakeWorkspace" PROJECT_DIR="$PROJECT_DIR$" />
-  <component name="CidrRootsConfiguration">
-    <sourceRoots>
-      <file path="$PROJECT_DIR$/src" />
-    </sourceRoots>
-    <libraryRoots>
-      <file path="$PROJECT_DIR$/lib" />
-      <file path="$PROJECT_DIR$/.piolibdeps" />
-    </libraryRoots>
-    <excludeRoots>
-      <file path="$PROJECT_DIR$/.pio" />
-    </excludeRoots>
-    <excludeRoots>
-      <file path="$PROJECT_DIR$/.pioenvs" />
-    </excludeRoots>
-  </component>
-</project>