Merge branch 'release/v4.0.0'

.appveyor.yml

@@ -1,19 +1,20 @@
 build: off
 
 platform:
-  - x86
   - x64
 
 environment:
   matrix:
     - TOXENV: "py27"
+      PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P2_{build}
+
+    - TOXENV: "py36"
+      PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P3_{build}
 
 install:
   - cmd: git submodule update --init --recursive
   - cmd: SET PATH=C:\MinGW\bin;%PATH%
-  - if %PLATFORM% == x64 SET PATH=C:\Python27-x64;C:\Python27-x64\Scripts;%PATH%
+  - cmd: pip install --force-reinstall tox
-  - if %PLATFORM% == x86 SET PATH=C:\Python27;C:\Python27\Scripts;%PATH%
-  - cmd: pip install tox
 
 test_script:
   - cmd: tox

.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: https://platformio.org/donate

.isort.cfg

@@ -1,3 +1,3 @@
 [settings]
 line_length=79
-known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial
+known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,jsonrpc

.pylintrc

@@ -1,23 +1,12 @@
 [MESSAGES CONTROL]
-
+disable=
-# Only show warnings with the listed confidence levels. Leave empty to show
+    missing-docstring,
-# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+    ungrouped-imports,
-confidence=
+    invalid-name,
-
+    cyclic-import,
-# Enable the message, report, category or checker with the given id(s). You can
+    duplicate-code,
-# either give multiple identifier separated by comma (,) or put this option
+    superfluous-parens,
-# multiple time. See also the "--disable" option for examples.
+    too-few-public-methods,
-#enable=
+    useless-object-inheritance,
-
+    useless-import-alias,
-# Disable the message, report, category or checker with the given id(s). You
+    fixme
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once).You can also use "--disable=all" to
-# disable everything first and then reenable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use"--disable=all --enable=classes
-# --disable=W"
-# disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
-
-disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens

.travis.yml

@@ -6,14 +6,14 @@ matrix:
       sudo: false
       python: 2.7
       env: TOX_ENV=docs
-    - os: linux
-      sudo: false
-      python: 2.7
-      env: TOX_ENV=lint
     - os: linux
       sudo: required
       python: 2.7
-      env: TOX_ENV=py27
+      env: TOX_ENV=py27 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d)
+    - os: linux
+      sudo: required
+      python: 3.6
+      env: TOX_ENV=py36 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d)
     - os: osx
       language: generic
       env: TOX_ENV=skipexamples
@@ -21,10 +21,10 @@ matrix:
 install:
   - git submodule update --init --recursive
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl -fsSL https://bootstrap.pypa.io/get-pip.py | sudo python; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox==3.0.0"; else pip install -U tox; fi
+  - pip install -U tox
 
   # ChipKIT issue: install 32-bit support for GCC PIC32
-  - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi
 
 script:
   - tox -e $TOX_ENV

.vscode/settings.json

@@ -1,15 +0,0 @@
-{
-  "python.pythonPath": "${workspaceRoot}/.tox/develop/bin/python",
-  "python.formatting.provider": "yapf",
-  "files.exclude": {
-    "**/*.pyc": true,
-    "*.egg-info": true,
-    ".cache": true,
-    "build": true,
-    "dist": true
-  },
-  "editor.rulers": [79],
-  "restructuredtext.builtDocumentationPath": "${workspaceRoot}/docs/_build/html",
-  "restructuredtext.confPath": "${workspaceRoot}/docs",
-  "restructuredtext.linter.executablePath": "${workspaceRoot}/.tox/docs/bin/restructuredtext-lint"
-}

HISTORY.rst

@@ -1,6 +1,78 @@
 Release Notes
 =============
 
+.. _release_notes_4_0:
+
+PlatformIO 4.0
+--------------
+
+4.0.0 (2019-07-10)
+~~~~~~~~~~~~~~~~~~
+
+`Migration Guide from 3.0 to 4.0 <http://docs.platformio.org/page/migration.html>`__.
+
+* `PlatformIO Plus Goes Open Source <https://community.platformio.org/t/platformio-plus-goes-open-source-improving-embedded-development-community-worldwide/8240/4>`__
+
+  - Built-in `PIO Unified Debugger <http://docs.platformio.org/page/plus/debugging.html>`__
+  - Built-in `PIO Unit Testing <http://docs.platformio.org/page/plus/unit-testing.html>`__
+
+* **Project Configuration**
+
+  - New project configuration parser with a strict options typing (`API <https://github.com/platformio/platformio-core/blob/develop/platformio/project/options.py>`__)
+  - Unified workspace storage (`workspace_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#workspace-dir>`__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 <https://github.com/platformio/platformio-core/issues/1778>`_)
+  - Share common (global) options between project environments using `[env] <http://docs.platformio.org/page/projectconf/section_env.html#global-scope-env>`__ section (`issue #1643 <https://github.com/platformio/platformio-core/issues/1643>`_)
+  - Include external configuration files with `extra_configs <http://docs.platformio.org/page/projectconf/section_platformio.html#extra-configs>`__ option (`issue #1590 <https://github.com/platformio/platformio-core/issues/1590>`_)
+  - Custom project ``***_dir`` options declared in `platformio <http://docs.platformio.org/page/projectconf/section_platformio.html>`__ section have higher priority than `Environment variables <http://docs.platformio.org/page/envvars.html>`__
+  - Added support for Unix shell-style wildcards for `monitor_port <http://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-port>`__ option (`issue #2541 <https://github.com/platformio/platformio-core/issues/2541>`_)
+  - Added new `monitor_flags <http://docs.platformio.org/page/projectconf/section_env_monitor.html#monitor-flags>`__ option which allows passing extra flags and options to `platformio device monitor <http://docs.platformio.org/page/userguide/cmd_device.html#cmd-device-monitor>`__ command (`issue #2165 <https://github.com/platformio/platformio-core/issues/2165>`_)
+  - Added support for `PLATFORMIO_DEFAULT_ENVS <http://docs.platformio.org/page/envvars.html#envvar-PLATFORMIO_DEFAULT_ENVS>`__ system environment variable (`issue #1967 <https://github.com/platformio/platformio-core/issues/1967>`_)
+  - Added support for `shared_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#shared-dir>`__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a `PIO Remote <http://docs.platformio.org/page/plus/pio-remote.html>`__ machine
+
+* **Library Management**
+
+  - Switched to workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps``
+  - Save libraries passed to `platformio lib install <http://docs.platformio.org/page/userguide/lib/cmd_install.html>`__ command into the project dependency list (`lib_deps <http://docs.platformio.org/page/projectconf/section_env_library.html#lib-deps>`__) with a new ``--save`` flag (`issue #1028 <https://github.com/platformio/platformio-core/issues/1028>`_)
+  - Install all project dependencies declared via `lib_deps <http://docs.platformio.org/page/projectconf/section_env_library.html#lib-deps>`__ option using a simple `platformio lib install <http://docs.platformio.org/page/userguide/lib/cmd_install.html>`__ command (`issue #2147 <https://github.com/platformio/platformio-core/issues/2147>`_)
+  - Use isolated library dependency storage per project build environment (`issue #1696 <https://github.com/platformio/platformio-core/issues/1696>`_)
+  - Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry (`issue #1654 <https://github.com/platformio/platformio-core/issues/1654>`_)
+  - Override default source and include directories for a library via `library.json <http://docs.platformio.org/page/librarymanager/config.html>`__ manifest using ``includeDir`` and ``srcDir`` fields
+  - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 <https://github.com/platformio/platformio-core/issues/1252>`_)
+  - Fixed an issue when `lib_compat_mode = strict <http://docs.platformio.org/page/librarymanager/ldf.html#ldf-compat-mode>`__ does not ignore libraries incompatible with a project framework
+
+* **Build System**
+
+  - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs``
+  - Switch between `Build Configurations <http://docs.platformio.org/page/projectconf/build_configurations.html>`__ (``release`` and ``debug``) with a new project configuration option `build_type <http://docs.platformio.org/page/projectconf/section_env_build.html#build-type>`__
+  - Custom `platform_packages <http://docs.platformio.org/page/projectconf/section_env_general.html#platform>`__ per a build environment with an option to override default (`issue #1367 <https://github.com/platformio/platformio-core/issues/1367>`_)
+  - Print platform package details, such as version, VSC source and commit (`issue #2155 <https://github.com/platformio/platformio-core/issues/2155>`_)
+  - Control a number of parallel build jobs with a new `-j, --jobs <http://docs.platformio.org/page/userguide/cmd_run.html#cmdoption-platformio-run-j>`__ option
+  - Override default `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ with a custom using ``-c, --project-conf`` option for `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__, `platformio debug <http://docs.platformio.org/page/userguide/cmd_debug.html>`__, or `platformio test <http://docs.platformio.org/page/userguide/cmd_test.html>`__ commands (`issue #1913 <https://github.com/platformio/platformio-core/issues/1913>`_)
+  - Override default development platform upload command with a custom `upload_command <http://docs.platformio.org/page/projectconf/section_env_upload.html#upload-command>`__ (`issue #2599 <https://github.com/platformio/platformio-core/issues/2599>`_)
+  - Configure a shared folder for the derived files (objects, firmwares, ELFs) from a build system using `build_cache_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#build-cache-dir>`__ option (`issue #2674 <https://github.com/platformio/platformio-core/issues/2674>`_)
+  - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 <https://github.com/platformio/platformio-core/issues/2508>`_)
+
+* **Infrastructure**
+
+  - Python 3 support (`issue #895 <https://github.com/platformio/platformio-core/issues/895>`_)
+  - Significantly speedup back-end for PIO Home. It works super fast now!
+  - Added support for the latest Python "Click" package (CLI) (`issue #349 <https://github.com/platformio/platformio-core/issues/349>`_)
+  - Added options to override default locations used by PlatformIO Core (`core_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#core-dir>`__, `globallib_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#globallib-dir>`__, `platforms_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#platforms-dir>`__, `packages_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#packages-dir>`__, `cache_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#cache-dir>`__) (`issue #1615 <https://github.com/platformio/platformio-core/issues/1615>`_)
+  - Removed line-buffering from `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__ command which was leading to omitting progress bar from upload tools (`issue #856 <https://github.com/platformio/platformio-core/issues/856>`_)
+  - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #143 <https://github.com/platformio/platformio-core/issues/143>`_, `issue #1342 <https://github.com/platformio/platformio-core/issues/1342>`_, `issue #1959 <https://github.com/platformio/platformio-core/issues/1959>`_, `issue #2100 <https://github.com/platformio/platformio-core/issues/2100>`_)
+
+* **Integration**
+
+  - Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file
+  - Fixed an issue with hardcoded C standard version when generating project for CLion IDE (`issue #2527 <https://github.com/platformio/platformio-core/issues/2527>`_)
+  - Fixed an issue with Project Generator when an include path search order is inconsistent to what passed to the compiler (`issue #2509 <https://github.com/platformio/platformio-core/issues/2509>`_)
+  - Fixed an issue when generating invalid "Eclipse CDT Cross GCC Built-in Compiler Settings" if a custom `PLATFORMIO_CORE_DIR <http://docs.platformio.org/page/envvars.html#envvar-PLATFORMIO_CORE_DIR>`__ is used (`issue #806 <https://github.com/platformio/platformio-core/issues/806>`_)
+
+* **Miscellaneous**
+
+  - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead
+  - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules <http://docs.platformio.org/page/faq.html#platformio-udev-rules>`__ (`issue #2442 <https://github.com/platformio/platformio-core/issues/2442>`_)
+  - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 <https://github.com/platformio/platformio-core/issues/2559>`_)
+
 PlatformIO 3.0
 --------------
 

Makefile

@@ -1,4 +1,3 @@
-
 lint:
 	pylint --rcfile=./.pylintrc ./platformio
 
@@ -10,7 +9,7 @@ yapf:
 	yapf --recursive --in-place platformio/
 
 test:
-	py.test -v -s -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py
+	py.test --verbose --capture=no --exitfirst -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py
 
 before-commit: isort yapf lint test
 
@@ -23,4 +22,9 @@ clean: clean-docs
 	rm -rf .cache
 	rm -rf build
 	rm -rf htmlcov
-	rm -f .coverage
+	rm -f .coverage
+
+profile:
+	# Usage $ > make PIOARGS="boards" profile
+	python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS}
+	snakeviz .tox/.tmp/cprofile.prof

README.rst

@@ -78,6 +78,7 @@ Registry
 Development Platforms
 ---------------------
 
+* `Aceinna IMU <https://platformio.org/platforms/aceinna_imu?utm_source=github&utm_medium=core>`_
 * `Atmel AVR <https://platformio.org/platforms/atmelavr?utm_source=github&utm_medium=core>`_
 * `Atmel SAM <https://platformio.org/platforms/atmelsam?utm_source=github&utm_medium=core>`_
 * `Espressif 32 <https://platformio.org/platforms/espressif32?utm_source=github&utm_medium=core>`_
@@ -86,6 +87,7 @@ Development Platforms
 * `Infineon XMC <https://platformio.org/platforms/infineonxmc?utm_source=github&utm_medium=core>`_
 * `Intel ARC32 <https://platformio.org/platforms/intel_arc32?utm_source=github&utm_medium=core>`_
 * `Intel MCS-51 (8051) <https://platformio.org/platforms/intel_mcs51?utm_source=github&utm_medium=core>`_
+* `Kendryte K210 <https://platformio.org/platforms/kendryte210?utm_source=github&utm_medium=core>`_
 * `Lattice iCE40 <https://platformio.org/platforms/lattice_ice40?utm_source=github&utm_medium=core>`_
 * `Maxim 32 <https://platformio.org/platforms/maxim32?utm_source=github&utm_medium=core>`_
 * `Microchip PIC32 <https://platformio.org/platforms/microchippic32?utm_source=github&utm_medium=core>`_
@@ -93,9 +95,11 @@ Development Platforms
 * `Nordic nRF52 <https://platformio.org/platforms/nordicnrf52?utm_source=github&utm_medium=core>`_
 * `NXP LPC <https://platformio.org/platforms/nxplpc?utm_source=github&utm_medium=core>`_
 * `RISC-V <https://platformio.org/platforms/riscv?utm_source=github&utm_medium=core>`_
+* `RISC-V GAP <https://platformio.org/platforms/riscv_gap?utm_source=github&utm_medium=core>`_
 * `Samsung ARTIK <https://platformio.org/platforms/samsung_artik?utm_source=github&utm_medium=core>`_
 * `Silicon Labs EFM32 <https://platformio.org/platforms/siliconlabsefm32?utm_source=github&utm_medium=core>`_
 * `ST STM32 <https://platformio.org/platforms/ststm32?utm_source=github&utm_medium=core>`_
+* `ST STM8 <https://platformio.org/platforms/ststm8?utm_source=github&utm_medium=core>`_
 * `Teensy <https://platformio.org/platforms/teensy?utm_source=github&utm_medium=core>`_
 * `TI MSP430 <https://platformio.org/platforms/timsp430?utm_source=github&utm_medium=core>`_
 * `TI Tiva <https://platformio.org/platforms/titiva?utm_source=github&utm_medium=core>`_
@@ -111,8 +115,11 @@ Frameworks
 * `ESP-IDF <https://platformio.org/frameworks/espidf?utm_source=github&utm_medium=core>`_
 * `ESP8266 Non-OS SDK <https://platformio.org/frameworks/esp8266-nonos-sdk?utm_source=github&utm_medium=core>`_
 * `ESP8266 RTOS SDK <https://platformio.org/frameworks/esp8266-rtos-sdk?utm_source=github&utm_medium=core>`_
+* `Freedom E SDK <https://platformio.org/frameworks/freedom-e-sdk?utm_source=github&utm_medium=core>`_
+* `Kendryte Standalone SDK <https://platformio.org/frameworks/kendryte-standalone-sdk?utm_source=github&utm_medium=core>`_
 * `libOpenCM3 <https://platformio.org/frameworks/libopencm3?utm_source=github&utm_medium=core>`_
 * `mbed <https://platformio.org/frameworks/mbed?utm_source=github&utm_medium=core>`_
+* `PULP OS <https://platformio.org/frameworks/pulp-os?utm_source=github&utm_medium=core>`_
 * `Pumbaa <https://platformio.org/frameworks/pumbaa?utm_source=github&utm_medium=core>`_
 * `Simba <https://platformio.org/frameworks/simba?utm_source=github&utm_medium=core>`_
 * `SPL <https://platformio.org/frameworks/spl?utm_source=github&utm_medium=core>`_

platformio/__init__.py

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
+VERSION = (4, 0, 0)
-
-VERSION = (3, 6, 7)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -33,10 +31,3 @@ __license__ = "Apache Software License"
 __copyright__ = "Copyright 2014-present PlatformIO"
 
 __apiurl__ = "https://api.platformio.org"
-
-if sys.version_info < (2, 7, 0) or sys.version_info >= (3, 0, 0):
-    msg = ("PlatformIO Core v%s does not run under Python version %s.\n"
-           "Minimum supported version is 2.7, please upgrade Python.\n"
-           "Python 3 is not yet supported.\n")
-    sys.stderr.write(msg % (__version__, sys.version))
-    sys.exit(1)

platformio/__main__.py

@@ -14,60 +14,22 @@
 
 import os
 import sys
-from os.path import join
-from platform import system
 from traceback import format_exc
 
 import click
 
-from platformio import __version__, exception, maintenance
+from platformio import __version__, exception, maintenance, util
-from platformio.util import get_source_dir
+from platformio.commands import PlatformioCLI
+from platformio.compat import CYGWIN
 
 
-class PlatformioCLI(click.MultiCommand):  # pylint: disable=R0904
+@click.command(cls=PlatformioCLI,
-
+               context_settings=dict(help_option_names=["-h", "--help"]))
-    def list_commands(self, ctx):
-        cmds = []
-        for filename in os.listdir(join(get_source_dir(), "commands")):
-            if filename.startswith("__init__"):
-                continue
-            if filename.endswith(".py"):
-                cmds.append(filename[:-3])
-        cmds.sort()
-        return cmds
-
-    def get_command(self, ctx, cmd_name):
-        mod = None
-        try:
-            mod = __import__("platformio.commands." + cmd_name, None, None,
-                             ["cli"])
-        except ImportError:
-            try:
-                return self._handle_obsolate_command(cmd_name)
-            except AttributeError:
-                raise click.UsageError('No such command "%s"' % cmd_name, ctx)
-        return mod.cli
-
-    @staticmethod
-    def _handle_obsolate_command(name):
-        if name == "platforms":
-            from platformio.commands import platform
-            return platform.cli
-        elif name == "serialports":
-            from platformio.commands import device
-            return device.cli
-        raise AttributeError()
-
-
-@click.command(
-    cls=PlatformioCLI,
-    context_settings=dict(help_option_names=["-h", "--help"]))
 @click.version_option(__version__, prog_name="PlatformIO")
-@click.option(
+@click.option("--force",
-    "--force",
+              "-f",
-    "-f",
+              is_flag=True,
-    is_flag=True,
+              help="Force to accept any confirmation prompts.")
-    help="Force to accept any confirmation prompts.")
 @click.option("--caller", "-c", help="Caller ID (service).")
 @click.pass_context
 def cli(ctx, force, caller):
@@ -80,8 +42,9 @@ def process_result(ctx, result, force, caller):  # pylint: disable=W0613
     maintenance.on_platformio_end(ctx, result)
 
 
+@util.memoized()
 def configure():
-    if "cygwin" in system().lower():
+    if CYGWIN:
         raise exception.CygwinEnvDetected()
 
     # https://urllib3.readthedocs.org
@@ -114,10 +77,17 @@ def configure():
     click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs)
 
 
-def main():
+def main(argv=None):
+    exit_code = 0
+    prev_sys_argv = sys.argv[:]
+    if argv:
+        assert isinstance(argv, list)
+        sys.argv = argv
     try:
         configure()
         cli(None, None, None)
+    except SystemExit:
+        pass
     except Exception as e:  # pylint: disable=broad-except
         if not isinstance(e, exception.ReturnErrorCode):
             maintenance.on_platformio_exception(e)
@@ -143,13 +113,13 @@ An unexpected error occurred. Further steps:
 ============================================================
 """
             click.secho(error_str, fg="red", err=True)
-        return int(str(e)) if str(e).isdigit() else 1
+        exit_code = int(str(e)) if str(e).isdigit() else 1
-    return 0
+    sys.argv = prev_sys_argv
+    return exit_code
 
 
 def debug_gdb_main():
-    sys.argv = [sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]
+    return main([sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:])
-    return main()
 
 
 if __name__ == "__main__":

platformio/app.py

@@ -14,10 +14,8 @@
 
 import codecs
 import hashlib
-import json
 import os
 import uuid
-from copy import deepcopy
 from os import environ, getenv, listdir, remove
 from os.path import abspath, dirname, expanduser, isdir, isfile, join
 from time import time
@@ -25,6 +23,24 @@ from time import time
 import requests
 
 from platformio import exception, lockfile, util
+from platformio.compat import (WINDOWS, dump_json_to_unicode,
+                               hashlib_encode_data)
+from platformio.proc import is_ci
+from platformio.project.helpers import (get_project_cache_dir,
+                                        get_project_core_dir)
+
+
+def get_default_projects_dir():
+    docs_dir = join(expanduser("~"), "Documents")
+    try:
+        assert WINDOWS
+        import ctypes.wintypes
+        buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH)
+        ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf)
+        docs_dir = buf.value
+    except:  # pylint: disable=bare-except
+        pass
+    return join(docs_dir, "PlatformIO", "Projects")
 
 
 def projects_dir_validate(projects_dir):
@@ -74,7 +90,7 @@ DEFAULT_SETTINGS = {
     },
     "projects_dir": {
         "description": "Default location for PlatformIO projects (PIO Home)",
-        "value": join(expanduser("~"), "Documents", "PlatformIO", "Projects"),
+        "value": get_default_projects_dir(),
         "validator": projects_dir_validate
     },
 }
@@ -88,28 +104,29 @@ class State(object):
         self.path = path
         self.lock = lock
         if not self.path:
-            self.path = join(util.get_home_dir(), "appstate.json")
+            self.path = join(get_project_core_dir(), "appstate.json")
-        self._state = {}
+        self._storage = {}
-        self._prev_state = {}
         self._lockfile = None
+        self.modified = False
 
     def __enter__(self):
         try:
             self._lock_state_file()
             if isfile(self.path):
-                self._state = util.load_json(self.path)
+                self._storage = util.load_json(self.path)
-        except exception.PlatformioException:
+            assert isinstance(self._storage, dict)
-            self._state = {}
+        except (AssertionError, ValueError, UnicodeDecodeError,
-        self._prev_state = deepcopy(self._state)
+                exception.InvalidJSONFile):
-        return self._state
+            self._storage = {}
+        return self
 
     def __exit__(self, type_, value, traceback):
-        if self._prev_state != self._state:
+        if self.modified:
             try:
-                with codecs.open(self.path, "w", encoding="utf8") as fp:
+                with open(self.path, "w") as fp:
-                    json.dump(self._state, fp)
+                    fp.write(dump_json_to_unicode(self._storage))
             except IOError:
-                raise exception.HomeDirPermissionsError(util.get_home_dir())
+                raise exception.HomeDirPermissionsError(get_project_core_dir())
         self._unlock_state_file()
 
     def _lock_state_file(self):
@@ -128,6 +145,32 @@ class State(object):
     def __del__(self):
         self._unlock_state_file()
 
+    # Dictionary Proxy
+
+    def as_dict(self):
+        return self._storage
+
+    def get(self, key, default=True):
+        return self._storage.get(key, default)
+
+    def update(self, *args, **kwargs):
+        self.modified = True
+        return self._storage.update(*args, **kwargs)
+
+    def __getitem__(self, key):
+        return self._storage[key]
+
+    def __setitem__(self, key, value):
+        self.modified = True
+        self._storage[key] = value
+
+    def __delitem__(self, key):
+        self.modified = True
+        del self._storage[key]
+
+    def __contains__(self, item):
+        return item in self._storage
+
 
 class ContentCache(object):
 
@@ -136,7 +179,7 @@ class ContentCache(object):
         self._db_path = None
         self._lockfile = None
 
-        self.cache_dir = cache_dir or util.get_cache_dir()
+        self.cache_dir = cache_dir or get_project_cache_dir()
         self._db_path = join(self.cache_dir, "db.data")
 
     def __enter__(self):
@@ -163,14 +206,16 @@ class ContentCache(object):
         return True
 
     def get_cache_path(self, key):
+        key = str(key)
         assert len(key) > 3
         return join(self.cache_dir, key[-2:], key)
 
     @staticmethod
     def key_from_args(*args):
         h = hashlib.md5()
-        for data in args:
+        for arg in args:
-            h.update(str(data))
+            if arg:
+                h.update(hashlib_encode_data(arg))
         return h.hexdigest()
 
     def get(self, key):
@@ -191,7 +236,7 @@ class ContentCache(object):
         if not isdir(self.cache_dir):
             os.makedirs(self.cache_dir)
         tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
-        assert valid.endswith(tuple(tdmap.keys()))
+        assert valid.endswith(tuple(tdmap))
         expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1]))
 
         if not self._lock_dbindex():
@@ -230,10 +275,13 @@ class ContentCache(object):
                 if "=" not in line:
                     continue
                 expire, path = line.split("=")
-                if time() < int(expire) and isfile(path) and \
+                try:
-                        path not in paths_for_delete:
+                    if time() < int(expire) and isfile(path) and \
-                    newlines.append(line)
+                            path not in paths_for_delete:
-                    continue
+                        newlines.append(line)
+                        continue
+                except ValueError:
+                    pass
                 found = True
                 if isfile(path):
                     try:
@@ -280,19 +328,20 @@ def sanitize_setting(name, value):
 
 
 def get_state_item(name, default=None):
-    with State() as data:
+    with State() as state:
-        return data.get(name, default)
+        return state.get(name, default)
 
 
 def set_state_item(name, value):
-    with State(lock=True) as data:
+    with State(lock=True) as state:
-        data[name] = value
+        state[name] = value
+        state.modified = True
 
 
 def delete_state_item(name):
-    with State(lock=True) as data:
+    with State(lock=True) as state:
-        if name in data:
+        if name in state:
-            del data[name]
+            del state[name]
 
 
 def get_setting(name):
@@ -300,24 +349,25 @@ def get_setting(name):
     if _env_name in environ:
         return sanitize_setting(name, getenv(_env_name))
 
-    with State() as data:
+    with State() as state:
-        if "settings" in data and name in data['settings']:
+        if "settings" in state and name in state['settings']:
-            return data['settings'][name]
+            return state['settings'][name]
 
     return DEFAULT_SETTINGS[name]['value']
 
 
 def set_setting(name, value):
-    with State(lock=True) as data:
+    with State(lock=True) as state:
-        if "settings" not in data:
+        if "settings" not in state:
-            data['settings'] = {}
+            state['settings'] = {}
-        data['settings'][name] = sanitize_setting(name, value)
+        state['settings'][name] = sanitize_setting(name, value)
+        state.modified = True
 
 
 def reset_settings():
-    with State(lock=True) as data:
+    with State(lock=True) as state:
-        if "settings" in data:
+        if "settings" in state:
-            del data['settings']
+            del state['settings']
 
 
 def get_session_var(name, default=None):
@@ -332,28 +382,29 @@ def set_session_var(name, value):
 def is_disabled_progressbar():
     return any([
         get_session_var("force_option"),
-        util.is_ci(),
+        is_ci(),
         getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true"
     ])
 
 
 def get_cid():
     cid = get_state_item("cid")
-    if not cid:
+    if cid:
-        _uid = None
+        return cid
-        if getenv("C9_UID"):
+    uid = None
-            _uid = getenv("C9_UID")
+    if getenv("C9_UID"):
-        elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")):
+        uid = getenv("C9_UID")
-            try:
+    elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")):
-                _uid = requests.get("{api}/user?token={token}".format(
+        try:
-                    api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
+            uid = requests.get("{api}/user?token={token}".format(
-                    token=getenv("USER_TOKEN"))).json().get("id")
+                api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
-            except:  # pylint: disable=bare-except
+                token=getenv("USER_TOKEN"))).json().get("id")
-                pass
+        except:  # pylint: disable=bare-except
-        cid = str(
+            pass
-            uuid.UUID(
+    if not uid:
-                bytes=hashlib.md5(str(
+        uid = uuid.getnode()
-                    _uid if _uid else uuid.getnode())).digest()))
+    cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest())
-        if "windows" in util.get_systype() or os.getuid() > 0:
+    cid = str(cid)
-            set_state_item("cid", cid)
+    if WINDOWS or os.getuid() > 0:  # yapf: disable pylint: disable=no-member
+        set_state_item("cid", cid)
     return cid

platformio/builder/main.py

@@ -12,108 +12,74 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import base64
+from os import environ, makedirs
-import json
+from os.path import isdir, join
-import sys
-from os import environ
-from os.path import expanduser, join
 from time import time
 
-from SCons.Script import (ARGUMENTS, COMMAND_LINE_TARGETS, DEFAULT_TARGETS,
+import click
-                          AllowSubstExceptions, AlwaysBuild, Default,
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
-                          DefaultEnvironment, Variables)
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
+from SCons.Script import DEFAULT_TARGETS  # pylint: disable=import-error
+from SCons.Script import AllowSubstExceptions  # pylint: disable=import-error
+from SCons.Script import AlwaysBuild  # pylint: disable=import-error
+from SCons.Script import Default  # pylint: disable=import-error
+from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
+from SCons.Script import Import  # pylint: disable=import-error
+from SCons.Script import Variables  # pylint: disable=import-error
 
 from platformio import util
+from platformio.compat import PY2, dump_json_to_unicode
+from platformio.managers.platform import PlatformBase
+from platformio.proc import get_pythonexe_path
+from platformio.project import helpers as project_helpers
 
 AllowSubstExceptions(NameError)
 
-# allow common variables from INI file
+# append CLI arguments to build environment
-commonvars = Variables(None)
+clivars = Variables(None)
-commonvars.AddVariables(
+clivars.AddVariables(
     ("PLATFORM_MANIFEST",),
     ("BUILD_SCRIPT",),
-    ("EXTRA_SCRIPTS",),
+    ("PROJECT_CONFIG",),
     ("PIOENV",),
-    ("PIOTEST",),
+    ("PIOTEST_RUNNING_NAME",),
-    ("PIOPLATFORM",),
+    ("UPLOAD_PORT",)
-    ("PIOFRAMEWORK",),
-
-    # build options
-    ("BUILD_FLAGS",),
-    ("SRC_BUILD_FLAGS",),
-    ("BUILD_UNFLAGS",),
-    ("SRC_FILTER",),
-
-    # library options
-    ("LIB_LDF_MODE",),
-    ("LIB_COMPAT_MODE",),
-    ("LIB_DEPS",),
-    ("LIB_IGNORE",),
-    ("LIB_EXTRA_DIRS",),
-    ("LIB_ARCHIVE",),
-
-    # board options
-    ("BOARD",),
-    # deprecated options, use board_{object.path} instead
-    ("BOARD_MCU",),
-    ("BOARD_F_CPU",),
-    ("BOARD_F_FLASH",),
-    ("BOARD_FLASH_MODE",),
-    # end of deprecated options
-
-    # upload options
-    ("UPLOAD_PORT",),
-    ("UPLOAD_PROTOCOL",),
-    ("UPLOAD_SPEED",),
-    ("UPLOAD_FLAGS",),
-    ("UPLOAD_RESETMETHOD",),
-
-    # test options
-    ("TEST_BUILD_PROJECT_SRC",),
-
-    # debug options
-    ("DEBUG_TOOL",),
-    ("DEBUG_SVD_PATH",),
-
 )  # yapf: disable
 
-MULTILINE_VARS = [
-    "EXTRA_SCRIPTS", "PIOFRAMEWORK", "BUILD_FLAGS", "SRC_BUILD_FLAGS",
-    "BUILD_UNFLAGS", "UPLOAD_FLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE",
-    "LIB_EXTRA_DIRS"
-]
-
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
         "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform",
-        "piowinhooks", "piolib", "pioupload", "piomisc", "pioide"
+        "pioproject", "piowinhooks", "piolib", "pioupload", "piomisc", "pioide"
-    ],  # yapf: disable
+    ],
     toolpath=[join(util.get_source_dir(), "builder", "tools")],
-    variables=commonvars,
+    variables=clivars,
 
     # Propagating External Environment
-    PIOVARIABLES=commonvars.keys(),
     ENV=environ,
     UNIX_TIME=int(time()),
-    PIOHOME_DIR=util.get_home_dir(),
+    PROJECT_DIR=project_helpers.get_project_dir(),
-    PROJECT_DIR=util.get_project_dir(),
+    PROJECTCORE_DIR=project_helpers.get_project_core_dir(),
-    PROJECTINCLUDE_DIR=util.get_projectinclude_dir(),
+    PROJECTPACKAGES_DIR=project_helpers.get_project_packages_dir(),
-    PROJECTSRC_DIR=util.get_projectsrc_dir(),
+    PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(),
-    PROJECTTEST_DIR=util.get_projecttest_dir(),
+    PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(),
-    PROJECTDATA_DIR=util.get_projectdata_dir(),
+    PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(),
-    PROJECTBUILD_DIR=util.get_projectbuild_dir(),
+    PROJECTSRC_DIR=project_helpers.get_project_src_dir(),
+    PROJECTTEST_DIR=project_helpers.get_project_test_dir(),
+    PROJECTDATA_DIR=project_helpers.get_project_data_dir(),
+    PROJECTBUILD_DIR=project_helpers.get_project_build_dir(),
+    BUILDCACHE_DIR=project_helpers.get_project_optional_dir("build_cache_dir"),
     BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"),
     BUILDSRC_DIR=join("$BUILD_DIR", "src"),
     BUILDTEST_DIR=join("$BUILD_DIR", "test"),
     LIBPATH=["$BUILD_DIR"],
     LIBSOURCE_DIRS=[
-        util.get_projectlib_dir(),
+        project_helpers.get_project_lib_dir(),
-        util.get_projectlibdeps_dir(),
+        join("$PROJECTLIBDEPS_DIR", "$PIOENV"),
-        join("$PIOHOME_DIR", "lib")
+        project_helpers.get_project_global_lib_dir()
     ],
     PROGNAME="program",
     PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
-    PYTHONEXE=util.get_pythonexe_path())
+    PYTHONEXE=get_pythonexe_path())
 
 if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     DEFAULT_ENV_OPTIONS['ARCOMSTR'] = "Archiving $TARGET"
@@ -124,12 +90,21 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
 
 env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
 
-# decode common variables
+# Load variables from CLI
-for k in commonvars.keys():
+env.Replace(
-    if k in env:
+    **{
-        env[k] = base64.b64decode(env[k])
+        key: PlatformBase.decode_scons_arg(env[key])
-        if k in MULTILINE_VARS:
+        for key in list(clivars.keys()) if key in env
-            env[k] = util.parse_conf_multi_values(env[k])
+    })
+
+if env.subst("$BUILDCACHE_DIR"):
+    if not isdir(env.subst("$BUILDCACHE_DIR")):
+        makedirs(env.subst("$BUILDCACHE_DIR"))
+    env.CacheDir("$BUILDCACHE_DIR")
+
+if int(ARGUMENTS.get("ISATTY", 0)):
+    # pylint: disable=protected-access
+    click._compat.isatty = lambda stream: True
 
 if env.GetOption('clean'):
     env.PioClean(env.subst("$BUILD_DIR"))
@@ -137,31 +112,13 @@ if env.GetOption('clean'):
 elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     print("Verbose mode can be enabled via `-v, --verbose` option")
 
-# Handle custom variables from system environment
+env.LoadProjectOptions()
-for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPTS",
+env.LoadPioPlatform()
-            "UPLOAD_PORT", "UPLOAD_FLAGS", "LIB_EXTRA_DIRS"):
-    k = "PLATFORMIO_%s" % var
-    if k not in environ:
-        continue
-    if var in ("UPLOAD_PORT", ):
-        env[var] = environ.get(k)
-        continue
-    env.Append(**{var: util.parse_conf_multi_values(environ.get(k))})
-
-# Configure extra library source directories for LDF
-if util.get_project_optional_dir("lib_extra_dirs"):
-    env.Prepend(
-        LIBSOURCE_DIRS=util.parse_conf_multi_values(
-            util.get_project_optional_dir("lib_extra_dirs")))
-env.Prepend(LIBSOURCE_DIRS=env.get("LIB_EXTRA_DIRS", []))
-env['LIBSOURCE_DIRS'] = [
-    expanduser(d) if d.startswith("~") else d for d in env['LIBSOURCE_DIRS']
-]
-
-env.LoadPioPlatform(commonvars)
 
 env.SConscriptChdir(0)
-env.SConsignFile(join("$PROJECTBUILD_DIR", ".sconsign.dblite"))
+env.SConsignFile(
+    join("$PROJECTBUILD_DIR",
+         ".sconsign.dblite" if PY2 else ".sconsign3.dblite"))
 
 for item in env.GetExtraScripts("pre"):
     env.SConscript(item, exports="env")
@@ -170,6 +127,8 @@ env.SConscript("$BUILD_SCRIPT")
 
 if "UPLOAD_FLAGS" in env:
     env.Prepend(UPLOADERFLAGS=["$UPLOAD_FLAGS"])
+if env.GetProjectOption("upload_command"):
+    env.Replace(UPLOADCMD=env.GetProjectOption("upload_command"))
 
 for item in env.GetExtraScripts("post"):
     env.SConscript(item, exports="env")
@@ -192,7 +151,6 @@ env.AddPreAction(
                       "Configuring upload protocol..."))
 
 AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS))
-AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS))
 AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS))
 
 ##############################################################################
@@ -202,14 +160,8 @@ if "envdump" in COMMAND_LINE_TARGETS:
     env.Exit(0)
 
 if "idedata" in COMMAND_LINE_TARGETS:
-    try:
+    Import("projenv")
-        print("\n%s\n" % util.path_to_unicode(
+    print("\n%s\n" % dump_json_to_unicode(
-            json.dumps(env.DumpIDEData(), ensure_ascii=False)))
+        env.DumpIDEData(projenv)  # pylint: disable=undefined-variable
-        env.Exit(0)
+    ))
-    except UnicodeDecodeError:
+    env.Exit(0)
-        sys.stderr.write(
-            "\nUnicodeDecodeError: Non-ASCII characters found in build "
-            "environment\n"
-            "See explanation in FAQ > Troubleshooting > Building\n"
-            "https://docs.platformio.org/page/faq.html\n\n")
-        env.Exit(1)

platformio/builder/tools/pioide.py

@@ -18,17 +18,18 @@ from glob import glob
 from os import environ
 from os.path import abspath, isfile, join
 
-from SCons.Defaults import processDefines
+from SCons.Defaults import processDefines  # pylint: disable=import-error
 
-from platformio import util
+from platformio.compat import glob_escape
 from platformio.managers.core import get_core_package_dir
+from platformio.proc import exec_command, where_is_program
 
 
-def _dump_includes(env):
+def _dump_includes(env, projenv):
     includes = []
 
-    for item in env.get("CPPPATH", []):
+    for item in projenv.get("CPPPATH", []):
-        includes.append(env.subst(item))
+        includes.append(projenv.subst(item))
 
     # installed libs
     for lb in env.GetLibBuilders():
@@ -39,7 +40,7 @@ def _dump_includes(env):
     for name in p.get_installed_packages():
         if p.get_package_type(name) != "toolchain":
             continue
-        toolchain_dir = util.glob_escape(p.get_package_dir(name))
+        toolchain_dir = glob_escape(p.get_package_dir(name))
         toolchain_incglobs = [
             join(toolchain_dir, "*", "include*"),
             join(toolchain_dir, "*", "include", "c++", "*"),
@@ -71,8 +72,9 @@ def _get_gcc_defines(env):
     try:
         sysenv = environ.copy()
         sysenv['PATH'] = str(env['ENV']['PATH'])
-        result = util.exec_command(
+        result = exec_command("echo | %s -dM -E -" % env.subst("$CC"),
-            "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True)
+                              env=sysenv,
+                              shell=True)
     except OSError:
         return items
     if result['returncode'] != 0:
@@ -112,7 +114,7 @@ def _dump_defines(env):
 
 
 def _get_svd_path(env):
-    svd_path = env.subst("$DEBUG_SVD_PATH")
+    svd_path = env.GetProjectOption("debug_svd_path")
     if svd_path:
         return abspath(svd_path)
 
@@ -133,27 +135,26 @@ def _get_svd_path(env):
     return None
 
 
-def DumpIDEData(env):
+def DumpIDEData(env, projenv):
     LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS"
     LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS"
 
     data = {
-        "libsource_dirs":
+        "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()],
-        [env.subst(l) for l in env.get("LIBSOURCE_DIRS", [])],
         "defines":
         _dump_defines(env),
         "includes":
-        _dump_includes(env),
+        _dump_includes(env, projenv),
         "cc_flags":
         env.subst(LINTCCOM),
         "cxx_flags":
         env.subst(LINTCXXCOM),
         "cc_path":
-        util.where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
+        where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
         "cxx_path":
-        util.where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
+        where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
         "gdb_path":
-        util.where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
+        where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
         "prog_path":
         env.subst("$PROG_PATH"),
         "flash_extra_images": [{

platformio/builder/tools/piolib.py

@@ -17,41 +17,46 @@
 
 from __future__ import absolute_import
 
+import codecs
 import hashlib
 import os
 import re
 import sys
-from glob import glob
+from os.path import (basename, commonprefix, expanduser, isdir, isfile, join,
-from os.path import (basename, commonprefix, dirname, isdir, isfile, join,
                      realpath, sep)
 
-import SCons.Scanner
+import click
-from SCons.Script import ARGUMENTS, COMMAND_LINE_TARGETS, DefaultEnvironment
+import SCons.Scanner  # pylint: disable=import-error
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
+from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import exception, util
 from platformio.builder.tools import platformio as piotool
+from platformio.compat import (WINDOWS, get_file_contents, hashlib_encode_data,
+                               string_types)
 from platformio.managers.lib import LibraryManager
-from platformio.managers.package import PackageManager
 
 
 class LibBuilderFactory(object):
 
     @staticmethod
-    def new(env, path, verbose=False):
+    def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))):
         clsname = "UnknownLibBuilder"
         if isfile(join(path, "library.json")):
             clsname = "PlatformIOLibBuilder"
         else:
             used_frameworks = LibBuilderFactory.get_used_frameworks(env, path)
-            common_frameworks = (
+            common_frameworks = (set(env.get("PIOFRAMEWORK", []))
-                set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks))
+                                 & set(used_frameworks))
             if common_frameworks:
                 clsname = "%sLibBuilder" % list(common_frameworks)[0].title()
             elif used_frameworks:
                 clsname = "%sLibBuilder" % used_frameworks[0].title()
 
-        obj = getattr(sys.modules[__name__], clsname)(
+        obj = getattr(sys.modules[__name__], clsname)(env,
-            env, path, verbose=verbose)
+                                                      path,
+                                                      verbose=verbose)
         assert isinstance(obj, LibBuilderBase)
         return obj
 
@@ -65,8 +70,8 @@ class LibBuilderFactory(object):
         if isfile(join(path, "module.json")):
             return ["mbed"]
 
-        include_re = re.compile(
+        include_re = re.compile(r'^#include\s+(<|")(Arduino|mbed)\.h(<|")',
-            r'^#include\s+(<|")(Arduino|mbed)\.h(<|")', flags=re.MULTILINE)
+                                flags=re.MULTILINE)
 
         # check source files
         for root, _, files in os.walk(path, followlinks=True):
@@ -76,19 +81,18 @@ class LibBuilderFactory(object):
                 if not env.IsFileWithExt(
                         fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT):
                     continue
-                with open(join(root, fname)) as f:
+                content = get_file_contents(join(root, fname))
-                    content = f.read()
+                if not content:
-                    if "Arduino.h" in content and include_re.search(content):
+                    continue
-                        return ["arduino"]
+                if "Arduino.h" in content and include_re.search(content):
-                    elif "mbed.h" in content and include_re.search(content):
+                    return ["arduino"]
-                        return ["mbed"]
+                if "mbed.h" in content and include_re.search(content):
+                    return ["mbed"]
         return []
 
 
 class LibBuilderBase(object):
 
-    IS_WINDOWS = "windows" in util.get_systype()
-
     LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"]
     LDF_MODE_DEFAULT = "chain"
 
@@ -131,9 +135,11 @@ class LibBuilderBase(object):
     def __contains__(self, path):
         p1 = self.path
         p2 = path
-        if self.IS_WINDOWS:
+        if WINDOWS:
             p1 = p1.lower()
             p2 = p2.lower()
+        if p1 == p2:
+            return True
         return commonprefix((p1 + sep, p2)) == p1 + sep
 
     @property
@@ -144,13 +150,6 @@ class LibBuilderBase(object):
     def version(self):
         return self._manifest.get("version")
 
-    @property
-    def vcs_info(self):
-        items = glob(join(self.path, ".*", PackageManager.SRC_MANIFEST_NAME))
-        if not items:
-            return None
-        return util.load_json(items[0])
-
     @property
     def dependencies(self):
         return LibraryManager.normalize_dependencies(
@@ -179,16 +178,15 @@ class LibBuilderBase(object):
     def get_include_dirs(self):
         items = []
         include_dir = self.include_dir
-        if include_dir and include_dir not in items:
+        if include_dir:
             items.append(include_dir)
         items.append(self.src_dir)
         return items
 
     @property
     def build_dir(self):
-        return join("$BUILD_DIR",
+        lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3]
-                    "lib%s" % hashlib.sha1(self.path).hexdigest()[:3],
+        return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path))
-                    basename(self.path))
 
     @property
     def build_flags(self):
@@ -204,17 +202,18 @@ class LibBuilderBase(object):
 
     @property
     def lib_archive(self):
-        return self.env.get("LIB_ARCHIVE", "") != "false"
+        return self.env.GetProjectOption("lib_archive", True)
 
     @property
     def lib_ldf_mode(self):
         return self.validate_ldf_mode(
-            self.env.get("LIB_LDF_MODE", self.LDF_MODE_DEFAULT))
+            self.env.GetProjectOption("lib_ldf_mode", self.LDF_MODE_DEFAULT))
 
     @property
     def lib_compat_mode(self):
         return self.validate_compat_mode(
-            self.env.get("LIB_COMPAT_MODE", self.COMPAT_MODE_DEFAULT))
+            self.env.GetProjectOption("lib_compat_mode",
+                                      self.COMPAT_MODE_DEFAULT))
 
     @property
     def depbuilders(self):
@@ -230,7 +229,7 @@ class LibBuilderBase(object):
 
     @staticmethod
     def validate_ldf_mode(mode):
-        if isinstance(mode, basestring):
+        if isinstance(mode, string_types):
             mode = mode.strip().lower()
         if mode in LibBuilderBase.LDF_MODES:
             return mode
@@ -242,7 +241,7 @@ class LibBuilderBase(object):
 
     @staticmethod
     def validate_compat_mode(mode):
-        if isinstance(mode, basestring):
+        if isinstance(mode, string_types):
             mode = mode.strip().lower()
         if mode in LibBuilderBase.COMPAT_MODES:
             return mode
@@ -266,51 +265,29 @@ class LibBuilderBase(object):
             self.env.ProcessFlags(self.build_flags)
             if self.extra_script:
                 self.env.SConscriptChdir(1)
-                self.env.SConscript(
+                self.env.SConscript(realpath(self.extra_script),
-                    realpath(self.extra_script),
+                                    exports={
-                    exports={
+                                        "env": self.env,
-                        "env": self.env,
+                                        "pio_lib_builder": self
-                        "pio_lib_builder": self
+                                    })
-                    })
             self.env.ProcessUnFlags(self.build_unflags)
 
     def process_dependencies(self):
         if not self.dependencies:
             return
         for item in self.dependencies:
-            skip = False
-            for key in ("platforms", "frameworks"):
-                env_key = "PIO" + key.upper()[:-1]
-                if env_key not in self.env:
-                    continue
-                if (key in item and
-                        not util.items_in_list(self.env[env_key], item[key])):
-                    if self.verbose:
-                        sys.stderr.write("Skip %s incompatible dependency %s\n"
-                                         % (key[:-1], item))
-                    skip = True
-            if skip:
-                continue
-
             found = False
             for lb in self.env.GetLibBuilders():
                 if item['name'] != lb.name:
                     continue
-                elif "frameworks" in item and \
-                     not lb.is_frameworks_compatible(item["frameworks"]):
-                    continue
-                elif "platforms" in item and \
-                     not lb.is_platforms_compatible(item["platforms"]):
-                    continue
                 found = True
-                self.depend_recursive(lb)
+                if lb not in self.depbuilders:
+                    self.depend_recursive(lb)
                 break
 
-            if not found:
+            if not found and self.verbose:
-                sys.stderr.write(
+                sys.stderr.write("Warning: Ignored `%s` dependency for `%s` "
-                    "Error: Could not find `%s` dependency for `%s` "
+                                 "library\n" % (item['name'], self.name))
-                    "library\n" % (item['name'], self.name))
-                self.env.Exit(1)
 
     def get_search_files(self):
         items = [
@@ -400,9 +377,9 @@ class LibBuilderBase(object):
         if self != lb:
             if _already_depends(lb):
                 if self.verbose:
-                    sys.stderr.write(
+                    sys.stderr.write("Warning! Circular dependencies detected "
-                        "Warning! Circular dependencies detected "
+                                     "between `%s` and `%s`\n" %
-                        "between `%s` and `%s`\n" % (self.path, lb.path))
+                                     (self.path, lb.path))
                 self._circular_deps.append(lb)
             elif lb not in self._depbuilders:
                 self._depbuilders.append(lb)
@@ -477,7 +454,8 @@ class ArduinoLibBuilder(LibBuilderBase):
         manifest = {}
         if not isfile(join(self.path, "library.properties")):
             return manifest
-        with open(join(self.path, "library.properties")) as fp:
+        manifest_path = join(self.path, "library.properties")
+        with codecs.open(manifest_path, encoding="utf-8") as fp:
             for line in fp.readlines():
                 if "=" not in line:
                     continue
@@ -528,22 +506,23 @@ class ArduinoLibBuilder(LibBuilderBase):
 
     def is_platforms_compatible(self, platforms):
         platforms_map = {
-            "avr": "atmelavr",
+            "avr": ["atmelavr"],
-            "sam": "atmelsam",
+            "sam": ["atmelsam"],
-            "samd": "atmelsam",
+            "samd": ["atmelsam"],
-            "esp8266": "espressif8266",
+            "esp8266": ["espressif8266"],
-            "esp32": "espressif32",
+            "esp32": ["espressif32"],
-            "arc32": "intel_arc32",
+            "arc32": ["intel_arc32"],
-            "stm32": "ststm32"
+            "stm32": ["ststm32"],
+            "nrf5": ["nordicnrf51", "nordicnrf52"]
         }
         items = []
         for arch in self._manifest.get("architectures", "").split(","):
-            arch = arch.strip()
+            arch = arch.strip().lower()
             if arch == "*":
                 items = "*"
                 break
             if arch in platforms_map:
-                items.append(platforms_map[arch])
+                items.extend(platforms_map[arch])
         if not items:
             return LibBuilderBase.is_platforms_compatible(self, platforms)
         return util.items_in_list(platforms, items)
@@ -643,8 +622,8 @@ class MbedLibBuilder(LibBuilderBase):
         for key, options in manifest.get("config", {}).items():
             if "value" not in options:
                 continue
-            macros[key] = dict(
+            macros[key] = dict(name=options.get("macro_name"),
-                name=options.get("macro_name"), value=options.get("value"))
+                               value=options.get("value"))
 
         # overrode items per target
         for target, options in manifest.get("target_overrides", {}).items():
@@ -664,8 +643,10 @@ class MbedLibBuilder(LibBuilderBase):
                 if "." not in macro['name']:
                     macro['name'] = "%s.%s" % (manifest.get("name"),
                                                macro['name'])
-                macro['name'] = re.sub(
+                macro['name'] = re.sub(r"[^a-z\d]+",
-                    r"[^a-z\d]+", "_", macro['name'], flags=re.I).upper()
+                                       "_",
+                                       macro['name'],
+                                       flags=re.I).upper()
                 macro['name'] = "MBED_CONF_" + macro['name']
             if isinstance(macro['value'], bool):
                 macro['value'] = 1 if macro['value'] else 0
@@ -681,8 +662,8 @@ class MbedLibBuilder(LibBuilderBase):
                     lines.append(
                         "// PlatformIO Library Dependency Finder (LDF)")
                     lines.extend([
-                        "#define %s %s" % (name,
+                        "#define %s %s" %
-                                           value if value is not None else "")
+                        (name, value if value is not None else "")
                         for name, value in macros.items()
                     ])
                     lines.append("")
@@ -716,13 +697,27 @@ class PlatformIOLibBuilder(LibBuilderBase):
     def _is_arduino_manifest(self):
         return isfile(join(self.path, "library.properties"))
 
+    @property
+    def include_dir(self):
+        if "includeDir" in self._manifest.get("build", {}):
+            with util.cd(self.path):
+                return realpath(self._manifest.get("build").get("includeDir"))
+        return LibBuilderBase.include_dir.fget(self)
+
+    @property
+    def src_dir(self):
+        if "srcDir" in self._manifest.get("build", {}):
+            with util.cd(self.path):
+                return realpath(self._manifest.get("build").get("srcDir"))
+        return LibBuilderBase.src_dir.fget(self)
+
     @property
     def src_filter(self):
         if "srcFilter" in self._manifest.get("build", {}):
             return self._manifest.get("build").get("srcFilter")
-        elif self.env['SRC_FILTER']:
+        if self.env['SRC_FILTER']:
             return self.env['SRC_FILTER']
-        elif self._is_arduino_manifest():
+        if self._is_arduino_manifest():
             return ArduinoLibBuilder.src_filter.fget(self)
         return LibBuilderBase.src_filter.fget(self)
 
@@ -788,6 +783,7 @@ class PlatformIOLibBuilder(LibBuilderBase):
         for path in self.env.get("CPPPATH", []):
             if path not in self.envorigin.get("CPPPATH", []):
                 include_dirs.append(self.env.subst(path))
+
         return include_dirs
 
 
@@ -813,7 +809,9 @@ class ProjectAsLibBuilder(LibBuilderBase):
         project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR")
         if isdir(project_include_dir):
             include_dirs.append(project_include_dir)
-        include_dirs.extend(LibBuilderBase.get_include_dirs(self))
+        for include_dir in LibBuilderBase.get_include_dirs(self):
+            if include_dir not in include_dirs:
+                include_dirs.append(include_dir)
         return include_dirs
 
     def get_search_files(self):
@@ -841,43 +839,80 @@ class ProjectAsLibBuilder(LibBuilderBase):
         return (self.env.get("SRC_FILTER")
                 or LibBuilderBase.src_filter.fget(self))
 
+    @property
+    def dependencies(self):
+        return self.env.GetProjectOption("lib_deps", [])
+
     def process_extra_options(self):
         # skip for project, options are already processed
         pass
 
-    def process_dependencies(self):  # pylint: disable=too-many-branches
+    def install_dependencies(self):
-        uris = self.env.get("LIB_DEPS", [])
+
-        if not uris:
+        def _is_builtin(uri):
-            return
+            for lb in self.env.GetLibBuilders():
-        storage_dirs = []
+                if lb.name == uri:
-        for lb in self.env.GetLibBuilders():
+                    return True
-            if dirname(lb.path) not in storage_dirs:
+            return False
-                storage_dirs.append(dirname(lb.path))
+
+        not_found_uri = []
+        for uri in self.dependencies:
+            # check if built-in library
+            if _is_builtin(uri):
+                continue
 
-        for uri in uris:
             found = False
-            for storage_dir in storage_dirs:
+            for storage_dir in self.env.GetLibSourceDirs():
+                lm = LibraryManager(storage_dir)
+                if lm.get_package_dir(*lm.parse_pkg_uri(uri)):
+                    found = True
+                    break
+            if not found:
+                not_found_uri.append(uri)
+
+        did_install = False
+        lm = LibraryManager(
+            self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV")))
+        for uri in not_found_uri:
+            try:
+                lm.install(uri)
+                did_install = True
+            except (exception.LibNotFound, exception.InternetIsOffline) as e:
+                click.secho("Warning! %s" % e, fg="yellow")
+
+        # reset cache
+        if did_install:
+            DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None)
+
+    def process_dependencies(self):  # pylint: disable=too-many-branches
+        for uri in self.dependencies:
+            found = False
+            for storage_dir in self.env.GetLibSourceDirs():
                 if found:
                     break
                 lm = LibraryManager(storage_dir)
-                pkg_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri))
+                lib_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri))
-                if not pkg_dir:
+                if not lib_dir:
                     continue
                 for lb in self.env.GetLibBuilders():
-                    if lb.path != pkg_dir:
+                    if lib_dir not in lb:
                         continue
                     if lb not in self.depbuilders:
                         self.depend_recursive(lb)
                     found = True
                     break
+            if found:
+                continue
 
-            if not found:
+            # look for built-in libraries by a name
-                for lb in self.env.GetLibBuilders():
+            # which don't have package manifest
-                    if lb.name != uri:
+            for lb in self.env.GetLibBuilders():
-                        continue
+                if lb.name != uri:
-                    if lb not in self.depbuilders:
+                    continue
-                        self.depend_recursive(lb)
+                if lb not in self.depbuilders:
-                    break
+                    self.depend_recursive(lb)
+                found = True
+                break
 
     def build(self):
         self._is_built = True  # do not build Project now
@@ -886,61 +921,69 @@ class ProjectAsLibBuilder(LibBuilderBase):
         return result
 
 
+def GetLibSourceDirs(env):
+    items = env.GetProjectOption("lib_extra_dirs", [])
+    items.extend(env['LIBSOURCE_DIRS'])
+    return [
+        env.subst(expanduser(item) if item.startswith("~") else item)
+        for item in items
+    ]
+
+
+def IsCompatibleLibBuilder(env,
+                           lb,
+                           verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))):
+    compat_mode = lb.lib_compat_mode
+    if lb.name in env.GetProjectOption("lib_ignore", []):
+        if verbose:
+            sys.stderr.write("Ignored library %s\n" % lb.path)
+        return None
+    if compat_mode == "strict" and not lb.is_platforms_compatible(
+            env['PIOPLATFORM']):
+        if verbose:
+            sys.stderr.write("Platform incompatible library %s\n" % lb.path)
+        return False
+    if (compat_mode in ("soft", "strict") and "PIOFRAMEWORK" in env
+            and not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", []))):
+        if verbose:
+            sys.stderr.write("Framework incompatible library %s\n" % lb.path)
+        return False
+    return True
+
+
 def GetLibBuilders(env):  # pylint: disable=too-many-branches
+    if DefaultEnvironment().get("__PIO_LIB_BUILDERS", None) is not None:
+        return sorted(DefaultEnvironment()['__PIO_LIB_BUILDERS'],
+                      key=lambda lb: 0 if lb.dependent else 1)
 
-    if "__PIO_LIB_BUILDERS" in DefaultEnvironment():
+    DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=[])
-        return sorted(
-            DefaultEnvironment()['__PIO_LIB_BUILDERS'],
-            key=lambda lb: 0 if lb.dependent else 1)
-
-    items = []
-    verbose = int(ARGUMENTS.get("PIOVERBOSE",
-                                0)) and not env.GetOption('clean')
-
-    def _check_lib_builder(lb):
-        compat_mode = lb.lib_compat_mode
-        if lb.name in env.get("LIB_IGNORE", []):
-            if verbose:
-                sys.stderr.write("Ignored library %s\n" % lb.path)
-            return None
-        if compat_mode == "strict" and not lb.is_platforms_compatible(
-                env['PIOPLATFORM']):
-            if verbose:
-                sys.stderr.write(
-                    "Platform incompatible library %s\n" % lb.path)
-            return False
-        if compat_mode == "soft" and "PIOFRAMEWORK" in env and \
-           not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])):
-            if verbose:
-                sys.stderr.write(
-                    "Framework incompatible library %s\n" % lb.path)
-            return False
-        return True
 
+    verbose = int(ARGUMENTS.get("PIOVERBOSE", 0))
     found_incompat = False
-    for libs_dir in env['LIBSOURCE_DIRS']:
+
-        libs_dir = env.subst(libs_dir)
+    for storage_dir in env.GetLibSourceDirs():
-        if not isdir(libs_dir):
+        storage_dir = realpath(storage_dir)
+        if not isdir(storage_dir):
             continue
-        for item in sorted(os.listdir(libs_dir)):
+        for item in sorted(os.listdir(storage_dir)):
-            if item == "__cores__" or not isdir(join(libs_dir, item)):
+            lib_dir = join(storage_dir, item)
+            if item == "__cores__" or not isdir(lib_dir):
                 continue
             try:
-                lb = LibBuilderFactory.new(
+                lb = LibBuilderFactory.new(env, lib_dir)
-                    env, join(libs_dir, item), verbose=verbose)
             except exception.InvalidJSONFile:
                 if verbose:
-                    sys.stderr.write("Skip library with broken manifest: %s\n"
+                    sys.stderr.write(
-                                     % join(libs_dir, item))
+                        "Skip library with broken manifest: %s\n" % lib_dir)
                 continue
-            if _check_lib_builder(lb):
+            if env.IsCompatibleLibBuilder(lb):
-                items.append(lb)
+                DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
             else:
                 found_incompat = True
 
     for lb in env.get("EXTRA_LIB_BUILDERS", []):
-        if _check_lib_builder(lb):
+        if env.IsCompatibleLibBuilder(lb):
-            items.append(lb)
+            DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
         else:
             found_incompat = True
 
@@ -950,13 +993,16 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             "https://docs.platformio.org/page/librarymanager/ldf.html#"
             "ldf-compat-mode\n")
 
-    DefaultEnvironment()['__PIO_LIB_BUILDERS'] = items
+    return DefaultEnvironment()['__PIO_LIB_BUILDERS']
-    return items
 
 
 def ConfigureProjectLibBuilder(env):
 
-    def correct_found_libs(lib_builders):
+    def _get_vcs_info(lb):
+        path = LibraryManager.get_src_manifest_path(lb.path)
+        return util.load_json(path) if path else None
+
+    def _correct_found_libs(lib_builders):
         # build full dependency graph
         found_lbs = [lb for lb in lib_builders if lb.dependent]
         for lb in lib_builders:
@@ -967,11 +1013,11 @@ def ConfigureProjectLibBuilder(env):
                 if deplb not in found_lbs:
                     lb.depbuilders.remove(deplb)
 
-    def print_deps_tree(root, level=0):
+    def _print_deps_tree(root, level=0):
         margin = "|   " * (level)
         for lb in root.depbuilders:
             title = "<%s>" % lb.name
-            vcs_info = lb.vcs_info
+            vcs_info = _get_vcs_info(lb)
             if lb.version:
                 title += " %s" % lb.version
             if vcs_info and vcs_info.get("version"):
@@ -985,27 +1031,29 @@ def ConfigureProjectLibBuilder(env):
                 sys.stdout.write(")")
             sys.stdout.write("\n")
             if lb.depbuilders:
-                print_deps_tree(lb, level + 1)
+                _print_deps_tree(lb, level + 1)
 
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
     ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project)
 
-    print("Library Dependency Finder -> http://bit.ly/configure-pio-ldf")
+    print("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf")
-    print("LDF MODES: FINDER(%s) COMPATIBILITY(%s)" %
+    print("LDF Modes: Finder ~ %s, Compatibility ~ %s" %
           (ldf_mode, project.lib_compat_mode))
 
+    project.install_dependencies()
+
     lib_builders = env.GetLibBuilders()
-    print("Collected %d compatible libraries" % len(lib_builders))
+    print("Found %d compatible libraries" % len(lib_builders))
 
     print("Scanning dependencies...")
     project.search_deps_recursive()
 
     if ldf_mode.startswith("chain") and project.depbuilders:
-        correct_found_libs(lib_builders)
+        _correct_found_libs(lib_builders)
 
     if project.depbuilders:
         print("Dependency Graph")
-        print_deps_tree(project)
+        _print_deps_tree(project)
     else:
         print("No dependencies")
 
@@ -1017,6 +1065,8 @@ def exists(_):
 
 
 def generate(env):
+    env.AddMethod(GetLibSourceDirs)
+    env.AddMethod(IsCompatibleLibBuilder)
     env.AddMethod(GetLibBuilders)
     env.AddMethod(ConfigureProjectLibBuilder)
     return env

platformio/builder/tools/piomisc.py

@@ -21,11 +21,13 @@ from os import environ, remove, walk
 from os.path import basename, isdir, isfile, join, realpath, relpath, sep
 from tempfile import mkstemp
 
-from SCons.Action import Action
+from SCons.Action import Action  # pylint: disable=import-error
-from SCons.Script import ARGUMENTS
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 
 from platformio import util
+from platformio.compat import get_file_contents, glob_escape
 from platformio.managers.core import get_core_package_dir
+from platformio.proc import exec_command
 
 
 class InoToCPPConverter(object):
@@ -58,7 +60,7 @@ class InoToCPPConverter(object):
         assert nodes
         lines = []
         for node in nodes:
-            contents = node.get_text_contents()
+            contents = get_file_contents(node.get_path())
             _lines = [
                 '# 1 "%s"' % node.get_path().replace("\\", "/"), contents
             ]
@@ -76,8 +78,7 @@ class InoToCPPConverter(object):
     def process(self, contents):
         out_file = self._main_ino + ".cpp"
         assert self._gcc_preprocess(contents, out_file)
-        with open(out_file) as fp:
+        contents = get_file_contents(out_file)
-            contents = fp.read()
         contents = self._join_multiline_strings(contents)
         with open(out_file, "w") as fp:
             fp.write(self.append_prototypes(contents))
@@ -158,9 +159,7 @@ class InoToCPPConverter(object):
         return total
 
     def append_prototypes(self, contents):
-        prototypes = self._parse_prototypes(contents)
+        prototypes = self._parse_prototypes(contents) or []
-        if not prototypes:
-            return contents
 
         # skip already declared prototypes
         declared = set(
@@ -169,6 +168,9 @@ class InoToCPPConverter(object):
             m for m in prototypes if m.group(1).strip() not in declared
         ]
 
+        if not prototypes:
+            return contents
+
         prototype_names = set(m.group(3).strip() for m in prototypes)
         split_pos = prototypes[0].start()
         match_ptrs = re.search(
@@ -187,9 +189,9 @@ class InoToCPPConverter(object):
 
 
 def ConvertInoToCpp(env):
-    src_dir = util.glob_escape(env.subst("$PROJECTSRC_DIR"))
+    src_dir = glob_escape(env.subst("$PROJECTSRC_DIR"))
-    ino_nodes = (
+    ino_nodes = (env.Glob(join(src_dir, "*.ino")) +
-        env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde")))
+                 env.Glob(join(src_dir, "*.pde")))
     if not ino_nodes:
         return
     c = InoToCPPConverter(env)
@@ -208,10 +210,12 @@ def _delete_file(path):
 
 @util.memoized()
 def _get_compiler_type(env):
+    if env.subst("$CC").endswith("-gcc"):
+        return "gcc"
     try:
         sysenv = environ.copy()
         sysenv['PATH'] = str(env['ENV']['PATH'])
-        result = util.exec_command([env.subst("$CC"), "-v"], env=sysenv)
+        result = exec_command([env.subst("$CC"), "-v"], env=sysenv)
     except OSError:
         return None
     if result['returncode'] != 0:
@@ -219,7 +223,7 @@ def _get_compiler_type(env):
     output = "".join([result['out'], result['err']]).lower()
     if "clang" in output and "LLVM" in output:
         return "clang"
-    elif "gcc" in output:
+    if "gcc" in output:
         return "gcc"
     return None
 
@@ -283,10 +287,13 @@ def PioClean(env, clean_dir):
     if not isdir(clean_dir):
         print("Build environment is clean")
         env.Exit(0)
+    clean_rel_path = relpath(clean_dir)
     for root, _, files in walk(clean_dir):
-        for file_ in files:
+        for f in files:
-            remove(join(root, file_))
+            dst = join(root, f)
-            print("Removed %s" % relpath(join(root, file_)))
+            remove(dst)
+            print("Removed %s" %
+                  (dst if clean_rel_path.startswith(".") else relpath(dst)))
     print("Done cleaning")
     util.rmtree_(clean_dir)
     env.Exit(0)
@@ -295,8 +302,8 @@ def PioClean(env, clean_dir):
 def ProcessDebug(env):
     if not env.subst("$PIODEBUGFLAGS"):
         env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"])
-    env.Append(
+    env.Append(BUILD_FLAGS=list(env['PIODEBUGFLAGS']) +
-        BUILD_FLAGS=list(env['PIODEBUGFLAGS']) + ["-D__PLATFORMIO_DEBUG__"])
+               ["-D__PLATFORMIO_BUILD_DEBUG__"])
     unflags = ["-Os"]
     for level in [0, 1, 2]:
         for flag in ("O", "g", "ggdb"):
@@ -305,22 +312,21 @@ def ProcessDebug(env):
 
 
 def ProcessTest(env):
-    env.Append(
+    env.Append(CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
-        CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
+               CPPPATH=[join("$BUILD_DIR", "UnityTestLib")])
-        CPPPATH=[join("$BUILD_DIR", "UnityTestLib")])
+    unitylib = env.BuildLibrary(join("$BUILD_DIR", "UnityTestLib"),
-    unitylib = env.BuildLibrary(
+                                get_core_package_dir("tool-unity"))
-        join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity"))
     env.Prepend(LIBS=[unitylib])
 
     src_filter = ["+<*.cpp>", "+<*.c>"]
-    if "PIOTEST" in env:
+    if "PIOTEST_RUNNING_NAME" in env:
-        src_filter.append("+<%s%s>" % (env['PIOTEST'], sep))
+        src_filter.append("+<%s%s>" % (env['PIOTEST_RUNNING_NAME'], sep))
     env.Replace(PIOTEST_SRC_FILTER=src_filter)
 
 
 def GetExtraScripts(env, scope):
     items = []
-    for item in env.get("EXTRA_SCRIPTS", []):
+    for item in env.GetProjectOption("extra_scripts", []):
         if scope == "post" and ":" not in item:
             items.append(item)
         elif item.startswith("%s:" % scope):

platformio/builder/tools/pioplatform.py

@@ -14,35 +14,33 @@
 
 from __future__ import absolute_import
 
-import base64
 import sys
 from os.path import isdir, isfile, join
 
-from SCons.Script import COMMAND_LINE_TARGETS
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 
 from platformio import exception, util
+from platformio.compat import WINDOWS
 from platformio.managers.platform import PlatformFactory
+from platformio.project.config import ProjectOptions
 
 # pylint: disable=too-many-branches, too-many-locals
 
 
 @util.memoized()
-def initPioPlatform(name):
-    return PlatformFactory.newPlatform(name)
-
-
 def PioPlatform(env):
-    variables = {}
+    variables = env.GetProjectOptions(as_dict=True)
-    for name in env['PIOVARIABLES']:
+    if "framework" in variables:
-        if name in env:
+        # support PIO Core 3.0 dev/platforms
-            variables[name.lower()] = env[name]
+        variables['pioframework'] = variables['framework']
-    p = initPioPlatform(env['PLATFORM_MANIFEST'])
+    p = PlatformFactory.newPlatform(env['PLATFORM_MANIFEST'])
     p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
     return p
 
 
 def BoardConfig(env, board=None):
-    p = initPioPlatform(env['PLATFORM_MANIFEST'])
+    p = env.PioPlatform()
     try:
         board = board or env.get("BOARD")
         assert board, "BoardConfig: Board is not defined"
@@ -62,7 +60,7 @@ def GetFrameworkScript(env, framework):
     return script_path
 
 
-def LoadPioPlatform(env, variables):
+def LoadPioPlatform(env):
     p = env.PioPlatform()
     installed_packages = p.get_installed_packages()
 
@@ -79,7 +77,7 @@ def LoadPioPlatform(env, variables):
         env.PrependENVPath(
             "PATH",
             join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir)
-        if ("windows" not in systype and isdir(join(pkg_dir, "lib"))
+        if (not WINDOWS and isdir(join(pkg_dir, "lib"))
                 and type_ != "toolchain"):
             env.PrependENVPath(
                 "DYLD_LIBRARY_PATH"
@@ -91,92 +89,122 @@ def LoadPioPlatform(env, variables):
         env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")])
 
     if "BOARD" not in env:
-        # handle _MCU and _F_CPU variables for AVR native
-        for key, value in variables.UnknownVariables().items():
-            if not key.startswith("BOARD_"):
-                continue
-            env.Replace(**{
-                key.upper().replace("BUILD.", ""):
-                base64.b64decode(value)
-            })
         return
 
-    # update board manifest with a custom data
+    # update board manifest with overridden data from INI config
     board_config = env.BoardConfig()
-    for key, value in variables.UnknownVariables().items():
+    for option, value in env.GetProjectOptions():
-        if not key.startswith("BOARD_"):
+        if option.startswith("board_"):
-            continue
+            board_config.update(option.lower()[6:], value)
-        board_config.update(key.lower()[6:], base64.b64decode(value))
 
-    # update default environment variables
+    # load default variables from board config
-    for key in variables.keys():
+    for option_meta in ProjectOptions.values():
-        if key in env or \
+        if not option_meta.buildenvvar or option_meta.buildenvvar in env:
-                not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]):
             continue
-        _opt, _val = key.lower().split("_", 1)
+        data_path = (option_meta.name[6:]
-        if _opt == "board":
+                     if option_meta.name.startswith("board_") else
-            _opt = "build"
+                     option_meta.name.replace("_", "."))
-        if _val in board_config.get(_opt):
+        try:
-            env.Replace(**{key: board_config.get("%s.%s" % (_opt, _val))})
+            env[option_meta.buildenvvar] = board_config.get(data_path)
+        except KeyError:
+            pass
 
     if "build.ldscript" in board_config:
         env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript"))
 
 
-def PrintConfiguration(env):
+def PrintConfiguration(env):  # pylint: disable=too-many-statements
     platform = env.PioPlatform()
-    platform_data = ["PLATFORM: %s >" % platform.title]
+    board_config = env.BoardConfig() if "BOARD" in env else None
-    hardware_data = ["HARDWARE:"]
-    configuration_data = ["CONFIGURATION:"]
-    mcu = env.subst("$BOARD_MCU")
-    f_cpu = env.subst("$BOARD_F_CPU")
-    if mcu:
-        hardware_data.append(mcu.upper())
-    if f_cpu:
-        f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()]))
-        hardware_data.append("%dMHz" % (f_cpu / 1000000))
 
-    debug_tools = None
+    def _get_configuration_data():
-    if "BOARD" in env:
+        return None if not board_config else [
-        board_config = env.BoardConfig()
+            "CONFIGURATION:",
-        platform_data.append(board_config.get("name"))
+            "https://docs.platformio.org/page/boards/%s/%s.html" %
+            (platform.name, board_config.id)
+        ]
 
-        debug_tools = board_config.get("debug", {}).get("tools")
+    def _get_plaform_data():
+        data = ["PLATFORM: %s %s" % (platform.title, platform.version)]
+        src_manifest_path = platform.pm.get_src_manifest_path(
+            platform.get_dir())
+        if src_manifest_path:
+            src_manifest = util.load_json(src_manifest_path)
+            if "version" in src_manifest:
+                data.append("#" + src_manifest['version'])
+            if int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                data.append("(%s)" % src_manifest['url'])
+        if board_config:
+            data.extend([">", board_config.get("name")])
+        return data
+
+    def _get_hardware_data():
+        data = ["HARDWARE:"]
+        mcu = env.subst("$BOARD_MCU")
+        f_cpu = env.subst("$BOARD_F_CPU")
+        if mcu:
+            data.append(mcu.upper())
+        if f_cpu:
+            f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()]))
+            data.append("%dMHz," % (f_cpu / 1000000))
+        if not board_config:
+            return data
         ram = board_config.get("upload", {}).get("maximum_ram_size")
         flash = board_config.get("upload", {}).get("maximum_size")
-        hardware_data.append(
+        data.append("%s RAM, %s Flash" %
-            "%s RAM (%s Flash)" % (util.format_filesize(ram),
+                    (util.format_filesize(ram), util.format_filesize(flash)))
-                                   util.format_filesize(flash)))
+        return data
-        configuration_data.append(
-            "https://docs.platformio.org/page/boards/%s/%s.html" %
-            (platform.name, board_config.id))
 
-    for data in (configuration_data, platform_data, hardware_data):
+    def _get_debug_data():
-        if len(data) > 1:
+        debug_tools = board_config.get(
+            "debug", {}).get("tools") if board_config else None
+        if not debug_tools:
+            return None
+        data = [
+            "DEBUG:", "Current",
+            "(%s)" % board_config.get_debug_tool_name(
+                env.GetProjectOption("debug_tool"))
+        ]
+        onboard = []
+        external = []
+        for key, value in debug_tools.items():
+            if value.get("onboard"):
+                onboard.append(key)
+            else:
+                external.append(key)
+        if onboard:
+            data.extend(["On-board", "(%s)" % ", ".join(sorted(onboard))])
+        if external:
+            data.extend(["External", "(%s)" % ", ".join(sorted(external))])
+        return data
+
+    def _get_packages_data():
+        data = []
+        for name, options in platform.packages.items():
+            if options.get("optional"):
+                continue
+            pkg_dir = platform.get_package_dir(name)
+            if not pkg_dir:
+                continue
+            manifest = platform.pm.load_manifest(pkg_dir)
+            original_version = util.get_original_version(manifest['version'])
+            info = "%s %s" % (manifest['name'], manifest['version'])
+            extra = []
+            if original_version:
+                extra.append(original_version)
+            if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                extra.append(manifest['__src_url'])
+            if extra:
+                info += " (%s)" % ", ".join(extra)
+            data.append(info)
+        return ["PACKAGES:", ", ".join(data)]
+
+    for data in (_get_configuration_data(), _get_plaform_data(),
+                 _get_hardware_data(), _get_debug_data(),
+                 _get_packages_data()):
+        if data and len(data) > 1:
             print(" ".join(data))
 
-    # Debugging
-    if not debug_tools:
-        return
-
-    data = [
-        "CURRENT(%s)" % board_config.get_debug_tool_name(
-            env.subst("$DEBUG_TOOL"))
-    ]
-    onboard = []
-    external = []
-    for key, value in debug_tools.items():
-        if value.get("onboard"):
-            onboard.append(key)
-        else:
-            external.append(key)
-    if onboard:
-        data.append("ON-BOARD(%s)" % ", ".join(sorted(onboard)))
-    if external:
-        data.append("EXTERNAL(%s)" % ", ".join(sorted(external)))
-
-    print("DEBUG: %s" % " ".join(data))
-
 
 def exists(_):
     return True

platformio/builder/tools/pioproject.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from platformio.project.config import ProjectConfig, ProjectOptions
+
+
+def GetProjectConfig(env):
+    return ProjectConfig.get_instance(env['PROJECT_CONFIG'])
+
+
+def GetProjectOptions(env, as_dict=False):
+    return env.GetProjectConfig().items(env=env['PIOENV'], as_dict=as_dict)
+
+
+def GetProjectOption(env, option, default=None):
+    return env.GetProjectConfig().get("env:" + env['PIOENV'], option, default)
+
+
+def LoadProjectOptions(env):
+    for option, value in env.GetProjectOptions():
+        option_meta = ProjectOptions.get("env." + option)
+        if not option_meta or not option_meta.buildenvvar:
+            continue
+        env[option_meta.buildenvvar] = value
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    env.AddMethod(GetProjectConfig)
+    env.AddMethod(GetProjectOptions)
+    env.AddMethod(GetProjectOption)
+    env.AddMethod(LoadProjectOptions)
+    return env

platformio/builder/tools/pioupload.py

@@ -22,10 +22,12 @@ from os.path import isfile, join
 from shutil import copyfile
 from time import sleep
 
-from SCons.Script import ARGUMENTS
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from serial import Serial, SerialException
 
 from platformio import exception, util
+from platformio.compat import WINDOWS
+from platformio.proc import exec_command
 
 # pylint: disable=unused-argument
 
@@ -134,8 +136,7 @@ def AutodetectUploadPort(*args, **kwargs):
                 continue
             port = item['port']
             if upload_protocol.startswith("blackmagic"):
-                if "windows" in util.get_systype() and \
+                if WINDOWS and port.startswith("COM") and len(port) > 4:
-                        port.startswith("COM") and len(port) > 4:
                     port = "\\\\.\\%s" % port
                 if "GDB" in item['description']:
                     return port
@@ -197,10 +198,9 @@ def CheckUploadSize(_, target, source, env):
         return
 
     def _configure_defaults():
-        env.Replace(
+        env.Replace(SIZECHECKCMD="$SIZETOOL -B -d $SOURCES",
-            SIZECHECKCMD="$SIZETOOL -B -d $SOURCES",
+                    SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s",
-            SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s",
+                    SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+")
-            SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+")
 
     def _get_size_output():
         cmd = env.get("SIZECHECKCMD")
@@ -211,7 +211,7 @@ def CheckUploadSize(_, target, source, env):
         cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
         sysenv = environ.copy()
         sysenv['PATH'] = str(env['ENV']['PATH'])
-        result = util.exec_command(env.subst(cmd), env=sysenv)
+        result = exec_command(env.subst(cmd), env=sysenv)
         if result['returncode'] != 0:
             return None
         return result['out'].strip()
@@ -250,8 +250,8 @@ def CheckUploadSize(_, target, source, env):
     if data_max_size and data_size > -1:
         print("DATA:    %s" % _format_availale_bytes(data_size, data_max_size))
     if program_size > -1:
-        print("PROGRAM: %s" % _format_availale_bytes(program_size,
+        print("PROGRAM: %s" %
-                                                     program_max_size))
+              _format_availale_bytes(program_size, program_max_size))
     if int(ARGUMENTS.get("PIOVERBOSE", 0)):
         print(output)
 
@@ -272,8 +272,8 @@ def PrintUploadInfo(env):
     configured = env.subst("$UPLOAD_PROTOCOL")
     available = [configured] if configured else []
     if "BOARD" in env:
-        available.extend(env.BoardConfig().get("upload", {}).get(
+        available.extend(env.BoardConfig().get("upload",
-            "protocols", []))
+                                               {}).get("protocols", []))
     if available:
         print("AVAILABLE: %s" % ", ".join(sorted(set(available))))
     if configured:

platformio/builder/tools/piowinhooks.py

@@ -12,10 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import absolute_import
+
 from hashlib import md5
 from os import makedirs
 from os.path import isdir, isfile, join
-from platform import system
+
+from platformio.compat import WINDOWS, hashlib_encode_data
 
 # Windows CLI has limit with command length to 8192
 # Leave 2000 chars for flags and other options
@@ -58,7 +61,8 @@ def _file_long_data(env, data):
     build_dir = env.subst("$BUILD_DIR")
     if not isdir(build_dir):
         makedirs(build_dir)
-    tmp_file = join(build_dir, "longcmd-%s" % md5(data).hexdigest())
+    tmp_file = join(build_dir,
+                    "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest())
     if isfile(tmp_file):
         return tmp_file
     with open(tmp_file, "w") as fp:
@@ -71,7 +75,7 @@ def exists(_):
 
 
 def generate(env):
-    if system() != "Windows":
+    if not WINDOWS:
         return None
 
     env.Replace(_long_sources_hook=long_sources_hook)

platformio/builder/tools/platformio.py

@@ -20,11 +20,15 @@ from glob import glob
 from os import sep, walk
 from os.path import basename, dirname, isdir, join, realpath
 
-from SCons import Builder, Util
+from SCons import Builder, Util  # pylint: disable=import-error
-from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild,
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
-                          DefaultEnvironment, Export, SConscript)
+from SCons.Script import AlwaysBuild  # pylint: disable=import-error
+from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
+from SCons.Script import Export  # pylint: disable=import-error
+from SCons.Script import SConscript  # pylint: disable=import-error
 
-from platformio.util import glob_escape, pioversion_to_intstr
+from platformio.compat import glob_escape, string_types
+from platformio.util import pioversion_to_intstr
 
 SRC_HEADER_EXT = ["h", "hpp"]
 SRC_C_EXT = ["c", "cc", "cpp"]
@@ -65,7 +69,7 @@ def _build_project_deps(env):
     if is_test:
         projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR",
                              "$PIOTEST_SRC_FILTER")
-    if not is_test or env.get("TEST_BUILD_PROJECT_SRC") == "true":
+    if not is_test or env.GetProjectOption("test_build_project_src", False):
         projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR",
                              env.get("SRC_FILTER"))
 
@@ -93,7 +97,8 @@ def BuildProgram(env):
     if not Util.case_sensitive_suffixes(".s", ".S"):
         env.Replace(AS="$CC", ASCOM="$ASPPCOM")
 
-    if set(["__debug", "debug"]) & set(COMMAND_LINE_TARGETS):
+    if ("debug" in COMMAND_LINE_TARGETS
+            or env.GetProjectOption("build_type") == "debug"):
         env.ProcessDebug()
 
     # process extra flags from board
@@ -128,8 +133,8 @@ def BuildProgram(env):
         env.Prepend(_LIBFLAGS="-Wl,--start-group ")
         env.Append(_LIBFLAGS=" -Wl,--end-group")
 
-    program = env.Program(
+    program = env.Program(join("$BUILD_DIR", env.subst("$PROGNAME")),
-        join("$BUILD_DIR", env.subst("$PROGNAME")), env['PIOBUILDFILES'])
+                          env['PIOBUILDFILES'])
     env.Replace(PIOMAINPROG=program)
 
     AlwaysBuild(
@@ -189,11 +194,13 @@ def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
     # provided with a -U option // Issue #191
     undefines = [
         u for u in env.get("CCFLAGS", [])
-        if isinstance(u, basestring) and u.startswith("-U")
+        if isinstance(u, string_types) and u.startswith("-U")
     ]
     if undefines:
         for undef in undefines:
             env['CCFLAGS'].remove(undef)
+            if undef[2:] in env['CPPDEFINES']:
+                env['CPPDEFINES'].remove(undef[2:])
         env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines))
 
 

platformio/commands/__init__.py

@@ -11,3 +11,59 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import os
+from os.path import dirname
+
+import click
+
+
+class PlatformioCLI(click.MultiCommand):
+
+    leftover_args = []
+
+    @staticmethod
+    def in_silence():
+        args = PlatformioCLI.leftover_args
+        return args and any([
+            args[0] == "debug" and "--interpreter" in " ".join(args),
+            args[0] == "upgrade", "--json-output" in args, "--version" in args
+        ])
+
+    def invoke(self, ctx):
+        PlatformioCLI.leftover_args = ctx.args
+        if hasattr(ctx, "protected_args"):
+            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
+        return super(PlatformioCLI, self).invoke(ctx)
+
+    def list_commands(self, ctx):
+        cmds = []
+        for filename in os.listdir(dirname(__file__)):
+            if filename.startswith("__init__"):
+                continue
+            if filename.endswith(".py"):
+                cmds.append(filename[:-3])
+        cmds.sort()
+        return cmds
+
+    def get_command(self, ctx, cmd_name):
+        mod = None
+        try:
+            mod = __import__("platformio.commands." + cmd_name, None, None,
+                             ["cli"])
+        except ImportError:
+            try:
+                return self._handle_obsolate_command(cmd_name)
+            except AttributeError:
+                raise click.UsageError('No such command "%s"' % cmd_name, ctx)
+        return mod.cli
+
+    @staticmethod
+    def _handle_obsolate_command(name):
+        if name == "platforms":
+            from platformio.commands import platform
+            return platform.cli
+        if name == "serialports":
+            from platformio.commands import device
+            return device.cli
+        raise AttributeError()

platformio/commands/boards.py

@@ -17,6 +17,7 @@ import json
 import click
 
 from platformio import util
+from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformManager
 
 
@@ -51,24 +52,22 @@ def print_boards(boards):
     BOARDLIST_TPL = ("{type:<30} {mcu:<14} {frequency:<8} "
                      " {flash:<7} {ram:<6} {name}")
     click.echo(
-        BOARDLIST_TPL.format(
+        BOARDLIST_TPL.format(type=click.style("ID", fg="cyan"),
-            type=click.style("ID", fg="cyan"),
+                             mcu="MCU",
-            mcu="MCU",
+                             frequency="Frequency",
-            frequency="Frequency",
+                             flash="Flash",
-            flash="Flash",
+                             ram="RAM",
-            ram="RAM",
+                             name="Name"))
-            name="Name"))
     click.echo("-" * terminal_width)
 
     for board in boards:
         click.echo(
-            BOARDLIST_TPL.format(
+            BOARDLIST_TPL.format(type=click.style(board['id'], fg="cyan"),
-                type=click.style(board['id'], fg="cyan"),
+                                 mcu=board['mcu'],
-                mcu=board['mcu'],
+                                 frequency="%dMHz" % (board['fcpu'] / 1000000),
-                frequency="%dMHz" % (board['fcpu'] / 1000000),
+                                 flash=util.format_filesize(board['rom']),
-                flash=util.format_filesize(board['rom']),
+                                 ram=util.format_filesize(board['ram']),
-                ram=util.format_filesize(board['ram']),
+                                 name=board['name']))
-                name=board['name']))
 
 
 def _get_boards(installed=False):
@@ -84,4 +83,4 @@ def _print_boards_json(query, installed=False):
             if query.lower() not in search_data.lower():
                 continue
         result.append(board)
-    click.echo(json.dumps(result))
+    click.echo(dump_json_to_unicode(result))

platformio/commands/ci.py

@@ -24,7 +24,9 @@ from platformio import app, util
 from platformio.commands.init import cli as cmd_init
 from platformio.commands.init import validate_boards
 from platformio.commands.run import cli as cmd_run
+from platformio.compat import glob_escape
 from platformio.exception import CIBuildEnvsEmpty
+from platformio.project.config import ProjectConfig
 
 
 def validate_path(ctx, param, value):  # pylint: disable=unused-argument
@@ -46,29 +48,31 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
 
 @click.command("ci", short_help="Continuous Integration")
 @click.argument("src", nargs=-1, callback=validate_path)
-@click.option(
+@click.option("-l",
-    "-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY")
+              "--lib",
+              multiple=True,
+              callback=validate_path,
+              metavar="DIRECTORY")
 @click.option("--exclude", multiple=True)
-@click.option(
+@click.option("-b",
-    "-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
+              "--board",
-@click.option(
+              multiple=True,
-    "--build-dir",
+              metavar="ID",
-    default=mkdtemp,
+              callback=validate_boards)
-    type=click.Path(
+@click.option("--build-dir",
-        file_okay=False,
+              default=mkdtemp,
-        dir_okay=True,
+              type=click.Path(file_okay=False,
-        writable=True,
+                              dir_okay=True,
-        resolve_path=True))
+                              writable=True,
+                              resolve_path=True))
 @click.option("--keep-build-dir", is_flag=True)
-@click.option(
+@click.option("-c",
-    "-C",
+              "--project-conf",
-    "--project-conf",
+              type=click.Path(exists=True,
-    type=click.Path(
+                              file_okay=True,
-        exists=True,
+                              dir_okay=False,
-        file_okay=True,
+                              readable=True,
-        dir_okay=False,
+                              resolve_path=True))
-        readable=True,
-        resolve_path=True))
 @click.option("-O", "--project-option", multiple=True)
 @click.option("-v", "--verbose", is_flag=True)
 @click.pass_context
@@ -106,11 +110,10 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
             _exclude_contents(build_dir, exclude)
 
         # initialise project
-        ctx.invoke(
+        ctx.invoke(cmd_init,
-            cmd_init,
+                   project_dir=build_dir,
-            project_dir=build_dir,
+                   board=board,
-            board=board,
+                   project_option=project_option)
-            project_option=project_option)
 
         # process project
         ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose)
@@ -154,7 +157,7 @@ def _copy_contents(dst_dir, contents):
 def _exclude_contents(dst_dir, patterns):
     contents = []
     for p in patterns:
-        contents += glob(join(util.glob_escape(dst_dir), p))
+        contents += glob(join(glob_escape(dst_dir), p))
     for path in contents:
         path = abspath(path)
         if isdir(path):
@@ -164,8 +167,7 @@ def _exclude_contents(dst_dir, patterns):
 
 
 def _copy_project_conf(build_dir, project_conf):
-    config = util.load_project_config(project_conf)
+    config = ProjectConfig(project_conf, parse_extra=False)
     if config.has_section("platformio"):
         config.remove_section("platformio")
-    with open(join(build_dir, "platformio.ini"), "w") as fp:
+    config.save(join(build_dir, "platformio.ini"))
-        config.write(fp)

platformio/commands/debug.py

@@ -1,42 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-from os import getcwd
-
-import click
-
-from platformio.managers.core import pioplus_call
-
-
-@click.command(
-    "debug",
-    context_settings=dict(ignore_unknown_options=True),
-    short_help="PIO Unified Debugger")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=getcwd,
-    type=click.Path(
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True))
-@click.option("--environment", "-e", metavar="<environment>")
-@click.option("--verbose", "-v", is_flag=True)
-@click.option("--interface", type=click.Choice(["gdb"]))
-@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
-def cli(*args, **kwargs):  # pylint: disable=unused-argument
-    pioplus_call(sys.argv[1:])

platformio/commands/debug/__init__.py

@@ -12,20 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
+from platformio.commands.debug.command import cli
-
-import click
-
-from platformio.managers.core import pioplus_call
-
-
-@click.command("home", short_help="PIO Home")
-@click.option("--port", type=int, default=8008, help="HTTP port, default=8008")
-@click.option(
-    "--host",
-    default="127.0.0.1",
-    help="HTTP host, default=127.0.0.1. "
-    "You can open PIO Home for inbound connections with --host=0.0.0.0")
-@click.option("--no-open", is_flag=True)
-def cli(*args, **kwargs):  # pylint: disable=unused-argument
-    pioplus_call(sys.argv[1:])

platformio/commands/debug/client.py

@@ -0,0 +1,290 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+import signal
+import time
+from hashlib import sha1
+from os.path import abspath, basename, dirname, isdir, join, splitext
+from tempfile import mkdtemp
+
+from twisted.internet import protocol  # pylint: disable=import-error
+from twisted.internet import reactor  # pylint: disable=import-error
+from twisted.internet import stdio  # pylint: disable=import-error
+from twisted.internet import task  # pylint: disable=import-error
+
+from platformio import app, exception, util
+from platformio.commands.debug import helpers, initcfgs
+from platformio.commands.debug.process import BaseProcess
+from platformio.commands.debug.server import DebugServer
+from platformio.compat import hashlib_encode_data
+from platformio.project.helpers import get_project_cache_dir
+from platformio.telemetry import MeasurementProtocol
+
+LOG_FILE = None
+
+
+class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
+
+    PIO_SRC_NAME = ".pioinit"
+    INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed"
+
+    def __init__(self, project_dir, args, debug_options, env_options):
+        self.project_dir = project_dir
+        self.args = list(args)
+        self.debug_options = debug_options
+        self.env_options = env_options
+
+        self._debug_server = DebugServer(debug_options, env_options)
+        self._session_id = None
+
+        if not isdir(get_project_cache_dir()):
+            os.makedirs(get_project_cache_dir())
+        self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(),
+                                   prefix=".piodebug-")
+
+        self._target_is_run = False
+        self._last_server_activity = 0
+        self._auto_continue_timer = None
+
+    def spawn(self, gdb_path, prog_path):
+        session_hash = gdb_path + prog_path
+        self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest()
+        self._kill_previous_session()
+
+        patterns = {
+            "PROJECT_DIR": helpers.escape_path(self.project_dir),
+            "PROG_PATH": helpers.escape_path(prog_path),
+            "PROG_DIR": helpers.escape_path(dirname(prog_path)),
+            "PROG_NAME": basename(splitext(prog_path)[0]),
+            "DEBUG_PORT": self.debug_options['port'],
+            "UPLOAD_PROTOCOL": self.debug_options['upload_protocol'],
+            "INIT_BREAK": self.debug_options['init_break'] or "",
+            "LOAD_CMDS": "\n".join(self.debug_options['load_cmds'] or []),
+        }
+
+        self._debug_server.spawn(patterns)
+
+        if not patterns['DEBUG_PORT']:
+            patterns['DEBUG_PORT'] = self._debug_server.get_debug_port()
+        self.generate_pioinit(self._gdbsrc_dir, patterns)
+
+        # start GDB client
+        args = [
+            "piogdb",
+            "-q",
+            "--directory", self._gdbsrc_dir,
+            "--directory", self.project_dir,
+            "-l", "10"
+        ]  # yapf: disable
+        args.extend(self.args)
+        if not gdb_path:
+            raise exception.DebugInvalidOptions("GDB client is not configured")
+        gdb_data_dir = self._get_data_dir(gdb_path)
+        if gdb_data_dir:
+            args.extend(["--data-directory", gdb_data_dir])
+        args.append(patterns['PROG_PATH'])
+
+        return reactor.spawnProcess(self,
+                                    gdb_path,
+                                    args,
+                                    path=self.project_dir,
+                                    env=os.environ)
+
+    @staticmethod
+    def _get_data_dir(gdb_path):
+        if "msp430" in gdb_path:
+            return None
+        gdb_data_dir = abspath(join(dirname(gdb_path), "..", "share", "gdb"))
+        return gdb_data_dir if isdir(gdb_data_dir) else None
+
+    def generate_pioinit(self, dst_dir, patterns):
+        server_exe = (self.debug_options.get("server")
+                      or {}).get("executable", "").lower()
+        if "jlink" in server_exe:
+            cfg = initcfgs.GDB_JLINK_INIT_CONFIG
+        elif "st-util" in server_exe:
+            cfg = initcfgs.GDB_STUTIL_INIT_CONFIG
+        elif "mspdebug" in server_exe:
+            cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG
+        elif "qemu" in server_exe:
+            cfg = initcfgs.GDB_QEMU_INIT_CONFIG
+        elif self.debug_options['require_debug_port']:
+            cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG
+        else:
+            cfg = initcfgs.GDB_DEFAULT_INIT_CONFIG
+        commands = cfg.split("\n")
+
+        if self.debug_options['init_cmds']:
+            commands = self.debug_options['init_cmds']
+        commands.extend(self.debug_options['extra_cmds'])
+
+        if not any("define pio_reset_target" in cmd for cmd in commands):
+            commands = [
+                "define pio_reset_target",
+                "   echo Warning! Undefined pio_reset_target command\\n",
+                "   mon reset",
+                "end"
+            ] + commands  # yapf: disable
+        if not any("define pio_reset_halt_target" in cmd for cmd in commands):
+            commands = [
+                "define pio_reset_halt_target",
+                "   echo Warning! Undefined pio_reset_halt_target command\\n",
+                "   mon reset halt",
+                "end"
+            ] + commands  # yapf: disable
+        if not any("define pio_restart_target" in cmd for cmd in commands):
+            commands += [
+                "define pio_restart_target",
+                "   pio_reset_halt_target",
+                "   $INIT_BREAK",
+                "   %s" % ("continue" if patterns['INIT_BREAK'] else "next"),
+                "end"
+            ]  # yapf: disable
+
+        banner = [
+            "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n",
+            "echo PlatformIO: Initializing remote target...\\n"
+        ]
+        footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER]
+        commands = banner + commands + footer
+
+        with open(join(dst_dir, self.PIO_SRC_NAME), "w") as fp:
+            fp.write("\n".join(self.apply_patterns(commands, patterns)))
+
+    def connectionMade(self):
+        self._lock_session(self.transport.pid)
+
+        p = protocol.Protocol()
+        p.dataReceived = self.onStdInData
+        stdio.StandardIO(p)
+
+    def onStdInData(self, data):
+        if LOG_FILE:
+            with open(LOG_FILE, "ab") as fp:
+                fp.write(data)
+
+        self._last_server_activity = time.time()
+
+        if b"-exec-run" in data:
+            if self._target_is_run:
+                token, _ = data.split(b"-", 1)
+                self.outReceived(token + b"^running\n")
+                return
+            data = data.replace(b"-exec-run", b"-exec-continue")
+
+        if b"-exec-continue" in data:
+            self._target_is_run = True
+        if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"):
+            # Allow terminating via SIGINT/CTRL+C
+            signal.signal(signal.SIGINT, signal.default_int_handler)
+            self.transport.write(b"pio_reset_target\n")
+        self.transport.write(data)
+
+    def processEnded(self, reason):  # pylint: disable=unused-argument
+        self._unlock_session()
+        if self._gdbsrc_dir and isdir(self._gdbsrc_dir):
+            util.rmtree_(self._gdbsrc_dir)
+        if self._debug_server:
+            self._debug_server.terminate()
+
+        reactor.stop()
+
+    def outReceived(self, data):
+        if LOG_FILE:
+            with open(LOG_FILE, "ab") as fp:
+                fp.write(data)
+
+        self._last_server_activity = time.time()
+        super(GDBClient, self).outReceived(data)
+        self._handle_error(data)
+        # go to init break automatically
+        if self.INIT_COMPLETED_BANNER.encode() in data:
+            self._auto_continue_timer = task.LoopingCall(
+                self._auto_exec_continue)
+            self._auto_continue_timer.start(0.1)
+
+    def errReceived(self, data):
+        super(GDBClient, self).errReceived(data)
+        self._handle_error(data)
+
+    def console_log(self, msg):
+        if helpers.is_mi_mode(self.args):
+            self.outReceived(('~"%s\\n"\n' % msg).encode())
+        else:
+            self.outReceived(("%s\n" % msg).encode())
+
+    def _auto_exec_continue(self):
+        auto_exec_delay = 0.5  # in seconds
+        if self._last_server_activity > (time.time() - auto_exec_delay):
+            return
+        if self._auto_continue_timer:
+            self._auto_continue_timer.stop()
+        self._auto_continue_timer = None
+
+        if not self.debug_options['init_break'] or self._target_is_run:
+            return
+        self.console_log(
+            "PlatformIO: Resume the execution to `debug_init_break = %s`" %
+            self.debug_options['init_break'])
+        self.console_log("PlatformIO: More configuration options -> "
+                         "http://bit.ly/pio-debug")
+        self.transport.write(b"0-exec-continue\n" if helpers.
+                             is_mi_mode(self.args) else b"continue\n")
+        self._target_is_run = True
+
+    def _handle_error(self, data):
+        if (self.PIO_SRC_NAME.encode() not in data
+                or b"Error in sourced" not in data):
+            return
+        configuration = {"debug": self.debug_options, "env": self.env_options}
+        exd = re.sub(r'\\(?!")', "/", json.dumps(configuration))
+        exd = re.sub(r'"(?:[a-z]\:)?((/[^"/]+)+)"', lambda m: '"%s"' % join(
+            *m.group(1).split("/")[-2:]), exd, re.I | re.M)
+        mp = MeasurementProtocol()
+        mp['exd'] = "DebugGDBPioInitError: %s" % exd
+        mp['exf'] = 1
+        mp.send("exception")
+        self.transport.loseConnection()
+
+    def _kill_previous_session(self):
+        assert self._session_id
+        pid = None
+        with app.ContentCache() as cc:
+            pid = cc.get(self._session_id)
+            cc.delete(self._session_id)
+        if not pid:
+            return
+        if "windows" in util.get_systype():
+            kill = ["Taskkill", "/PID", pid, "/F"]
+        else:
+            kill = ["kill", pid]
+        try:
+            util.exec_command(kill)
+        except:  # pylint: disable=bare-except
+            pass
+
+    def _lock_session(self, pid):
+        if not self._session_id:
+            return
+        with app.ContentCache() as cc:
+            cc.set(self._session_id, str(pid), "1h")
+
+    def _unlock_session(self):
+        if not self._session_id:
+            return
+        with app.ContentCache() as cc:
+            cc.delete(self._session_id)

platformio/commands/debug/command.py

@@ -0,0 +1,153 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-arguments, too-many-statements
+# pylint: disable=too-many-locals, too-many-branches
+
+import os
+import signal
+from os.path import isfile, join
+
+import click
+
+from platformio import exception, util
+from platformio.commands.debug import helpers
+from platformio.managers.core import inject_contrib_pysite
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import (is_platformio_project,
+                                        load_project_ide_data)
+
+
+@click.command("debug",
+               context_settings=dict(ignore_unknown_options=True),
+               short_help="PIO Unified Debugger")
+@click.option("-d",
+              "--project-dir",
+              default=os.getcwd,
+              type=click.Path(exists=True,
+                              file_okay=False,
+                              dir_okay=True,
+                              writable=True,
+                              resolve_path=True))
+@click.option("-c",
+              "--project-conf",
+              type=click.Path(exists=True,
+                              file_okay=True,
+                              dir_okay=False,
+                              readable=True,
+                              resolve_path=True))
+@click.option("--environment", "-e", metavar="<environment>")
+@click.option("--verbose", "-v", is_flag=True)
+@click.option("--interface", type=click.Choice(["gdb"]))
+@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
+@click.pass_context
+def cli(ctx, project_dir, project_conf, environment, verbose, interface,
+        __unprocessed):
+    # use env variables from Eclipse or CLion
+    for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"):
+        if is_platformio_project(project_dir):
+            break
+        if os.getenv(sysenv):
+            project_dir = os.getenv(sysenv)
+
+    with util.cd(project_dir):
+        config = ProjectConfig.get_instance(
+            project_conf or join(project_dir, "platformio.ini"))
+        config.validate(envs=[environment] if environment else None)
+
+        env_name = environment or helpers.get_default_debug_env(config)
+        env_options = config.items(env=env_name, as_dict=True)
+        if not set(env_options.keys()) >= set(["platform", "board"]):
+            raise exception.ProjectEnvsNotAvailable()
+        debug_options = helpers.validate_debug_options(ctx, env_options)
+        assert debug_options
+
+    if not interface:
+        return helpers.predebug_project(ctx, project_dir, env_name, False,
+                                        verbose)
+
+    configuration = load_project_ide_data(project_dir, env_name)
+    if not configuration:
+        raise exception.DebugInvalidOptions(
+            "Could not load debug configuration")
+
+    if "--version" in __unprocessed:
+        result = util.exec_command([configuration['gdb_path'], "--version"])
+        if result['returncode'] == 0:
+            return click.echo(result['out'])
+        raise exception.PlatformioException("\n".join(
+            [result['out'], result['err']]))
+
+    try:
+        util.ensure_udev_rules()
+    except NameError:
+        pass
+    except exception.InvalidUdevRules as e:
+        for line in str(e).split("\n") + [""]:
+            click.echo(
+                ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") %
+                line)
+
+    debug_options['load_cmds'] = helpers.configure_esp32_load_cmds(
+        debug_options, configuration)
+
+    rebuild_prog = False
+    preload = debug_options['load_cmds'] == ["preload"]
+    load_mode = debug_options['load_mode']
+    if load_mode == "always":
+        rebuild_prog = (
+            preload
+            or not helpers.has_debug_symbols(configuration['prog_path']))
+    elif load_mode == "modified":
+        rebuild_prog = (
+            helpers.is_prog_obsolete(configuration['prog_path'])
+            or not helpers.has_debug_symbols(configuration['prog_path']))
+    else:
+        rebuild_prog = not isfile(configuration['prog_path'])
+
+    if preload or (not rebuild_prog and load_mode != "always"):
+        # don't load firmware through debug server
+        debug_options['load_cmds'] = []
+
+    if rebuild_prog:
+        if helpers.is_mi_mode(__unprocessed):
+            click.echo('~"Preparing firmware for debugging...\\n"')
+            output = helpers.GDBBytesIO()
+            with util.capture_std_streams(output):
+                helpers.predebug_project(ctx, project_dir, env_name, preload,
+                                         verbose)
+            output.close()
+        else:
+            click.echo("Preparing firmware for debugging...")
+            helpers.predebug_project(ctx, project_dir, env_name, preload,
+                                     verbose)
+
+        # save SHA sum of newly created prog
+        if load_mode == "modified":
+            helpers.is_prog_obsolete(configuration['prog_path'])
+
+    if not isfile(configuration['prog_path']):
+        raise exception.DebugInvalidOptions("Program/firmware is missed")
+
+    # run debugging client
+    inject_contrib_pysite()
+    from platformio.commands.debug.client import GDBClient, reactor
+
+    client = GDBClient(project_dir, __unprocessed, debug_options, env_options)
+    client.spawn(configuration['gdb_path'], configuration['prog_path'])
+
+    signal.signal(signal.SIGINT, lambda *args, **kwargs: None)
+    reactor.run()
+
+    return True

platformio/commands/debug/helpers.py

@@ -0,0 +1,269 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import time
+from fnmatch import fnmatch
+from hashlib import sha1
+from io import BytesIO
+from os.path import isfile
+
+from platformio import exception, util
+from platformio.commands.platform import \
+    platform_install as cmd_platform_install
+from platformio.commands.run import cli as cmd_run
+from platformio.managers.platform import PlatformFactory
+from platformio.project.config import ProjectConfig
+
+
+class GDBBytesIO(BytesIO):  # pylint: disable=too-few-public-methods
+
+    STDOUT = sys.stdout
+
+    def write(self, text):
+        if "\n" in text:
+            for line in text.strip().split("\n"):
+                self.STDOUT.write('~"%s\\n"\n' % line)
+        else:
+            self.STDOUT.write('~"%s"' % text)
+        self.STDOUT.flush()
+
+
+def is_mi_mode(args):
+    return "--interpreter" in " ".join(args)
+
+
+def escape_path(path):
+    return path.replace("\\", "/")
+
+
+def get_default_debug_env(config):
+    default_envs = config.default_envs()
+    all_envs = config.envs()
+    for env in default_envs:
+        if config.get("env:" + env, "build_type") == "debug":
+            return env
+    for env in all_envs:
+        if config.get("env:" + env, "build_type") == "debug":
+            return env
+    return default_envs[0] if default_envs else all_envs[0]
+
+
+def predebug_project(ctx, project_dir, env_name, preload, verbose):
+    ctx.invoke(cmd_run,
+               project_dir=project_dir,
+               environment=[env_name],
+               target=["debug"] + (["upload"] if preload else []),
+               verbose=verbose)
+    if preload:
+        time.sleep(5)
+
+
+def validate_debug_options(cmd_ctx, env_options):
+
+    def _cleanup_cmds(items):
+        items = ProjectConfig.parse_multi_values(items)
+        return [
+            "$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items
+        ]
+
+    try:
+        platform = PlatformFactory.newPlatform(env_options['platform'])
+    except exception.UnknownPlatform:
+        cmd_ctx.invoke(cmd_platform_install,
+                       platforms=[env_options['platform']],
+                       skip_default_package=True)
+        platform = PlatformFactory.newPlatform(env_options['platform'])
+
+    board_config = platform.board_config(env_options['board'])
+    tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool"))
+    tool_settings = board_config.get("debug", {}).get("tools",
+                                                      {}).get(tool_name, {})
+    server_options = None
+
+    # specific server per a system
+    if isinstance(tool_settings.get("server", {}), list):
+        for item in tool_settings['server'][:]:
+            tool_settings['server'] = item
+            if util.get_systype() in item.get("system", []):
+                break
+
+    # user overwrites debug server
+    if env_options.get("debug_server"):
+        server_options = {
+            "cwd": None,
+            "executable": None,
+            "arguments": env_options.get("debug_server")
+        }
+        server_options['executable'] = server_options['arguments'][0]
+        server_options['arguments'] = server_options['arguments'][1:]
+    elif "server" in tool_settings:
+        server_package = tool_settings['server'].get("package")
+        server_package_dir = platform.get_package_dir(
+            server_package) if server_package else None
+        if server_package and not server_package_dir:
+            platform.install_packages(with_packages=[server_package],
+                                      skip_default_package=True,
+                                      silent=True)
+            server_package_dir = platform.get_package_dir(server_package)
+        server_options = dict(
+            cwd=server_package_dir if server_package else None,
+            executable=tool_settings['server'].get("executable"),
+            arguments=[
+                a.replace("$PACKAGE_DIR", escape_path(server_package_dir))
+                if server_package_dir else a
+                for a in tool_settings['server'].get("arguments", [])
+            ])
+
+    extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds"))
+    extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds")))
+    result = dict(
+        tool=tool_name,
+        upload_protocol=env_options.get(
+            "upload_protocol",
+            board_config.get("upload", {}).get("protocol")),
+        load_cmds=_cleanup_cmds(
+            env_options.get(
+                "debug_load_cmds",
+                tool_settings.get("load_cmds",
+                                  tool_settings.get("load_cmd", "load")))),
+        load_mode=env_options.get("debug_load_mode",
+                                  tool_settings.get("load_mode", "always")),
+        init_break=env_options.get(
+            "debug_init_break", tool_settings.get("init_break",
+                                                  "tbreak main")),
+        init_cmds=_cleanup_cmds(
+            env_options.get("debug_init_cmds",
+                            tool_settings.get("init_cmds"))),
+        extra_cmds=extra_cmds,
+        require_debug_port=tool_settings.get("require_debug_port", False),
+        port=reveal_debug_port(
+            env_options.get("debug_port", tool_settings.get("port")),
+            tool_name, tool_settings),
+        server=server_options)
+    return result
+
+
+def configure_esp32_load_cmds(debug_options, configuration):
+    ignore_conds = [
+        debug_options['load_cmds'] != ["load"],
+        "xtensa-esp32" not in configuration.get("cc_path", ""),
+        not configuration.get("flash_extra_images"), not all([
+            isfile(item['path'])
+            for item in configuration.get("flash_extra_images")
+        ])
+    ]
+    if any(ignore_conds):
+        return debug_options['load_cmds']
+
+    mon_cmds = [
+        'monitor program_esp32 "{{{path}}}" {offset} verify'.format(
+            path=escape_path(item['path']), offset=item['offset'])
+        for item in configuration.get("flash_extra_images")
+    ]
+    mon_cmds.append('monitor program_esp32 "{%s.bin}" 0x10000 verify' %
+                    escape_path(configuration['prog_path'][:-4]))
+    return mon_cmds
+
+
+def has_debug_symbols(prog_path):
+    if not isfile(prog_path):
+        return False
+    matched = {
+        b".debug_info": False,
+        b".debug_abbrev": False,
+        b" -Og": False,
+        b" -g": False,
+        b"__PLATFORMIO_BUILD_DEBUG__": False
+    }
+    with open(prog_path, "rb") as fp:
+        last_data = b""
+        while True:
+            data = fp.read(1024)
+            if not data:
+                break
+            for pattern, found in matched.items():
+                if found:
+                    continue
+                if pattern in last_data + data:
+                    matched[pattern] = True
+            last_data = data
+    return all(matched.values())
+
+
+def is_prog_obsolete(prog_path):
+    prog_hash_path = prog_path + ".sha1"
+    if not isfile(prog_path):
+        return True
+    shasum = sha1()
+    with open(prog_path, "rb") as fp:
+        while True:
+            data = fp.read(1024)
+            if not data:
+                break
+            shasum.update(data)
+    new_digest = shasum.hexdigest()
+    old_digest = None
+    if isfile(prog_hash_path):
+        with open(prog_hash_path, "r") as fp:
+            old_digest = fp.read()
+    if new_digest == old_digest:
+        return False
+    with open(prog_hash_path, "w") as fp:
+        fp.write(new_digest)
+    return True
+
+
+def reveal_debug_port(env_debug_port, tool_name, tool_settings):
+
+    def _get_pattern():
+        if not env_debug_port:
+            return None
+        if set(["*", "?", "[", "]"]) & set(env_debug_port):
+            return env_debug_port
+        return None
+
+    def _is_match_pattern(port):
+        pattern = _get_pattern()
+        if not pattern:
+            return True
+        return fnmatch(port, pattern)
+
+    def _look_for_serial_port(hwids):
+        for item in util.get_serialports(filter_hwid=True):
+            if not _is_match_pattern(item['port']):
+                continue
+            port = item['port']
+            if tool_name.startswith("blackmagic"):
+                if "windows" in util.get_systype() and \
+                        port.startswith("COM") and len(port) > 4:
+                    port = "\\\\.\\%s" % port
+                if "GDB" in item['description']:
+                    return port
+            for hwid in hwids:
+                hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
+                if hwid_str in item['hwid']:
+                    return port
+        return None
+
+    if env_debug_port and not _get_pattern():
+        return env_debug_port
+    if not tool_settings.get("require_debug_port"):
+        return None
+
+    debug_port = _look_for_serial_port(tool_settings.get("hwids", []))
+    if not debug_port:
+        raise exception.DebugInvalidOptions(
+            "Please specify `debug_port` for environment")
+    return debug_port

platformio/commands/debug/initcfgs.py

@@ -0,0 +1,124 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GDB_DEFAULT_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor reset halt
+end
+
+define pio_reset_target
+    monitor reset
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+pio_reset_halt_target
+$LOAD_CMDS
+monitor init
+pio_reset_halt_target
+"""
+
+GDB_STUTIL_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor halt
+    monitor reset
+end
+
+define pio_reset_target
+    monitor reset
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+pio_reset_halt_target
+$LOAD_CMDS
+pio_reset_halt_target
+"""
+
+GDB_JLINK_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor halt
+    monitor reset
+end
+
+define pio_reset_target
+    monitor reset
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+pio_reset_halt_target
+$LOAD_CMDS
+pio_reset_halt_target
+"""
+
+GDB_BLACKMAGIC_INIT_CONFIG = """
+define pio_reset_halt_target
+    set language c
+    set *0xE000ED0C = 0x05FA0004
+    set $busy = (*0xE000ED0C & 0x4)
+    while ($busy)
+        set $busy = (*0xE000ED0C & 0x4)
+    end
+    set language auto
+end
+
+define pio_reset_target
+    pio_reset_halt_target
+end
+
+target extended-remote $DEBUG_PORT
+monitor swdp_scan
+attach 1
+set mem inaccessible-by-default off
+$INIT_BREAK
+$LOAD_CMDS
+
+set language c
+set *0xE000ED0C = 0x05FA0004
+set $busy = (*0xE000ED0C & 0x4)
+while ($busy)
+    set $busy = (*0xE000ED0C & 0x4)
+end
+set language auto
+"""
+
+GDB_MSPDEBUG_INIT_CONFIG = """
+define pio_reset_halt_target
+end
+
+define pio_reset_target
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+monitor erase
+$LOAD_CMDS
+pio_reset_halt_target
+"""
+
+GDB_QEMU_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor system_reset
+end
+
+define pio_reset_target
+    pio_reset_halt_target
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+$LOAD_CMDS
+pio_reset_halt_target
+"""

platformio/commands/debug/process.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import signal
+
+import click
+from twisted.internet import protocol  # pylint: disable=import-error
+
+from platformio.commands.debug import helpers
+from platformio.compat import string_types
+from platformio.proc import get_pythonexe_path
+from platformio.project.helpers import get_project_core_dir
+
+LOG_FILE = None
+
+
+class BaseProcess(protocol.ProcessProtocol, object):
+
+    STDOUT_CHUNK_SIZE = 2048
+
+    COMMON_PATTERNS = {
+        "PLATFORMIO_HOME_DIR": helpers.escape_path(get_project_core_dir()),
+        "PLATFORMIO_CORE_DIR": helpers.escape_path(get_project_core_dir()),
+        "PYTHONEXE": get_pythonexe_path()
+    }
+
+    def apply_patterns(self, source, patterns=None):
+        _patterns = self.COMMON_PATTERNS.copy()
+        _patterns.update(patterns or {})
+
+        def _replace(text):
+            for key, value in _patterns.items():
+                pattern = "$%s" % key
+                text = text.replace(pattern, value or "")
+            return text
+
+        if isinstance(source, string_types):
+            source = _replace(source)
+        elif isinstance(source, (list, dict)):
+            items = enumerate(source) if isinstance(source,
+                                                    list) else source.items()
+            for key, value in items:
+                if isinstance(value, string_types):
+                    source[key] = _replace(value)
+                elif isinstance(value, (list, dict)):
+                    source[key] = self.apply_patterns(value, patterns)
+
+        return source
+
+    def outReceived(self, data):
+        if LOG_FILE:
+            with open(LOG_FILE, "ab") as fp:
+                fp.write(data)
+        while data:
+            chunk = data[:self.STDOUT_CHUNK_SIZE]
+            click.echo(chunk, nl=False)
+            data = data[self.STDOUT_CHUNK_SIZE:]
+
+    @staticmethod
+    def errReceived(data):
+        if LOG_FILE:
+            with open(LOG_FILE, "ab") as fp:
+                fp.write(data)
+        click.echo(data, nl=False, err=True)
+
+    @staticmethod
+    def processEnded(_):
+        # Allow terminating via SIGINT/CTRL+C
+        signal.signal(signal.SIGINT, signal.default_int_handler)

platformio/commands/debug/server.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from os.path import isdir, isfile, join
+
+from twisted.internet import error  # pylint: disable=import-error
+from twisted.internet import reactor  # pylint: disable=import-error
+
+from platformio import exception, util
+from platformio.commands.debug import helpers
+from platformio.commands.debug.process import BaseProcess
+from platformio.proc import where_is_program
+
+
+class DebugServer(BaseProcess):
+
+    def __init__(self, debug_options, env_options):
+        self.debug_options = debug_options
+        self.env_options = env_options
+
+        self._debug_port = None
+        self._transport = None
+        self._process_ended = False
+
+    def spawn(self, patterns):  # pylint: disable=too-many-branches
+        systype = util.get_systype()
+        server = self.debug_options.get("server")
+        if not server:
+            return None
+        server = self.apply_patterns(server, patterns)
+        server_executable = server['executable']
+        if not server_executable:
+            return None
+        if server['cwd']:
+            server_executable = join(server['cwd'], server_executable)
+        if ("windows" in systype and not server_executable.endswith(".exe")
+                and isfile(server_executable + ".exe")):
+            server_executable = server_executable + ".exe"
+
+        if not isfile(server_executable):
+            server_executable = where_is_program(server_executable)
+        if not isfile(server_executable):
+            raise exception.DebugInvalidOptions(
+                "\nCould not launch Debug Server '%s'. Please check that it "
+                "is installed and is included in a system PATH\n\n"
+                "See documentation or contact contact@platformio.org:\n"
+                "http://docs.platformio.org/page/plus/debugging.html\n" %
+                server_executable)
+
+        self._debug_port = ":3333"
+        openocd_pipe_allowed = all([
+            not self.debug_options['port'],
+            "openocd" in server_executable
+        ])  # yapf: disable
+        if openocd_pipe_allowed:
+            args = []
+            if server['cwd']:
+                args.extend(["-s", helpers.escape_path(server['cwd'])])
+            args.extend([
+                "-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"
+            ])
+            args.extend(server['arguments'])
+            str_args = " ".join(
+                [arg if arg.startswith("-") else '"%s"' % arg for arg in args])
+            self._debug_port = '| "%s" %s' % (
+                helpers.escape_path(server_executable), str_args)
+        else:
+            env = os.environ.copy()
+            # prepend server "lib" folder to LD path
+            if ("windows" not in systype and server['cwd']
+                    and isdir(join(server['cwd'], "lib"))):
+                ld_key = ("DYLD_LIBRARY_PATH"
+                          if "darwin" in systype else "LD_LIBRARY_PATH")
+                env[ld_key] = join(server['cwd'], "lib")
+                if os.environ.get(ld_key):
+                    env[ld_key] = "%s:%s" % (env[ld_key],
+                                             os.environ.get(ld_key))
+            # prepend BIN to PATH
+            if server['cwd'] and isdir(join(server['cwd'], "bin")):
+                env['PATH'] = "%s%s%s" % (
+                    join(server['cwd'], "bin"), os.pathsep,
+                    os.environ.get("PATH", os.environ.get("Path", "")))
+
+            self._transport = reactor.spawnProcess(
+                self,
+                server_executable, [server_executable] + server['arguments'],
+                path=server['cwd'],
+                env=env)
+            if "mspdebug" in server_executable.lower():
+                self._debug_port = ":2000"
+            elif "jlink" in server_executable.lower():
+                self._debug_port = ":2331"
+            elif "qemu" in server_executable.lower():
+                self._debug_port = ":1234"
+
+        return self._transport
+
+    def get_debug_port(self):
+        return self._debug_port
+
+    def processEnded(self, reason):
+        self._process_ended = True
+        super(DebugServer, self).processEnded(reason)
+
+    def terminate(self):
+        if self._process_ended or not self._transport:
+            return
+        try:
+            self._transport.signalProcess("KILL")
+        except (OSError, error.ProcessExitedAlready):
+            pass

platformio/commands/device.py

@@ -12,14 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import sys
+from fnmatch import fnmatch
 from os import getcwd
+from os.path import join
 
 import click
 from serial.tools import miniterm
 
 from platformio import exception, util
+from platformio.compat import dump_json_to_unicode
+from platformio.project.config import ProjectConfig
 
 
 @click.group(short_help="Monitor device or list existing")
@@ -44,10 +47,11 @@ def device_list(  # pylint: disable=too-many-branches
     if mdns:
         data['mdns'] = util.get_mdns_services()
 
-    single_key = data.keys()[0] if len(data.keys()) == 1 else None
+    single_key = list(data)[0] if len(list(data)) == 1 else None
 
     if json_output:
-        return click.echo(json.dumps(data[single_key] if single_key else data))
+        return click.echo(
+            dump_json_to_unicode(data[single_key] if single_key else data))
 
     titles = {
         "serial": "Serial Ports",
@@ -98,81 +102,74 @@ def device_list(  # pylint: disable=too-many-branches
 @cli.command("monitor", short_help="Monitor device (Serial)")
 @click.option("--port", "-p", help="Port, a number or a device name")
 @click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
-@click.option(
+@click.option("--parity",
-    "--parity",
+              default="N",
-    default="N",
+              type=click.Choice(["N", "E", "O", "S", "M"]),
-    type=click.Choice(["N", "E", "O", "S", "M"]),
+              help="Set parity, default=N")
-    help="Set parity, default=N")
+@click.option("--rtscts",
-@click.option(
+              is_flag=True,
-    "--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
+              help="Enable RTS/CTS flow control, default=Off")
-@click.option(
+@click.option("--xonxoff",
-    "--xonxoff",
+              is_flag=True,
-    is_flag=True,
+              help="Enable software flow control, default=Off")
-    help="Enable software flow control, default=Off")
+@click.option("--rts",
-@click.option(
+              default=None,
-    "--rts",
+              type=click.IntRange(0, 1),
-    default=None,
+              help="Set initial RTS line state")
-    type=click.IntRange(0, 1),
+@click.option("--dtr",
-    help="Set initial RTS line state")
+              default=None,
-@click.option(
+              type=click.IntRange(0, 1),
-    "--dtr",
+              help="Set initial DTR line state")
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial DTR line state")
 @click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
-@click.option(
+@click.option("--encoding",
-    "--encoding",
+              default="UTF-8",
-    default="UTF-8",
+              help="Set the encoding for the serial port (e.g. hexlify, "
-    help="Set the encoding for the serial port (e.g. hexlify, "
+              "Latin1, UTF-8), default: UTF-8")
-    "Latin1, UTF-8), default: UTF-8")
 @click.option("--filter", "-f", multiple=True, help="Add text transformation")
-@click.option(
+@click.option("--eol",
-    "--eol",
+              default="CRLF",
-    default="CRLF",
+              type=click.Choice(["CR", "LF", "CRLF"]),
-    type=click.Choice(["CR", "LF", "CRLF"]),
+              help="End of line mode, default=CRLF")
-    help="End of line mode, default=CRLF")
+@click.option("--raw",
-@click.option(
+              is_flag=True,
-    "--raw", is_flag=True, help="Do not apply any encodings/transformations")
+              help="Do not apply any encodings/transformations")
-@click.option(
+@click.option("--exit-char",
-    "--exit-char",
+              type=int,
-    type=int,
+              default=3,
-    default=3,
+              help="ASCII code of special character that is used to exit "
-    help="ASCII code of special character that is used to exit "
+              "the application, default=3 (Ctrl+C)")
-    "the application, default=3 (Ctrl+C)")
+@click.option("--menu-char",
-@click.option(
+              type=int,
-    "--menu-char",
+              default=20,
-    type=int,
+              help="ASCII code of special character that is used to "
-    default=20,
+              "control miniterm (menu), default=20 (DEC)")
-    help="ASCII code of special character that is used to "
+@click.option("--quiet",
-    "control miniterm (menu), default=20 (DEC)")
+              is_flag=True,
-@click.option(
+              help="Diagnostics: suppress non-error messages, default=Off")
-    "--quiet",
+@click.option("-d",
-    is_flag=True,
+              "--project-dir",
-    help="Diagnostics: suppress non-error messages, default=Off")
+              default=getcwd,
-@click.option(
+              type=click.Path(exists=True,
-    "-d",
+                              file_okay=False,
-    "--project-dir",
+                              dir_okay=True,
-    default=getcwd,
+                              resolve_path=True))
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, resolve_path=True))
 @click.option(
     "-e",
     "--environment",
     help="Load configuration from `platformio.ini` and specified environment")
 def device_monitor(**kwargs):  # pylint: disable=too-many-branches
+    env_options = {}
     try:
-        project_options = get_project_options(kwargs['project_dir'],
+        env_options = get_project_options(kwargs['project_dir'],
-                                              kwargs['environment'])
+                                          kwargs['environment'])
-        monitor_options = {k: v for k, v in project_options or []}
+        for k in ("port", "speed", "rts", "dtr"):
-        if monitor_options:
+            k2 = "monitor_%s" % k
-            for k in ("port", "baud", "speed", "rts", "dtr"):
+            if k == "speed":
-                k2 = "monitor_%s" % k
+                k = "baud"
-                if k == "speed":
+            if kwargs[k] is None and k2 in env_options:
-                    k = "baud"
+                kwargs[k] = env_options[k2]
-                if kwargs[k] is None and k2 in monitor_options:
+                if k != "port":
-                    kwargs[k] = monitor_options[k2]
+                    kwargs[k] = int(kwargs[k])
-                    if k != "port":
-                        kwargs[k] = int(kwargs[k])
     except exception.NotPlatformIOProject:
         pass
 
@@ -181,11 +178,13 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
         if len(ports) == 1:
             kwargs['port'] = ports[0]['port']
 
-    sys.argv = ["monitor"]
+    sys.argv = ["monitor"] + env_options.get("monitor_flags", [])
     for k, v in kwargs.items():
         if k in ("port", "baud", "rts", "dtr", "environment", "project_dir"):
             continue
         k = "--" + k.replace("_", "-")
+        if k in env_options.get("monitor_flags", []):
+            continue
         if isinstance(v, bool):
             if v:
                 sys.argv.append(k)
@@ -195,34 +194,28 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
         else:
             sys.argv.extend([k, str(v)])
 
+    if kwargs['port'] and (set(["*", "?", "[", "]"]) & set(kwargs['port'])):
+        for item in util.get_serial_ports():
+            if fnmatch(item['port'], kwargs['port']):
+                kwargs['port'] = item['port']
+                break
+
     try:
-        miniterm.main(
+        miniterm.main(default_port=kwargs['port'],
-            default_port=kwargs['port'],
+                      default_baudrate=kwargs['baud'] or 9600,
-            default_baudrate=kwargs['baud'] or 9600,
+                      default_rts=kwargs['rts'],
-            default_rts=kwargs['rts'],
+                      default_dtr=kwargs['dtr'])
-            default_dtr=kwargs['dtr'])
     except Exception as e:
         raise exception.MinitermException(e)
 
 
-def get_project_options(project_dir, environment):
+def get_project_options(project_dir, environment=None):
-    config = util.load_project_config(project_dir)
+    config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
-    if not config.sections():
+    config.validate(envs=[environment] if environment else None)
-        return None
+    if not environment:
-
+        default_envs = config.default_envs()
-    known_envs = [s[4:] for s in config.sections() if s.startswith("env:")]
+        if default_envs:
-    if environment:
+            environment = default_envs[0]
-        if environment in known_envs:
+        else:
-            return config.items("env:%s" % environment)
+            environment = config.envs()[0]
-        raise exception.UnknownEnvNames(environment, ", ".join(known_envs))
+    return config.items(env=environment, as_dict=True)
-
-    if not known_envs:
-        return None
-
-    if config.has_option("platformio", "env_default"):
-        env_default = config.get("platformio",
-                                 "env_default").split(", ")[0].strip()
-        if env_default and env_default in known_envs:
-            return config.items("env:%s" % env_default)
-
-    return config.items("env:%s" % known_envs[0])

platformio/commands/home/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.home.command import cli

platformio/commands/home/command.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mimetypes
+import socket
+from os.path import isdir
+
+import click
+
+from platformio import exception
+from platformio.managers.core import (get_core_package_dir,
+                                      inject_contrib_pysite)
+
+
+@click.command("home", short_help="PIO Home")
+@click.option("--port", type=int, default=8008, help="HTTP port, default=8008")
+@click.option(
+    "--host",
+    default="127.0.0.1",
+    help="HTTP host, default=127.0.0.1. "
+    "You can open PIO Home for inbound connections with --host=0.0.0.0")
+@click.option("--no-open", is_flag=True)  # pylint: disable=too-many-locals
+def cli(port, host, no_open):
+    # import contrib modules
+    inject_contrib_pysite()
+    # pylint: disable=import-error
+    from autobahn.twisted.resource import WebSocketResource
+    from twisted.internet import reactor
+    from twisted.web import server
+    # pylint: enable=import-error
+    from platformio.commands.home.rpc.handlers.app import AppRPC
+    from platformio.commands.home.rpc.handlers.ide import IDERPC
+    from platformio.commands.home.rpc.handlers.misc import MiscRPC
+    from platformio.commands.home.rpc.handlers.os import OSRPC
+    from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
+    from platformio.commands.home.rpc.handlers.project import ProjectRPC
+    from platformio.commands.home.rpc.server import JSONRPCServerFactory
+    from platformio.commands.home.web import WebRoot
+
+    factory = JSONRPCServerFactory()
+    factory.addHandler(AppRPC(), namespace="app")
+    factory.addHandler(IDERPC(), namespace="ide")
+    factory.addHandler(MiscRPC(), namespace="misc")
+    factory.addHandler(OSRPC(), namespace="os")
+    factory.addHandler(PIOCoreRPC(), namespace="core")
+    factory.addHandler(ProjectRPC(), namespace="project")
+
+    contrib_dir = get_core_package_dir("contrib-piohome")
+    if not isdir(contrib_dir):
+        raise exception.PlatformioException("Invalid path to PIO Home Contrib")
+
+    # Ensure PIO Home mimetypes are known
+    mimetypes.add_type("text/html", ".html")
+    mimetypes.add_type("text/css", ".css")
+    mimetypes.add_type("application/javascript", ".js")
+
+    root = WebRoot(contrib_dir)
+    root.putChild(b"wsrpc", WebSocketResource(factory))
+    site = server.Site(root)
+
+    # hook for `platformio-node-helpers`
+    if host == "__do_not_start__":
+        return
+
+    # if already started
+    already_started = False
+    socket.setdefaulttimeout(1)
+    try:
+        socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
+        already_started = True
+    except:  # pylint: disable=bare-except
+        pass
+
+    home_url = "http://%s:%d" % (host, port)
+    if not no_open:
+        if already_started:
+            click.launch(home_url)
+        else:
+            reactor.callLater(1, lambda: click.launch(home_url))
+
+    click.echo("\n".join([
+        "",
+        "  ___I_",
+        " /\\-_--\\   PlatformIO Home",
+        "/  \\_-__\\",
+        "|[]| [] |  %s" % home_url,
+        "|__|____|______________%s" % ("_" * len(host)),
+    ]))
+    click.echo("")
+    click.echo("Open PIO Home in your browser by this URL => %s" % home_url)
+
+    if already_started:
+        return
+
+    click.echo("PIO Home has been started. Press Ctrl+C to shutdown.")
+
+    reactor.listenTCP(port, site, interface=host)
+    reactor.run()

platformio/commands/home/helpers.py

@@ -0,0 +1,71 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=keyword-arg-before-vararg, arguments-differ
+
+import os
+import socket
+
+import requests
+from twisted.internet import defer  # pylint: disable=import-error
+from twisted.internet import reactor  # pylint: disable=import-error
+from twisted.internet import threads  # pylint: disable=import-error
+
+from platformio import util
+from platformio.proc import where_is_program
+
+
+class AsyncSession(requests.Session):
+
+    def __init__(self, n=None, *args, **kwargs):
+        if n:
+            pool = reactor.getThreadPool()
+            pool.adjustPoolsize(0, n)
+
+        super(AsyncSession, self).__init__(*args, **kwargs)
+
+    def request(self, *args, **kwargs):
+        func = super(AsyncSession, self).request
+        return threads.deferToThread(func, *args, **kwargs)
+
+    def wrap(self, *args, **kwargs):  # pylint: disable=no-self-use
+        return defer.ensureDeferred(*args, **kwargs)
+
+
+@util.memoized(expire="60s")
+def requests_session():
+    return AsyncSession(n=5)
+
+
+@util.memoized(expire="60s")
+def get_core_fullpath():
+    return where_is_program(
+        "platformio" + (".exe" if "windows" in util.get_systype() else ""))
+
+
+@util.memoized(expire="10s")
+def is_twitter_blocked():
+    ip = "104.244.42.1"
+    timeout = 2
+    try:
+        if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")):
+            requests.get("http://%s" % ip,
+                         allow_redirects=False,
+                         timeout=timeout)
+        else:
+            socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80))
+        return False
+    except:  # pylint: disable=bare-except
+        pass
+    return True

platformio/commands/home/rpc/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/home/rpc/handlers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/commands/home/rpc/handlers/app.py

@@ -0,0 +1,71 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from os.path import expanduser, join
+
+from platformio import __version__, app, util
+from platformio.project.helpers import (get_project_core_dir,
+                                        is_platformio_project)
+
+
+class AppRPC(object):
+
+    APPSTATE_PATH = join(get_project_core_dir(), "homestate.json")
+
+    @staticmethod
+    def load_state():
+        with app.State(AppRPC.APPSTATE_PATH, lock=True) as state:
+            storage = state.get("storage", {})
+
+            # base data
+            caller_id = app.get_session_var("caller_id")
+            storage['cid'] = app.get_cid()
+            storage['coreVersion'] = __version__
+            storage['coreSystype'] = util.get_systype()
+            storage['coreCaller'] = (str(caller_id).lower()
+                                     if caller_id else None)
+            storage['coreSettings'] = {
+                name: {
+                    "description": data['description'],
+                    "default_value": data['value'],
+                    "value": app.get_setting(name)
+                }
+                for name, data in app.DEFAULT_SETTINGS.items()
+            }
+
+            storage['homeDir'] = expanduser("~")
+            storage['projectsDir'] = storage['coreSettings']['projects_dir'][
+                'value']
+
+            # skip non-existing recent projects
+            storage['recentProjects'] = [
+                p for p in storage.get("recentProjects", [])
+                if is_platformio_project(p)
+            ]
+
+            state['storage'] = storage
+            return state.as_dict()
+
+    @staticmethod
+    def get_state():
+        return AppRPC.load_state()
+
+    @staticmethod
+    def save_state(state):
+        with app.State(AppRPC.APPSTATE_PATH, lock=True) as s:
+            # s.clear()
+            s.update(state)
+        return True

platformio/commands/home/rpc/handlers/ide.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+import jsonrpc  # pylint: disable=import-error
+from twisted.internet import defer  # pylint: disable=import-error
+
+
+class IDERPC(object):
+
+    def __init__(self):
+        self._queue = []
+
+    def send_command(self, command, params):
+        if not self._queue:
+            raise jsonrpc.exceptions.JSONRPCDispatchException(
+                code=4005, message="PIO Home IDE agent is not started")
+        while self._queue:
+            self._queue.pop().callback({
+                "id": time.time(),
+                "method": command,
+                "params": params
+            })
+
+    def listen_commands(self):
+        self._queue.append(defer.Deferred())
+        return self._queue[-1]
+
+    def open_project(self, project_dir):
+        return self.send_command("open_project", project_dir)

platformio/commands/home/rpc/handlers/misc.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import time
+
+from twisted.internet import defer, reactor  # pylint: disable=import-error
+
+from platformio import app
+from platformio.commands.home.rpc.handlers.os import OSRPC
+
+
+class MiscRPC(object):
+
+    def load_latest_tweets(self, username):
+        cache_key = "piohome_latest_tweets_" + str(username)
+        cache_valid = "7d"
+        with app.ContentCache() as cc:
+            cache_data = cc.get(cache_key)
+            if cache_data:
+                cache_data = json.loads(cache_data)
+                # automatically update cache in background every 12 hours
+                if cache_data['time'] < (time.time() - (3600 * 12)):
+                    reactor.callLater(5, self._preload_latest_tweets, username,
+                                      cache_key, cache_valid)
+                return cache_data['result']
+
+        result = self._preload_latest_tweets(username, cache_key, cache_valid)
+        return result
+
+    @staticmethod
+    @defer.inlineCallbacks
+    def _preload_latest_tweets(username, cache_key, cache_valid):
+        result = yield OSRPC.fetch_content(
+            "https://api.platformio.org/tweets/" + username)
+        result = json.loads(result)
+        with app.ContentCache() as cc:
+            cc.set(cache_key,
+                   json.dumps({
+                       "time": int(time.time()),
+                       "result": result
+                   }), cache_valid)
+        defer.returnValue(result)

platformio/commands/home/rpc/handlers/os.py

@@ -0,0 +1,152 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import codecs
+import glob
+import os
+import shutil
+from functools import cmp_to_key
+from os.path import expanduser, isdir, isfile, join
+
+import click
+from twisted.internet import defer  # pylint: disable=import-error
+
+from platformio import app, util
+from platformio.commands.home import helpers
+from platformio.compat import PY2, get_filesystem_encoding
+
+
+class OSRPC(object):
+
+    @staticmethod
+    @defer.inlineCallbacks
+    def fetch_content(uri, data=None, headers=None, cache_valid=None):
+        if not headers:
+            headers = {
+                "User-Agent":
+                ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) "
+                 "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 "
+                 "Safari/603.3.8")
+            }
+        cache_key = (app.ContentCache.key_from_args(uri, data)
+                     if cache_valid else None)
+        with app.ContentCache() as cc:
+            if cache_key:
+                result = cc.get(cache_key)
+                if result is not None:
+                    defer.returnValue(result)
+
+        # check internet before and resolve issue with 60 seconds timeout
+        util.internet_on(raise_exception=True)
+
+        session = helpers.requests_session()
+        if data:
+            r = yield session.post(uri, data=data, headers=headers)
+        else:
+            r = yield session.get(uri, headers=headers)
+
+        r.raise_for_status()
+        result = r.text
+        if cache_valid:
+            with app.ContentCache() as cc:
+                cc.set(cache_key, result, cache_valid)
+        defer.returnValue(result)
+
+    def request_content(self, uri, data=None, headers=None, cache_valid=None):
+        if uri.startswith('http'):
+            return self.fetch_content(uri, data, headers, cache_valid)
+        if not isfile(uri):
+            return None
+        with codecs.open(uri, encoding="utf-8") as fp:
+            return fp.read()
+
+    @staticmethod
+    def open_url(url):
+        return click.launch(url)
+
+    @staticmethod
+    def reveal_file(path):
+        return click.launch(
+            path.encode(get_filesystem_encoding()) if PY2 else path,
+            locate=True)
+
+    @staticmethod
+    def is_file(path):
+        return isfile(path)
+
+    @staticmethod
+    def is_dir(path):
+        return isdir(path)
+
+    @staticmethod
+    def make_dirs(path):
+        return os.makedirs(path)
+
+    @staticmethod
+    def rename(src, dst):
+        return os.rename(src, dst)
+
+    @staticmethod
+    def copy(src, dst):
+        return shutil.copytree(src, dst)
+
+    @staticmethod
+    def glob(pathnames, root=None):
+        if not isinstance(pathnames, list):
+            pathnames = [pathnames]
+        result = set()
+        for pathname in pathnames:
+            result |= set(
+                glob.glob(join(root, pathname) if root else pathname))
+        return list(result)
+
+    @staticmethod
+    def list_dir(path):
+
+        def _cmp(x, y):
+            if x[1] and not y[1]:
+                return -1
+            if not x[1] and y[1]:
+                return 1
+            if x[0].lower() > y[0].lower():
+                return 1
+            if x[0].lower() < y[0].lower():
+                return -1
+            return 0
+
+        items = []
+        if path.startswith("~"):
+            path = expanduser(path)
+        if not isdir(path):
+            return items
+        for item in os.listdir(path):
+            try:
+                item_is_dir = isdir(join(path, item))
+                if item_is_dir:
+                    os.listdir(join(path, item))
+                items.append((item, item_is_dir))
+            except OSError:
+                pass
+        return sorted(items, key=cmp_to_key(_cmp))
+
+    @staticmethod
+    def get_logical_devices():
+        items = []
+        for item in util.get_logical_devices():
+            if item['name']:
+                item['name'] = item['name']
+            items.append(item)
+        return items

platformio/commands/home/rpc/handlers/piocore.py

@@ -0,0 +1,142 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import json
+import os
+import sys
+from io import BytesIO, StringIO
+
+import click
+import jsonrpc  # pylint: disable=import-error
+from twisted.internet import threads  # pylint: disable=import-error
+from twisted.internet import utils  # pylint: disable=import-error
+
+from platformio import __main__, __version__, util
+from platformio.commands.home import helpers
+from platformio.compat import (PY2, get_filesystem_encoding, is_bytes,
+                               string_types)
+
+try:
+    from thread import get_ident as thread_get_ident
+except ImportError:
+    from threading import get_ident as thread_get_ident
+
+
+class MultiThreadingStdStream(object):
+
+    def __init__(self, parent_stream):
+        self._buffers = {thread_get_ident(): parent_stream}
+
+    def __getattr__(self, name):
+        thread_id = thread_get_ident()
+        self._ensure_thread_buffer(thread_id)
+        return getattr(self._buffers[thread_id], name)
+
+    def _ensure_thread_buffer(self, thread_id):
+        if thread_id not in self._buffers:
+            self._buffers[thread_id] = BytesIO() if PY2 else StringIO()
+
+    def write(self, value):
+        thread_id = thread_get_ident()
+        self._ensure_thread_buffer(thread_id)
+        return self._buffers[thread_id].write(
+            value.decode() if is_bytes(value) else value)
+
+    def get_value_and_reset(self):
+        result = ""
+        try:
+            result = self.getvalue()
+            self.truncate(0)
+            self.seek(0)
+        except AttributeError:
+            pass
+        return result
+
+
+class PIOCoreRPC(object):
+
+    @staticmethod
+    def setup_multithreading_std_streams():
+        if isinstance(sys.stdout, MultiThreadingStdStream):
+            return
+        PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout)
+        PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr)
+        sys.stdout = PIOCoreRPC.thread_stdout
+        sys.stderr = PIOCoreRPC.thread_stderr
+
+    @staticmethod
+    def call(args, options=None):
+        PIOCoreRPC.setup_multithreading_std_streams()
+        cwd = (options or {}).get("cwd") or os.getcwd()
+        for i, arg in enumerate(args):
+            if isinstance(arg, string_types):
+                args[i] = arg.encode(get_filesystem_encoding()) if PY2 else arg
+            else:
+                args[i] = str(arg)
+
+        def _call_inline():
+            with util.cd(cwd):
+                exit_code = __main__.main(["-c"] + args)
+            return (PIOCoreRPC.thread_stdout.get_value_and_reset(),
+                    PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code)
+
+        if args and args[0] in ("account", "remote"):
+            d = utils.getProcessOutputAndValue(
+                helpers.get_core_fullpath(),
+                args,
+                path=cwd,
+                env={k: v
+                     for k, v in os.environ.items() if "%" not in k})
+        else:
+            d = threads.deferToThread(_call_inline)
+
+        d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args)
+        d.addErrback(PIOCoreRPC._call_errback)
+        return d
+
+    @staticmethod
+    def _call_callback(result, json_output=False):
+        out, err, code = result
+        text = ("%s\n\n%s" % (out, err)).strip()
+        if code != 0:
+            raise Exception(text)
+        if not json_output:
+            return text
+        try:
+            return json.loads(out)
+        except ValueError as e:
+            click.secho("%s => `%s`" % (e, out), fg="red", err=True)
+            # if PIO Core prints unhandled warnings
+            for line in out.split("\n"):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    return json.loads(line)
+                except ValueError:
+                    pass
+            raise e
+
+    @staticmethod
+    def _call_errback(failure):
+        raise jsonrpc.exceptions.JSONRPCDispatchException(
+            code=4003,
+            message="PIO Core Call Error",
+            data=failure.getErrorMessage())
+
+    @staticmethod
+    def version():
+        return __version__

platformio/commands/home/rpc/handlers/project.py

@@ -0,0 +1,277 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import os
+import shutil
+import time
+from os.path import (basename, expanduser, getmtime, isdir, isfile, join,
+                     realpath, sep)
+
+import jsonrpc  # pylint: disable=import-error
+
+from platformio import exception, util
+from platformio.commands.home.rpc.handlers.app import AppRPC
+from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
+from platformio.compat import PY2, get_filesystem_encoding
+from platformio.ide.projectgenerator import ProjectGenerator
+from platformio.managers.platform import PlatformManager
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import (get_project_libdeps_dir,
+                                        get_project_src_dir,
+                                        is_platformio_project)
+
+
+class ProjectRPC(object):
+
+    @staticmethod
+    def _get_projects(project_dirs=None):
+
+        def _get_project_data(project_dir):
+            data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []}
+            config = ProjectConfig(join(project_dir, "platformio.ini"))
+            libdeps_dir = get_project_libdeps_dir()
+
+            data['libExtraDirs'].extend(
+                config.get("platformio", "lib_extra_dirs", []))
+
+            for section in config.sections():
+                if not section.startswith("env:"):
+                    continue
+                data['envLibdepsDirs'].append(join(libdeps_dir, section[4:]))
+                if config.has_option(section, "board"):
+                    data['boards'].append(config.get(section, "board"))
+                data['libExtraDirs'].extend(
+                    config.get(section, "lib_extra_dirs", []))
+
+            # skip non existing folders and resolve full path
+            for key in ("envLibdepsDirs", "libExtraDirs"):
+                data[key] = [
+                    expanduser(d) if d.startswith("~") else realpath(d)
+                    for d in data[key] if isdir(d)
+                ]
+
+            return data
+
+        def _path_to_name(path):
+            return (sep).join(path.split(sep)[-2:])
+
+        if not project_dirs:
+            project_dirs = AppRPC.load_state()['storage']['recentProjects']
+
+        result = []
+        pm = PlatformManager()
+        for project_dir in project_dirs:
+            data = {}
+            boards = []
+            try:
+                with util.cd(project_dir):
+                    data = _get_project_data(project_dir)
+            except exception.PlatformIOProjectException:
+                continue
+
+            for board_id in data.get("boards", []):
+                name = board_id
+                try:
+                    name = pm.board_config(board_id)['name']
+                except (exception.UnknownBoard, exception.UnknownPlatform):
+                    pass
+                boards.append({"id": board_id, "name": name})
+
+            result.append({
+                "path":
+                project_dir,
+                "name":
+                _path_to_name(project_dir),
+                "modified":
+                int(getmtime(project_dir)),
+                "boards":
+                boards,
+                "envLibStorages": [{
+                    "name": basename(d),
+                    "path": d
+                } for d in data.get("envLibdepsDirs", [])],
+                "extraLibStorages": [{
+                    "name": _path_to_name(d),
+                    "path": d
+                } for d in data.get("libExtraDirs", [])]
+            })
+        return result
+
+    def get_projects(self, project_dirs=None):
+        return self._get_projects(project_dirs)
+
+    @staticmethod
+    def get_project_examples():
+        result = []
+        for manifest in PlatformManager().get_installed():
+            examples_dir = join(manifest['__pkg_dir'], "examples")
+            if not isdir(examples_dir):
+                continue
+            items = []
+            for project_dir, _, __ in os.walk(examples_dir):
+                project_description = None
+                try:
+                    config = ProjectConfig(join(project_dir, "platformio.ini"))
+                    config.validate(silent=True)
+                    project_description = config.get("platformio",
+                                                     "description")
+                except exception.PlatformIOProjectException:
+                    continue
+
+                path_tokens = project_dir.split(sep)
+                items.append({
+                    "name":
+                    "/".join(path_tokens[path_tokens.index("examples") + 1:]),
+                    "path":
+                    project_dir,
+                    "description":
+                    project_description
+                })
+            result.append({
+                "platform": {
+                    "title": manifest['title'],
+                    "version": manifest['version']
+                },
+                "items": sorted(items, key=lambda item: item['name'])
+            })
+        return sorted(result, key=lambda data: data['platform']['title'])
+
+    def init(self, board, framework, project_dir):
+        assert project_dir
+        state = AppRPC.load_state()
+        if not isdir(project_dir):
+            os.makedirs(project_dir)
+        args = ["init", "--board", board]
+        if framework:
+            args.extend(["--project-option", "framework = %s" % framework])
+        if (state['storage']['coreCaller'] and state['storage']['coreCaller']
+                in ProjectGenerator.get_supported_ides()):
+            args.extend(["--ide", state['storage']['coreCaller']])
+        d = PIOCoreRPC.call(args, options={"cwd": project_dir})
+        d.addCallback(self._generate_project_main, project_dir, framework)
+        return d
+
+    @staticmethod
+    def _generate_project_main(_, project_dir, framework):
+        main_content = None
+        if framework == "arduino":
+            main_content = "\n".join([
+                "#include <Arduino.h>",
+                "",
+                "void setup() {",
+                "  // put your setup code here, to run once:",
+                "}",
+                "",
+                "void loop() {",
+                "  // put your main code here, to run repeatedly:",
+                "}"
+                ""
+            ])   # yapf: disable
+        elif framework == "mbed":
+            main_content = "\n".join([
+                "#include <mbed.h>",
+                "",
+                "int main() {",
+                "",
+                "  // put your setup code here, to run once:",
+                "",
+                "  while(1) {",
+                "    // put your main code here, to run repeatedly:",
+                "  }",
+                "}",
+                ""
+            ])   # yapf: disable
+        if not main_content:
+            return project_dir
+        with util.cd(project_dir):
+            src_dir = get_project_src_dir()
+            main_path = join(src_dir, "main.cpp")
+            if isfile(main_path):
+                return project_dir
+            if not isdir(src_dir):
+                os.makedirs(src_dir)
+            with open(main_path, "w") as f:
+                f.write(main_content.strip())
+        return project_dir
+
+    def import_arduino(self, board, use_arduino_libs, arduino_project_dir):
+        board = str(board)
+        if arduino_project_dir and PY2:
+            arduino_project_dir = arduino_project_dir.encode(
+                get_filesystem_encoding())
+        # don't import PIO Project
+        if is_platformio_project(arduino_project_dir):
+            return arduino_project_dir
+
+        is_arduino_project = any([
+            isfile(
+                join(arduino_project_dir,
+                     "%s.%s" % (basename(arduino_project_dir), ext)))
+            for ext in ("ino", "pde")
+        ])
+        if not is_arduino_project:
+            raise jsonrpc.exceptions.JSONRPCDispatchException(
+                code=4000,
+                message="Not an Arduino project: %s" % arduino_project_dir)
+
+        state = AppRPC.load_state()
+        project_dir = join(state['storage']['projectsDir'],
+                           time.strftime("%y%m%d-%H%M%S-") + board)
+        if not isdir(project_dir):
+            os.makedirs(project_dir)
+        args = ["init", "--board", board]
+        args.extend(["--project-option", "framework = arduino"])
+        if use_arduino_libs:
+            args.extend([
+                "--project-option",
+                "lib_extra_dirs = ~/Documents/Arduino/libraries"
+            ])
+        if (state['storage']['coreCaller'] and state['storage']['coreCaller']
+                in ProjectGenerator.get_supported_ides()):
+            args.extend(["--ide", state['storage']['coreCaller']])
+        d = PIOCoreRPC.call(args, options={"cwd": project_dir})
+        d.addCallback(self._finalize_arduino_import, project_dir,
+                      arduino_project_dir)
+        return d
+
+    @staticmethod
+    def _finalize_arduino_import(_, project_dir, arduino_project_dir):
+        with util.cd(project_dir):
+            src_dir = get_project_src_dir()
+            if isdir(src_dir):
+                util.rmtree_(src_dir)
+            shutil.copytree(arduino_project_dir, src_dir)
+        return project_dir
+
+    @staticmethod
+    def import_pio(project_dir):
+        if not project_dir or not is_platformio_project(project_dir):
+            raise jsonrpc.exceptions.JSONRPCDispatchException(
+                code=4001,
+                message="Not an PlatformIO project: %s" % project_dir)
+        new_project_dir = join(
+            AppRPC.load_state()['storage']['projectsDir'],
+            time.strftime("%y%m%d-%H%M%S-") + basename(project_dir))
+        shutil.copytree(project_dir, new_project_dir)
+
+        state = AppRPC.load_state()
+        args = ["init"]
+        if (state['storage']['coreCaller'] and state['storage']['coreCaller']
+                in ProjectGenerator.get_supported_ides()):
+            args.extend(["--ide", state['storage']['coreCaller']])
+        d = PIOCoreRPC.call(args, options={"cwd": new_project_dir})
+        d.addCallback(lambda _: new_project_dir)
+        return d

platformio/commands/home/rpc/server.py

@@ -0,0 +1,77 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=import-error
+
+import click
+import jsonrpc
+from autobahn.twisted.websocket import (WebSocketServerFactory,
+                                        WebSocketServerProtocol)
+from jsonrpc.exceptions import JSONRPCDispatchException
+from twisted.internet import defer
+
+from platformio.compat import PY2, dump_json_to_unicode, is_bytes
+
+
+class JSONRPCServerProtocol(WebSocketServerProtocol):
+
+    def onMessage(self, payload, isBinary):  # pylint: disable=unused-argument
+        # click.echo("> %s" % payload)
+        response = jsonrpc.JSONRPCResponseManager.handle(
+            payload, self.factory.dispatcher).data
+        # if error
+        if "result" not in response:
+            self.sendJSONResponse(response)
+            return None
+
+        d = defer.maybeDeferred(lambda: response['result'])
+        d.addCallback(self._callback, response)
+        d.addErrback(self._errback, response)
+
+        return None
+
+    def _callback(self, result, response):
+        response['result'] = result
+        self.sendJSONResponse(response)
+
+    def _errback(self, failure, response):
+        if isinstance(failure.value, JSONRPCDispatchException):
+            e = failure.value
+        else:
+            e = JSONRPCDispatchException(code=4999,
+                                         message=failure.getErrorMessage())
+        del response["result"]
+        response['error'] = e.error._data  # pylint: disable=protected-access
+        self.sendJSONResponse(response)
+
+    def sendJSONResponse(self, response):
+        # click.echo("< %s" % response)
+        if "error" in response:
+            click.secho("Error: %s" % response['error'], fg="red", err=True)
+        response = dump_json_to_unicode(response)
+        if not PY2 and not is_bytes(response):
+            response = response.encode("utf-8")
+        self.sendMessage(response)
+
+
+class JSONRPCServerFactory(WebSocketServerFactory):
+
+    protocol = JSONRPCServerProtocol
+
+    def __init__(self):
+        super(JSONRPCServerFactory, self).__init__()
+        self.dispatcher = jsonrpc.Dispatcher()
+
+    def addHandler(self, handler, namespace):
+        self.dispatcher.build_method_map(handler, prefix="%s." % namespace)

platformio/commands/home/web.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import reactor  # pylint: disable=import-error
+from twisted.web import static  # pylint: disable=import-error
+
+
+class WebRoot(static.File):
+
+    def render_GET(self, request):
+        if request.args.get("__shutdown__", False):
+            reactor.stop()
+            return "Server has been stopped"
+
+        request.setHeader("cache-control",
+                          "no-cache, no-store, must-revalidate")
+        request.setHeader("pragma", "no-cache")
+        request.setHeader("expires", "0")
+        return static.File.render_GET(self, request)

platformio/commands/init.py

@@ -16,16 +16,20 @@
 
 from os import getcwd, makedirs
 from os.path import isdir, isfile, join
-from shutil import copyfile
 
 import click
 
 from platformio import exception, util
 from platformio.commands.platform import \
     platform_install as cli_platform_install
-from platformio.commands.run import check_project_envs
 from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.managers.platform import PlatformManager
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import (get_project_include_dir,
+                                        get_project_lib_dir,
+                                        get_project_src_dir,
+                                        get_project_test_dir,
+                                        is_platformio_project)
 
 
 def validate_boards(ctx, param, value):  # pylint: disable=W0613
@@ -40,22 +44,23 @@ def validate_boards(ctx, param, value):  # pylint: disable=W0613
     return value
 
 
-@click.command(
+@click.command("init",
-    "init", short_help="Initialize PlatformIO project or update existing")
+               short_help="Initialize PlatformIO project or update existing")
-@click.option(
+@click.option("--project-dir",
-    "--project-dir",
+              "-d",
-    "-d",
+              default=getcwd,
-    default=getcwd,
+              type=click.Path(exists=True,
-    type=click.Path(
+                              file_okay=False,
-        exists=True,
+                              dir_okay=True,
-        file_okay=False,
+                              writable=True,
-        dir_okay=True,
+                              resolve_path=True))
-        writable=True,
+@click.option("-b",
-        resolve_path=True))
+              "--board",
-@click.option(
+              multiple=True,
-    "-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
+              metavar="ID",
-@click.option(
+              callback=validate_boards)
-    "--ide", type=click.Choice(ProjectGenerator.get_supported_ides()))
+@click.option("--ide",
+              type=click.Choice(ProjectGenerator.get_supported_ides()))
 @click.option("-O", "--project-option", multiple=True)
 @click.option("--env-prefix", default="")
 @click.option("-s", "--silent", is_flag=True)
@@ -68,38 +73,37 @@ def cli(
         project_option,
         env_prefix,
         silent):
-
     if not silent:
         if project_dir == getcwd():
-            click.secho(
+            click.secho("\nThe current working directory",
-                "\nThe current working directory", fg="yellow", nl=False)
+                        fg="yellow",
+                        nl=False)
             click.secho(" %s " % project_dir, fg="cyan", nl=False)
             click.secho("will be used for the project.", fg="yellow")
             click.echo("")
 
         click.echo("The next files/directories have been created in %s" %
                    click.style(project_dir, fg="cyan"))
-        click.echo("%s - Put project header files here" % click.style(
+        click.echo("%s - Put project header files here" %
-            "include", fg="cyan"))
+                   click.style("include", fg="cyan"))
         click.echo("%s - Put here project specific (private) libraries" %
                    click.style("lib", fg="cyan"))
-        click.echo("%s - Put project source files here" % click.style(
+        click.echo("%s - Put project source files here" %
-            "src", fg="cyan"))
+                   click.style("src", fg="cyan"))
-        click.echo("%s - Project Configuration File" % click.style(
+        click.echo("%s - Project Configuration File" %
-            "platformio.ini", fg="cyan"))
+                   click.style("platformio.ini", fg="cyan"))
 
-    is_new_project = not util.is_platformio_project(project_dir)
+    is_new_project = not is_platformio_project(project_dir)
-    init_base_project(project_dir)
+    if is_new_project:
+        init_base_project(project_dir)
 
     if board:
         fill_project_envs(ctx, project_dir, board, project_option, env_prefix,
                           ide is not None)
 
     if ide:
-        env_name = get_best_envname(project_dir, board)
+        pg = ProjectGenerator(project_dir, ide,
-        if not env_name:
+                              get_best_envname(project_dir, board))
-            raise exception.BoardNotDefined()
-        pg = ProjectGenerator(project_dir, ide, env_name)
         pg.generate()
 
     if is_new_project:
@@ -112,8 +116,8 @@ def cli(
     if ide:
         click.secho(
             "\nProject has been successfully %s including configuration files "
-            "for `%s` IDE." % ("initialized" if is_new_project else "updated",
+            "for `%s` IDE." %
-                               ide),
+            ("initialized" if is_new_project else "updated", ide),
             fg="green")
     else:
         click.secho(
@@ -128,38 +132,36 @@ def cli(
 
 
 def get_best_envname(project_dir, boards=None):
-    config = util.load_project_config(project_dir)
+    config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
-    env_default = None
+    config.validate()
-    if config.has_option("platformio", "env_default"):
+
-        env_default = util.parse_conf_multi_values(
+    envname = None
-            config.get("platformio", "env_default"))
+    default_envs = config.default_envs()
-        check_project_envs(config, env_default)
+    if default_envs:
-    if env_default:
+        envname = default_envs[0]
-        return env_default[0]
+        if not boards:
-    section = None
+            return envname
-    for section in config.sections():
+
-        if not section.startswith("env:"):
+    for env in config.envs():
-            continue
+        if not boards:
-        elif config.has_option(section, "board") and (not boards or config.get(
+            return env
-                section, "board") in boards):
+        if not envname:
-            break
+            envname = env
-    return section[4:] if section else None
+        items = config.items(env=env, as_dict=True)
+        if "board" in items and items.get("board") in boards:
+            return env
+
+    return envname
 
 
 def init_base_project(project_dir):
-    if util.is_platformio_project(project_dir):
+    ProjectConfig(join(project_dir, "platformio.ini")).save()
-        return
-
-    copyfile(
-        join(util.get_source_dir(), "projectconftpl.ini"),
-        join(project_dir, "platformio.ini"))
-
     with util.cd(project_dir):
         dir_to_readme = [
-            (util.get_projectsrc_dir(), None),
+            (get_project_src_dir(), None),
-            (util.get_projectinclude_dir(), init_include_readme),
+            (get_project_include_dir(), init_include_readme),
-            (util.get_projectlib_dir(), init_lib_readme),
+            (get_project_lib_dir(), init_lib_readme),
-            (util.get_projecttest_dir(), init_test_readme),
+            (get_project_test_dir(), init_test_readme),
         ]
         for (path, cb) in dir_to_readme:
             if isdir(path):
@@ -360,16 +362,14 @@ def init_cvs_ignore(project_dir):
     if isfile(conf_path):
         return
     with open(conf_path, "w") as fp:
-        fp.writelines([".pio\n", ".pioenvs\n", ".piolibdeps\n"])
+        fp.write(".pio\n")
 
 
 def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix,
                       force_download):
-    content = []
+    config = ProjectConfig(join(project_dir, "platformio.ini"),
+                           parse_extra=False)
     used_boards = []
-    used_platforms = []
-
-    config = util.load_project_config(project_dir)
     for section in config.sections():
         cond = [
             section.startswith("env:"),
@@ -379,12 +379,15 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix,
             used_boards.append(config.get(section, "board"))
 
     pm = PlatformManager()
+    used_platforms = []
+    modified = False
     for id_ in board_ids:
         board_config = pm.board_config(id_)
         used_platforms.append(board_config['platform'])
         if id_ in used_boards:
             continue
         used_boards.append(id_)
+        modified = True
 
         envopts = {"platform": board_config['platform'], "board": id_}
         # find default framework for board
@@ -398,20 +401,18 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix,
             _name, _value = item.split("=", 1)
             envopts[_name.strip()] = _value.strip()
 
-        content.append("")
+        section = "env:%s%s" % (env_prefix, id_)
-        content.append("[env:%s%s]" % (env_prefix, id_))
+        config.add_section(section)
-        for name, value in envopts.items():
+
-            content.append("%s = %s" % (name, value))
+        for option, value in envopts.items():
+            config.set(section, option, value)
 
     if force_download and used_platforms:
         _install_dependent_platforms(ctx, used_platforms)
 
-    if not content:
+    if modified:
-        return
+        config.save()
-
+        config.reset_instances()
-    with open(join(project_dir, "platformio.ini"), "a") as f:
-        content.append("")
-        f.write("\n".join(content))
 
 
 def _install_dependent_platforms(ctx, platforms):
@@ -420,6 +421,5 @@ def _install_dependent_platforms(ctx, platforms):
     ]
     if set(platforms) <= set(installed_platforms):
         return
-    ctx.invoke(
+    ctx.invoke(cli_platform_install,
-        cli_platform_install,
+               platforms=list(set(platforms) - set(installed_platforms)))
-        platforms=list(set(platforms) - set(installed_platforms)))

platformio/commands/lib.py

@@ -14,172 +14,276 @@
 
 # pylint: disable=too-many-branches, too-many-locals
 
-import json
 import time
 from os.path import isdir, join
-from urllib import quote
 
 import click
+import semantic_version
 
 from platformio import exception, util
-from platformio.managers.lib import LibraryManager, get_builtin_libs
+from platformio.commands import PlatformioCLI
-from platformio.util import get_api_result
+from platformio.compat import dump_json_to_unicode
+from platformio.managers.lib import (LibraryManager, get_builtin_libs,
+                                     is_builtin_lib)
+from platformio.proc import is_ci
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import (get_project_dir,
+                                        get_project_global_lib_dir,
+                                        get_project_libdeps_dir,
+                                        is_platformio_project)
+
+try:
+    from urllib.parse import quote
+except ImportError:
+    from urllib import quote
+
+CTX_META_INPUT_DIRS_KEY = __name__ + ".input_dirs"
+CTX_META_PROJECT_ENVIRONMENTS_KEY = __name__ + ".project_environments"
+CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs"
+CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps"
 
 
 @click.group(short_help="Library Manager")
+@click.option("-d",
+              "--storage-dir",
+              multiple=True,
+              default=None,
+              type=click.Path(exists=True,
+                              file_okay=False,
+                              dir_okay=True,
+                              writable=True,
+                              resolve_path=True),
+              help="Manage custom library storage")
+@click.option("-g",
+              "--global",
+              is_flag=True,
+              help="Manage global PlatformIO library storage")
 @click.option(
-    "-g",
+    "-e",
-    "--global",
+    "--environment",
-    is_flag=True,
+    multiple=True,
-    help="Manage global PlatformIO library storage")
+    help=("Manage libraries for the specific project build environments "
-@click.option(
+          "declared in `platformio.ini`"))
-    "-d",
-    "--storage-dir",
-    default=None,
-    type=click.Path(
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True),
-    help="Manage custom library storage")
 @click.pass_context
 def cli(ctx, **options):
-    non_storage_cmds = ("search", "show", "register", "stats", "builtin")
+    storage_cmds = ("install", "uninstall", "update", "list")
     # skip commands that don't need storage folder
-    if ctx.invoked_subcommand in non_storage_cmds or \
+    if ctx.invoked_subcommand not in storage_cmds or \
             (len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")):
         return
-    storage_dir = options['storage_dir']
+    storage_dirs = list(options['storage_dir'])
-    if not storage_dir:
+    if options['global']:
-        if options['global']:
+        storage_dirs.append(get_project_global_lib_dir())
-            storage_dir = join(util.get_home_dir(), "lib")
+    if not storage_dirs:
-        elif util.is_platformio_project():
+        if is_platformio_project():
-            storage_dir = util.get_projectlibdeps_dir()
+            storage_dirs = [get_project_dir()]
-        elif util.is_ci():
+        elif is_ci():
-            storage_dir = join(util.get_home_dir(), "lib")
+            storage_dirs = [get_project_global_lib_dir()]
             click.secho(
                 "Warning! Global library storage is used automatically. "
                 "Please use `platformio lib --global %s` command to remove "
                 "this warning." % ctx.invoked_subcommand,
                 fg="yellow")
-    elif util.is_platformio_project(storage_dir):
-        with util.cd(storage_dir):
-            storage_dir = util.get_projectlibdeps_dir()
 
-    if not storage_dir and not util.is_platformio_project():
+    if not storage_dirs:
-        raise exception.NotGlobalLibDir(util.get_project_dir(),
+        raise exception.NotGlobalLibDir(get_project_dir(),
-                                        join(util.get_home_dir(), "lib"),
+                                        get_project_global_lib_dir(),
                                         ctx.invoked_subcommand)
 
-    ctx.obj = LibraryManager(storage_dir)
+    in_silence = PlatformioCLI.in_silence()
-    if "--json-output" not in ctx.args:
+    ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options['environment']
-        click.echo("Library Storage: " + storage_dir)
+    ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs
+    ctx.meta[CTX_META_STORAGE_DIRS_KEY] = []
+    ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] = {}
+    for storage_dir in storage_dirs:
+        if not is_platformio_project(storage_dir):
+            ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
+            continue
+        with util.cd(storage_dir):
+            libdeps_dir = get_project_libdeps_dir()
+        config = ProjectConfig.get_instance(join(storage_dir,
+                                                 "platformio.ini"))
+        config.validate(options['environment'], silent=in_silence)
+        for env in config.envs():
+            if options['environment'] and env not in options['environment']:
+                continue
+            storage_dir = join(libdeps_dir, env)
+            ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
+            ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get(
+                "env:" + env, "lib_deps", [])
 
 
 @cli.command("install", short_help="Install library")
 @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]")
-# @click.option(
-#     "--save",
-#     is_flag=True,
-#     help="Save installed libraries into the project's platformio.ini "
-#     "library dependencies")
 @click.option(
-    "-s", "--silent", is_flag=True, help="Suppress progress reporting")
+    "--save",
-@click.option(
-    "--interactive",
     is_flag=True,
-    help="Allow to make a choice for all prompts")
+    help="Save installed libraries into the `platformio.ini` dependency list")
-@click.option(
+@click.option("-s",
-    "-f",
+              "--silent",
-    "--force",
+              is_flag=True,
-    is_flag=True,
+              help="Suppress progress reporting")
-    help="Reinstall/redownload library if exists")
+@click.option("--interactive",
-@click.pass_obj
+              is_flag=True,
-def lib_install(lm, libraries, silent, interactive, force):
+              help="Allow to make a choice for all prompts")
-    # @TODO: "save" option
+@click.option("-f",
-    for library in libraries:
+              "--force",
-        lm.install(
+              is_flag=True,
-            library, silent=silent, interactive=interactive, force=force)
+              help="Reinstall/redownload library if exists")
+@click.pass_context
+def lib_install(  # pylint: disable=too-many-arguments
+        ctx, libraries, save, silent, interactive, force):
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
+    storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, [])
+
+    installed_manifests = {}
+    for storage_dir in storage_dirs:
+        if not silent and (libraries or storage_dir in storage_libdeps):
+            print_storage_header(storage_dirs, storage_dir)
+        lm = LibraryManager(storage_dir)
+        if libraries:
+            for library in libraries:
+                pkg_dir = lm.install(library,
+                                     silent=silent,
+                                     interactive=interactive,
+                                     force=force)
+                installed_manifests[library] = lm.load_manifest(pkg_dir)
+        elif storage_dir in storage_libdeps:
+            builtin_lib_storages = None
+            for library in storage_libdeps[storage_dir]:
+                try:
+                    pkg_dir = lm.install(library,
+                                         silent=silent,
+                                         interactive=interactive,
+                                         force=force)
+                    installed_manifests[library] = lm.load_manifest(pkg_dir)
+                except exception.LibNotFound as e:
+                    if builtin_lib_storages is None:
+                        builtin_lib_storages = get_builtin_libs()
+                    if not silent or not is_builtin_lib(
+                            builtin_lib_storages, library):
+                        click.secho("Warning! %s" % e, fg="yellow")
+
+    if not save or not libraries:
+        return
+
+    input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
+    project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
+    for input_dir in input_dirs:
+        config = ProjectConfig.get_instance(join(input_dir, "platformio.ini"))
+        config.validate(project_environments)
+        for env in config.envs():
+            if project_environments and env not in project_environments:
+                continue
+            config.expand_interpolations = False
+            lib_deps = config.get("env:" + env, "lib_deps", [])
+            for library in libraries:
+                if library in lib_deps:
+                    continue
+                manifest = installed_manifests[library]
+                try:
+                    assert library.lower() == manifest['name'].lower()
+                    assert semantic_version.Version(manifest['version'])
+                    lib_deps.append("{name}@^{version}".format(**manifest))
+                except (AssertionError, ValueError):
+                    lib_deps.append(library)
+            config.set("env:" + env, "lib_deps", lib_deps)
+            config.save()
 
 
 @cli.command("uninstall", short_help="Uninstall libraries")
 @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]")
-@click.pass_obj
+@click.pass_context
-def lib_uninstall(lm, libraries):
+def lib_uninstall(ctx, libraries):
-    for library in libraries:
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
-        lm.uninstall(library)
+    for storage_dir in storage_dirs:
+        print_storage_header(storage_dirs, storage_dir)
+        lm = LibraryManager(storage_dir)
+        for library in libraries:
+            lm.uninstall(library)
 
 
 @cli.command("update", short_help="Update installed libraries")
 @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]")
-@click.option(
+@click.option("-c",
-    "-c",
+              "--only-check",
-    "--only-check",
+              is_flag=True,
-    is_flag=True,
+              help="DEPRECATED. Please use `--dry-run` instead")
-    help="Do not update, only check for new version")
+@click.option("--dry-run",
+              is_flag=True,
+              help="Do not update, only check for the new versions")
 @click.option("--json-output", is_flag=True)
-@click.pass_obj
+@click.pass_context
-def lib_update(lm, libraries, only_check, json_output):
+def lib_update(ctx, libraries, only_check, dry_run, json_output):
-    if not libraries:
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
-        libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()]
+    only_check = dry_run or only_check
+    json_result = {}
+    for storage_dir in storage_dirs:
+        if not json_output:
+            print_storage_header(storage_dirs, storage_dir)
+        lm = LibraryManager(storage_dir)
 
-    if only_check and json_output:
+        _libraries = libraries
-        result = []
+        if not _libraries:
-        for library in libraries:
+            _libraries = [
-            pkg_dir = library if isdir(library) else None
+                manifest['__pkg_dir'] for manifest in lm.get_installed()
-            requirements = None
+            ]
-            url = None
+
-            if not pkg_dir:
+        if only_check and json_output:
-                name, requirements, url = lm.parse_pkg_uri(library)
+            result = []
-                pkg_dir = lm.get_package_dir(name, requirements, url)
+            for library in _libraries:
-            if not pkg_dir:
+                pkg_dir = library if isdir(library) else None
-                continue
+                requirements = None
-            latest = lm.outdated(pkg_dir, requirements)
+                url = None
-            if not latest:
+                if not pkg_dir:
-                continue
+                    name, requirements, url = lm.parse_pkg_uri(library)
-            manifest = lm.load_manifest(pkg_dir)
+                    pkg_dir = lm.get_package_dir(name, requirements, url)
-            manifest['versionLatest'] = latest
+                if not pkg_dir:
-            result.append(manifest)
+                    continue
-        return click.echo(json.dumps(result))
+                latest = lm.outdated(pkg_dir, requirements)
-    else:
+                if not latest:
-        for library in libraries:
+                    continue
-            lm.update(library, only_check=only_check)
+                manifest = lm.load_manifest(pkg_dir)
+                manifest['versionLatest'] = latest
+                result.append(manifest)
+            json_result[storage_dir] = result
+        else:
+            for library in _libraries:
+                lm.update(library, only_check=only_check)
+
+    if json_output:
+        return click.echo(
+            dump_json_to_unicode(json_result[storage_dirs[0]]
+                                 if len(storage_dirs) == 1 else json_result))
 
     return True
 
 
-def print_lib_item(item):
+@cli.command("list", short_help="List installed libraries")
-    click.secho(item['name'], fg="cyan")
+@click.option("--json-output", is_flag=True)
-    click.echo("=" * len(item['name']))
+@click.pass_context
-    if "id" in item:
+def lib_list(ctx, json_output):
-        click.secho("#ID: %d" % item['id'], bold=True)
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
-    if "description" in item or "url" in item:
+    json_result = {}
-        click.echo(item.get("description", item.get("url", "")))
+    for storage_dir in storage_dirs:
-    click.echo()
+        if not json_output:
-
+            print_storage_header(storage_dirs, storage_dir)
-    for key in ("version", "homepage", "license", "keywords"):
+        lm = LibraryManager(storage_dir)
-        if key not in item or not item[key]:
+        items = lm.get_installed()
-            continue
+        if json_output:
-        if isinstance(item[key], list):
+            json_result[storage_dir] = items
-            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
+        elif items:
+            for item in sorted(items, key=lambda i: i['name']):
+                print_lib_item(item)
         else:
-            click.echo("%s: %s" % (key.title(), item[key]))
+            click.echo("No items found")
 
-    for key in ("frameworks", "platforms"):
+    if json_output:
-        if key not in item:
+        return click.echo(
-            continue
+            dump_json_to_unicode(json_result[storage_dirs[0]]
-        click.echo("Compatible %s: %s" % (key, ", ".join(
+                                 if len(storage_dirs) == 1 else json_result))
-            [i['title'] if isinstance(i, dict) else i for i in item[key]])))
 
-    if "authors" in item or "authornames" in item:
+    return True
-        click.echo("Authors: %s" % ", ".join(
-            item.get("authornames",
-                     [a.get("name", "") for a in item.get("authors", [])])))
-
-    if "__src_url" in item:
-        click.secho("Source: %s" % item['__src_url'])
-    click.echo()
 
 
 @cli.command("search", short_help="Search for a library")
@@ -193,10 +297,9 @@ def print_lib_item(item):
 @click.option("-f", "--framework", multiple=True)
 @click.option("-p", "--platform", multiple=True)
 @click.option("-i", "--header", multiple=True)
-@click.option(
+@click.option("--noninteractive",
-    "--noninteractive",
+              is_flag=True,
-    is_flag=True,
+              help="Do not prompt, automatically paginate with delay")
-    help="Do not prompt, automatically paginate with delay")
 def lib_search(query, json_output, page, noninteractive, **filters):
     if not query:
         query = []
@@ -207,13 +310,12 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         for value in values:
             query.append('%s:"%s"' % (key, value))
 
-    result = get_api_result(
+    result = util.get_api_result("/v2/lib/search",
-        "/v2/lib/search",
+                                 dict(query=" ".join(query), page=page),
-        dict(query=" ".join(query), page=page),
+                                 cache_valid="1d")
-        cache_valid="1d")
 
     if json_output:
-        click.echo(json.dumps(result))
+        click.echo(dump_json_to_unicode(result))
         return
 
     if result['total'] == 0:
@@ -232,9 +334,8 @@ def lib_search(query, json_output, page, noninteractive, **filters):
             fg="cyan")
         return
 
-    click.secho(
+    click.secho("Found %d libraries:\n" % result['total'],
-        "Found %d libraries:\n" % result['total'],
+                fg="green" if result['total'] else "yellow")
-        fg="green" if result['total'] else "yellow")
 
     while True:
         for item in result['items']:
@@ -246,38 +347,18 @@ def lib_search(query, json_output, page, noninteractive, **filters):
 
         if noninteractive:
             click.echo()
-            click.secho(
+            click.secho("Loading next %d libraries... Press Ctrl+C to stop!" %
-                "Loading next %d libraries... Press Ctrl+C to stop!" %
+                        result['perpage'],
-                result['perpage'],
+                        fg="yellow")
-                fg="yellow")
             click.echo()
             time.sleep(5)
         elif not click.confirm("Show next libraries?"):
             break
-        result = get_api_result(
+        result = util.get_api_result("/v2/lib/search", {
-            "/v2/lib/search", {
+            "query": " ".join(query),
-                "query": " ".join(query),
+            "page": int(result['page']) + 1
-                "page": int(result['page']) + 1
+        },
-            },
+                                     cache_valid="1d")
-            cache_valid="1d")
-
-
-@cli.command("list", short_help="List installed libraries")
-@click.option("--json-output", is_flag=True)
-@click.pass_obj
-def lib_list(lm, json_output):
-    items = lm.get_installed()
-
-    if json_output:
-        return click.echo(json.dumps(items))
-
-    if not items:
-        return None
-
-    for item in sorted(items, key=lambda i: i['name']):
-        print_lib_item(item)
-
-    return True
 
 
 @cli.command("builtin", short_help="List built-in libraries")
@@ -286,7 +367,7 @@ def lib_list(lm, json_output):
 def lib_builtin(storage, json_output):
     items = get_builtin_libs(storage)
     if json_output:
-        return click.echo(json.dumps(items))
+        return click.echo(dump_json_to_unicode(items))
 
     for storage_ in items:
         if not storage_['items']:
@@ -313,9 +394,9 @@ def lib_show(library, json_output):
     },
                               silent=json_output,
                               interactive=not json_output)
-    lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
+    lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
     if json_output:
-        return click.echo(json.dumps(lib))
+        return click.echo(dump_json_to_unicode(lib))
 
     click.secho(lib['name'], fg="cyan")
     click.echo("=" * len(lib['name']))
@@ -389,21 +470,21 @@ def lib_register(config_url):
             and not config_url.startswith("https://")):
         raise exception.InvalidLibConfURL(config_url)
 
-    result = get_api_result("/lib/register", data=dict(config_url=config_url))
+    result = util.get_api_result("/lib/register",
+                                 data=dict(config_url=config_url))
     if "message" in result and result['message']:
-        click.secho(
+        click.secho(result['message'],
-            result['message'],
+                    fg="green" if "successed" in result and result['successed']
-            fg="green"
+                    else "red")
-            if "successed" in result and result['successed'] else "red")
 
 
 @cli.command("stats", short_help="Library Registry Statistics")
 @click.option("--json-output", is_flag=True)
 def lib_stats(json_output):
-    result = get_api_result("/lib/stats", cache_valid="1h")
+    result = util.get_api_result("/lib/stats", cache_valid="1h")
 
     if json_output:
-        return click.echo(json.dumps(result))
+        return click.echo(dump_json_to_unicode(result))
 
     printitem_tpl = "{name:<33} {url}"
     printitemdate_tpl = "{name:<33} {date:23} {url}"
@@ -425,10 +506,9 @@ def lib_stats(json_output):
         date = str(
             time.strftime("%c", util.parse_date(item['date'])) if "date" in
             item else "")
-        url = click.style(
+        url = click.style("https://platformio.org/lib/show/%s/%s" %
-            "https://platformio.org/lib/show/%s/%s" % (item['id'],
+                          (item['id'], quote(item['name'])),
-                                                       quote(item['name'])),
+                          fg="blue")
-            fg="blue")
         click.echo(
             (printitemdate_tpl if "date" in item else printitem_tpl).format(
                 name=click.style(item['name'], fg="cyan"), date=date, url=url))
@@ -437,10 +517,9 @@ def lib_stats(json_output):
         click.echo(
             printitem_tpl.format(
                 name=click.style(name, fg="cyan"),
-                url=click.style(
+                url=click.style("https://platformio.org/lib/search?query=" +
-                    "https://platformio.org/lib/search?query=" + quote(
+                                quote("keyword:%s" % name),
-                        "keyword:%s" % name),
+                                fg="blue")))
-                    fg="blue")))
 
     for key in ("updated", "added"):
         _print_title("Recently " + key)
@@ -470,3 +549,44 @@ def lib_stats(json_output):
         click.echo()
 
     return True
+
+
+def print_storage_header(storage_dirs, storage_dir):
+    if storage_dirs and storage_dirs[0] != storage_dir:
+        click.echo("")
+    click.echo(
+        click.style("Library Storage: ", bold=True) +
+        click.style(storage_dir, fg="blue"))
+
+
+def print_lib_item(item):
+    click.secho(item['name'], fg="cyan")
+    click.echo("=" * len(item['name']))
+    if "id" in item:
+        click.secho("#ID: %d" % item['id'], bold=True)
+    if "description" in item or "url" in item:
+        click.echo(item.get("description", item.get("url", "")))
+    click.echo()
+
+    for key in ("version", "homepage", "license", "keywords"):
+        if key not in item or not item[key]:
+            continue
+        if isinstance(item[key], list):
+            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
+        else:
+            click.echo("%s: %s" % (key.title(), item[key]))
+
+    for key in ("frameworks", "platforms"):
+        if key not in item:
+            continue
+        click.echo("Compatible %s: %s" % (key, ", ".join(
+            [i['title'] if isinstance(i, dict) else i for i in item[key]])))
+
+    if "authors" in item or "authornames" in item:
+        click.echo("Authors: %s" % ", ".join(
+            item.get("authornames",
+                     [a.get("name", "") for a in item.get("authors", [])])))
+
+    if "__src_url" in item:
+        click.secho("Source: %s" % item['__src_url'])
+    click.echo()

platformio/commands/platform.py

@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 from os.path import dirname, isdir
 
 import click
 
 from platformio import app, exception, util
 from platformio.commands.boards import print_boards
+from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformFactory, PlatformManager
 
 
@@ -29,9 +29,9 @@ def cli():
 
 def _print_platforms(platforms):
     for platform in platforms:
-        click.echo("{name} ~ {title}".format(
+        click.echo("{name} ~ {title}".format(name=click.style(platform['name'],
-            name=click.style(platform['name'], fg="cyan"),
+                                                              fg="cyan"),
-            title=platform['title']))
+                                             title=platform['title']))
         click.echo("=" * (3 + len(platform['name'] + platform['title'])))
         click.echo(platform['description'])
         click.echo()
@@ -42,7 +42,11 @@ def _print_platforms(platforms):
         if "packages" in platform:
             click.echo("Packages: %s" % ", ".join(platform['packages']))
         if "version" in platform:
-            click.echo("Version: " + platform['version'])
+            if "__src_url" in platform:
+                click.echo("Version: #%s (%s)" %
+                           (platform['version'], platform['__src_url']))
+            else:
+                click.echo("Version: " + platform['version'])
         click.echo()
 
 
@@ -54,18 +58,6 @@ def _get_registry_platforms():
     return platforms
 
 
-def _original_version(version):
-    if version.count(".") != 2:
-        return None
-    _, y = version.split(".")[:2]
-    if int(y) < 100:
-        return None
-    if len(y) % 2 != 0:
-        y = "0" + y
-    parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(len(y) / 2)]
-    return ".".join(parts)
-
-
 def _get_platform_data(*args, **kwargs):
     try:
         return _get_installed_platform_data(*args, **kwargs)
@@ -77,19 +69,18 @@ def _get_installed_platform_data(platform,
                                  with_boards=True,
                                  expose_packages=True):
     p = PlatformFactory.newPlatform(platform)
-    data = dict(
+    data = dict(name=p.name,
-        name=p.name,
+                title=p.title,
-        title=p.title,
+                description=p.description,
-        description=p.description,
+                version=p.version,
-        version=p.version,
+                homepage=p.homepage,
-        homepage=p.homepage,
+                repository=p.repository_url,
-        repository=p.repository_url,
+                url=p.vendor_url,
-        url=p.vendor_url,
+                docs=p.docs_url,
-        docs=p.docs_url,
+                license=p.license,
-        license=p.license,
+                forDesktop=not p.is_embedded(),
-        forDesktop=not p.is_embedded(),
+                frameworks=sorted(list(p.frameworks) if p.frameworks else []),
-        frameworks=sorted(p.frameworks.keys() if p.frameworks else []),
+                packages=list(p.packages) if p.packages else [])
-        packages=p.packages.keys() if p.packages else [])
 
     # if dump to API
     # del data['version']
@@ -111,18 +102,17 @@ def _get_installed_platform_data(platform,
     data['packages'] = []
     installed_pkgs = p.get_installed_packages()
     for name, opts in p.packages.items():
-        item = dict(
+        item = dict(name=name,
-            name=name,
+                    type=p.get_package_type(name),
-            type=p.get_package_type(name),
+                    requirements=opts.get("version"),
-            requirements=opts.get("version"),
+                    optional=opts.get("optional") is True)
-            optional=opts.get("optional") is True)
         if name in installed_pkgs:
             for key, value in installed_pkgs[name].items():
                 if key not in ("url", "version", "description"):
                     continue
                 item[key] = value
                 if key == "version":
-                    item["originalVersion"] = _original_version(value)
+                    item["originalVersion"] = util.get_original_version(value)
         data['packages'].append(item)
 
     return data
@@ -141,18 +131,17 @@ def _get_registry_platform_data(  # pylint: disable=unused-argument
     if not _data:
         return None
 
-    data = dict(
+    data = dict(name=_data['name'],
-        name=_data['name'],
+                title=_data['title'],
-        title=_data['title'],
+                description=_data['description'],
-        description=_data['description'],
+                homepage=_data['homepage'],
-        homepage=_data['homepage'],
+                repository=_data['repository'],
-        repository=_data['repository'],
+                url=_data['url'],
-        url=_data['url'],
+                license=_data['license'],
-        license=_data['license'],
+                forDesktop=_data['forDesktop'],
-        forDesktop=_data['forDesktop'],
+                frameworks=_data['frameworks'],
-        frameworks=_data['frameworks'],
+                packages=_data['packages'],
-        packages=_data['packages'],
+                versions=_data['versions'])
-        versions=_data['versions'])
 
     if with_boards:
         data['boards'] = [
@@ -171,15 +160,16 @@ def platform_search(query, json_output):
     for platform in _get_registry_platforms():
         if query == "all":
             query = ""
-        search_data = json.dumps(platform)
+        search_data = dump_json_to_unicode(platform)
         if query and query.lower() not in search_data.lower():
             continue
         platforms.append(
-            _get_registry_platform_data(
+            _get_registry_platform_data(platform['name'],
-                platform['name'], with_boards=False, expose_packages=False))
+                                        with_boards=False,
+                                        expose_packages=False))
 
     if json_output:
-        click.echo(json.dumps(platforms))
+        click.echo(dump_json_to_unicode(platforms))
     else:
         _print_platforms(platforms)
 
@@ -192,11 +182,11 @@ def platform_frameworks(query, json_output):
     for framework in util.get_api_result("/frameworks", cache_valid="7d"):
         if query == "all":
             query = ""
-        search_data = json.dumps(framework)
+        search_data = dump_json_to_unicode(framework)
         if query and query.lower() not in search_data.lower():
             continue
-        framework['homepage'] = (
+        framework['homepage'] = ("https://platformio.org/frameworks/" +
-            "https://platformio.org/frameworks/" + framework['name'])
+                                 framework['name'])
         framework['platforms'] = [
             platform['name'] for platform in _get_registry_platforms()
             if framework['name'] in platform['frameworks']
@@ -205,7 +195,7 @@ def platform_frameworks(query, json_output):
 
     frameworks = sorted(frameworks, key=lambda manifest: manifest['name'])
     if json_output:
-        click.echo(json.dumps(frameworks))
+        click.echo(dump_json_to_unicode(frameworks))
     else:
         _print_platforms(frameworks)
 
@@ -217,14 +207,13 @@ def platform_list(json_output):
     pm = PlatformManager()
     for manifest in pm.get_installed():
         platforms.append(
-            _get_installed_platform_data(
+            _get_installed_platform_data(manifest['__pkg_dir'],
-                manifest['__pkg_dir'],
+                                         with_boards=False,
-                with_boards=False,
+                                         expose_packages=False))
-                expose_packages=False))
 
     platforms = sorted(platforms, key=lambda manifest: manifest['name'])
     if json_output:
-        click.echo(json.dumps(platforms))
+        click.echo(dump_json_to_unicode(platforms))
     else:
         _print_platforms(platforms)
 
@@ -237,10 +226,11 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     if not data:
         raise exception.UnknownPlatform(platform)
     if json_output:
-        return click.echo(json.dumps(data))
+        return click.echo(dump_json_to_unicode(data))
 
-    click.echo("{name} ~ {title}".format(
+    click.echo("{name} ~ {title}".format(name=click.style(data['name'],
-        name=click.style(data['name'], fg="cyan"), title=data['title']))
+                                                          fg="cyan"),
+                                         title=data['title']))
     click.echo("=" * (3 + len(data['name'] + data['title'])))
     click.echo(data['description'])
     click.echo()
@@ -305,17 +295,15 @@ def platform_install(platforms, with_package, without_package,
                      skip_default_package, force):
     pm = PlatformManager()
     for platform in platforms:
-        if pm.install(
+        if pm.install(name=platform,
-                name=platform,
+                      with_packages=with_package,
-                with_packages=with_package,
+                      without_packages=without_package,
-                without_packages=without_package,
+                      skip_default_package=skip_default_package,
-                skip_default_package=skip_default_package,
+                      force=force):
-                force=force):
+            click.secho("The platform '%s' has been successfully installed!\n"
-            click.secho(
+                        "The rest of packages will be installed automatically "
-                "The platform '%s' has been successfully installed!\n"
+                        "depending on your build environment." % platform,
-                "The rest of packages will be installed automatically "
+                        fg="green")
-                "depending on your build environment." % platform,
-                fg="green")
 
 
 @cli.command("uninstall", short_help="Uninstall development platform")
@@ -324,26 +312,27 @@ def platform_uninstall(platforms):
     pm = PlatformManager()
     for platform in platforms:
         if pm.uninstall(platform):
-            click.secho(
+            click.secho("The platform '%s' has been successfully "
-                "The platform '%s' has been successfully "
+                        "uninstalled!" % platform,
-                "uninstalled!" % platform,
+                        fg="green")
-                fg="green")
 
 
 @cli.command("update", short_help="Update installed development platforms")
 @click.argument("platforms", nargs=-1, required=False, metavar="[PLATFORM...]")
-@click.option(
+@click.option("-p",
-    "-p",
+              "--only-packages",
-    "--only-packages",
+              is_flag=True,
-    is_flag=True,
+              help="Update only the platform packages")
-    help="Update only the platform packages")
+@click.option("-c",
-@click.option(
+              "--only-check",
-    "-c",
+              is_flag=True,
-    "--only-check",
+              help="DEPRECATED. Please use `--dry-run` instead")
-    is_flag=True,
+@click.option("--dry-run",
-    help="Do not update, only check for a new version")
+              is_flag=True,
+              help="Do not update, only check for the new versions")
 @click.option("--json-output", is_flag=True)
-def platform_update(platforms, only_packages, only_check, json_output):
+def platform_update(  # pylint: disable=too-many-locals
+        platforms, only_packages, only_check, dry_run, json_output):
     pm = PlatformManager()
     pkg_dir_to_name = {}
     if not platforms:
@@ -353,6 +342,8 @@ def platform_update(platforms, only_packages, only_check, json_output):
             pkg_dir_to_name[manifest['__pkg_dir']] = manifest.get(
                 "title", manifest['name'])
 
+    only_check = dry_run or only_check
+
     if only_check and json_output:
         result = []
         for platform in platforms:
@@ -368,21 +359,22 @@ def platform_update(platforms, only_packages, only_check, json_output):
             if (not latest and not PlatformFactory.newPlatform(
                     pkg_dir).are_outdated_packages()):
                 continue
-            data = _get_installed_platform_data(
+            data = _get_installed_platform_data(pkg_dir,
-                pkg_dir, with_boards=False, expose_packages=False)
+                                                with_boards=False,
+                                                expose_packages=False)
             if latest:
                 data['versionLatest'] = latest
             result.append(data)
-        return click.echo(json.dumps(result))
+        return click.echo(dump_json_to_unicode(result))
-    else:
+
-        # cleanup cached board and platform lists
+    # cleanup cached board and platform lists
-        app.clean_cache()
+    app.clean_cache()
-        for platform in platforms:
+    for platform in platforms:
-            click.echo("Platform %s" % click.style(
+        click.echo(
-                pkg_dir_to_name.get(platform, platform), fg="cyan"))
+            "Platform %s" %
-            click.echo("--------")
+            click.style(pkg_dir_to_name.get(platform, platform), fg="cyan"))
-            pm.update(
+        click.echo("--------")
-                platform, only_packages=only_packages, only_check=only_check)
+        pm.update(platform, only_packages=only_packages, only_check=only_check)
-            click.echo()
+        click.echo()
 
     return True

platformio/commands/remote.py

@@ -23,6 +23,7 @@ import click
 
 from platformio import exception, util
 from platformio.commands.device import device_monitor as cmd_device_monitor
+from platformio.compat import get_file_contents
 from platformio.managers.core import pioplus_call
 
 # pylint: disable=unused-argument
@@ -42,12 +43,13 @@ def remote_agent():
 @remote_agent.command("start", short_help="Start agent")
 @click.option("-n", "--name")
 @click.option("-s", "--share", multiple=True, metavar="E-MAIL")
-@click.option(
+@click.option("-d",
-    "-d",
+              "--working-dir",
-    "--working-dir",
+              envvar="PLATFORMIO_REMOTE_AGENT_DIR",
-    envvar="PLATFORMIO_REMOTE_AGENT_DIR",
+              type=click.Path(file_okay=False,
-    type=click.Path(
+                              dir_okay=True,
-        file_okay=False, dir_okay=True, writable=True, resolve_path=True))
+                              writable=True,
+                              resolve_path=True))
 def remote_agent_start(**kwargs):
     pioplus_call(sys.argv[1:])
 
@@ -62,14 +64,16 @@ def remote_agent_list():
     pioplus_call(sys.argv[1:])
 
 
-@cli.command(
+@cli.command("update",
-    "update", short_help="Update installed Platforms, Packages and Libraries")
+             short_help="Update installed Platforms, Packages and Libraries")
-@click.option(
+@click.option("-c",
-    "-c",
+              "--only-check",
-    "--only-check",
+              is_flag=True,
-    is_flag=True,
+              help="DEPRECATED. Please use `--dry-run` instead")
-    help="Do not update, only check for new version")
+@click.option("--dry-run",
-def remote_update(only_check):
+              is_flag=True,
+              help="Do not update, only check for the new versions")
+def remote_update(only_check, dry_run):
     pioplus_call(sys.argv[1:])
 
 
@@ -77,16 +81,14 @@ def remote_update(only_check):
 @click.option("-e", "--environment", multiple=True)
 @click.option("-t", "--target", multiple=True)
 @click.option("--upload-port")
-@click.option(
+@click.option("-d",
-    "-d",
+              "--project-dir",
-    "--project-dir",
+              default=getcwd,
-    default=getcwd,
+              type=click.Path(exists=True,
-    type=click.Path(
+                              file_okay=True,
-        exists=True,
+                              dir_okay=True,
-        file_okay=True,
+                              writable=True,
-        dir_okay=True,
+                              resolve_path=True))
-        writable=True,
-        resolve_path=True))
 @click.option("--disable-auto-clean", is_flag=True)
 @click.option("-r", "--force-remote", is_flag=True)
 @click.option("-s", "--silent", is_flag=True)
@@ -100,16 +102,14 @@ def remote_run(**kwargs):
 @click.option("--ignore", "-i", multiple=True, metavar="<pattern>")
 @click.option("--upload-port")
 @click.option("--test-port")
-@click.option(
+@click.option("-d",
-    "-d",
+              "--project-dir",
-    "--project-dir",
+              default=getcwd,
-    default=getcwd,
+              type=click.Path(exists=True,
-    type=click.Path(
+                              file_okay=False,
-        exists=True,
+                              dir_okay=True,
-        file_okay=False,
+                              writable=True,
-        dir_okay=True,
+                              resolve_path=True))
-        writable=True,
-        resolve_path=True))
 @click.option("-r", "--force-remote", is_flag=True)
 @click.option("--without-building", is_flag=True)
 @click.option("--without-uploading", is_flag=True)
@@ -131,59 +131,55 @@ def device_list(json_output):
 
 @remote_device.command("monitor", short_help="Monitor remote device")
 @click.option("--port", "-p", help="Port, a number or a device name")
-@click.option(
+@click.option("--baud",
-    "--baud", "-b", type=int, default=9600, help="Set baud rate, default=9600")
+              "-b",
-@click.option(
+              type=int,
-    "--parity",
+              default=9600,
-    default="N",
+              help="Set baud rate, default=9600")
-    type=click.Choice(["N", "E", "O", "S", "M"]),
+@click.option("--parity",
-    help="Set parity, default=N")
+              default="N",
-@click.option(
+              type=click.Choice(["N", "E", "O", "S", "M"]),
-    "--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
+              help="Set parity, default=N")
-@click.option(
+@click.option("--rtscts",
-    "--xonxoff",
+              is_flag=True,
-    is_flag=True,
+              help="Enable RTS/CTS flow control, default=Off")
-    help="Enable software flow control, default=Off")
+@click.option("--xonxoff",
-@click.option(
+              is_flag=True,
-    "--rts",
+              help="Enable software flow control, default=Off")
-    default=None,
+@click.option("--rts",
-    type=click.IntRange(0, 1),
+              default=None,
-    help="Set initial RTS line state")
+              type=click.IntRange(0, 1),
-@click.option(
+              help="Set initial RTS line state")
-    "--dtr",
+@click.option("--dtr",
-    default=None,
+              default=None,
-    type=click.IntRange(0, 1),
+              type=click.IntRange(0, 1),
-    help="Set initial DTR line state")
+              help="Set initial DTR line state")
 @click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
-@click.option(
+@click.option("--encoding",
-    "--encoding",
+              default="UTF-8",
-    default="UTF-8",
+              help="Set the encoding for the serial port (e.g. hexlify, "
-    help="Set the encoding for the serial port (e.g. hexlify, "
+              "Latin1, UTF-8), default: UTF-8")
-    "Latin1, UTF-8), default: UTF-8")
 @click.option("--filter", "-f", multiple=True, help="Add text transformation")
-@click.option(
+@click.option("--eol",
-    "--eol",
+              default="CRLF",
-    default="CRLF",
+              type=click.Choice(["CR", "LF", "CRLF"]),
-    type=click.Choice(["CR", "LF", "CRLF"]),
+              help="End of line mode, default=CRLF")
-    help="End of line mode, default=CRLF")
+@click.option("--raw",
-@click.option(
+              is_flag=True,
-    "--raw", is_flag=True, help="Do not apply any encodings/transformations")
+              help="Do not apply any encodings/transformations")
-@click.option(
+@click.option("--exit-char",
-    "--exit-char",
+              type=int,
-    type=int,
+              default=3,
-    default=3,
+              help="ASCII code of special character that is used to exit "
-    help="ASCII code of special character that is used to exit "
+              "the application, default=3 (Ctrl+C)")
-    "the application, default=3 (Ctrl+C)")
+@click.option("--menu-char",
-@click.option(
+              type=int,
-    "--menu-char",
+              default=20,
-    type=int,
+              help="ASCII code of special character that is used to "
-    default=20,
+              "control miniterm (menu), default=20 (DEC)")
-    help="ASCII code of special character that is used to "
+@click.option("--quiet",
-    "control miniterm (menu), default=20 (DEC)")
+              is_flag=True,
-@click.option(
+              help="Diagnostics: suppress non-error messages, default=Off")
-    "--quiet",
-    is_flag=True,
-    help="Diagnostics: suppress non-error messages, default=Off")
 @click.pass_context
 def device_monitor(ctx, **kwargs):
 
@@ -202,7 +198,7 @@ def device_monitor(ctx, **kwargs):
             sleep(0.1)
         if not t.is_alive():
             return
-        kwargs['port'] = open(sock_file).read()
+        kwargs['port'] = get_file_contents(sock_file)
         ctx.invoke(cmd_device_monitor, **kwargs)
         t.join(2)
     finally:

platformio/commands/run.py

@@ -1,435 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from hashlib import sha1
-from os import getcwd, makedirs, walk
-from os.path import getmtime, isdir, isfile, join
-from time import time
-
-import click
-
-from platformio import __version__, exception, telemetry, util
-from platformio.commands.device import device_monitor as cmd_device_monitor
-from platformio.commands.lib import lib_install as cmd_lib_install
-from platformio.commands.platform import \
-    platform_install as cmd_platform_install
-from platformio.managers.lib import LibraryManager, is_builtin_lib
-from platformio.managers.platform import PlatformFactory
-
-# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
-
-
-@click.command("run", short_help="Process project environments")
-@click.option("-e", "--environment", multiple=True)
-@click.option("-t", "--target", multiple=True)
-@click.option("--upload-port")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=getcwd,
-    type=click.Path(
-        exists=True,
-        file_okay=True,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True))
-@click.option("-s", "--silent", is_flag=True)
-@click.option("-v", "--verbose", is_flag=True)
-@click.option("--disable-auto-clean", is_flag=True)
-@click.pass_context
-def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
-        disable_auto_clean):
-    # find project directory on upper level
-    if isfile(project_dir):
-        project_dir = util.find_project_dir_above(project_dir)
-
-    if not util.is_platformio_project(project_dir):
-        raise exception.NotPlatformIOProject(project_dir)
-
-    with util.cd(project_dir):
-        # clean obsolete build dir
-        if not disable_auto_clean:
-            try:
-                _clean_build_dir(util.get_projectbuild_dir())
-            except:  # pylint: disable=bare-except
-                click.secho(
-                    "Can not remove temporary directory `%s`. Please remove "
-                    "it manually to avoid build issues" %
-                    util.get_projectbuild_dir(force=True),
-                    fg="yellow")
-
-        config = util.load_project_config()
-        env_default = None
-        if config.has_option("platformio", "env_default"):
-            env_default = util.parse_conf_multi_values(
-                config.get("platformio", "env_default"))
-
-        check_project_defopts(config)
-        check_project_envs(config, environment or env_default)
-
-        results = []
-        start_time = time()
-        for section in config.sections():
-            if not section.startswith("env:"):
-                continue
-
-            envname = section[4:]
-            skipenv = any([
-                environment and envname not in environment, not environment
-                and env_default and envname not in env_default
-            ])
-            if skipenv:
-                results.append((envname, None))
-                continue
-
-            if not silent and results:
-                click.echo()
-
-            options = {}
-            for k, v in config.items(section):
-                options[k] = v
-            if "piotest" not in options and "piotest" in ctx.meta:
-                options['piotest'] = ctx.meta['piotest']
-
-            ep = EnvironmentProcessor(ctx, envname, options, target,
-                                      upload_port, silent, verbose)
-            result = (envname, ep.process())
-            results.append(result)
-            if result[1] and "monitor" in ep.get_build_targets() and \
-                    "nobuild" not in ep.get_build_targets():
-                ctx.invoke(
-                    cmd_device_monitor,
-                    environment=environment[0] if environment else None)
-
-        found_error = any(status is False for (_, status) in results)
-
-        if (found_error or not silent) and len(results) > 1:
-            click.echo()
-            print_summary(results, start_time)
-
-        if found_error:
-            raise exception.ReturnErrorCode(1)
-        return True
-
-
-class EnvironmentProcessor(object):
-
-    DEFAULT_DUMP_OPTIONS = ("platform", "framework", "board")
-
-    KNOWN_PLATFORMIO_OPTIONS = [
-        "description", "env_default", "home_dir", "lib_dir", "libdeps_dir",
-        "include_dir", "src_dir", "build_dir", "data_dir", "test_dir",
-        "boards_dir", "lib_extra_dirs"
-    ]
-
-    KNOWN_ENV_OPTIONS = [
-        "platform", "framework", "board", "build_flags", "src_build_flags",
-        "build_unflags", "src_filter", "extra_scripts", "targets",
-        "upload_port", "upload_protocol", "upload_speed", "upload_flags",
-        "upload_resetmethod", "lib_deps", "lib_ignore", "lib_extra_dirs",
-        "lib_ldf_mode", "lib_compat_mode", "lib_archive", "piotest",
-        "test_transport", "test_filter", "test_ignore", "test_port",
-        "test_speed", "test_build_project_src", "debug_tool", "debug_port",
-        "debug_init_cmds", "debug_extra_cmds", "debug_server",
-        "debug_init_break", "debug_load_cmd", "debug_load_mode",
-        "debug_svd_path", "monitor_port", "monitor_speed", "monitor_rts",
-        "monitor_dtr"
-    ]
-
-    IGNORE_BUILD_OPTIONS = [
-        "test_transport", "test_filter", "test_ignore", "test_port",
-        "test_speed", "debug_port", "debug_init_cmds", "debug_extra_cmds",
-        "debug_server", "debug_init_break", "debug_load_cmd",
-        "debug_load_mode", "monitor_port", "monitor_speed", "monitor_rts",
-        "monitor_dtr"
-    ]
-
-    REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"}
-
-    RENAMED_OPTIONS = {
-        "lib_use": "lib_deps",
-        "lib_force": "lib_deps",
-        "extra_script": "extra_scripts",
-        "monitor_baud": "monitor_speed",
-        "board_mcu": "board_build.mcu",
-        "board_f_cpu": "board_build.f_cpu",
-        "board_f_flash": "board_build.f_flash",
-        "board_flash_mode": "board_build.flash_mode"
-    }
-
-    RENAMED_PLATFORMS = {"espressif": "espressif8266"}
-
-    def __init__(
-            self,  # pylint: disable=R0913
-            cmd_ctx,
-            name,
-            options,
-            targets,
-            upload_port,
-            silent,
-            verbose):
-        self.cmd_ctx = cmd_ctx
-        self.name = name
-        self.options = options
-        self.targets = targets
-        self.upload_port = upload_port
-        self.silent = silent
-        self.verbose = verbose
-
-    def process(self):
-        terminal_width, _ = click.get_terminal_size()
-        start_time = time()
-        env_dump = []
-
-        for k, v in self.options.items():
-            self.options[k] = self.options[k].strip()
-            if self.verbose or k in self.DEFAULT_DUMP_OPTIONS:
-                env_dump.append(
-                    "%s: %s" % (k, ", ".join(util.parse_conf_multi_values(v))))
-
-        if not self.silent:
-            click.echo("Processing %s (%s)" % (click.style(
-                self.name, fg="cyan", bold=True), "; ".join(env_dump)))
-            click.secho("-" * terminal_width, bold=True)
-
-        self.options = self._validate_options(self.options)
-        result = self._run()
-        is_error = result['returncode'] != 0
-
-        if self.silent and not is_error:
-            return True
-
-        if is_error or "piotest_processor" not in self.cmd_ctx.meta:
-            print_header(
-                "[%s] Took %.2f seconds" % (
-                    (click.style("ERROR", fg="red", bold=True) if is_error else
-                     click.style("SUCCESS", fg="green", bold=True)),
-                    time() - start_time),
-                is_error=is_error)
-
-        return not is_error
-
-    def _validate_options(self, options):
-        result = {}
-        for k, v in options.items():
-            # process obsolete options
-            if k in self.RENAMED_OPTIONS:
-                click.secho(
-                    "Warning! `%s` option is deprecated and will be "
-                    "removed in the next release! Please use "
-                    "`%s` instead." % (k, self.RENAMED_OPTIONS[k]),
-                    fg="yellow")
-                k = self.RENAMED_OPTIONS[k]
-            # process renamed platforms
-            if k == "platform" and v in self.RENAMED_PLATFORMS:
-                click.secho(
-                    "Warning! Platform `%s` is deprecated and will be "
-                    "removed in the next release! Please use "
-                    "`%s` instead." % (v, self.RENAMED_PLATFORMS[v]),
-                    fg="yellow")
-                v = self.RENAMED_PLATFORMS[v]
-
-            # warn about unknown options
-            unknown_conditions = [
-                k not in self.KNOWN_ENV_OPTIONS, not k.startswith("custom_"),
-                not k.startswith("board_")
-            ]
-            if all(unknown_conditions):
-                click.secho(
-                    "Detected non-PlatformIO `%s` option in `[env:%s]` section"
-                    % (k, self.name),
-                    fg="yellow")
-            result[k] = v
-        return result
-
-    def get_build_variables(self):
-        variables = {"pioenv": self.name}
-        if self.upload_port:
-            variables['upload_port'] = self.upload_port
-        for k, v in self.options.items():
-            if k in self.REMAPED_OPTIONS:
-                k = self.REMAPED_OPTIONS[k]
-            if k in self.IGNORE_BUILD_OPTIONS:
-                continue
-            if k == "targets" or (k == "upload_port" and self.upload_port):
-                continue
-            variables[k] = v
-        return variables
-
-    def get_build_targets(self):
-        targets = []
-        if self.targets:
-            targets = [t for t in self.targets]
-        elif "targets" in self.options:
-            targets = self.options['targets'].split(", ")
-        return targets
-
-    def _run(self):
-        if "platform" not in self.options:
-            raise exception.UndefinedEnvPlatform(self.name)
-
-        build_vars = self.get_build_variables()
-        build_targets = self.get_build_targets()
-
-        telemetry.on_run_environment(self.options, build_targets)
-
-        # skip monitor target, we call it above
-        if "monitor" in build_targets:
-            build_targets.remove("monitor")
-        if "nobuild" not in build_targets:
-            # install dependent libraries
-            if "lib_install" in self.options:
-                _autoinstall_libdeps(self.cmd_ctx, [
-                    int(d.strip())
-                    for d in self.options['lib_install'].split(",")
-                    if d.strip()
-                ], self.verbose)
-            if "lib_deps" in self.options:
-                _autoinstall_libdeps(
-                    self.cmd_ctx,
-                    util.parse_conf_multi_values(self.options['lib_deps']),
-                    self.verbose)
-
-        try:
-            p = PlatformFactory.newPlatform(self.options['platform'])
-        except exception.UnknownPlatform:
-            self.cmd_ctx.invoke(
-                cmd_platform_install,
-                platforms=[self.options['platform']],
-                skip_default_package=True)
-            p = PlatformFactory.newPlatform(self.options['platform'])
-
-        return p.run(build_vars, build_targets, self.silent, self.verbose)
-
-
-def _autoinstall_libdeps(ctx, libraries, verbose=False):
-    if not libraries:
-        return
-    storage_dir = util.get_projectlibdeps_dir()
-    ctx.obj = LibraryManager(storage_dir)
-    if verbose:
-        click.echo("Library Storage: " + storage_dir)
-    for lib in libraries:
-        try:
-            ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose)
-        except exception.LibNotFound as e:
-            if verbose or not is_builtin_lib(lib):
-                click.secho("Warning! %s" % e, fg="yellow")
-        except exception.InternetIsOffline as e:
-            click.secho(str(e), fg="yellow")
-
-
-def _clean_build_dir(build_dir):
-    structhash_file = join(build_dir, "structure.hash")
-    proj_hash = calculate_project_hash()
-
-    # if project's config is modified
-    if (isdir(build_dir)
-            and getmtime(join(util.get_project_dir(),
-                              "platformio.ini")) > getmtime(build_dir)):
-        util.rmtree_(build_dir)
-
-    # check project structure
-    if isdir(build_dir) and isfile(structhash_file):
-        with open(structhash_file) as f:
-            if f.read() == proj_hash:
-                return
-        util.rmtree_(build_dir)
-
-    if not isdir(build_dir):
-        makedirs(build_dir)
-
-    with open(structhash_file, "w") as f:
-        f.write(proj_hash)
-
-
-def print_header(label, is_error=False):
-    terminal_width, _ = click.get_terminal_size()
-    width = len(click.unstyle(label))
-    half_line = "=" * ((terminal_width - width - 2) / 2)
-    click.echo("%s %s %s" % (half_line, label, half_line), err=is_error)
-
-
-def print_summary(results, start_time):
-    print_header("[%s]" % click.style("SUMMARY"))
-
-    envname_max_len = 0
-    for (envname, _) in results:
-        if len(envname) > envname_max_len:
-            envname_max_len = len(envname)
-
-    successed = True
-    for (envname, status) in results:
-        status_str = click.style("SUCCESS", fg="green")
-        if status is False:
-            successed = False
-            status_str = click.style("ERROR", fg="red")
-        elif status is None:
-            status_str = click.style("SKIP", fg="yellow")
-
-        format_str = (
-            "Environment {0:<" + str(envname_max_len + 9) + "}\t[{1}]")
-        click.echo(
-            format_str.format(click.style(envname, fg="cyan"), status_str),
-            err=status is False)
-
-    print_header(
-        "[%s] Took %.2f seconds" % (
-            (click.style("SUCCESS", fg="green", bold=True) if successed else
-             click.style("ERROR", fg="red", bold=True)), time() - start_time),
-        is_error=not successed)
-
-
-def check_project_defopts(config):
-    if not config.has_section("platformio"):
-        return True
-    unknown = set([k for k, _ in config.items("platformio")]) - set(
-        EnvironmentProcessor.KNOWN_PLATFORMIO_OPTIONS)
-    if not unknown:
-        return True
-    click.secho(
-        "Warning! Ignore unknown `%s` option in `[platformio]` section" %
-        ", ".join(unknown),
-        fg="yellow")
-    return False
-
-
-def check_project_envs(config, environments=None):
-    if not config.sections():
-        raise exception.ProjectEnvsNotAvailable()
-
-    known = set([s[4:] for s in config.sections() if s.startswith("env:")])
-    unknown = set(environments or []) - known
-    if unknown:
-        raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known))
-    return True
-
-
-def calculate_project_hash():
-    check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
-    chunks = [__version__]
-    for d in (util.get_projectsrc_dir(), util.get_projectlib_dir()):
-        if not isdir(d):
-            continue
-        for root, _, files in walk(d):
-            for f in files:
-                path = join(root, f)
-                if path.endswith(check_suffixes):
-                    chunks.append(path)
-    chunks_to_str = ",".join(sorted(chunks))
-    if "windows" in util.get_systype():
-        # Fix issue with useless project rebuilding for case insensitive FS.
-        # A case of disk drive can differ...
-        chunks_to_str = chunks_to_str.lower()
-    return sha1(chunks_to_str).hexdigest()

platformio/commands/run/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.run.command import cli
+from platformio.commands.run.helpers import print_header

platformio/commands/run/command.py

@@ -0,0 +1,128 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from multiprocessing import cpu_count
+from os import getcwd
+from os.path import isfile, join
+from time import time
+
+import click
+
+from platformio import exception, util
+from platformio.commands.device import device_monitor as cmd_device_monitor
+from platformio.commands.run.helpers import (clean_build_dir,
+                                             handle_legacy_libdeps,
+                                             print_summary)
+from platformio.commands.run.processor import EnvironmentProcessor
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import (find_project_dir_above,
+                                        get_project_build_dir)
+
+# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
+
+try:
+    DEFAULT_JOB_NUMS = cpu_count()
+except NotImplementedError:
+    DEFAULT_JOB_NUMS = 1
+
+
+@click.command("run", short_help="Process project environments")
+@click.option("-e", "--environment", multiple=True)
+@click.option("-t", "--target", multiple=True)
+@click.option("--upload-port")
+@click.option("-d",
+              "--project-dir",
+              default=getcwd,
+              type=click.Path(exists=True,
+                              file_okay=True,
+                              dir_okay=True,
+                              writable=True,
+                              resolve_path=True))
+@click.option("-c",
+              "--project-conf",
+              type=click.Path(exists=True,
+                              file_okay=True,
+                              dir_okay=False,
+                              readable=True,
+                              resolve_path=True))
+@click.option("-j",
+              "--jobs",
+              type=int,
+              default=DEFAULT_JOB_NUMS,
+              help=("Allow N jobs at once. "
+                    "Default is a number of CPUs in a system (N=%d)" %
+                    DEFAULT_JOB_NUMS))
+@click.option("-s", "--silent", is_flag=True)
+@click.option("-v", "--verbose", is_flag=True)
+@click.option("--disable-auto-clean", is_flag=True)
+@click.pass_context
+def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs,
+        silent, verbose, disable_auto_clean):
+    # find project directory on upper level
+    if isfile(project_dir):
+        project_dir = find_project_dir_above(project_dir)
+
+    with util.cd(project_dir):
+        # clean obsolete build dir
+        if not disable_auto_clean:
+            try:
+                clean_build_dir(get_project_build_dir())
+            except:  # pylint: disable=bare-except
+                click.secho(
+                    "Can not remove temporary directory `%s`. Please remove "
+                    "it manually to avoid build issues" %
+                    get_project_build_dir(force=True),
+                    fg="yellow")
+
+        config = ProjectConfig.get_instance(
+            project_conf or join(project_dir, "platformio.ini"))
+        config.validate(environment)
+
+        handle_legacy_libdeps(project_dir, config)
+
+        results = []
+        start_time = time()
+        default_envs = config.default_envs()
+        for envname in config.envs():
+            skipenv = any([
+                environment and envname not in environment, not environment
+                and default_envs and envname not in default_envs
+            ])
+            if skipenv:
+                results.append((envname, None))
+                continue
+
+            if not silent and any(status is not None
+                                  for (_, status) in results):
+                click.echo()
+
+            ep = EnvironmentProcessor(ctx, envname, config, target,
+                                      upload_port, silent, verbose, jobs)
+            result = (envname, ep.process())
+            results.append(result)
+
+            if result[1] and "monitor" in ep.get_build_targets() and \
+                    "nobuild" not in ep.get_build_targets():
+                ctx.invoke(cmd_device_monitor,
+                           environment=environment[0] if environment else None)
+
+        found_error = any(status is False for (_, status) in results)
+
+        if (found_error or not silent) and len(results) > 1:
+            click.echo()
+            print_summary(results, start_time)
+
+        if found_error:
+            raise exception.ReturnErrorCode(1)
+        return True

platformio/commands/run/helpers.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os import makedirs
+from os.path import getmtime, isdir, isfile, join
+from time import time
+
+import click
+
+from platformio import util
+from platformio.project.helpers import (calculate_project_hash,
+                                        get_project_dir,
+                                        get_project_libdeps_dir)
+
+
+def handle_legacy_libdeps(project_dir, config):
+    legacy_libdeps_dir = join(project_dir, ".piolibdeps")
+    if (not isdir(legacy_libdeps_dir)
+            or legacy_libdeps_dir == get_project_libdeps_dir()):
+        return
+    if not config.has_section("env"):
+        config.add_section("env")
+    lib_extra_dirs = config.get("env", "lib_extra_dirs", [])
+    lib_extra_dirs.append(legacy_libdeps_dir)
+    config.set("env", "lib_extra_dirs", lib_extra_dirs)
+    click.secho(
+        "DEPRECATED! A legacy library storage `{0}` has been found in a "
+        "project. \nPlease declare project dependencies in `platformio.ini`"
+        " file using `lib_deps` option and remove `{0}` folder."
+        "\nMore details -> http://docs.platformio.org/page/projectconf/"
+        "section_env_library.html#lib-deps".format(legacy_libdeps_dir),
+        fg="yellow")
+
+
+def clean_build_dir(build_dir):
+    # remove legacy ".pioenvs" folder
+    legacy_build_dir = join(get_project_dir(), ".pioenvs")
+    if isdir(legacy_build_dir) and legacy_build_dir != build_dir:
+        util.rmtree_(legacy_build_dir)
+
+    structhash_file = join(build_dir, "structure.hash")
+    proj_hash = calculate_project_hash()
+
+    # if project's config is modified
+    if (isdir(build_dir) and getmtime(join(
+            get_project_dir(), "platformio.ini")) > getmtime(build_dir)):
+        util.rmtree_(build_dir)
+
+    # check project structure
+    if isdir(build_dir) and isfile(structhash_file):
+        with open(structhash_file) as f:
+            if f.read() == proj_hash:
+                return
+        util.rmtree_(build_dir)
+
+    if not isdir(build_dir):
+        makedirs(build_dir)
+
+    with open(structhash_file, "w") as f:
+        f.write(proj_hash)
+
+
+def print_header(label, is_error=False, fg=None):
+    terminal_width, _ = click.get_terminal_size()
+    width = len(click.unstyle(label))
+    half_line = "=" * int((terminal_width - width - 2) / 2)
+    click.secho("%s %s %s" % (half_line, label, half_line),
+                fg=fg,
+                err=is_error)
+
+
+def print_summary(results, start_time):
+    print_header("[%s]" % click.style("SUMMARY"))
+
+    succeeded_nums = 0
+    failed_nums = 0
+    envname_max_len = max(
+        [len(click.style(envname, fg="cyan")) for (envname, _) in results])
+    for (envname, status) in results:
+        if status is False:
+            failed_nums += 1
+            status_str = click.style("FAILED", fg="red")
+        elif status is None:
+            status_str = click.style("IGNORED", fg="yellow")
+        else:
+            succeeded_nums += 1
+            status_str = click.style("SUCCESS", fg="green")
+
+        format_str = "Environment {0:<%d}\t[{1}]" % envname_max_len
+        click.echo(format_str.format(click.style(envname, fg="cyan"),
+                                     status_str),
+                   err=status is False)
+
+    print_header("%s%d succeeded in %.2f seconds" %
+                 ("%d failed, " % failed_nums if failed_nums else "",
+                  succeeded_nums, time() - start_time),
+                 is_error=failed_nums,
+                 fg="red" if failed_nums else "green")

platformio/commands/run/processor.py

@@ -0,0 +1,117 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from time import time
+
+import click
+
+from platformio import exception, telemetry
+from platformio.commands.platform import \
+    platform_install as cmd_platform_install
+from platformio.commands.run.helpers import print_header
+from platformio.commands.test.processor import (CTX_META_TEST_IS_RUNNING,
+                                                CTX_META_TEST_RUNNING_NAME)
+from platformio.managers.platform import PlatformFactory
+
+# pylint: disable=too-many-instance-attributes
+
+
+class EnvironmentProcessor(object):
+
+    DEFAULT_PRINT_OPTIONS = ("platform", "framework", "board")
+
+    def __init__(  # pylint: disable=too-many-arguments
+            self, cmd_ctx, name, config, targets, upload_port, silent, verbose,
+            jobs):
+        self.cmd_ctx = cmd_ctx
+        self.name = name
+        self.config = config
+        self.targets = [str(t) for t in targets]
+        self.upload_port = upload_port
+        self.silent = silent
+        self.verbose = verbose
+        self.jobs = jobs
+        self.options = config.items(env=name, as_dict=True)
+
+    def process(self):
+        terminal_width, _ = click.get_terminal_size()
+        start_time = time()
+        env_dump = []
+
+        for k, v in self.options.items():
+            if self.verbose or k in self.DEFAULT_PRINT_OPTIONS:
+                env_dump.append(
+                    "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v))
+
+        if not self.silent:
+            click.echo("Processing %s (%s)" % (click.style(
+                self.name, fg="cyan", bold=True), "; ".join(env_dump)))
+            click.secho("-" * terminal_width, bold=True)
+
+        result = self._run_platform()
+        is_error = result['returncode'] != 0
+
+        if self.silent and not is_error:
+            return True
+
+        if is_error or CTX_META_TEST_IS_RUNNING not in self.cmd_ctx.meta:
+            print_header(
+                "[%s] Took %.2f seconds" %
+                ((click.style("ERROR", fg="red", bold=True) if
+                  is_error else click.style("SUCCESS", fg="green", bold=True)),
+                 time() - start_time),
+                is_error=is_error)
+
+        return not is_error
+
+    def get_build_variables(self):
+        variables = {"pioenv": self.name, "project_config": self.config.path}
+
+        if CTX_META_TEST_RUNNING_NAME in self.cmd_ctx.meta:
+            variables['piotest_running_name'] = self.cmd_ctx.meta[
+                CTX_META_TEST_RUNNING_NAME]
+
+        if self.upload_port:
+            # override upload port with a custom from CLI
+            variables['upload_port'] = self.upload_port
+        return variables
+
+    def get_build_targets(self):
+        if self.targets:
+            return [t for t in self.targets]
+        return self.config.get("env:" + self.name, "targets", [])
+
+    def _run_platform(self):
+        if "platform" not in self.options:
+            raise exception.UndefinedEnvPlatform(self.name)
+
+        build_vars = self.get_build_variables()
+        build_targets = self.get_build_targets()
+
+        telemetry.on_run_environment(self.options, build_targets)
+
+        # skip monitor target, we call it above
+        if "monitor" in build_targets:
+            build_targets.remove("monitor")
+
+        try:
+            p = PlatformFactory.newPlatform(self.options['platform'])
+        except exception.UnknownPlatform:
+            self.cmd_ctx.invoke(cmd_platform_install,
+                                platforms=[self.options['platform']],
+                                skip_default_package=True)
+            p = PlatformFactory.newPlatform(self.options['platform'])
+
+        return p.run(build_vars, build_targets, self.silent, self.verbose,
+                     self.jobs)

platformio/commands/settings.py

@@ -15,6 +15,7 @@
 import click
 
 from platformio import app
+from platformio.compat import string_types
 
 
 @click.group(short_help="Manage PlatformIO settings")
@@ -26,15 +27,14 @@ def cli():
 @click.argument("name", required=False)
 def settings_get(name):
 
-    list_tpl = "{name:<40} {value:<35} {description}"
+    list_tpl = u"{name:<40} {value:<35} {description}"
     terminal_width, _ = click.get_terminal_size()
 
     click.echo(
-        list_tpl.format(
+        list_tpl.format(name=click.style("Name", fg="cyan"),
-            name=click.style("Name", fg="cyan"),
+                        value=(click.style("Value", fg="green") +
-            value=(click.style("Value", fg="green") + click.style(
+                               click.style(" [Default]", fg="yellow")),
-                " [Default]", fg="yellow")),
+                        description="Description"))
-            description="Description"))
     click.echo("-" * terminal_width)
 
     for _name, _data in sorted(app.DEFAULT_SETTINGS.items()):
@@ -42,7 +42,8 @@ def settings_get(name):
             continue
         _value = app.get_setting(_name)
 
-        _value_str = str(_value)
+        _value_str = (str(_value)
+                      if not isinstance(_value, string_types) else _value)
         if isinstance(_value, bool):
             _value_str = "Yes" if _value else "No"
         _value_str = click.style(_value_str, fg="green")
@@ -56,10 +57,9 @@ def settings_get(name):
             _value_str += click.style(" ", fg="yellow")
 
         click.echo(
-            list_tpl.format(
+            list_tpl.format(name=click.style(_name, fg="cyan"),
-                name=click.style(_name, fg="cyan"),
+                            value=_value_str,
-                value=_value_str,
+                            description=_data['description']))
-                description=_data['description']))
 
 
 @cli.command("set", short_help="Set new value for the setting")

platformio/commands/test.py

@@ -1,67 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-from os import getcwd
-
-import click
-
-from platformio.managers.core import pioplus_call
-
-
-@click.command("test", short_help="Local Unit Testing")
-@click.option("--environment", "-e", multiple=True, metavar="<environment>")
-@click.option(
-    "--filter",
-    "-f",
-    multiple=True,
-    metavar="<pattern>",
-    help="Filter tests by a pattern")
-@click.option(
-    "--ignore",
-    "-i",
-    multiple=True,
-    metavar="<pattern>",
-    help="Ignore tests by a pattern")
-@click.option("--upload-port")
-@click.option("--test-port")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=getcwd,
-    type=click.Path(
-        exists=True,
-        file_okay=False,
-        dir_okay=True,
-        writable=True,
-        resolve_path=True))
-@click.option("--without-building", is_flag=True)
-@click.option("--without-uploading", is_flag=True)
-@click.option(
-    "--no-reset",
-    is_flag=True,
-    help="Disable software reset via Serial.DTR/RST")
-@click.option(
-    "--monitor-rts",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial RTS line state for Serial Monitor")
-@click.option(
-    "--monitor-dtr",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial DTR line state for Serial Monitor")
-@click.option("--verbose", "-v", is_flag=True)
-def cli(*args, **kwargs):  # pylint: disable=unused-argument
-    pioplus_call(sys.argv[1:])

platformio/commands/test/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.test.command import cli

platformio/commands/test/command.py

@@ -0,0 +1,183 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
+
+from fnmatch import fnmatch
+from os import getcwd, listdir
+from os.path import isdir, join
+from time import time
+
+import click
+
+from platformio import exception, util
+from platformio.commands.run.helpers import print_header
+from platformio.commands.test.embedded import EmbeddedTestProcessor
+from platformio.commands.test.native import NativeTestProcessor
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import get_project_test_dir
+
+
+@click.command("test", short_help="Unit Testing")
+@click.option("--environment", "-e", multiple=True, metavar="<environment>")
+@click.option("--filter",
+              "-f",
+              multiple=True,
+              metavar="<pattern>",
+              help="Filter tests by a pattern")
+@click.option("--ignore",
+              "-i",
+              multiple=True,
+              metavar="<pattern>",
+              help="Ignore tests by a pattern")
+@click.option("--upload-port")
+@click.option("--test-port")
+@click.option("-d",
+              "--project-dir",
+              default=getcwd,
+              type=click.Path(exists=True,
+                              file_okay=False,
+                              dir_okay=True,
+                              writable=True,
+                              resolve_path=True))
+@click.option("-c",
+              "--project-conf",
+              type=click.Path(exists=True,
+                              file_okay=True,
+                              dir_okay=False,
+                              readable=True,
+                              resolve_path=True))
+@click.option("--without-building", is_flag=True)
+@click.option("--without-uploading", is_flag=True)
+@click.option("--without-testing", is_flag=True)
+@click.option("--no-reset", is_flag=True)
+@click.option("--monitor-rts",
+              default=None,
+              type=click.IntRange(0, 1),
+              help="Set initial RTS line state for Serial Monitor")
+@click.option("--monitor-dtr",
+              default=None,
+              type=click.IntRange(0, 1),
+              help="Set initial DTR line state for Serial Monitor")
+@click.option("--verbose", "-v", is_flag=True)
+@click.pass_context
+def cli(  # pylint: disable=redefined-builtin
+        ctx, environment, ignore, filter, upload_port, test_port, project_dir,
+        project_conf, without_building, without_uploading, without_testing,
+        no_reset, monitor_rts, monitor_dtr, verbose):
+    with util.cd(project_dir):
+        test_dir = get_project_test_dir()
+        if not isdir(test_dir):
+            raise exception.TestDirNotExists(test_dir)
+        test_names = get_test_names(test_dir)
+
+        config = ProjectConfig.get_instance(
+            project_conf or join(project_dir, "platformio.ini"))
+        config.validate(envs=environment)
+
+        click.echo("Verbose mode can be enabled via `-v, --verbose` option")
+        click.echo("Collected %d items" % len(test_names))
+
+        results = []
+        start_time = time()
+        default_envs = config.default_envs()
+        for testname in test_names:
+            for envname in config.envs():
+                section = "env:%s" % envname
+
+                # filter and ignore patterns
+                patterns = dict(filter=list(filter), ignore=list(ignore))
+                for key in patterns:
+                    patterns[key].extend(
+                        config.get(section, "test_%s" % key, []))
+
+                skip_conditions = [
+                    environment and envname not in environment,
+                    not environment and default_envs
+                    and envname not in default_envs,
+                    testname != "*" and patterns['filter'] and
+                    not any([fnmatch(testname, p)
+                             for p in patterns['filter']]),
+                    testname != "*"
+                    and any([fnmatch(testname, p)
+                             for p in patterns['ignore']]),
+                ]
+                if any(skip_conditions):
+                    results.append((None, testname, envname))
+                    continue
+
+                cls = (NativeTestProcessor
+                       if config.get(section, "platform") == "native" else
+                       EmbeddedTestProcessor)
+                tp = cls(
+                    ctx, testname, envname,
+                    dict(project_config=config,
+                         project_dir=project_dir,
+                         upload_port=upload_port,
+                         test_port=test_port,
+                         without_building=without_building,
+                         without_uploading=without_uploading,
+                         without_testing=without_testing,
+                         no_reset=no_reset,
+                         monitor_rts=monitor_rts,
+                         monitor_dtr=monitor_dtr,
+                         verbose=verbose))
+                results.append((tp.process(), testname, envname))
+
+    if without_testing:
+        return
+
+    passed_nums = 0
+    failed_nums = 0
+    testname_max_len = max([len(r[1]) for r in results])
+    envname_max_len = max([len(click.style(r[2], fg="cyan")) for r in results])
+
+    print_header("[%s]" % click.style("TEST SUMMARY"))
+    click.echo()
+
+    for result in results:
+        status, testname, envname = result
+        if status is False:
+            failed_nums += 1
+            status_str = click.style("FAILED", fg="red")
+        elif status is None:
+            status_str = click.style("IGNORED", fg="yellow")
+        else:
+            passed_nums += 1
+            status_str = click.style("PASSED", fg="green")
+
+        format_str = "test/{:<%d} > {:<%d}\t[{}]" % (testname_max_len,
+                                                     envname_max_len)
+        click.echo(format_str.format(testname, click.style(envname, fg="cyan"),
+                                     status_str),
+                   err=status is False)
+
+    print_header("%s%d passed in %.2f seconds" %
+                 ("%d failed, " % failed_nums if failed_nums else "",
+                  passed_nums, time() - start_time),
+                 is_error=failed_nums,
+                 fg="red" if failed_nums else "green")
+
+    if failed_nums:
+        raise exception.ReturnErrorCode(1)
+
+
+def get_test_names(test_dir):
+    names = []
+    for item in sorted(listdir(test_dir)):
+        if isdir(join(test_dir, item)):
+            names.append(item)
+    if not names:
+        names = ["*"]
+    return names

platformio/commands/test/embedded.py

@@ -0,0 +1,135 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from time import sleep
+
+import click
+import serial
+
+from platformio import exception, util
+from platformio.commands.test.processor import TestProcessorBase
+from platformio.managers.platform import PlatformFactory
+
+
+class EmbeddedTestProcessor(TestProcessorBase):
+
+    SERIAL_TIMEOUT = 600
+
+    def process(self):
+        if not self.options['without_building']:
+            self.print_progress("Building... (1/3)")
+            target = ["__test"]
+            if self.options['without_uploading']:
+                target.append("checkprogsize")
+            if not self.build_or_upload(target):
+                return False
+
+        if not self.options['without_uploading']:
+            self.print_progress("Uploading... (2/3)")
+            target = ["upload"]
+            if self.options['without_building']:
+                target.append("nobuild")
+            else:
+                target.append("__test")
+            if not self.build_or_upload(target):
+                return False
+
+        if self.options['without_testing']:
+            return None
+
+        self.print_progress("Testing... (3/3)")
+        return self.run()
+
+    def run(self):
+        click.echo("If you don't see any output for the first 10 secs, "
+                   "please reset board (press reset button)")
+        click.echo()
+
+        try:
+            ser = serial.Serial(baudrate=self.get_baudrate(),
+                                timeout=self.SERIAL_TIMEOUT)
+            ser.port = self.get_test_port()
+            ser.rts = self.options['monitor_rts']
+            ser.dtr = self.options['monitor_dtr']
+            ser.open()
+        except serial.SerialException as e:
+            click.secho(str(e), fg="red", err=True)
+            return False
+
+        if not self.options['no_reset']:
+            ser.flushInput()
+            ser.setDTR(False)
+            ser.setRTS(False)
+            sleep(0.1)
+            ser.setDTR(True)
+            ser.setRTS(True)
+            sleep(0.1)
+
+        while True:
+            line = ser.readline().strip()
+
+            # fix non-ascii output from device
+            for i, c in enumerate(line[::-1]):
+                if not isinstance(c, int):
+                    c = ord(c)
+                if c > 127:
+                    line = line[-i:]
+                    break
+
+            if not line:
+                continue
+            if isinstance(line, bytes):
+                line = line.decode("utf8")
+            self.on_run_out(line)
+            if all([l in line for l in ("Tests", "Failures", "Ignored")]):
+                break
+        ser.close()
+        return not self._run_failed
+
+    def get_test_port(self):
+        # if test port is specified manually or in config
+        if self.options.get("test_port"):
+            return self.options.get("test_port")
+        if self.env_options.get("test_port"):
+            return self.env_options.get("test_port")
+
+        assert set(["platform", "board"]) & set(self.env_options.keys())
+        p = PlatformFactory.newPlatform(self.env_options['platform'])
+        board_hwids = p.board_config(self.env_options['board']).get(
+            "build.hwids", [])
+        port = None
+        elapsed = 0
+        while elapsed < 5 and not port:
+            for item in util.get_serialports():
+                port = item['port']
+                for hwid in board_hwids:
+                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
+                    if hwid_str in item['hwid']:
+                        return port
+
+            # check if port is already configured
+            try:
+                serial.Serial(port, timeout=self.SERIAL_TIMEOUT).close()
+            except serial.SerialException:
+                port = None
+
+            if not port:
+                sleep(0.25)
+                elapsed += 0.25
+
+        if not port:
+            raise exception.PlatformioException(
+                "Please specify `test_port` for environment or use "
+                "global `--test-port` option.")
+        return port

platformio/commands/test/native.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os.path import join
+
+from platformio import util
+from platformio.commands.test.processor import TestProcessorBase
+from platformio.proc import LineBufferedAsyncPipe
+from platformio.project.helpers import get_project_build_dir
+
+
+class NativeTestProcessor(TestProcessorBase):
+
+    def process(self):
+        if not self.options['without_building']:
+            self.print_progress("Building... (1/2)")
+            if not self.build_or_upload(["__test"]):
+                return False
+        if self.options['without_testing']:
+            return None
+        self.print_progress("Testing... (2/2)")
+        return self.run()
+
+    def run(self):
+        with util.cd(self.options['project_dir']):
+            build_dir = get_project_build_dir()
+        result = util.exec_command(
+            [join(build_dir, self.env_name, "program")],
+            stdout=LineBufferedAsyncPipe(self.on_run_out),
+            stderr=LineBufferedAsyncPipe(self.on_run_out))
+        assert "returncode" in result
+        return result['returncode'] == 0 and not self._run_failed

platformio/commands/test/processor.py

@@ -0,0 +1,206 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import atexit
+from os import remove
+from os.path import isdir, isfile, join
+from string import Template
+
+import click
+
+from platformio import exception
+from platformio.commands.run.helpers import print_header
+from platformio.project.helpers import get_project_test_dir
+
+TRANSPORT_OPTIONS = {
+    "arduino": {
+        "include": "#include <Arduino.h>",
+        "object": "",
+        "putchar": "Serial.write(c)",
+        "flush": "Serial.flush()",
+        "begin": "Serial.begin($baudrate)",
+        "end": "Serial.end()"
+    },
+    "mbed": {
+        "include": "#include <mbed.h>",
+        "object": "Serial pc(USBTX, USBRX);",
+        "putchar": "pc.putc(c)",
+        "flush": "",
+        "begin": "pc.baud($baudrate)",
+        "end": ""
+    },
+    "energia": {
+        "include": "#include <Energia.h>",
+        "object": "",
+        "putchar": "Serial.write(c)",
+        "flush": "Serial.flush()",
+        "begin": "Serial.begin($baudrate)",
+        "end": "Serial.end()"
+    },
+    "espidf": {
+        "include": "#include <stdio.h>",
+        "object": "",
+        "putchar": "putchar(c)",
+        "flush": "fflush(stdout)",
+        "begin": "",
+        "end": ""
+    },
+    "native": {
+        "include": "#include <stdio.h>",
+        "object": "",
+        "putchar": "putchar(c)",
+        "flush": "fflush(stdout)",
+        "begin": "",
+        "end": ""
+    },
+    "custom": {
+        "include": '#include "unittest_transport.h"',
+        "object": "",
+        "putchar": "unittest_uart_putchar(c)",
+        "flush": "unittest_uart_flush()",
+        "begin": "unittest_uart_begin()",
+        "end": "unittest_uart_end()"
+    }
+}
+
+CTX_META_TEST_IS_RUNNING = __name__ + ".test_running"
+CTX_META_TEST_RUNNING_NAME = __name__ + ".test_running_name"
+
+
+class TestProcessorBase(object):
+
+    DEFAULT_BAUDRATE = 115200
+
+    def __init__(self, cmd_ctx, testname, envname, options):
+        self.cmd_ctx = cmd_ctx
+        self.cmd_ctx.meta[CTX_META_TEST_IS_RUNNING] = True
+        self.test_name = testname
+        self.options = options
+        self.env_name = envname
+        self.env_options = options['project_config'].items(env=envname,
+                                                           as_dict=True)
+        self._run_failed = False
+        self._outputcpp_generated = False
+
+    def get_transport(self):
+        if self.env_options.get("platform") == "native":
+            transport = "native"
+        elif "framework" in self.env_options:
+            transport = self.env_options.get("framework")[0]
+        if "test_transport" in self.env_options:
+            transport = self.env_options['test_transport']
+        if transport not in TRANSPORT_OPTIONS:
+            raise exception.PlatformioException(
+                "Unknown Unit Test transport `%s`" % transport)
+        return transport.lower()
+
+    def get_baudrate(self):
+        return int(self.env_options.get("test_speed", self.DEFAULT_BAUDRATE))
+
+    def print_progress(self, text, is_error=False):
+        click.echo()
+        print_header("[test/%s > %s] %s" %
+                     (click.style(self.test_name, fg="yellow"),
+                      click.style(self.env_name, fg="cyan"), text),
+                     is_error=is_error)
+
+    def build_or_upload(self, target):
+        if not self._outputcpp_generated:
+            self.generate_outputcpp(get_project_test_dir())
+            self._outputcpp_generated = True
+
+        if self.test_name != "*":
+            self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name
+
+        if not self.options['verbose']:
+            click.echo("Please wait...")
+
+        try:
+            from platformio.commands.run import cli as cmd_run
+            return self.cmd_ctx.invoke(cmd_run,
+                                       project_dir=self.options['project_dir'],
+                                       upload_port=self.options['upload_port'],
+                                       silent=not self.options['verbose'],
+                                       environment=[self.env_name],
+                                       disable_auto_clean="nobuild" in target,
+                                       target=target)
+        except exception.ReturnErrorCode:
+            return False
+
+    def process(self):
+        raise NotImplementedError
+
+    def run(self):
+        raise NotImplementedError
+
+    def on_run_out(self, line):
+        line = line.strip()
+        if line.endswith(":PASS"):
+            click.echo("%s\t[%s]" %
+                       (line[:-5], click.style("PASSED", fg="green")))
+        elif ":FAIL" in line:
+            self._run_failed = True
+            click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red")))
+        else:
+            click.echo(line)
+
+    def generate_outputcpp(self, test_dir):
+        assert isdir(test_dir)
+
+        cpp_tpl = "\n".join([
+            "$include",
+            "#include <output_export.h>",
+            "",
+            "$object",
+            "",
+            "void output_start(unsigned int baudrate)",
+            "{",
+            "    $begin;",
+            "}",
+            "",
+            "void output_char(int c)",
+            "{",
+            "    $putchar;",
+            "}",
+            "",
+            "void output_flush(void)",
+            "{",
+            "    $flush;",
+            "}",
+            "",
+            "void output_complete(void)",
+            "{",
+            "   $end;",
+            "}"
+        ])  # yapf: disable
+
+        def delete_tmptest_file(file_):
+            try:
+                remove(file_)
+            except:  # pylint: disable=bare-except
+                if isfile(file_):
+                    click.secho(
+                        "Warning: Could not remove temporary file '%s'. "
+                        "Please remove it manually." % file_,
+                        fg="yellow")
+
+        tpl = Template(cpp_tpl).substitute(
+            TRANSPORT_OPTIONS[self.get_transport()])
+        data = Template(tpl).substitute(baudrate=self.get_baudrate())
+
+        tmp_file = join(test_dir, "output_export.cpp")
+        with open(tmp_file, "w") as f:
+            f.write(data)
+
+        atexit.register(delete_tmptest_file, tmp_file)

platformio/commands/update.py

@@ -15,26 +15,32 @@
 import click
 
 from platformio import app
+from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY
 from platformio.commands.lib import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.managers.core import update_core_packages
 from platformio.managers.lib import LibraryManager
 
 
-@click.command(
+@click.command("update",
-    "update", short_help="Update installed platforms, packages and libraries")
+               short_help="Update installed platforms, packages and libraries")
-@click.option(
+@click.option("--core-packages",
-    "--core-packages", is_flag=True, help="Update only the core packages")
+              is_flag=True,
-@click.option(
+              help="Update only the core packages")
-    "-c",
+@click.option("-c",
-    "--only-check",
+              "--only-check",
-    is_flag=True,
+              is_flag=True,
-    help="Do not update, only check for new version")
+              help="DEPRECATED. Please use `--dry-run` instead")
+@click.option("--dry-run",
+              is_flag=True,
+              help="Do not update, only check for the new versions")
 @click.pass_context
-def cli(ctx, core_packages, only_check):
+def cli(ctx, core_packages, only_check, dry_run):
     # cleanup lib search results, cached board and platform lists
     app.clean_cache()
 
+    only_check = dry_run or only_check
+
     update_core_packages(only_check)
 
     if core_packages:
@@ -48,5 +54,5 @@ def cli(ctx, core_packages, only_check):
     click.echo()
     click.echo("Library Manager")
     click.echo("===============")
-    ctx.obj = LibraryManager()
+    ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryManager().package_dir]
     ctx.invoke(cmd_lib_update, only_check=only_check)

platformio/commands/upgrade.py

@@ -20,11 +20,14 @@ import click
 import requests
 
 from platformio import VERSION, __version__, exception, util
+from platformio.compat import WINDOWS
 from platformio.managers.core import shutdown_piohome_servers
+from platformio.proc import exec_command, get_pythonexe_path
+from platformio.project.helpers import get_project_cache_dir
 
 
-@click.command(
+@click.command("upgrade",
-    "upgrade", short_help="Upgrade PlatformIO to the latest version")
+               short_help="Upgrade PlatformIO to the latest version")
 @click.option("--dev", is_flag=True, help="Use development branch")
 def cli(dev):
     if not dev and __version__ == get_latest_version():
@@ -43,35 +46,33 @@ def cli(dev):
              get_pip_package(to_develop)], ["platformio", "--version"])
 
     cmd = None
-    r = None
+    r = {}
     try:
         for cmd in cmds:
-            cmd = [util.get_pythonexe_path(), "-m"] + cmd
+            cmd = [get_pythonexe_path(), "-m"] + cmd
-            r = None
+            r = exec_command(cmd)
-            r = util.exec_command(cmd)
 
             # try pip with disabled cache
             if r['returncode'] != 0 and cmd[2] == "pip":
                 cmd.insert(3, "--no-cache-dir")
-                r = util.exec_command(cmd)
+                r = exec_command(cmd)
 
             assert r['returncode'] == 0
         assert "version" in r['out']
         actual_version = r['out'].strip().split("version", 1)[1].strip()
-        click.secho(
+        click.secho("PlatformIO has been successfully upgraded to %s" %
-            "PlatformIO has been successfully upgraded to %s" % actual_version,
+                    actual_version,
-            fg="green")
+                    fg="green")
         click.echo("Release notes: ", nl=False)
-        click.secho(
+        click.secho("https://docs.platformio.org/en/latest/history.html",
-            "https://docs.platformio.org/en/latest/history.html", fg="cyan")
+                    fg="cyan")
     except Exception as e:  # pylint: disable=broad-except
         if not r:
             raise exception.UpgradeError("\n".join([str(cmd), str(e)]))
         permission_errors = ("permission denied", "not permitted")
         if (any(m in r['err'].lower() for m in permission_errors)
-                and "windows" not in util.get_systype()):
+                and not WINDOWS):
-            click.secho(
+            click.secho("""
-                """
 -----------------
 Permission denied
 -----------------
@@ -81,12 +82,10 @@ You need the `sudo` permission to install Python packages. Try
 
 WARNING! Don't use `sudo` for the rest PlatformIO commands.
 """,
-                fg="yellow",
+                        fg="yellow",
-                err=True)
+                        err=True)
             raise exception.ReturnErrorCode(1)
-        else:
+        raise exception.UpgradeError("\n".join([str(cmd), r['out'], r['err']]))
-            raise exception.UpgradeError("\n".join(
-                [str(cmd), r['out'], r['err']]))
 
     return True
 
@@ -96,15 +95,15 @@ def get_pip_package(to_develop):
         return "platformio"
     dl_url = ("https://github.com/platformio/"
               "platformio-core/archive/develop.zip")
-    cache_dir = util.get_cache_dir()
+    cache_dir = get_project_cache_dir()
     if not os.path.isdir(cache_dir):
         os.makedirs(cache_dir)
     pkg_name = os.path.join(cache_dir, "piocoredevelop.zip")
     try:
         with open(pkg_name, "w") as fp:
-            r = util.exec_command(["curl", "-fsSL", dl_url],
+            r = exec_command(["curl", "-fsSL", dl_url],
-                                  stdout=fp,
+                             stdout=fp,
-                                  universal_newlines=True)
+                             universal_newlines=True)
             assert r['returncode'] == 0
         # check ZIP structure
         with ZipFile(pkg_name) as zp:
@@ -150,8 +149,7 @@ def get_develop_latest_version():
 
 
 def get_pypi_latest_version():
-    r = requests.get(
+    r = requests.get("https://pypi.org/pypi/platformio/json",
-        "https://pypi.org/pypi/platformio/json",
+                     headers=util.get_request_defheaders())
-        headers=util.get_request_defheaders())
     r.raise_for_status()
     return r.json()['info']['version']

platformio/compat.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=unused-import
+
+import json
+import os
+import re
+import sys
+
+PY2 = sys.version_info[0] == 2
+CYGWIN = sys.platform.startswith('cygwin')
+WINDOWS = sys.platform.startswith('win')
+
+
+def get_filesystem_encoding():
+    return sys.getfilesystemencoding() or sys.getdefaultencoding()
+
+
+if PY2:
+    # pylint: disable=undefined-variable
+    string_types = (str, unicode)
+
+    def is_bytes(x):
+        return isinstance(x, (buffer, bytearray))
+
+    def path_to_unicode(path):
+        if isinstance(path, unicode):
+            return path
+        return path.decode(get_filesystem_encoding()).encode("utf-8")
+
+    def get_file_contents(path):
+        with open(path) as f:
+            return f.read()
+
+    def hashlib_encode_data(data):
+        if is_bytes(data):
+            return data
+        if isinstance(data, unicode):
+            data = data.encode(get_filesystem_encoding())
+        elif not isinstance(data, string_types):
+            data = str(data)
+        return data
+
+    def dump_json_to_unicode(obj):
+        if isinstance(obj, unicode):
+            return obj
+        return json.dumps(obj,
+                          encoding=get_filesystem_encoding(),
+                          ensure_ascii=False,
+                          sort_keys=True).encode("utf8")
+
+    _magic_check = re.compile('([*?[])')
+    _magic_check_bytes = re.compile(b'([*?[])')
+
+    def glob_escape(pathname):
+        """Escape all special characters."""
+        # https://github.com/python/cpython/blob/master/Lib/glob.py#L161
+        # Escaping is done by wrapping any of "*?[" between square brackets.
+        # Metacharacters do not work in the drive part and shouldn't be
+        # escaped.
+        drive, pathname = os.path.splitdrive(pathname)
+        if isinstance(pathname, bytes):
+            pathname = _magic_check_bytes.sub(br'[\1]', pathname)
+        else:
+            pathname = _magic_check.sub(r'[\1]', pathname)
+        return drive + pathname
+else:
+    from glob import escape as glob_escape  # pylint: disable=no-name-in-module
+
+    string_types = (str, )
+
+    def is_bytes(x):
+        return isinstance(x, (bytes, memoryview, bytearray))
+
+    def path_to_unicode(path):
+        return path
+
+    def get_file_contents(path):
+        try:
+            with open(path) as f:
+                return f.read()
+        except UnicodeDecodeError:
+            with open(path, encoding="latin-1") as f:
+                return f.read()
+
+    def hashlib_encode_data(data):
+        if is_bytes(data):
+            return data
+        if not isinstance(data, string_types):
+            data = str(data)
+        return data.encode()
+
+    def dump_json_to_unicode(obj):
+        if isinstance(obj, string_types):
+            return obj
+        return json.dumps(obj, ensure_ascii=False, sort_keys=True)

platformio/downloader.py

@@ -15,7 +15,7 @@
 from email.utils import parsedate_tz
 from math import ceil
 from os.path import getsize, join
-from sys import getfilesystemencoding, version_info
+from sys import version_info
 from time import mktime
 
 import click
@@ -24,6 +24,7 @@ import requests
 from platformio import util
 from platformio.exception import (FDSHASumMismatch, FDSizeMismatch,
                                   FDUnrecognizedStatusCode)
+from platformio.proc import exec_command
 
 
 class FileDownloader(object):
@@ -33,11 +34,10 @@ class FileDownloader(object):
     def __init__(self, url, dest_dir=None):
         self._request = None
         # make connection
-        self._request = requests.get(
+        self._request = requests.get(url,
-            url,
+                                     stream=True,
-            stream=True,
+                                     headers=util.get_request_defheaders(),
-            headers=util.get_request_defheaders(),
+                                     verify=version_info >= (2, 7, 9))
-            verify=version_info >= (2, 7, 9))
         if self._request.status_code != 200:
             raise FDUnrecognizedStatusCode(self._request.status_code, url)
 
@@ -45,14 +45,12 @@ class FileDownloader(object):
         if disposition and "filename=" in disposition:
             self._fname = disposition[disposition.index("filename=") +
                                       9:].replace('"', "").replace("'", "")
-            self._fname = self._fname.encode("utf8")
         else:
             self._fname = [p for p in url.split("/") if p][-1]
-
+        self._fname = str(self._fname)
         self._destination = self._fname
         if dest_dir:
-            self.set_destination(
+            self.set_destination(join(dest_dir, self._fname))
-                join(dest_dir.decode(getfilesystemencoding()), self._fname))
 
     def set_destination(self, destination):
         self._destination = destination
@@ -98,24 +96,24 @@ class FileDownloader(object):
             raise FDSizeMismatch(_dlsize, self._fname, self.get_size())
 
         if not sha1:
-            return
+            return None
 
         dlsha1 = None
         try:
-            result = util.exec_command(["sha1sum", self._destination])
+            result = exec_command(["sha1sum", self._destination])
             dlsha1 = result['out']
         except (OSError, ValueError):
             try:
-                result = util.exec_command(
+                result = exec_command(["shasum", "-a", "1", self._destination])
-                    ["shasum", "-a", "1", self._destination])
                 dlsha1 = result['out']
             except (OSError, ValueError):
                 pass
-
+        if not dlsha1:
-        if dlsha1:
+            return None
-            dlsha1 = dlsha1[1:41] if dlsha1.startswith("\\") else dlsha1[:40]
+        dlsha1 = dlsha1[1:41] if dlsha1.startswith("\\") else dlsha1[:40]
-            if sha1 != dlsha1:
+        if sha1.lower() != dlsha1.lower():
-                raise FDSHASumMismatch(dlsha1, self._fname, sha1)
+            raise FDSHASumMismatch(dlsha1, self._fname, sha1)
+        return True
 
     def _preserve_filemtime(self, lmdate):
         timedata = parsedate_tz(lmdate)

platformio/exception.py

@@ -19,8 +19,10 @@ class PlatformioException(Exception):
 
     def __str__(self):  # pragma: no cover
         if self.MESSAGE:
+            # pylint: disable=not-an-iterable
             return self.MESSAGE.format(*self.args)
-        return Exception.__str__(self)
+
+        return super(PlatformioException, self).__str__()
 
 
 class ReturnErrorCode(PlatformioException):
@@ -40,11 +42,16 @@ class UserSideException(PlatformioException):
     pass
 
 
-class AbortedByUser(PlatformioException):
+class AbortedByUser(UserSideException):
 
     MESSAGE = "Aborted by user"
 
 
+#
+# Development Platform
+#
+
+
 class UnknownPlatform(PlatformioException):
 
     MESSAGE = "Unknown development platform '{0}'"
@@ -61,13 +68,6 @@ class PlatformNotInstalledYet(PlatformioException):
                "Use `platformio platform install {0}` command")
 
 
-class BoardNotDefined(PlatformioException):
-
-    MESSAGE = (
-        "You need to specify board ID using `-b` or `--board` option. "
-        "Supported boards list is available via `platformio boards` command")
-
-
 class UnknownBoard(PlatformioException):
 
     MESSAGE = "Unknown board ID '{0}'"
@@ -83,54 +83,75 @@ class UnknownFramework(PlatformioException):
     MESSAGE = "Unknown framework '{0}'"
 
 
-class UnknownPackage(PlatformioException):
+# Package Manager
+
+
+class PlatformIOPackageException(PlatformioException):
+    pass
+
+
+class UnknownPackage(PlatformIOPackageException):
 
     MESSAGE = "Detected unknown package '{0}'"
 
 
-class MissingPackageManifest(PlatformioException):
+class MissingPackageManifest(PlatformIOPackageException):
 
     MESSAGE = "Could not find one of '{0}' manifest files in the package"
 
 
-class UndefinedPackageVersion(PlatformioException):
+class UndefinedPackageVersion(PlatformIOPackageException):
 
     MESSAGE = ("Could not find a version that satisfies the requirement '{0}'"
                " for your system '{1}'")
 
 
-class PackageInstallError(PlatformioException):
+class PackageInstallError(PlatformIOPackageException):
 
     MESSAGE = ("Could not install '{0}' with version requirements '{1}' "
                "for your system '{2}'.\n\n"
                "Please try this solution -> http://bit.ly/faq-package-manager")
 
 
-class ExtractArchiveItemError(PlatformioException):
+class ExtractArchiveItemError(PlatformIOPackageException):
 
     MESSAGE = (
         "Could not extract `{0}` to `{1}`. Try to disable antivirus "
         "tool or check this solution -> http://bit.ly/faq-package-manager")
 
 
-class FDUnrecognizedStatusCode(PlatformioException):
+class UnsupportedArchiveType(PlatformIOPackageException):
+
+    MESSAGE = "Can not unpack file '{0}'"
+
+
+class FDUnrecognizedStatusCode(PlatformIOPackageException):
 
     MESSAGE = "Got an unrecognized status code '{0}' when downloaded {1}"
 
 
-class FDSizeMismatch(PlatformioException):
+class FDSizeMismatch(PlatformIOPackageException):
 
     MESSAGE = ("The size ({0:d} bytes) of downloaded file '{1}' "
                "is not equal to remote size ({2:d} bytes)")
 
 
-class FDSHASumMismatch(PlatformioException):
+class FDSHASumMismatch(PlatformIOPackageException):
 
     MESSAGE = ("The 'sha1' sum '{0}' of downloaded file '{1}' "
                "is not equal to remote '{2}'")
 
 
-class NotPlatformIOProject(PlatformioException):
+#
+# Project
+#
+
+
+class PlatformIOProjectException(PlatformioException):
+    pass
+
+
+class NotPlatformIOProject(PlatformIOProjectException):
 
     MESSAGE = (
         "Not a PlatformIO project. `platformio.ini` file has not been "
@@ -138,26 +159,87 @@ class NotPlatformIOProject(PlatformioException):
         "please use `platformio init` command")
 
 
-class UndefinedEnvPlatform(PlatformioException):
+class InvalidProjectConf(PlatformIOProjectException):
+
+    MESSAGE = ("Invalid '{0}' (project configuration file): '{1}'")
+
+
+class UndefinedEnvPlatform(PlatformIOProjectException):
 
     MESSAGE = "Please specify platform for '{0}' environment"
 
 
-class UnsupportedArchiveType(PlatformioException):
+class ProjectEnvsNotAvailable(PlatformIOProjectException):
-
-    MESSAGE = "Can not unpack file '{0}'"
-
-
-class ProjectEnvsNotAvailable(PlatformioException):
 
     MESSAGE = "Please setup environments in `platformio.ini` file"
 
 
-class UnknownEnvNames(PlatformioException):
+class UnknownEnvNames(PlatformIOProjectException):
 
     MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'"
 
 
+class ProjectOptionValueError(PlatformIOProjectException):
+
+    MESSAGE = "{0} for option `{1}` in section [{2}]"
+
+
+#
+# Library
+#
+
+
+class LibNotFound(PlatformioException):
+
+    MESSAGE = ("Library `{0}` has not been found in PlatformIO Registry.\n"
+               "You can ignore this message, if `{0}` is a built-in library "
+               "(included in framework, SDK). E.g., SPI, Wire, etc.")
+
+
+class NotGlobalLibDir(UserSideException):
+
+    MESSAGE = (
+        "The `{0}` is not a PlatformIO project.\n\n"
+        "To manage libraries in global storage `{1}`,\n"
+        "please use `platformio lib --global {2}` or specify custom storage "
+        "`platformio lib --storage-dir /path/to/storage/ {2}`.\n"
+        "Check `platformio lib --help` for details.")
+
+
+class InvalidLibConfURL(PlatformioException):
+
+    MESSAGE = "Invalid library config URL '{0}'"
+
+
+#
+# UDEV Rules
+#
+
+
+class InvalidUdevRules(PlatformioException):
+    pass
+
+
+class MissedUdevRules(InvalidUdevRules):
+
+    MESSAGE = (
+        "Warning! Please install `99-platformio-udev.rules`. \nMode details: "
+        "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules")
+
+
+class OutdatedUdevRules(InvalidUdevRules):
+
+    MESSAGE = (
+        "Warning! Your `{0}` are outdated. Please update or reinstall them."
+        "\n Mode details: https://docs.platformio.org"
+        "/en/latest/faq.html#platformio-udev-rules")
+
+
+#
+# Misc
+#
+
+
 class GetSerialPortsError(PlatformioException):
 
     MESSAGE = "No implementation for your platform ('{0}') available"
@@ -173,7 +255,7 @@ class APIRequestError(PlatformioException):
     MESSAGE = "[API] {0}"
 
 
-class InternetIsOffline(PlatformioException):
+class InternetIsOffline(UserSideException):
 
     MESSAGE = (
         "You are not connected to the Internet.\n"
@@ -181,33 +263,6 @@ class InternetIsOffline(PlatformioException):
         "to install all dependencies and toolchains.")
 
 
-class LibNotFound(PlatformioException):
-
-    MESSAGE = ("Library `{0}` has not been found in PlatformIO Registry.\n"
-               "You can ignore this message, if `{0}` is a built-in library "
-               "(included in framework, SDK). E.g., SPI, Wire, etc.")
-
-
-class NotGlobalLibDir(PlatformioException):
-
-    MESSAGE = (
-        "The `{0}` is not a PlatformIO project.\n\n"
-        "To manage libraries in global storage `{1}`,\n"
-        "please use `platformio lib --global {2}` or specify custom storage "
-        "`platformio lib --storage-dir /path/to/storage/ {2}`.\n"
-        "Check `platformio lib --help` for details.")
-
-
-class InvalidLibConfURL(PlatformioException):
-
-    MESSAGE = "Invalid library config URL '{0}'"
-
-
-class InvalidProjectConf(PlatformioException):
-
-    MESSAGE = "Invalid `platformio.ini`, project configuration file: '{0}'"
-
-
 class BuildScriptNotFound(PlatformioException):
 
     MESSAGE = "Invalid path '{0}' to build script"
@@ -235,25 +290,6 @@ class CIBuildEnvsEmpty(PlatformioException):
                "predefined environments using `--project-conf` option")
 
 
-class InvalidUdevRules(PlatformioException):
-    pass
-
-
-class MissedUdevRules(InvalidUdevRules):
-
-    MESSAGE = (
-        "Warning! Please install `99-platformio-udev.rules`. \nMode details: "
-        "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules")
-
-
-class OutdatedUdevRules(InvalidUdevRules):
-
-    MESSAGE = (
-        "Warning! Your `{0}` are outdated. Please update or reinstall them."
-        "\n Mode details: https://docs.platformio.org"
-        "/en/latest/faq.html#platformio-udev-rules")
-
-
 class UpgradeError(PlatformioException):
 
     MESSAGE = """{0}
@@ -282,11 +318,21 @@ class CygwinEnvDetected(PlatformioException):
 
 class DebugSupportError(PlatformioException):
 
-    MESSAGE = ("Currently, PlatformIO does not support debugging for `{0}`.\n"
+    MESSAGE = (
-               "Please contact support@pioplus.com or visit "
+        "Currently, PlatformIO does not support debugging for `{0}`.\n"
-               "< https://docs.platformio.org/page/plus/debugging.html >")
+        "Please request support at https://github.com/platformio/"
+        "platformio-core/issues \nor visit -> https://docs.platformio.org"
+        "/page/plus/debugging.html")
 
 
 class DebugInvalidOptions(PlatformioException):
-
     pass
+
+
+class TestDirNotExists(PlatformioException):
+
+    MESSAGE = "A test folder '{0}' does not exist.\nPlease create 'test' "\
+              "directory in project's root and put a test set.\n"\
+              "More details about Unit "\
+              "Testing: http://docs.platformio.org/page/plus/"\
+              "unit-testing.html"

platformio/ide/projectgenerator.py

@@ -12,28 +12,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
+import codecs
 import os
 import re
 import sys
 from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath
 
 import bottle
-from click.testing import CliRunner
 
-from platformio import exception, util
+from platformio import util
-from platformio.commands.run import cli as cmd_run
+from platformio.compat import WINDOWS, get_file_contents
+from platformio.proc import where_is_program
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import (get_project_lib_dir,
+                                        get_project_libdeps_dir,
+                                        get_project_src_dir,
+                                        load_project_ide_data)
 
 
 class ProjectGenerator(object):
 
     def __init__(self, project_dir, ide, env_name):
         self.project_dir = project_dir
-        self.ide = ide
+        self.ide = str(ide)
-        self.env_name = env_name
+        self.env_name = str(env_name)
-
-        self._tplvars = {}
-        self._gather_tplvars()
 
     @staticmethod
     def get_supported_ides():
@@ -41,55 +43,44 @@ class ProjectGenerator(object):
         return sorted(
             [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))])
 
-    @util.memoized()
+    def _load_tplvars(self):
-    def get_project_env(self):
+        tpl_vars = {"env_name": self.env_name}
-        data = {}
+        # default env configuration
-        config = util.load_project_config(self.project_dir)
+        tpl_vars.update(
-        for section in config.sections():
+            ProjectConfig.get_instance(join(
-            if not section.startswith("env:"):
+                self.project_dir, "platformio.ini")).items(env=self.env_name,
-                continue
+                                                           as_dict=True))
-            if self.env_name != section[4:]:
+        # build data
-                continue
+        tpl_vars.update(
-            data = {"env_name": section[4:]}
+            load_project_ide_data(self.project_dir, self.env_name) or {})
-            for k, v in config.items(section):
-                data[k] = v
-        return data
 
-    def get_project_build_data(self):
+        with util.cd(self.project_dir):
-        data = {
+            tpl_vars.update({
-            "defines": [],
+                "project_name": basename(self.project_dir),
-            "includes": [],
+                "src_files": self.get_src_files(),
-            "cxx_path": None,
+                "user_home_dir": abspath(expanduser("~")),
-            "prog_path": None
+                "project_dir": self.project_dir,
-        }
+                "project_src_dir": get_project_src_dir(),
-        envdata = self.get_project_env()
+                "project_lib_dir": get_project_lib_dir(),
-        if not envdata:
+                "project_libdeps_dir": join(
-            return data
+                    get_project_libdeps_dir(), self.env_name),
+                "systype": util.get_systype(),
+                "platformio_path": self._fix_os_path(
+                    sys.argv[0] if isfile(sys.argv[0])
+                    else where_is_program("platformio")),
+                "env_pathsep": os.pathsep,
+                "env_path": self._fix_os_path(os.getenv("PATH"))
+            })  # yapf: disable
+        return tpl_vars
 
-        result = CliRunner().invoke(cmd_run, [
+    @staticmethod
-            "--project-dir", self.project_dir, "--environment",
+    def _fix_os_path(path):
-            envdata['env_name'], "--target", "idedata"
+        return (re.sub(r"[\\]+", '\\' * 4, path) if WINDOWS else path)
-        ])
-
-        if result.exit_code != 0 and not isinstance(result.exception,
-                                                    exception.ReturnErrorCode):
-            raise result.exception
-        if '"includes":' not in result.output:
-            raise exception.PlatformioException(result.output)
-
-        for line in result.output.split("\n"):
-            line = line.strip()
-            if line.startswith('{"') and line.endswith("}"):
-                data = json.loads(line)
-        return data
-
-    def get_project_name(self):
-        return basename(self.project_dir)
 
     def get_src_files(self):
         result = []
         with util.cd(self.project_dir):
-            for root, _, files in os.walk(util.get_projectsrc_dir()):
+            for root, _, files in os.walk(get_project_src_dir()):
                 for f in files:
                     result.append(relpath(join(root, f)))
         return result
@@ -108,6 +99,7 @@ class ProjectGenerator(object):
         return tpls
 
     def generate(self):
+        tpl_vars = self._load_tplvars()
         for tpl_relpath, tpl_path in self.get_tpls():
             dst_dir = self.project_dir
             if tpl_relpath:
@@ -116,44 +108,16 @@ class ProjectGenerator(object):
                     os.makedirs(dst_dir)
 
             file_name = basename(tpl_path)[:-4]
-            self._merge_contents(
+            contents = self._render_tpl(tpl_path, tpl_vars)
-                join(dst_dir, file_name),
+            self._merge_contents(join(dst_dir, file_name), contents)
-                self._render_tpl(tpl_path).encode("utf8"))
 
-    def _render_tpl(self, tpl_path):
+    @staticmethod
-        content = ""
+    def _render_tpl(tpl_path, tpl_vars):
-        with open(tpl_path) as f:
+        return bottle.template(get_file_contents(tpl_path), **tpl_vars)
-            content = f.read()
-        return bottle.template(content, **self._tplvars)
 
     @staticmethod
     def _merge_contents(dst_path, contents):
         if basename(dst_path) == ".gitignore" and isfile(dst_path):
             return
-        with open(dst_path, "w") as f:
+        with codecs.open(dst_path, "w", encoding="utf8") as fp:
-            f.write(contents)
+            fp.write(contents)
-
-    def _gather_tplvars(self):
-        self._tplvars.update(self.get_project_env())
-        self._tplvars.update(self.get_project_build_data())
-        with util.cd(self.project_dir):
-            self._tplvars.update({
-                "project_name": self.get_project_name(),
-                "src_files": self.get_src_files(),
-                "user_home_dir": abspath(expanduser("~")),
-                "project_dir": self.project_dir,
-                "project_src_dir": util.get_projectsrc_dir(),
-                "project_lib_dir": util.get_projectlib_dir(),
-                "project_libdeps_dir": util.get_projectlibdeps_dir(),
-                "systype": util.get_systype(),
-                "platformio_path": self._fix_os_path(
-                    sys.argv[0] if isfile(sys.argv[0])
-                    else util.where_is_program("platformio")),
-                "env_pathsep": os.pathsep,
-                "env_path": self._fix_os_path(os.getenv("PATH"))
-            })  # yapf: disable
-
-    @staticmethod
-    def _fix_os_path(path):
-        return (re.sub(r"[\\]+", '\\' * 4, path)
-                if "windows" in util.get_systype() else path)

platformio/ide/tpls/atom/.gitignore.tpl

@@ -1,5 +1,3 @@
 .pio
-.pioenvs
-.piolibdeps
 .clang_complete
 .gcc-flags.json

platformio/ide/tpls/clion/.gitignore.tpl

@@ -1,4 +1,2 @@
 .pio
-.pioenvs
-.piolibdeps
 CMakeListsPrivate.txt

platformio/ide/tpls/clion/.idea/misc.xml.tpl

@@ -7,13 +7,10 @@
     </sourceRoots>
     <libraryRoots>
       <file path="$PROJECT_DIR$/lib" />
-      <file path="$PROJECT_DIR$/.piolibdeps" />
+      <file path="$PROJECT_DIR$/.pio/libdeps" />
     </libraryRoots>
     <excludeRoots>
       <file path="$PROJECT_DIR$/.pio" />
     </excludeRoots>
-    <excludeRoots>
-      <file path="$PROJECT_DIR$/.pioenvs" />
-    </excludeRoots>
   </component>
 </project>

platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl

@@ -15,7 +15,7 @@
           <FilterInfo>
             <option name="description" value="" />
             <option name="name" value="PIO Conf" />
-            <option name="regExp" value="$FILE_PATH$:^platofrmio" />
+            <option name="regExp" value="$FILE_PATH$:^platformio" />
           </FilterInfo>
         </array>
       </option>

platformio/ide/tpls/clion/CMakeLists.txt.tpl

@@ -1,8 +1,19 @@
+# !!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE
+# https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags
+#
+# If you need to override existing CMake configuration or add extra,
+# please create `CMakeListsUser.txt` in the root of project.
+# The `CMakeListsUser.txt` will not be overwritten by PlatformIO.
+
 cmake_minimum_required(VERSION 3.2)
 project({{project_name}})
 
 include(CMakeListsPrivate.txt)
 
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/CMakeListsUser.txt)
+include(CMakeListsUser.txt)
+endif()
+
 add_custom_target(
     PLATFORMIO_BUILD ALL
     COMMAND ${PLATFORMIO_CMD} -f -c clion run

platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -1,7 +1,12 @@
-# !!! WARNING !!!
+# !!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE
-# PLEASE DO NOT MODIFY THIS FILE!
+# https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags
-# USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags
+#
+# If you need to override existing CMake configuration or add extra,
+# please create `CMakeListsUser.txt` in the root of project.
+# The `CMakeListsUser.txt` will not be overwritten by PlatformIO.
 
+% import re
+%
 % def _normalize_path(path):
 %   if project_dir in path:
 %     path = path.replace(project_dir, "${CMAKE_CURRENT_LIST_DIR}")
@@ -21,9 +26,17 @@ SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}")
 SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}")
 SET(CMAKE_CXX_FLAGS_DISTRIBUTION "{{cxx_flags}}")
 SET(CMAKE_C_FLAGS_DISTRIBUTION "{{cc_flags}}")
-set(CMAKE_CXX_STANDARD 11)
 
-% import re
+% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
+% cc_stds = STD_RE.findall(cc_flags)
+% cxx_stds = STD_RE.findall(cxx_flags)
+% if cc_stds:
+SET(CMAKE_C_STANDARD {{ cc_stds[-1] }})
+% end
+% if cxx_stds:
+set(CMAKE_CXX_STANDARD {{ cxx_stds[-1] }})
+% end
+
 % for define in defines:
 add_definitions(-D'{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}')
 % end

platformio/ide/tpls/eclipse/.cproject.tpl

@@ -219,17 +219,17 @@
 			<target name="PlatformIO: Upload using Programmer" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
 				<buildCommand>platformio</buildCommand>
 				<buildArguments>-f -c eclipse</buildArguments>
-				<buildTarget>run -t program</buildTarget>
+				<buildTarget>run --target program</buildTarget>
 				<stopOnError>true</stopOnError>
-				<useDefaultCommand>true</useDefaultCommand>
+				<useDefaultCommand>false</useDefaultCommand>
 				<runAllBuilders>false</runAllBuilders>
 			</target>
 			<target name="PlatformIO: Upload SPIFFS image" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
 				<buildCommand>platformio</buildCommand>
 				<buildArguments>-f -c eclipse</buildArguments>
-				<buildTarget>run -t uploadfs</buildTarget>
+				<buildTarget>run --target uploadfs</buildTarget>
 				<stopOnError>true</stopOnError>
-				<useDefaultCommand>true</useDefaultCommand>
+				<useDefaultCommand>false</useDefaultCommand>
 				<runAllBuilders>false</runAllBuilders>
 			</target>
 			<target name="PlatformIO: Build" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
@@ -237,23 +237,39 @@
 				<buildArguments>-f -c eclipse</buildArguments>
 				<buildTarget>run</buildTarget>
 				<stopOnError>true</stopOnError>
-				<useDefaultCommand>true</useDefaultCommand>
+				<useDefaultCommand>false</useDefaultCommand>
+				<runAllBuilders>false</runAllBuilders>
+			</target>
+			<target name="PlatformIO: Verbose Build" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
+				<buildCommand>platformio</buildCommand>
+				<buildArguments>-f -c eclipse</buildArguments>
+				<buildTarget>run --verbose</buildTarget>
+				<stopOnError>true</stopOnError>
+				<useDefaultCommand>false</useDefaultCommand>
 				<runAllBuilders>false</runAllBuilders>
 			</target>
 			<target name="PlatformIO: Upload" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
 				<buildCommand>platformio</buildCommand>
 				<buildArguments>-f -c eclipse</buildArguments>
-				<buildTarget>run -t upload</buildTarget>
+				<buildTarget>run --target upload</buildTarget>
 				<stopOnError>true</stopOnError>
-				<useDefaultCommand>true</useDefaultCommand>
+				<useDefaultCommand>false</useDefaultCommand>
+				<runAllBuilders>false</runAllBuilders>
+			</target>
+			<target name="PlatformIO: Verbose Upload" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
+				<buildCommand>platformio</buildCommand>
+				<buildArguments>-f -c eclipse</buildArguments>
+				<buildTarget>run --target upload --verbose</buildTarget>
+				<stopOnError>true</stopOnError>
+				<useDefaultCommand>false</useDefaultCommand>
 				<runAllBuilders>false</runAllBuilders>
 			</target>
 			<target name="PlatformIO: Clean" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
 				<buildCommand>platformio</buildCommand>
 				<buildArguments>-f -c eclipse</buildArguments>
-				<buildTarget>run -t clean</buildTarget>
+				<buildTarget>run --target clean</buildTarget>
 				<stopOnError>true</stopOnError>
-				<useDefaultCommand>true</useDefaultCommand>
+				<useDefaultCommand>false</useDefaultCommand>
 				<runAllBuilders>false</runAllBuilders>
 			</target>
 			<target name="PlatformIO: Test" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
@@ -261,15 +277,15 @@
 				<buildArguments>-f -c eclipse</buildArguments>
 				<buildTarget>test</buildTarget>
 				<stopOnError>true</stopOnError>
-				<useDefaultCommand>true</useDefaultCommand>
+				<useDefaultCommand>false</useDefaultCommand>
 				<runAllBuilders>false</runAllBuilders>
 			</target>
-			<target name="PlatformIO: Update platforms and libraries" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
+			<target name="PlatformIO: Remote" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
 				<buildCommand>platformio</buildCommand>
 				<buildArguments>-f -c eclipse</buildArguments>
-				<buildTarget>update</buildTarget>
+				<buildTarget>remote run --target upload</buildTarget>
 				<stopOnError>true</stopOnError>
-				<useDefaultCommand>true</useDefaultCommand>
+				<useDefaultCommand>false</useDefaultCommand>
 				<runAllBuilders>false</runAllBuilders>
 			</target>
 			<target name="PlatformIO: Rebuild C/C++ Project Index" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
@@ -277,7 +293,39 @@
 				<buildArguments>-f -c eclipse</buildArguments>
 				<buildTarget>init --ide eclipse</buildTarget>
 				<stopOnError>true</stopOnError>
-				<useDefaultCommand>true</useDefaultCommand>
+				<useDefaultCommand>false</useDefaultCommand>
+				<runAllBuilders>false</runAllBuilders>
+			</target>
+			<target name="PlatformIO: List Devices" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
+				<buildCommand>platformio</buildCommand>
+				<buildArguments>-f -c eclipse</buildArguments>
+				<buildTarget>device list</buildTarget>
+				<stopOnError>true</stopOnError>
+				<useDefaultCommand>false</useDefaultCommand>
+				<runAllBuilders>false</runAllBuilders>
+			</target>
+			<target name="PlatformIO: Update Project Libraries" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
+				<buildCommand>platformio</buildCommand>
+				<buildArguments>-f -c eclipse</buildArguments>
+				<buildTarget>lib update</buildTarget>
+				<stopOnError>true</stopOnError>
+				<useDefaultCommand>false</useDefaultCommand>
+				<runAllBuilders>false</runAllBuilders>
+			</target>
+			<target name="PlatformIO: Update All" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
+				<buildCommand>platformio</buildCommand>
+				<buildArguments>-f -c eclipse</buildArguments>
+				<buildTarget>update</buildTarget>
+				<stopOnError>true</stopOnError>
+				<useDefaultCommand>false</useDefaultCommand>
+				<runAllBuilders>false</runAllBuilders>
+			</target>
+			<target name="PlatformIO: Upgrade Core" path="" targetID="org.eclipse.cdt.build.MakeTargetBuilder">
+				<buildCommand>platformio</buildCommand>
+				<buildArguments>-f -c eclipse</buildArguments>
+				<buildTarget>upgrade</buildTarget>
+				<stopOnError>true</stopOnError>
+				<useDefaultCommand>false</useDefaultCommand>
 				<runAllBuilders>false</runAllBuilders>
 			</target>
 		</buildTargets>

platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl

@@ -3,6 +3,14 @@
 % cxx_stds = STD_RE.findall(cxx_flags)
 % cxx_std = cxx_stds[-1] if cxx_stds else ""
 %
+% if cxx_path.startswith(user_home_dir):
+% if "windows" in systype:
+%    cxx_path = "${USERPROFILE}" + cxx_path.replace(user_home_dir, "")
+% else:
+%    cxx_path = "${HOME}" + cxx_path.replace(user_home_dir, "")
+% end
+% end
+%
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <project>
 	<configuration id="0.910961921" name="Default">
@@ -10,11 +18,7 @@
 			<provider copy-of="extension" id="org.eclipse.cdt.ui.UserLanguageSettingsProvider"/>
 			<provider-reference id="org.eclipse.cdt.core.ReferencedProjectsLanguageSettingsProvider" ref="shared-provider"/>
 			<provider-reference id="org.eclipse.cdt.managedbuilder.core.MBSLanguageSettingsProvider" ref="shared-provider"/>
-			% if "windows" in systype:
+			<provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="1291887707783033084" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="{{ cxx_path }} ${FLAGS} {{ cxx_std }} -E -P -v -dD &quot;${INPUTS}&quot;" prefer-non-shared="true">
-			<provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="1291887707783033084" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="${USERPROFILE}{{cxx_path.replace(user_home_dir, '')}} ${FLAGS} {{ cxx_std }} -E -P -v -dD &quot;${INPUTS}&quot;" prefer-non-shared="true">
-			% else:
-			<provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="-869785120007741010" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="${HOME}{{cxx_path.replace(user_home_dir, '')}} ${FLAGS} {{ cxx_std }} -E -P -v -dD &quot;${INPUTS}&quot;" prefer-non-shared="true">
-			% end
 				<language-scope id="org.eclipse.cdt.core.gcc"/>
 				<language-scope id="org.eclipse.cdt.core.g++"/>
 			</provider>
@@ -25,11 +29,7 @@
 			<provider copy-of="extension" id="org.eclipse.cdt.ui.UserLanguageSettingsProvider"/>
 			<provider-reference id="org.eclipse.cdt.core.ReferencedProjectsLanguageSettingsProvider" ref="shared-provider"/>
 			<provider-reference id="org.eclipse.cdt.managedbuilder.core.MBSLanguageSettingsProvider" ref="shared-provider"/>
-			% if "windows" in systype:
+			<provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="1291887707783033084" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="{{ cxx_path }} ${FLAGS} {{ cxx_std }} -E -P -v -dD &quot;${INPUTS}&quot;" prefer-non-shared="true">
-			<provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="1291887707783033084" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="${USERPROFILE}{{cxx_path.replace(user_home_dir, '')}} ${FLAGS} {{ cxx_std }} -E -P -v -dD &quot;${INPUTS}&quot;" prefer-non-shared="true">
-			% else:
-			<provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="-869785120007741010" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="${HOME}{{cxx_path.replace(user_home_dir, '')}} ${FLAGS} {{ cxx_std }} -E -P -v -dD &quot;${INPUTS}&quot;" prefer-non-shared="true">
-			% end
 				<language-scope id="org.eclipse.cdt.core.gcc"/>
 				<language-scope id="org.eclipse.cdt.core.g++"/>
 			</provider>

platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl

@@ -1,11 +1,11 @@
 eclipse.preferences.version=1
 environment/project/0.910961921/PATH/delimiter={{env_pathsep.replace(":", "\\:")}}
 environment/project/0.910961921/PATH/operation=replace
-environment/project/0.910961921/PATH/value={{env_path.replace(":", "\\:")}}
+environment/project/0.910961921/PATH/value={{env_path.replace(":", "\\:")}}${PathDelimiter}${PATH}
 environment/project/0.910961921/append=true
 environment/project/0.910961921/appendContributed=true
 environment/project/0.910961921.1363900502/PATH/delimiter={{env_pathsep.replace(":", "\\:")}}
 environment/project/0.910961921.1363900502/PATH/operation=replace
-environment/project/0.910961921.1363900502/PATH/value={{env_path.replace(":", "\\:")}}
+environment/project/0.910961921.1363900502/PATH/value={{env_path.replace(":", "\\:")}}${PathDelimiter}${PATH}
 environment/project/0.910961921.1363900502/append=true
 environment/project/0.910961921.1363900502/appendContributed=true

platformio/ide/tpls/emacs/.gitignore.tpl

@@ -1,4 +1,2 @@
 .pio
-.pioenvs
-.piolibdeps
 .clang_complete

platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl

@@ -11,7 +11,7 @@
       <itemPath>nbproject/private/launcher.properties</itemPath>
     </logicalFolder>
   </logicalFolder>
-  <sourceFolderFilter>^(nbproject|.pio|.pioenvs)$</sourceFolderFilter>
+  <sourceFolderFilter>^(nbproject|.pio)$</sourceFolderFilter>
   <sourceRootList>
     <Elem>.</Elem>
   </sourceRootList>

platformio/ide/tpls/vim/.gitignore.tpl

@@ -1,5 +1,3 @@
 .pio
-.pioenvs
-.piolibdeps
 .clang_complete
 .gcc-flags.json

platformio/ide/tpls/vscode/.gitignore.tpl

@@ -1,6 +1,5 @@
 .pio
-.pioenvs
-.piolibdeps
 .vscode/.browse.c_cpp.db*
 .vscode/c_cpp_properties.json
 .vscode/launch.json
+.vscode/ipch

platformio/maintenance.py

@@ -12,26 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
-import os
 from os import getenv
-from os.path import isdir, join
+from os.path import join
 from time import time
 
 import click
 import semantic_version
 
 from platformio import __version__, app, exception, telemetry, util
+from platformio.commands import PlatformioCLI
+from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY
 from platformio.commands.lib import lib_update as cmd_lib_update
-from platformio.commands.platform import \
-    platform_install as cmd_platform_install
-from platformio.commands.platform import \
-    platform_uninstall as cmd_platform_uninstall
 from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.commands.upgrade import get_latest_version
 from platformio.managers.core import update_core_packages
 from platformio.managers.lib import LibraryManager
 from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.proc import is_ci, is_container
 
 
 def on_platformio_start(ctx, force, caller):
@@ -40,12 +37,12 @@ def on_platformio_start(ctx, force, caller):
     set_caller(caller)
     telemetry.on_command()
 
-    if not in_silence(ctx):
+    if not PlatformioCLI.in_silence():
         after_upgrade(ctx)
 
 
-def on_platformio_end(ctx, result):  # pylint: disable=W0613
+def on_platformio_end(ctx, result):  # pylint: disable=unused-argument
-    if in_silence(ctx):
+    if PlatformioCLI.in_silence():
         return
 
     try:
@@ -64,24 +61,13 @@ def on_platformio_exception(e):
     telemetry.on_exception(e)
 
 
-def in_silence(ctx=None):
-    ctx = ctx or app.get_session_var("command_ctx")
-    if not ctx:
-        return True
-    return ctx.args and any([
-        ctx.args[0] == "debug" and "--interpreter" in " ".join(ctx.args),
-        ctx.args[0] == "upgrade", "--json-output" in ctx.args,
-        "--version" in ctx.args
-    ])
-
-
 def set_caller(caller=None):
     if not caller:
         if getenv("PLATFORMIO_CALLER"):
             caller = getenv("PLATFORMIO_CALLER")
         elif getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"):
             caller = "vscode"
-        elif util.is_container():
+        elif is_container():
             if getenv("C9_UID"):
                 caller = "C9"
             elif getenv("USER") == "cabox":
@@ -99,11 +85,7 @@ class Upgrader(object):
         self.to_version = semantic_version.Version.coerce(
             util.pepver_to_semver(to_version))
 
-        self._upgraders = [(semantic_version.Version("3.0.0-a.1"),
+        self._upgraders = [(semantic_version.Version("3.5.0-a.2"),
-                            self._upgrade_to_3_0_0),
-                           (semantic_version.Version("3.0.0-b.11"),
-                            self._upgrade_to_3_0_0b11),
-                           (semantic_version.Version("3.5.0-a.2"),
                             self._update_dev_platforms)]
 
     def run(self, ctx):
@@ -118,43 +100,6 @@ class Upgrader(object):
 
         return all(result)
 
-    @staticmethod
-    def _upgrade_to_3_0_0(ctx):
-        # convert custom board configuration
-        boards_dir = join(util.get_home_dir(), "boards")
-        if isdir(boards_dir):
-            for item in os.listdir(boards_dir):
-                if not item.endswith(".json"):
-                    continue
-                data = util.load_json(join(boards_dir, item))
-                if set(["name", "url", "vendor"]) <= set(data.keys()):
-                    continue
-                os.remove(join(boards_dir, item))
-                for key, value in data.items():
-                    with open(join(boards_dir, "%s.json" % key), "w") as f:
-                        json.dump(value, f, sort_keys=True, indent=2)
-
-        # re-install PlatformIO 2.0 development platforms
-        installed_platforms = app.get_state_item("installed_platforms", [])
-        if installed_platforms:
-            if "espressif" in installed_platforms:
-                installed_platforms[installed_platforms.index(
-                    "espressif")] = "espressif8266"
-            ctx.invoke(cmd_platform_install, platforms=installed_platforms)
-
-        return True
-
-    @staticmethod
-    def _upgrade_to_3_0_0b11(ctx):
-        current_platforms = [
-            m['name'] for m in PlatformManager().get_installed()
-        ]
-        if "espressif" not in current_platforms:
-            return True
-        ctx.invoke(cmd_platform_install, platforms=["espressif8266"])
-        ctx.invoke(cmd_platform_uninstall, platforms=["espressif"])
-        return True
-
     @staticmethod
     def _update_dev_platforms(ctx):
         ctx.invoke(cmd_platform_update)
@@ -173,12 +118,11 @@ def after_upgrade(ctx):
             last_version)) > semantic_version.Version.coerce(
                 util.pepver_to_semver(__version__)):
         click.secho("*" * terminal_width, fg="yellow")
-        click.secho(
+        click.secho("Obsolete PIO Core v%s is used (previous was %s)" %
-            "Obsolete PIO Core v%s is used (previous was %s)" % (__version__,
+                    (__version__, last_version),
-                                                                 last_version),
+                    fg="yellow")
-            fg="yellow")
+        click.secho("Please remove multiple PIO Cores from a system:",
-        click.secho(
+                    fg="yellow")
-            "Please remove multiple PIO Cores from a system:", fg="yellow")
         click.secho(
             "https://docs.platformio.org/page/faq.html"
             "#multiple-pio-cores-in-a-system",
@@ -195,22 +139,20 @@ def after_upgrade(ctx):
         u = Upgrader(last_version, __version__)
         if u.run(ctx):
             app.set_state_item("last_version", __version__)
-            click.secho(
+            click.secho("PlatformIO has been successfully upgraded to %s!\n" %
-                "PlatformIO has been successfully upgraded to %s!\n" %
+                        __version__,
-                __version__,
+                        fg="green")
-                fg="green")
+            telemetry.on_event(category="Auto",
-            telemetry.on_event(
+                               action="Upgrade",
-                category="Auto",
+                               label="%s > %s" % (last_version, __version__))
-                action="Upgrade",
-                label="%s > %s" % (last_version, __version__))
         else:
             raise exception.UpgradeError("Auto upgrading...")
         click.echo("")
 
     # PlatformIO banner
     click.echo("*" * terminal_width)
-    click.echo(
+    click.echo("If you like %s, please:" %
-        "If you like %s, please:" % (click.style("PlatformIO", fg="cyan")))
+               (click.style("PlatformIO", fg="cyan")))
     click.echo("- %s us on Twitter to stay up-to-date "
                "on the latest project news > %s" %
                (click.style("follow", fg="cyan"),
@@ -224,10 +166,10 @@ def after_upgrade(ctx):
             "- %s PlatformIO IDE for IoT development > %s" %
             (click.style("try", fg="cyan"),
              click.style("https://platformio.org/platformio-ide", fg="cyan")))
-    if not util.is_ci():
+    if not is_ci():
-        click.echo("- %s us with PlatformIO Plus > %s" % (click.style(
+        click.echo("- %s us with PlatformIO Plus > %s" %
-            "support", fg="cyan"), click.style(
+                   (click.style("support", fg="cyan"),
-                "https://pioplus.com", fg="cyan")))
+                    click.style("https://pioplus.com", fg="cyan")))
 
     click.echo("*" * terminal_width)
     click.echo("")
@@ -257,14 +199,14 @@ def check_platformio_upgrade():
 
     click.echo("")
     click.echo("*" * terminal_width)
-    click.secho(
+    click.secho("There is a new version %s of PlatformIO available.\n"
-        "There is a new version %s of PlatformIO available.\n"
+                "Please upgrade it via `" % latest_version,
-        "Please upgrade it via `" % latest_version,
+                fg="yellow",
-        fg="yellow",
+                nl=False)
-        nl=False)
     if getenv("PLATFORMIO_IDE"):
-        click.secho(
+        click.secho("PlatformIO IDE Menu: Upgrade PlatformIO",
-            "PlatformIO IDE Menu: Upgrade PlatformIO", fg="cyan", nl=False)
+                    fg="cyan",
+                    nl=False)
         click.secho("`.", fg="yellow")
     elif join("Cellar", "platformio") in util.get_source_dir():
         click.secho("brew update && brew upgrade", fg="cyan", nl=False)
@@ -275,8 +217,8 @@ def check_platformio_upgrade():
         click.secho("pip install -U platformio", fg="cyan", nl=False)
         click.secho("` command.", fg="yellow")
     click.secho("Changes: ", fg="yellow", nl=False)
-    click.secho(
+    click.secho("https://docs.platformio.org/en/latest/history.html",
-        "https://docs.platformio.org/en/latest/history.html", fg="cyan")
+                fg="cyan")
     click.echo("*" * terminal_width)
     click.echo("")
 
@@ -312,41 +254,39 @@ def check_internal_updates(ctx, what):
 
     click.echo("")
     click.echo("*" * terminal_width)
-    click.secho(
+    click.secho("There are the new updates for %s (%s)" %
-        "There are the new updates for %s (%s)" % (what,
+                (what, ", ".join(outdated_items)),
-                                                   ", ".join(outdated_items)),
+                fg="yellow")
-        fg="yellow")
 
     if not app.get_setting("auto_update_" + what):
         click.secho("Please update them via ", fg="yellow", nl=False)
-        click.secho(
+        click.secho("`platformio %s update`" %
-            "`platformio %s update`" %
+                    ("lib --global" if what == "libraries" else "platform"),
-            ("lib --global" if what == "libraries" else "platform"),
+                    fg="cyan",
-            fg="cyan",
+                    nl=False)
-            nl=False)
         click.secho(" command.\n", fg="yellow")
         click.secho(
             "If you want to manually check for the new versions "
             "without updating, please use ",
             fg="yellow",
             nl=False)
-        click.secho(
+        click.secho("`platformio %s update --dry-run`" %
-            "`platformio %s update --only-check`" %
+                    ("lib --global" if what == "libraries" else "platform"),
-            ("lib --global" if what == "libraries" else "platform"),
+                    fg="cyan",
-            fg="cyan",
+                    nl=False)
-            nl=False)
         click.secho(" command.", fg="yellow")
     else:
         click.secho("Please wait while updating %s ..." % what, fg="yellow")
         if what == "platforms":
             ctx.invoke(cmd_platform_update, platforms=outdated_items)
         elif what == "libraries":
-            ctx.obj = pm
+            ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [pm.package_dir]
             ctx.invoke(cmd_lib_update, libraries=outdated_items)
         click.echo()
 
-        telemetry.on_event(
+        telemetry.on_event(category="Auto",
-            category="Auto", action="Update", label=what.title())
+                           action="Update",
+                           label=what.title())
 
     click.echo("*" * terminal_width)
     click.echo("")

platformio/managers/core.py

@@ -21,15 +21,18 @@ from time import sleep
 import requests
 
 from platformio import __version__, exception, util
+from platformio.compat import PY2, WINDOWS
 from platformio.managers.package import PackageManager
+from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
+from platformio.project.helpers import get_project_packages_dir
 
 CORE_PACKAGES = {
-    "contrib-piohome": "^2.0.1",
+    "contrib-piohome": "^2.1.0",
     "contrib-pysite":
     "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]),
-    "tool-pioplus": "^2.1.4",
+    "tool-pioplus": "^2.5.2",
     "tool-unity": "~1.20403.0",
-    "tool-scons": "~2.20501.7"
+    "tool-scons": "~2.20501.7" if PY2 else "~3.30005.0"
 }
 
 PIOPLUS_AUTO_UPDATES_MAX = 100
@@ -40,12 +43,11 @@ PIOPLUS_AUTO_UPDATES_MAX = 100
 class CorePackageManager(PackageManager):
 
     def __init__(self):
-        super(CorePackageManager, self).__init__(
+        super(CorePackageManager, self).__init__(get_project_packages_dir(), [
-            join(util.get_home_dir(), "packages"), [
+            "https://dl.bintray.com/platformio/dl-packages/manifest.json",
-                "https://dl.bintray.com/platformio/dl-packages/manifest.json",
+            "http%s://dl.platformio.org/packages/manifest.json" %
-                "http%s://dl.platformio.org/packages/manifest.json" %
+            ("" if sys.version_info < (2, 7, 9) else "s")
-                ("" if sys.version_info < (2, 7, 9) else "s")
+        ])
-            ])
 
     def install(  # pylint: disable=keyword-arg-before-vararg
             self,
@@ -99,7 +101,7 @@ def update_core_packages(only_check=False, silent=False):
         if not silent or pm.outdated(pkg_dir, requirements):
             if name == "tool-pioplus" and not only_check:
                 shutdown_piohome_servers()
-                if "windows" in util.get_systype():
+                if WINDOWS:
                     sleep(1)
             pm.update(name, requirements, only_check=only_check)
     return True
@@ -109,28 +111,38 @@ def shutdown_piohome_servers():
     port = 8010
     while port < 8050:
         try:
-            requests.get(
+            requests.get("http://127.0.0.1:%d?__shutdown__=1" % port,
-                "http://127.0.0.1:%d?__shutdown__=1" % port, timeout=0.01)
+                         timeout=0.01)
         except:  # pylint: disable=bare-except
             pass
         port += 1
 
 
+def inject_contrib_pysite():
+    from site import addsitedir
+    contrib_pysite_dir = get_core_package_dir("contrib-pysite")
+    if contrib_pysite_dir in sys.path:
+        return
+    addsitedir(contrib_pysite_dir)
+    sys.path.insert(0, contrib_pysite_dir)
+
+
 def pioplus_call(args, **kwargs):
-    if "windows" in util.get_systype() and sys.version_info < (2, 7, 6):
+    if WINDOWS and sys.version_info < (2, 7, 6):
         raise exception.PlatformioException(
             "PlatformIO Core Plus v%s does not run under Python version %s.\n"
             "Minimum supported version is 2.7.6, please upgrade Python.\n"
             "Python 3 is not yet supported.\n" % (__version__, sys.version))
 
     pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus")
-    pythonexe_path = util.get_pythonexe_path()
+    pythonexe_path = get_pythonexe_path()
     os.environ['PYTHONEXEPATH'] = pythonexe_path
     os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite")
     os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "")
-    os.environ['PATH'] = (os.pathsep).join(
+    if dirname(pythonexe_path) not in os.environ['PATH'].split(os.pathsep):
-        [dirname(pythonexe_path), os.environ['PATH']])
+        os.environ['PATH'] = (os.pathsep).join(
-    util.copy_pythonpath_to_osenv()
+            [dirname(pythonexe_path), os.environ['PATH']])
+    copy_pythonpath_to_osenv()
     code = subprocess.call([pioplus_path] + args, **kwargs)
 
     # handle remote update request

platformio/managers/lib.py

@@ -22,16 +22,20 @@ from os.path import isdir, join
 
 import click
 
-from platformio import app, commands, exception, util
+from platformio import app, exception, util
+from platformio.compat import glob_escape, string_types
 from platformio.managers.package import BasePkgManager
 from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.project.helpers import get_project_global_lib_dir
 
 
 class LibraryManager(BasePkgManager):
 
+    FILE_CACHE_VALID = "30d"  # 1 month
+
     def __init__(self, package_dir=None):
         if not package_dir:
-            package_dir = join(util.get_home_dir(), "lib")
+            package_dir = get_project_global_lib_dir()
         super(LibraryManager, self).__init__(package_dir)
 
     @property
@@ -47,7 +51,7 @@ class LibraryManager(BasePkgManager):
             return path
 
         # if library without manifest, returns first source file
-        src_dir = join(util.glob_escape(pkg_dir))
+        src_dir = join(glob_escape(pkg_dir))
         if isdir(join(pkg_dir, "src")):
             src_dir = join(src_dir, "src")
         chs_files = glob(join(src_dir, "*.[chS]"))
@@ -120,7 +124,7 @@ class LibraryManager(BasePkgManager):
         # convert listed items via comma to array
         for key in ("keywords", "frameworks", "platforms"):
             if key not in manifest or \
-                    not isinstance(manifest[key], basestring):
+                    not isinstance(manifest[key], string_types):
                 continue
             manifest[key] = [
                 i.strip() for i in manifest[key].split(",") if i.strip()
@@ -147,7 +151,7 @@ class LibraryManager(BasePkgManager):
                     continue
                 if item[k] == "*":
                     del item[k]
-                elif isinstance(item[k], basestring):
+                elif isinstance(item[k], string_types):
                     item[k] = [
                         i.strip() for i in item[k].split(",") if i.strip()
                     ]
@@ -164,7 +168,7 @@ class LibraryManager(BasePkgManager):
 
         semver_spec = self.parse_semver_spec(
             requirements) if requirements else None
-        item = None
+        item = {}
 
         for v in versions:
             semver_new = self.parse_semver_version(v['name'])
@@ -186,14 +190,12 @@ class LibraryManager(BasePkgManager):
 
     def get_latest_repo_version(self, name, requirements, silent=False):
         item = self.max_satisfying_repo_version(
-            util.get_api_result(
+            util.get_api_result("/lib/info/%d" % self.search_lib_id(
-                "/lib/info/%d" % self.search_lib_id(
+                {
-                    {
+                    "name": name,
-                        "name": name,
+                    "requirements": requirements
-                        "requirements": requirements
+                }, silent=silent),
-                    },
+                                cache_valid="1h")['versions'], requirements)
-                    silent=silent),
-                cache_valid="1h")['versions'], requirements)
         return item['name'] if item else None
 
     def _install_from_piorepo(self, name, requirements):
@@ -202,10 +204,9 @@ class LibraryManager(BasePkgManager):
         if not version:
             raise exception.UndefinedPackageVersion(requirements or "latest",
                                                     util.get_systype())
-        dl_data = util.get_api_result(
+        dl_data = util.get_api_result("/lib/download/" + str(name[3:]),
-            "/lib/download/" + str(name[3:]),
+                                      dict(version=version),
-            dict(version=version),
+                                      cache_valid="30d")
-            cache_valid="30d")
         assert dl_data
 
         return self._install_from_url(
@@ -227,8 +228,8 @@ class LibraryManager(BasePkgManager):
 
         # looking in PIO Library Registry
         if not silent:
-            click.echo("Looking for %s library in registry" % click.style(
+            click.echo("Looking for %s library in registry" %
-                filters['name'], fg="cyan"))
+                       click.style(filters['name'], fg="cyan"))
         query = []
         for key in filters:
             if key not in ("name", "authors", "frameworks", "platforms"):
@@ -241,21 +242,22 @@ class LibraryManager(BasePkgManager):
                              (key[:-1] if key.endswith("s") else key, value))
 
         lib_info = None
-        result = util.get_api_result(
+        result = util.get_api_result("/v2/lib/search",
-            "/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h")
+                                     dict(query=" ".join(query)),
+                                     cache_valid="1h")
         if result['total'] == 1:
             lib_info = result['items'][0]
         elif result['total'] > 1:
             if silent and not interactive:
                 lib_info = result['items'][0]
             else:
-                click.secho(
+                click.secho("Conflict: More than one library has been found "
-                    "Conflict: More than one library has been found "
+                            "by request %s:" % json.dumps(filters),
-                    "by request %s:" % json.dumps(filters),
+                            fg="yellow",
-                    fg="yellow",
+                            err=True)
-                    err=True)
+                from platformio.commands.lib import print_lib_item
                 for item in result['items']:
-                    commands.lib.print_lib_item(item)
+                    print_lib_item(item)
 
                 if not interactive:
                     click.secho(
@@ -265,20 +267,20 @@ class LibraryManager(BasePkgManager):
                         err=True)
                     lib_info = result['items'][0]
                 else:
-                    deplib_id = click.prompt(
+                    deplib_id = click.prompt("Please choose library ID",
-                        "Please choose library ID",
+                                             type=click.Choice([
-                        type=click.Choice(
+                                                 str(i['id'])
-                            [str(i['id']) for i in result['items']]))
+                                                 for i in result['items']
+                                             ]))
                     for item in result['items']:
                         if item['id'] == int(deplib_id):
                             lib_info = item
                             break
 
         if not lib_info:
-            if filters.keys() == ["name"]:
+            if list(filters) == ["name"]:
                 raise exception.LibNotFound(filters['name'])
-            else:
+            raise exception.LibNotFound(str(filters))
-                raise exception.LibNotFound(str(filters))
         if not silent:
             click.echo("Found: %s" % click.style(
                 "https://platformio.org/lib/show/{id}/{name}".format(
@@ -303,9 +305,8 @@ class LibraryManager(BasePkgManager):
                 continue
             if key not in manifest:
                 return None
-            if not util.items_in_list(
+            if not util.items_in_list(util.items_to_list(filters[key]),
-                    util.items_to_list(filters[key]),
+                                      util.items_to_list(manifest[key])):
-                    util.items_to_list(manifest[key])):
                 return None
 
         if "authors" in filters:
@@ -336,20 +337,20 @@ class LibraryManager(BasePkgManager):
             force=False):
         _name, _requirements, _url = self.parse_pkg_uri(name, requirements)
         if not _url:
-            name = "id=%d" % self.search_lib_id({
+            name = "id=%d" % self.search_lib_id(
-                "name": _name,
+                {
-                "requirements": _requirements
+                    "name": _name,
-            },
+                    "requirements": _requirements
-                                                silent=silent,
+                },
-                                                interactive=interactive)
+                silent=silent,
+                interactive=interactive)
             requirements = _requirements
-        pkg_dir = BasePkgManager.install(
+        pkg_dir = BasePkgManager.install(self,
-            self,
+                                         name,
-            name,
+                                         requirements,
-            requirements,
+                                         silent=silent,
-            silent=silent,
+                                         after_update=after_update,
-            after_update=after_update,
+                                         force=force)
-            force=force)
 
         if not pkg_dir:
             return None
@@ -361,6 +362,7 @@ class LibraryManager(BasePkgManager):
         if not silent:
             click.secho("Installing dependencies", fg="yellow")
 
+        builtin_lib_storages = None
         for filters in self.normalize_dependencies(manifest['dependencies']):
             assert "name" in filters
 
@@ -373,39 +375,38 @@ class LibraryManager(BasePkgManager):
             self.INSTALL_HISTORY.append(history_key)
 
             if any(s in filters.get("version", "") for s in ("\\", "/")):
-                self.install(
+                self.install("{name}={version}".format(**filters),
-                    "{name}={version}".format(**filters),
+                             silent=silent,
-                    silent=silent,
+                             after_update=after_update,
-                    after_update=after_update,
+                             interactive=interactive,
-                    interactive=interactive,
+                             force=force)
-                    force=force)
             else:
                 try:
                     lib_id = self.search_lib_id(filters, silent, interactive)
                 except exception.LibNotFound as e:
-                    if not silent or is_builtin_lib(filters['name']):
+                    if builtin_lib_storages is None:
+                        builtin_lib_storages = get_builtin_libs()
+                    if not silent or is_builtin_lib(builtin_lib_storages,
+                                                    filters['name']):
                         click.secho("Warning! %s" % e, fg="yellow")
                     continue
 
                 if filters.get("version"):
-                    self.install(
+                    self.install(lib_id,
-                        lib_id,
+                                 filters.get("version"),
-                        filters.get("version"),
+                                 silent=silent,
-                        silent=silent,
+                                 after_update=after_update,
-                        after_update=after_update,
+                                 interactive=interactive,
-                        interactive=interactive,
+                                 force=force)
-                        force=force)
                 else:
-                    self.install(
+                    self.install(lib_id,
-                        lib_id,
+                                 silent=silent,
-                        silent=silent,
+                                 after_update=after_update,
-                        after_update=after_update,
+                                 interactive=interactive,
-                        interactive=interactive,
+                                 force=force)
-                        force=force)
         return pkg_dir
 
 
-@util.memoized()
 def get_builtin_libs(storage_names=None):
     items = []
     storage_names = storage_names or []
@@ -424,9 +425,8 @@ def get_builtin_libs(storage_names=None):
     return items
 
 
-@util.memoized()
+def is_builtin_lib(storages, name):
-def is_builtin_lib(name):
+    for storage in storages or []:
-    for storage in get_builtin_libs():
         if any(l.get("name") == name for l in storage['items']):
             return True
     return False

platformio/managers/package.py

@@ -26,6 +26,7 @@ import requests
 import semantic_version
 
 from platformio import __version__, app, exception, telemetry, util
+from platformio.compat import hashlib_encode_data
 from platformio.downloader import FileDownloader
 from platformio.lockfile import LockFile
 from platformio.unpacker import FileUnpacker
@@ -36,8 +37,6 @@ from platformio.vcsclient import VCSClientFactory
 
 class PackageRepoIterator(object):
 
-    _MANIFEST_CACHE = {}
-
     def __init__(self, package, repositories):
         assert isinstance(repositories, list)
         self.package = package
@@ -49,29 +48,27 @@ class PackageRepoIterator(object):
     def __next__(self):
         return self.next()
 
-    def next(self):
+    @staticmethod
-        manifest = {}
+    @util.memoized(expire="60s")
-        repo = next(self.repositories)
+    def load_manifest(url):
-        if isinstance(repo, dict):
+        r = None
-            manifest = repo
+        try:
-        elif repo in PackageRepoIterator._MANIFEST_CACHE:
+            r = requests.get(url, headers=util.get_request_defheaders())
-            manifest = PackageRepoIterator._MANIFEST_CACHE[repo]
+            r.raise_for_status()
-        else:
+            return r.json()
-            r = None
+        except:  # pylint: disable=bare-except
-            try:
+            pass
-                r = requests.get(repo, headers=util.get_request_defheaders())
+        finally:
-                r.raise_for_status()
+            if r:
-                manifest = r.json()
+                r.close()
-            except:  # pylint: disable=bare-except
+        return None
-                pass
-            finally:
-                if r:
-                    r.close()
-            PackageRepoIterator._MANIFEST_CACHE[repo] = manifest
 
-        if self.package in manifest:
+    def next(self):
+        repo = next(self.repositories)
+        manifest = repo if isinstance(repo, dict) else self.load_manifest(repo)
+        if manifest and self.package in manifest:
             return manifest[self.package]
-        return self.next()
+        return next(self)
 
 
 class PkgRepoMixin(object):
@@ -91,18 +88,18 @@ class PkgRepoMixin(object):
         reqspec = None
         if requirements:
             try:
-                reqspec = self.parse_semver_spec(
+                reqspec = self.parse_semver_spec(requirements,
-                    requirements, raise_exception=True)
+                                                 raise_exception=True)
             except ValueError:
                 pass
 
         for v in versions:
             if not self.is_system_compatible(v.get("system")):
                 continue
-            if "platformio" in v.get("engines", {}):
+            # if "platformio" in v.get("engines", {}):
-                if PkgRepoMixin.PIO_VERSION not in self.parse_semver_spec(
+            #     if PkgRepoMixin.PIO_VERSION not in self.parse_semver_spec(
-                        v['engines']['platformio'], raise_exception=True):
+            #             v['engines']['platformio'], raise_exception=True):
-                    continue
+            #         continue
             specver = semantic_version.Version(v['version'])
             if reqspec and specver not in reqspec:
                 continue
@@ -138,22 +135,19 @@ class PkgInstallerMixin(object):
     SRC_MANIFEST_NAME = ".piopkgmanager.json"
     TMP_FOLDER_PREFIX = "_tmp_installing-"
 
-    FILE_CACHE_VALID = "1m"  # 1 month
+    FILE_CACHE_VALID = None  # for example, 1 week = "7d"
-    FILE_CACHE_MAX_SIZE = 1024 * 1024
+    FILE_CACHE_MAX_SIZE = 1024 * 1024 * 50  # 50 Mb
 
-    MEMORY_CACHE = {}
+    MEMORY_CACHE = {}  # cache for package manifests and read dirs
 
-    @staticmethod
+    def cache_get(self, key, default=None):
-    def cache_get(key, default=None):
+        return self.MEMORY_CACHE.get(key, default)
-        return PkgInstallerMixin.MEMORY_CACHE.get(key, default)
 
-    @staticmethod
+    def cache_set(self, key, value):
-    def cache_set(key, value):
+        self.MEMORY_CACHE[key] = value
-        PkgInstallerMixin.MEMORY_CACHE[key] = value
 
-    @staticmethod
+    def cache_reset(self):
-    def cache_reset():
+        self.MEMORY_CACHE.clear()
-        PkgInstallerMixin.MEMORY_CACHE = {}
 
     def read_dirs(self, src_dir):
         cache_key = "read_dirs-%s" % src_dir
@@ -172,11 +166,12 @@ class PkgInstallerMixin(object):
         cache_key_data = app.ContentCache.key_from_args(url, "data")
         if self.FILE_CACHE_VALID:
             with app.ContentCache() as cc:
-                fname = cc.get(cache_key_fname)
+                fname = str(cc.get(cache_key_fname))
                 cache_path = cc.get_cache_path(cache_key_data)
                 if fname and isfile(cache_path):
                     dst_path = join(dest_dir, fname)
                     shutil.copy(cache_path, dst_path)
+                    click.echo("Using cache: %s" % cache_path)
                     return dst_path
 
         with_progress = not app.is_disabled_progressbar()
@@ -329,14 +324,15 @@ class PkgInstallerMixin(object):
             name += "_ID%d" % manifest['id']
         return str(name)
 
-    def get_src_manifest_path(self, pkg_dir):
+    @classmethod
+    def get_src_manifest_path(cls, pkg_dir):
         if not isdir(pkg_dir):
             return None
         for item in os.listdir(pkg_dir):
             if not isdir(join(pkg_dir, item)):
                 continue
-            if isfile(join(pkg_dir, item, self.SRC_MANIFEST_NAME)):
+            if isfile(join(pkg_dir, item, cls.SRC_MANIFEST_NAME)):
-                return join(pkg_dir, item, self.SRC_MANIFEST_NAME)
+                return join(pkg_dir, item, cls.SRC_MANIFEST_NAME)
         return None
 
     def get_manifest_path(self, pkg_dir):
@@ -392,7 +388,7 @@ class PkgInstallerMixin(object):
         if "version" not in manifest:
             manifest['version'] = "0.0.0"
 
-        manifest['__pkg_dir'] = util.path_to_unicode(pkg_dir)
+        manifest['__pkg_dir'] = pkg_dir
         self.cache_set(cache_key, manifest)
         return manifest
 
@@ -429,8 +425,8 @@ class PkgInstallerMixin(object):
             try:
                 if requirements and not self.parse_semver_spec(
                         requirements, raise_exception=True).match(
-                            self.parse_semver_version(
+                            self.parse_semver_version(manifest['version'],
-                                manifest['version'], raise_exception=True)):
+                                                      raise_exception=True)):
                     continue
                 elif not best or (self.parse_semver_version(
                         manifest['version'], raise_exception=True) >
@@ -449,7 +445,7 @@ class PkgInstallerMixin(object):
 
     def get_package_by_dir(self, pkg_dir):
         for manifest in self.get_installed():
-            if manifest['__pkg_dir'] == util.path_to_unicode(abspath(pkg_dir)):
+            if manifest['__pkg_dir'] == abspath(pkg_dir):
                 return manifest
         return None
 
@@ -481,7 +477,7 @@ class PkgInstallerMixin(object):
         if versions is None:
             util.internet_on(raise_exception=True)
             raise exception.UnknownPackage(name)
-        elif not pkgdata:
+        if not pkgdata:
             raise exception.UndefinedPackageVersion(requirements or "latest",
                                                     util.get_systype())
         return pkg_dir
@@ -544,7 +540,7 @@ class PkgInstallerMixin(object):
     def _install_from_tmp_dir(  # pylint: disable=too-many-branches
             self, tmp_dir, requirements=None):
         tmp_manifest = self.load_manifest(tmp_dir)
-        assert set(["name", "version"]) <= set(tmp_manifest.keys())
+        assert set(["name", "version"]) <= set(tmp_manifest)
 
         pkg_dirname = self.get_install_dirname(tmp_manifest)
         pkg_dir = join(self.package_dir, pkg_dirname)
@@ -587,8 +583,10 @@ class PkgInstallerMixin(object):
                                             cur_manifest['version'])
                 if "__src_url" in cur_manifest:
                     target_dirname = "%s@src-%s" % (
-                        pkg_dirname, hashlib.md5(
+                        pkg_dirname,
-                            cur_manifest['__src_url']).hexdigest())
+                        hashlib.md5(
+                            hashlib_encode_data(
+                                cur_manifest['__src_url'])).hexdigest())
                 shutil.move(pkg_dir, join(self.package_dir, target_dirname))
             # fix to a version
             elif action == 2:
@@ -596,8 +594,10 @@ class PkgInstallerMixin(object):
                                             tmp_manifest['version'])
                 if "__src_url" in tmp_manifest:
                     target_dirname = "%s@src-%s" % (
-                        pkg_dirname, hashlib.md5(
+                        pkg_dirname,
-                            tmp_manifest['__src_url']).hexdigest())
+                        hashlib.md5(
+                            hashlib_encode_data(
+                                tmp_manifest['__src_url'])).hexdigest())
                 pkg_dir = join(self.package_dir, target_dirname)
 
         # remove previous/not-satisfied package
@@ -645,8 +645,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
 
         if "__src_url" in manifest:
             try:
-                vcs = VCSClientFactory.newClient(
+                vcs = VCSClientFactory.newClient(pkg_dir,
-                    pkg_dir, manifest['__src_url'], silent=True)
+                                                 manifest['__src_url'],
+                                                 silent=True)
             except (AttributeError, exception.PlatformioException):
                 return None
             if not vcs.can_be_updated:
@@ -655,8 +656,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         else:
             try:
                 latest = self.get_latest_repo_version(
-                    "id=%d" % manifest['id']
+                    "id=%d" %
-                    if "id" in manifest else manifest['name'],
+                    manifest['id'] if "id" in manifest else manifest['name'],
                     requirements,
                     silent=True)
             except (exception.PlatformioException, ValueError):
@@ -668,10 +669,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         up_to_date = False
         try:
             assert "__src_url" not in manifest
-            up_to_date = (self.parse_semver_version(
+            up_to_date = (self.parse_semver_version(manifest['version'],
-                manifest['version'], raise_exception=True) >=
+                                                    raise_exception=True) >=
-                          self.parse_semver_version(
+                          self.parse_semver_version(latest,
-                              latest, raise_exception=True))
+                                                    raise_exception=True))
         except (AssertionError, ValueError):
             up_to_date = latest == manifest['version']
 
@@ -717,8 +718,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 return package_dir
 
             if url:
-                pkg_dir = self._install_from_url(
+                pkg_dir = self._install_from_url(name,
-                    name, url, requirements, track=True)
+                                                 url,
+                                                 requirements,
+                                                 track=True)
             else:
                 pkg_dir = self._install_from_piorepo(name, requirements)
 
@@ -730,16 +733,14 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             assert manifest
 
             if not after_update:
-                telemetry.on_event(
+                telemetry.on_event(category=self.__class__.__name__,
-                    category=self.__class__.__name__,
+                                   action="Install",
-                    action="Install",
+                                   label=manifest['name'])
-                    label=manifest['name'])
 
-            if not silent:
+            click.secho(
-                click.secho(
+                "{name} @ {version} has been successfully installed!".format(
-                    "{name} @ {version} has been successfully installed!".
+                    **manifest),
-                    format(**manifest),
+                fg="green")
-                    fg="green")
 
         return pkg_dir
 
@@ -756,14 +757,13 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 pkg_dir = self.get_package_dir(name, requirements, url)
 
             if not pkg_dir:
-                raise exception.UnknownPackage(
+                raise exception.UnknownPackage("%s @ %s" %
-                    "%s @ %s" % (package, requirements or "*"))
+                                               (package, requirements or "*"))
 
             manifest = self.load_manifest(pkg_dir)
-            click.echo(
+            click.echo("Uninstalling %s @ %s: \t" % (click.style(
-                "Uninstalling %s @ %s: \t" % (click.style(
+                manifest['name'], fg="cyan"), manifest['version']),
-                    manifest['name'], fg="cyan"), manifest['version']),
+                       nl=False)
-                nl=False)
 
             if islink(pkg_dir):
                 os.unlink(pkg_dir)
@@ -782,31 +782,30 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             click.echo("[%s]" % click.style("OK", fg="green"))
 
             if not after_update:
-                telemetry.on_event(
+                telemetry.on_event(category=self.__class__.__name__,
-                    category=self.__class__.__name__,
+                                   action="Uninstall",
-                    action="Uninstall",
+                                   label=manifest['name'])
-                    label=manifest['name'])
 
         return True
 
     def update(self, package, requirements=None, only_check=False):
+        self.cache_reset()
         if isdir(package) and self.get_package_by_dir(package):
             pkg_dir = package
         else:
             pkg_dir = self.get_package_dir(*self.parse_pkg_uri(package))
 
         if not pkg_dir:
-            raise exception.UnknownPackage(
+            raise exception.UnknownPackage("%s @ %s" %
-                "%s @ %s" % (package, requirements or "*"))
+                                           (package, requirements or "*"))
 
         manifest = self.load_manifest(pkg_dir)
         name = manifest['name']
 
-        click.echo(
+        click.echo("{} {:<40} @ {:<15}".format(
-            "{} {:<40} @ {:<15}".format(
+            "Checking" if only_check else "Updating",
-                "Checking" if only_check else "Updating",
+            click.style(manifest['name'], fg="cyan"), manifest['version']),
-                click.style(manifest['name'], fg="cyan"), manifest['version']),
+                   nl=False)
-            nl=False)
         if not util.internet_on():
             click.echo("[%s]" % (click.style("Off-line", fg="yellow")))
             return None
@@ -825,23 +824,20 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         if "__src_url" in manifest:
             vcs = VCSClientFactory.newClient(pkg_dir, manifest['__src_url'])
             assert vcs.update()
-            self._update_src_manifest(
+            self._update_src_manifest(dict(version=vcs.get_current_revision()),
-                dict(version=vcs.get_current_revision()), vcs.storage_dir)
+                                      vcs.storage_dir)
         else:
             self.uninstall(pkg_dir, after_update=True)
             self.install(name, latest, after_update=True)
 
-        telemetry.on_event(
+        telemetry.on_event(category=self.__class__.__name__,
-            category=self.__class__.__name__,
+                           action="Update",
-            action="Update",
+                           label=manifest['name'])
-            label=manifest['name'])
         return True
 
 
 class PackageManager(BasePkgManager):
 
-    FILE_CACHE_VALID = None  # disable package caching
-
     @property
     def manifest_names(self):
         return ["package.json"]

platformio/managers/platform.py

@@ -15,23 +15,33 @@
 import base64
 import os
 import re
+import sys
 from imp import load_source
-from multiprocessing import cpu_count
 from os.path import basename, dirname, isdir, isfile, join
-from urllib import quote
 
 import click
 import semantic_version
 
 from platformio import __version__, app, exception, util
+from platformio.compat import PY2, hashlib_encode_data, is_bytes
 from platformio.managers.core import get_core_package_dir
 from platformio.managers.package import BasePkgManager, PackageManager
+from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv,
+                             exec_command, get_pythonexe_path)
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import (get_project_boards_dir,
+                                        get_project_core_dir,
+                                        get_project_packages_dir,
+                                        get_project_platforms_dir)
+
+try:
+    from urllib.parse import quote
+except ImportError:
+    from urllib import quote
 
 
 class PlatformManager(BasePkgManager):
 
-    FILE_CACHE_VALID = None  # disable platform download caching
-
     def __init__(self, package_dir=None, repositories=None):
         if not repositories:
             repositories = [
@@ -39,9 +49,8 @@ class PlatformManager(BasePkgManager):
                 "{0}://dl.platformio.org/platforms/manifest.json".format(
                     "https" if app.get_setting("enable_ssl") else "http")
             ]
-        BasePkgManager.__init__(
+        BasePkgManager.__init__(self, package_dir
-            self, package_dir or join(util.get_home_dir(), "platforms"),
+                                or get_project_platforms_dir(), repositories)
-            repositories)
 
     @property
     def manifest_names(self):
@@ -66,8 +75,11 @@ class PlatformManager(BasePkgManager):
                 silent=False,
                 force=False,
                 **_):  # pylint: disable=too-many-arguments, arguments-differ
-        platform_dir = BasePkgManager.install(
+        platform_dir = BasePkgManager.install(self,
-            self, name, requirements, silent=silent, force=force)
+                                              name,
+                                              requirements,
+                                              silent=silent,
+                                              force=force)
         p = PlatformFactory.newPlatform(platform_dir)
 
         # don't cleanup packages or install them after update
@@ -75,13 +87,12 @@ class PlatformManager(BasePkgManager):
         if after_update:
             return True
 
-        p.install_packages(
+        p.install_packages(with_packages,
-            with_packages,
+                           without_packages,
-            without_packages,
+                           skip_default_package,
-            skip_default_package,
+                           silent=silent,
-            silent=silent,
+                           force=force)
-            force=force)
+        return self.cleanup_packages(list(p.packages))
-        return self.cleanup_packages(p.packages.keys())
 
     def uninstall(self, package, requirements=None, after_update=False):
         if isdir(package):
@@ -101,7 +112,7 @@ class PlatformManager(BasePkgManager):
         if after_update:
             return True
 
-        return self.cleanup_packages(p.packages.keys())
+        return self.cleanup_packages(list(p.packages))
 
     def update(  # pylint: disable=arguments-differ
             self,
@@ -119,21 +130,21 @@ class PlatformManager(BasePkgManager):
             raise exception.UnknownPlatform(package)
 
         p = PlatformFactory.newPlatform(pkg_dir)
-        pkgs_before = p.get_installed_packages().keys()
+        pkgs_before = list(p.get_installed_packages())
 
         missed_pkgs = set()
         if not only_packages:
             BasePkgManager.update(self, pkg_dir, requirements, only_check)
             p = PlatformFactory.newPlatform(pkg_dir)
-            missed_pkgs = set(pkgs_before) & set(p.packages.keys())
+            missed_pkgs = set(pkgs_before) & set(p.packages)
-            missed_pkgs -= set(p.get_installed_packages().keys())
+            missed_pkgs -= set(p.get_installed_packages())
 
         p.update_packages(only_check)
-        self.cleanup_packages(p.packages.keys())
+        self.cleanup_packages(list(p.packages))
 
         if missed_pkgs:
-            p.install_packages(
+            p.install_packages(with_packages=list(missed_pkgs),
-                with_packages=list(missed_pkgs), skip_default_package=True)
+                               skip_default_package=True)
 
         return True
 
@@ -147,7 +158,7 @@ class PlatformManager(BasePkgManager):
                     deppkgs[pkgname] = set()
                 deppkgs[pkgname].add(pkgmanifest['version'])
 
-        pm = PackageManager(join(util.get_home_dir(), "packages"))
+        pm = PackageManager(get_project_packages_dir())
         for manifest in pm.get_installed():
             if manifest['name'] not in names:
                 continue
@@ -161,7 +172,7 @@ class PlatformManager(BasePkgManager):
         self.cache_reset()
         return True
 
-    @util.memoized(expire=5000)
+    @util.memoized(expire="5s")
     def get_installed_boards(self):
         boards = []
         for manifest in self.get_installed():
@@ -173,7 +184,6 @@ class PlatformManager(BasePkgManager):
         return boards
 
     @staticmethod
-    @util.memoized()
     def get_registered_boards():
         return util.get_api_result("/boards", cache_valid="7d")
 
@@ -244,8 +254,8 @@ class PlatformFactory(object):
                 cls.load_module(name, join(platform_dir, "platform.py")),
                 cls.get_clsname(name))
         else:
-            platform_cls = type(
+            platform_cls = type(str(cls.get_clsname(name)), (PlatformBase, ),
-                str(cls.get_clsname(name)), (PlatformBase, ), {})
+                                {})
 
         _instance = platform_cls(join(platform_dir, "platform.json"))
         assert isinstance(_instance, PlatformBase)
@@ -265,7 +275,7 @@ class PlatformPackagesMixin(object):
         without_packages = set(self.find_pkg_names(without_packages or []))
 
         upkgs = with_packages | without_packages
-        ppkgs = set(self.packages.keys())
+        ppkgs = set(self.packages)
         if not upkgs.issubset(ppkgs):
             raise exception.UnknownPackage(", ".join(upkgs - ppkgs))
 
@@ -276,8 +286,9 @@ class PlatformPackagesMixin(object):
             elif (name in with_packages or
                   not (skip_default_package or opts.get("optional", False))):
                 if ":" in version:
-                    self.pm.install(
+                    self.pm.install("%s=%s" % (name, version),
-                        "%s=%s" % (name, version), silent=silent, force=force)
+                                    silent=silent,
+                                    force=force)
                 else:
                     self.pm.install(name, version, silent=silent, force=force)
 
@@ -346,11 +357,27 @@ class PlatformRunMixin(object):
 
     LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I)
 
-    def run(self, variables, targets, silent, verbose):
+    @staticmethod
+    def encode_scons_arg(value):
+        data = base64.urlsafe_b64encode(hashlib_encode_data(value))
+        return data.decode() if is_bytes(data) else data
+
+    @staticmethod
+    def decode_scons_arg(data):
+        value = base64.urlsafe_b64decode(data)
+        return value.decode() if is_bytes(value) else value
+
+    def run(  # pylint: disable=too-many-arguments
+            self, variables, targets, silent, verbose, jobs):
         assert isinstance(variables, dict)
         assert isinstance(targets, list)
 
-        self.configure_default_packages(variables, targets)
+        config = ProjectConfig.get_instance(variables['project_config'])
+        options = config.items(env=variables['pioenv'], as_dict=True)
+        if "framework" in options:
+            # support PIO Core 3.0 dev/platforms
+            options['pioframework'] = options['framework']
+        self.configure_default_packages(options, targets)
         self.install_packages(silent=True)
 
         self.silent = silent
@@ -366,39 +393,53 @@ class PlatformRunMixin(object):
         if not isfile(variables['build_script']):
             raise exception.BuildScriptNotFound(variables['build_script'])
 
-        result = self._run_scons(variables, targets)
+        result = self._run_scons(variables, targets, jobs)
         assert "returncode" in result
 
         return result
 
-    def _run_scons(self, variables, targets):
+    def _run_scons(self, variables, targets, jobs):
-        cmd = [
+        args = [
-            util.get_pythonexe_path(),
+            get_pythonexe_path(),
-            join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q",
+            join(get_core_package_dir("tool-scons"), "script", "scons"),
-            "-j %d" % self.get_job_nums(), "--warn=no-no-parallel-support",
+            "-Q", "--warn=no-no-parallel-support",
-            "-f",
+            "--jobs", str(jobs),
-            join(util.get_source_dir(), "builder", "main.py")
+            "--sconstruct", join(util.get_source_dir(), "builder", "main.py")
-        ]
+        ]  # yapf: disable
-        cmd.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
+        args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
-        cmd += targets
+        # pylint: disable=protected-access
+        args.append("ISATTY=%d" %
+                    (1 if click._compat.isatty(sys.stdout) else 0))
+        args += targets
 
         # encode and append variables
         for key, value in variables.items():
-            cmd.append("%s=%s" % (key.upper(), base64.b64encode(value)))
+            args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value)))
 
-        util.copy_pythonpath_to_osenv()
+        def _write_and_flush(stream, data):
-        result = util.exec_command(
+            try:
-            cmd,
+                stream.write(data)
-            stdout=util.AsyncPipe(self.on_run_out),
+                stream.flush()
-            stderr=util.AsyncPipe(self.on_run_err))
+            except IOError:
+                pass
+
+        copy_pythonpath_to_osenv()
+        result = exec_command(
+            args,
+            stdout=BuildAsyncPipe(
+                line_callback=self._on_stdout_line,
+                data_callback=lambda data: _write_and_flush(sys.stdout, data)),
+            stderr=BuildAsyncPipe(
+                line_callback=self._on_stderr_line,
+                data_callback=lambda data: _write_and_flush(sys.stderr, data)))
         return result
 
-    def on_run_out(self, line):
+    def _on_stdout_line(self, line):
         if "`buildprog' is up to date." in line:
             return
         self._echo_line(line, level=1)
 
-    def on_run_err(self, line):
+    def _on_stderr_line(self, line):
         is_error = self.LINE_ERROR_RE.search(line) is not None
         self._echo_line(line, level=3 if is_error else 2)
 
@@ -417,7 +458,7 @@ class PlatformRunMixin(object):
         fg = (None, "yellow", "red")[level - 1]
         if level == 1 and "is up to date" in line:
             fg = "green"
-        click.secho(line, fg=fg, err=level > 1)
+        click.secho(line, fg=fg, err=level > 1, nl=False)
 
     @staticmethod
     def _echo_missed_dependency(filename):
@@ -434,19 +475,12 @@ class PlatformRunMixin(object):
 """.format(filename=filename,
            filename_styled=click.style(filename, fg="cyan"),
            link=click.style(
-               "https://platformio.org/lib/search?query=header:%s" % quote(
+               "https://platformio.org/lib/search?query=header:%s" %
-                   filename, safe=""),
+               quote(filename, safe=""),
                fg="blue"),
            dots="*" * (56 + len(filename)))
         click.echo(banner, err=True)
 
-    @staticmethod
-    def get_job_nums():
-        try:
-            return cpu_count()
-        except NotImplementedError:
-            return 1
-
 
 class PlatformBase(  # pylint: disable=too-many-public-methods
         PlatformPackagesMixin, PlatformRunMixin):
@@ -455,21 +489,22 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
     _BOARDS_CACHE = {}
 
     def __init__(self, manifest_path):
-        self._BOARDS_CACHE = {}
         self.manifest_path = manifest_path
-        self._manifest = util.load_json(manifest_path)
-
-        self.pm = PackageManager(
-            join(util.get_home_dir(), "packages"), self.package_repositories)
-
         self.silent = False
         self.verbose = False
 
-        if self.engines and "platformio" in self.engines:
+        self._BOARDS_CACHE = {}
-            if self.PIO_VERSION not in semantic_version.Spec(
+        self._manifest = util.load_json(manifest_path)
-                    self.engines['platformio']):
+        self._custom_packages = None
-                raise exception.IncompatiblePlatform(self.name,
+
-                                                     str(self.PIO_VERSION))
+        self.pm = PackageManager(get_project_packages_dir(),
+                                 self.package_repositories)
+
+        # if self.engines and "platformio" in self.engines:
+        #     if self.PIO_VERSION not in semantic_version.Spec(
+        #             self.engines['platformio']):
+        #         raise exception.IncompatiblePlatform(self.name,
+        #                                              str(self.PIO_VERSION))
 
     @property
     def name(self):
@@ -525,9 +560,20 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
 
     @property
     def packages(self):
-        if "packages" not in self._manifest:
+        packages = self._manifest.get("packages", {})
-            self._manifest['packages'] = {}
+        for item in (self._custom_packages or []):
-        return self._manifest['packages']
+            name = item
+            version = "*"
+            if "@" in item:
+                name, version = item.split("@", 2)
+            name = name.strip()
+            if name not in packages:
+                packages[name] = {}
+            packages[name].update({
+                "version": version.strip(),
+                "optional": False
+            })
+        return packages
 
     def get_dir(self):
         return dirname(self.manifest_path)
@@ -550,15 +596,15 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
             config = PlatformBoardConfig(manifest_path)
             if "platform" in config and config.get("platform") != self.name:
                 return
-            elif "platforms" in config \
+            if "platforms" in config \
                     and self.name not in config.get("platforms"):
                 return
             config.manifest['platform'] = self.name
             self._BOARDS_CACHE[board_id] = config
 
         bdirs = [
-            util.get_projectboards_dir(),
+            get_project_boards_dir(),
-            join(util.get_home_dir(), "boards"),
+            join(get_project_core_dir(), "boards"),
             join(self.get_dir(), "boards"),
         ]
 
@@ -590,12 +636,12 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
     def get_package_type(self, name):
         return self.packages[name].get("type")
 
-    def configure_default_packages(self, variables, targets):
+    def configure_default_packages(self, options, targets):
+        # override user custom packages
+        self._custom_packages = options.get("platform_packages")
+
         # enable used frameworks
-        frameworks = variables.get("pioframework", [])
+        for framework in options.get("framework", []):
-        if not isinstance(frameworks, list):
-            frameworks = frameworks.split(", ")
-        for framework in frameworks:
             if not self.frameworks:
                 continue
             framework = framework.lower().strip()
@@ -650,7 +696,7 @@ class PlatformBoardConfig(object):
             self._manifest = util.load_json(manifest_path)
         except ValueError:
             raise exception.InvalidBoardManifest(manifest_path)
-        if not set(["name", "url", "vendor"]) <= set(self._manifest.keys()):
+        if not set(["name", "url", "vendor"]) <= set(self._manifest):
             raise exception.PlatformioException(
                 "Please specify name, url and vendor fields for " +
                 manifest_path)
@@ -660,12 +706,20 @@ class PlatformBoardConfig(object):
             value = self._manifest
             for k in path.split("."):
                 value = value[k]
+            # pylint: disable=undefined-variable
+            if PY2 and isinstance(value, unicode):
+                # cast to plain string from unicode for PY2, resolves issue in
+                # dev/platform when BoardConfig.get() is used in pair with
+                # os.path.join(file_encoding, unicode_encoding)
+                try:
+                    value = value.encode("utf-8")
+                except UnicodeEncodeError:
+                    pass
             return value
         except KeyError:
             if default is not None:
                 return default
-            else:
+        raise KeyError("Invalid board option '%s'" % path)
-                raise KeyError("Invalid board option '%s'" % path)
 
     def update(self, path, value):
         newdict = None
@@ -750,7 +804,7 @@ class PlatformBoardConfig(object):
                 return tool_name
             raise exception.DebugInvalidOptions(
                 "Unknown debug tool `%s`. Please use one of `%s` or `custom`" %
-                (tool_name, ", ".join(sorted(debug_tools.keys()))))
+                (tool_name, ", ".join(sorted(list(debug_tools)))))
 
         # automatically select best tool
         data = {"default": [], "onboard": [], "external": []}

platformio/proc.py

@@ -0,0 +1,191 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+from os.path import isdir, isfile, join, normpath
+from threading import Thread
+
+from platformio import exception
+from platformio.compat import WINDOWS, string_types
+
+
+class AsyncPipeBase(object):
+
+    def __init__(self):
+        self._fd_read, self._fd_write = os.pipe()
+        self._pipe_reader = os.fdopen(self._fd_read)
+        self._buffer = ""
+        self._thread = Thread(target=self.run)
+        self._thread.start()
+
+    def get_buffer(self):
+        return self._buffer
+
+    def fileno(self):
+        return self._fd_write
+
+    def run(self):
+        try:
+            self.do_reading()
+        except (KeyboardInterrupt, SystemExit, IOError):
+            self.close()
+
+    def do_reading(self):
+        raise NotImplementedError()
+
+    def close(self):
+        self._buffer = ""
+        os.close(self._fd_write)
+        self._thread.join()
+
+
+class BuildAsyncPipe(AsyncPipeBase):
+
+    def __init__(self, line_callback, data_callback):
+        self.line_callback = line_callback
+        self.data_callback = data_callback
+        super(BuildAsyncPipe, self).__init__()
+
+    def do_reading(self):
+        line = ""
+        print_immediately = False
+
+        for byte in iter(lambda: self._pipe_reader.read(1), ""):
+            self._buffer += byte
+
+            if line and byte.strip() and line[-3:] == (byte * 3):
+                print_immediately = True
+
+            if print_immediately:
+                # leftover bytes
+                if line:
+                    self.data_callback(line)
+                    line = ""
+                self.data_callback(byte)
+                if byte == "\n":
+                    print_immediately = False
+            else:
+                line += byte
+                if byte != "\n":
+                    continue
+                self.line_callback(line)
+                line = ""
+
+        self._pipe_reader.close()
+
+
+class LineBufferedAsyncPipe(AsyncPipeBase):
+
+    def __init__(self, line_callback):
+        self.line_callback = line_callback
+        super(LineBufferedAsyncPipe, self).__init__()
+
+    def do_reading(self):
+        for line in iter(self._pipe_reader.readline, ""):
+            self._buffer += line
+            self.line_callback(line)
+        self._pipe_reader.close()
+
+
+def exec_command(*args, **kwargs):
+    result = {"out": None, "err": None, "returncode": None}
+
+    default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    default.update(kwargs)
+    kwargs = default
+
+    p = subprocess.Popen(*args, **kwargs)
+    try:
+        result['out'], result['err'] = p.communicate()
+        result['returncode'] = p.returncode
+    except KeyboardInterrupt:
+        raise exception.AbortedByUser()
+    finally:
+        for s in ("stdout", "stderr"):
+            if isinstance(kwargs[s], AsyncPipeBase):
+                kwargs[s].close()
+
+    for s in ("stdout", "stderr"):
+        if isinstance(kwargs[s], AsyncPipeBase):
+            result[s[3:]] = kwargs[s].get_buffer()
+
+    for k, v in result.items():
+        if isinstance(result[k], bytes):
+            try:
+                result[k] = result[k].decode(sys.getdefaultencoding())
+            except UnicodeDecodeError:
+                result[k] = result[k].decode("latin-1")
+        if v and isinstance(v, string_types):
+            result[k] = result[k].strip()
+
+    return result
+
+
+def is_ci():
+    return os.getenv("CI", "").lower() == "true"
+
+
+def is_container():
+    if not isfile("/proc/1/cgroup"):
+        return False
+    with open("/proc/1/cgroup") as fp:
+        for line in fp:
+            line = line.strip()
+            if ":" in line and not line.endswith(":/"):
+                return True
+    return False
+
+
+def get_pythonexe_path():
+    return os.environ.get("PYTHONEXEPATH", normpath(sys.executable))
+
+
+def copy_pythonpath_to_osenv():
+    _PYTHONPATH = []
+    if "PYTHONPATH" in os.environ:
+        _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep)
+    for p in os.sys.path:
+        conditions = [p not in _PYTHONPATH]
+        if not WINDOWS:
+            conditions.append(
+                isdir(join(p, "click")) or isdir(join(p, "platformio")))
+        if all(conditions):
+            _PYTHONPATH.append(p)
+    os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH)
+
+
+def where_is_program(program, envpath=None):
+    env = os.environ
+    if envpath:
+        env['PATH'] = envpath
+
+    # try OS's built-in commands
+    try:
+        result = exec_command(["where" if WINDOWS else "which", program],
+                              env=env)
+        if result['returncode'] == 0 and isfile(result['out'].strip()):
+            return result['out'].strip()
+    except OSError:
+        pass
+
+    # look up in $PATH
+    for bin_dir in env.get("PATH", "").split(os.pathsep):
+        if isfile(join(bin_dir, program)):
+            return join(bin_dir, program)
+        if isfile(join(bin_dir, "%s.exe" % program)):
+            return join(bin_dir, "%s.exe" % program)
+
+    return program

platformio/project/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/project/config.py

@@ -0,0 +1,314 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import json
+import os
+import re
+from os.path import isfile
+
+import click
+
+from platformio import exception
+from platformio.project.options import ProjectOptions
+
+try:
+    import ConfigParser as ConfigParser
+except ImportError:
+    import configparser as ConfigParser
+
+CONFIG_HEADER = """;PlatformIO Project Configuration File
+;
+;   Build options: build flags, source filter
+;   Upload options: custom upload port, speed and extra flags
+;   Library options: dependencies, extra library storages
+;   Advanced options: extra scripting
+;
+; Please visit documentation for the other options and examples
+; https://docs.platformio.org/page/projectconf.html
+
+"""
+
+
+class ProjectConfig(object):
+
+    INLINE_COMMENT_RE = re.compile(r"\s+;.*$")
+    VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}")
+
+    expand_interpolations = True
+    warnings = []
+
+    _instances = {}
+    _parser = None
+    _parsed = []
+
+    @staticmethod
+    def parse_multi_values(items):
+        result = []
+        if not items:
+            return result
+        if not isinstance(items, (list, tuple)):
+            items = items.split("\n" if "\n" in items else ", ")
+        for item in items:
+            item = item.strip()
+            # comment
+            if not item or item.startswith((";", "#")):
+                continue
+            if ";" in item:
+                item = ProjectConfig.INLINE_COMMENT_RE.sub("", item).strip()
+            result.append(item)
+        return result
+
+    @staticmethod
+    def get_instance(path):
+        if path not in ProjectConfig._instances:
+            ProjectConfig._instances[path] = ProjectConfig(path)
+        return ProjectConfig._instances[path]
+
+    @staticmethod
+    def reset_instances():
+        ProjectConfig._instances = {}
+
+    def __init__(self, path, parse_extra=True, expand_interpolations=True):
+        self.path = path
+        self.expand_interpolations = expand_interpolations
+        self.warnings = []
+        self._parsed = []
+        self._parser = ConfigParser.ConfigParser()
+        if isfile(path):
+            self.read(path, parse_extra)
+
+    def __getattr__(self, name):
+        return getattr(self._parser, name)
+
+    def read(self, path, parse_extra=True):
+        if path in self._parsed:
+            return
+        self._parsed.append(path)
+        try:
+            self._parser.read(path)
+        except ConfigParser.Error as e:
+            raise exception.InvalidProjectConf(path, str(e))
+
+        if not parse_extra:
+            return
+
+        # load extra configs
+        for pattern in self.get("platformio", "extra_configs", []):
+            for item in glob.glob(pattern):
+                self.read(item)
+
+        self._maintain_renaimed_options()
+
+    def _maintain_renaimed_options(self):
+        # legacy `lib_extra_dirs` in [platformio]
+        if (self._parser.has_section("platformio")
+                and self._parser.has_option("platformio", "lib_extra_dirs")):
+            if not self._parser.has_section("env"):
+                self._parser.add_section("env")
+            self._parser.set("env", "lib_extra_dirs",
+                             self._parser.get("platformio", "lib_extra_dirs"))
+            self._parser.remove_option("platformio", "lib_extra_dirs")
+            self.warnings.append(
+                "`lib_extra_dirs` configuration option is deprecated in "
+                "section [platformio]! Please move it to global `env` section")
+
+        renamed_options = {}
+        for option in ProjectOptions.values():
+            if option.oldnames:
+                renamed_options.update(
+                    {name: option.name
+                     for name in option.oldnames})
+
+        for section in self._parser.sections():
+            scope = section.split(":", 1)[0]
+            if scope not in ("platformio", "env"):
+                continue
+            for option in self._parser.options(section):
+                if option in renamed_options:
+                    self.warnings.append(
+                        "`%s` configuration option in section [%s] is "
+                        "deprecated and will be removed in the next release! "
+                        "Please use `%s` instead" %
+                        (option, section, renamed_options[option]))
+                    # rename on-the-fly
+                    self._parser.set(section, renamed_options[option],
+                                     self._parser.get(section, option))
+                    self._parser.remove_option(section, option)
+                    continue
+
+                # unknown
+                unknown_conditions = [
+                    ("%s.%s" % (scope, option)) not in ProjectOptions,
+                    scope != "env" or
+                    not option.startswith(("custom_", "board_"))
+                ]  # yapf: disable
+                if all(unknown_conditions):
+                    self.warnings.append(
+                        "Ignore unknown configuration option `%s` "
+                        "in section [%s]" % (option, section))
+        return True
+
+    def options(self, section=None, env=None):
+        assert section or env
+        if not section:
+            section = "env:" + env
+        options = self._parser.options(section)
+
+        # handle global options from [env]
+        if ((env or section.startswith("env:"))
+                and self._parser.has_section("env")):
+            for option in self._parser.options("env"):
+                if option not in options:
+                    options.append(option)
+
+        # handle system environment variables
+        scope = section.split(":", 1)[0]
+        for option_meta in ProjectOptions.values():
+            if option_meta.scope != scope or option_meta.name in options:
+                continue
+            if option_meta.sysenvvar and option_meta.sysenvvar in os.environ:
+                options.append(option_meta.name)
+
+        return options
+
+    def has_option(self, section, option):
+        if self._parser.has_option(section, option):
+            return True
+        return (section.startswith("env:") and self._parser.has_section("env")
+                and self._parser.has_option("env", option))
+
+    def items(self, section=None, env=None, as_dict=False):
+        assert section or env
+        if not section:
+            section = "env:" + env
+        if as_dict:
+            return {
+                option: self.get(section, option)
+                for option in self.options(section)
+            }
+        return [(option, self.get(section, option))
+                for option in self.options(section)]
+
+    def set(self, section, option, value):
+        if isinstance(value, (list, tuple)):
+            value = "\n".join(value)
+            if value:
+                value = "\n" + value  # start from a new line
+        self._parser.set(section, option, value)
+
+    def getraw(self, section, option):
+        if not self.expand_interpolations:
+            return self._parser.get(section, option)
+
+        try:
+            value = self._parser.get(section, option)
+        except ConfigParser.NoOptionError as e:
+            if not section.startswith("env:"):
+                raise e
+            value = self._parser.get("env", option)
+
+        if "${" not in value or "}" not in value:
+            return value
+        return self.VARTPL_RE.sub(self._re_interpolation_handler, value)
+
+    def _re_interpolation_handler(self, match):
+        section, option = match.group(1), match.group(2)
+        if section == "sysenv":
+            return os.getenv(option)
+        return self.getraw(section, option)
+
+    def get(self, section, option, default=None):
+        value = None
+        try:
+            value = self.getraw(section, option)
+        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            pass  # handle value from system environment
+        except ConfigParser.Error as e:
+            raise exception.InvalidProjectConf(self.path, str(e))
+
+        option_meta = ProjectOptions.get("%s.%s" %
+                                         (section.split(":", 1)[0], option))
+        if not option_meta:
+            return value or default
+
+        if option_meta.multiple:
+            value = self.parse_multi_values(value)
+
+        if option_meta.sysenvvar:
+            envvar_value = os.getenv(option_meta.sysenvvar)
+            if not envvar_value and option_meta.oldnames:
+                for oldoption in option_meta.oldnames:
+                    envvar_value = os.getenv("PLATFORMIO_" + oldoption.upper())
+                    if envvar_value:
+                        break
+            if envvar_value and option_meta.multiple:
+                value = value or []
+                value.extend(self.parse_multi_values(envvar_value))
+            elif envvar_value and not value:
+                value = envvar_value
+
+        # option is not specified by user
+        if value is None:
+            return default
+
+        try:
+            return self._covert_value(value, option_meta.type)
+        except click.BadParameter as e:
+            raise exception.ProjectOptionValueError(e.format_message(), option,
+                                                    section)
+
+    @staticmethod
+    def _covert_value(value, to_type):
+        items = value
+        if not isinstance(value, (list, tuple)):
+            items = [value]
+        items = [
+            to_type(item) if isinstance(to_type, click.ParamType) else item
+            for item in items
+        ]
+        return items if isinstance(value, (list, tuple)) else items[0]
+
+    def envs(self):
+        return [s[4:] for s in self._parser.sections() if s.startswith("env:")]
+
+    def default_envs(self):
+        return self.get("platformio", "default_envs", [])
+
+    def validate(self, envs=None, silent=False):
+        if not isfile(self.path):
+            raise exception.NotPlatformIOProject(self.path)
+        # check envs
+        known = set(self.envs())
+        if not known:
+            raise exception.ProjectEnvsNotAvailable()
+        unknown = set(list(envs or []) + self.default_envs()) - known
+        if unknown:
+            raise exception.UnknownEnvNames(", ".join(unknown),
+                                            ", ".join(known))
+        if not silent:
+            for warning in self.warnings:
+                click.secho("Warning! %s" % warning, fg="yellow")
+        return True
+
+    def to_json(self):
+        result = {}
+        for section in self.sections():
+            result[section] = self.items(section, as_dict=True)
+        return json.dumps(result)
+
+    def save(self, path=None):
+        with open(path or self.path, "w") as fp:
+            fp.write(CONFIG_HEADER)
+            self._parser.write(fp)

platformio/project/helpers.py

@@ -0,0 +1,203 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+from hashlib import sha1
+from os import walk
+from os.path import (basename, dirname, expanduser, isdir, isfile, join,
+                     realpath, splitdrive)
+
+from click.testing import CliRunner
+
+from platformio import __version__, exception
+from platformio.compat import WINDOWS, hashlib_encode_data
+from platformio.project.config import ProjectConfig
+
+
+def get_project_dir():
+    return os.getcwd()
+
+
+def is_platformio_project(project_dir=None):
+    if not project_dir:
+        project_dir = get_project_dir()
+    return isfile(join(project_dir, "platformio.ini"))
+
+
+def find_project_dir_above(path):
+    if isfile(path):
+        path = dirname(path)
+    if is_platformio_project(path):
+        return path
+    if isdir(dirname(path)):
+        return find_project_dir_above(dirname(path))
+    return None
+
+
+def get_project_optional_dir(name, default=None):
+    project_dir = get_project_dir()
+    config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
+    optional_dir = config.get("platformio", name)
+
+    if not optional_dir:
+        return default
+
+    if "$PROJECT_HASH" in optional_dir:
+        optional_dir = optional_dir.replace(
+            "$PROJECT_HASH", "%s-%s" %
+            (basename(project_dir), sha1(
+                hashlib_encode_data(project_dir)).hexdigest()[:10]))
+
+    if optional_dir.startswith("~"):
+        optional_dir = expanduser(optional_dir)
+
+    return realpath(optional_dir)
+
+
+def get_project_core_dir():
+    default = join(expanduser("~"), ".platformio")
+    core_dir = get_project_optional_dir(
+        "core_dir", get_project_optional_dir("home_dir", default))
+    win_core_dir = None
+    if WINDOWS and core_dir == default:
+        win_core_dir = splitdrive(core_dir)[0] + "\\.platformio"
+        if isdir(win_core_dir):
+            core_dir = win_core_dir
+
+    if not isdir(core_dir):
+        try:
+            os.makedirs(core_dir)
+        except OSError as e:
+            if win_core_dir:
+                os.makedirs(win_core_dir)
+                core_dir = win_core_dir
+            else:
+                raise e
+
+    assert isdir(core_dir)
+    return core_dir
+
+
+def get_project_global_lib_dir():
+    return get_project_optional_dir("globallib_dir",
+                                    join(get_project_core_dir(), "lib"))
+
+
+def get_project_platforms_dir():
+    return get_project_optional_dir("platforms_dir",
+                                    join(get_project_core_dir(), "platforms"))
+
+
+def get_project_packages_dir():
+    return get_project_optional_dir("packages_dir",
+                                    join(get_project_core_dir(), "packages"))
+
+
+def get_project_cache_dir():
+    return get_project_optional_dir("cache_dir",
+                                    join(get_project_core_dir(), ".cache"))
+
+
+def get_project_workspace_dir():
+    return get_project_optional_dir("workspace_dir",
+                                    join(get_project_dir(), ".pio"))
+
+
+def get_project_build_dir(force=False):
+    path = get_project_optional_dir("build_dir",
+                                    join(get_project_workspace_dir(), "build"))
+    try:
+        if not isdir(path):
+            os.makedirs(path)
+    except Exception as e:  # pylint: disable=broad-except
+        if not force:
+            raise Exception(e)
+    return path
+
+
+def get_project_libdeps_dir():
+    return get_project_optional_dir(
+        "libdeps_dir", join(get_project_workspace_dir(), "libdeps"))
+
+
+def get_project_lib_dir():
+    return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib"))
+
+
+def get_project_include_dir():
+    return get_project_optional_dir("include_dir",
+                                    join(get_project_dir(), "include"))
+
+
+def get_project_src_dir():
+    return get_project_optional_dir("src_dir", join(get_project_dir(), "src"))
+
+
+def get_project_test_dir():
+    return get_project_optional_dir("test_dir", join(get_project_dir(),
+                                                     "test"))
+
+
+def get_project_boards_dir():
+    return get_project_optional_dir("boards_dir",
+                                    join(get_project_dir(), "boards"))
+
+
+def get_project_data_dir():
+    return get_project_optional_dir("data_dir", join(get_project_dir(),
+                                                     "data"))
+
+
+def get_project_shared_dir():
+    return get_project_optional_dir("shared_dir",
+                                    join(get_project_dir(), "shared"))
+
+
+def calculate_project_hash():
+    check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
+    chunks = [__version__]
+    for d in (get_project_src_dir(), get_project_lib_dir()):
+        if not isdir(d):
+            continue
+        for root, _, files in walk(d):
+            for f in files:
+                path = join(root, f)
+                if path.endswith(check_suffixes):
+                    chunks.append(path)
+    chunks_to_str = ",".join(sorted(chunks))
+    if WINDOWS:
+        # Fix issue with useless project rebuilding for case insensitive FS.
+        # A case of disk drive can differ...
+        chunks_to_str = chunks_to_str.lower()
+    return sha1(hashlib_encode_data(chunks_to_str)).hexdigest()
+
+
+def load_project_ide_data(project_dir, env_name):
+    from platformio.commands.run import cli as cmd_run
+    result = CliRunner().invoke(cmd_run, [
+        "--project-dir", project_dir, "--environment", env_name, "--target",
+        "idedata"
+    ])
+    if result.exit_code != 0 and not isinstance(result.exception,
+                                                exception.ReturnErrorCode):
+        raise result.exception
+    if '"includes":' not in result.output:
+        raise exception.PlatformioException(result.output)
+
+    for line in result.output.split("\n"):
+        line = line.strip()
+        if line.startswith('{"') and line.endswith("}"):
+            return json.loads(line)
+    return None

platformio/project/options.py

@@ -0,0 +1,203 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=redefined-builtin, too-many-arguments
+
+from collections import OrderedDict, namedtuple
+
+import click
+
+ConfigOptionClass = namedtuple("ConfigOption", [
+    "scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames"
+])
+
+
+def ConfigOption(scope,
+                 name,
+                 type=str,
+                 multiple=False,
+                 sysenvvar=None,
+                 buildenvvar=None,
+                 oldnames=None):
+    return ConfigOptionClass(scope, name, type, multiple, sysenvvar,
+                             buildenvvar, oldnames)
+
+
+def ConfigPlatformioOption(*args, **kwargs):
+    return ConfigOption("platformio", *args, **kwargs)
+
+
+def ConfigEnvOption(*args, **kwargs):
+    return ConfigOption("env", *args, **kwargs)
+
+
+ProjectOptions = OrderedDict([
+    ("%s.%s" % (option.scope, option.name), option) for option in [
+        #
+        # [platformio]
+        #
+        ConfigPlatformioOption(name="description"),
+        ConfigPlatformioOption(name="default_envs",
+                               oldnames=["env_default"],
+                               multiple=True,
+                               sysenvvar="PLATFORMIO_DEFAULT_ENVS"),
+        ConfigPlatformioOption(name="extra_configs", multiple=True),
+
+        # Dirs
+        ConfigPlatformioOption(name="core_dir",
+                               oldnames=["home_dir"],
+                               sysenvvar="PLATFORMIO_CORE_DIR"),
+        ConfigPlatformioOption(name="globallib_dir",
+                               sysenvvar="PLATFORMIO_GLOBALLIB_DIR"),
+        ConfigPlatformioOption(name="platforms_dir",
+                               sysenvvar="PLATFORMIO_PLATFORMS_DIR"),
+        ConfigPlatformioOption(name="packages_dir",
+                               sysenvvar="PLATFORMIO_PACKAGES_DIR"),
+        ConfigPlatformioOption(name="cache_dir",
+                               sysenvvar="PLATFORMIO_CACHE_DIR"),
+        ConfigPlatformioOption(name="build_cache_dir",
+                               sysenvvar="PLATFORMIO_BUILD_CACHE_DIR"),
+        ConfigPlatformioOption(name="workspace_dir",
+                               sysenvvar="PLATFORMIO_WORKSPACE_DIR"),
+        ConfigPlatformioOption(name="build_dir",
+                               sysenvvar="PLATFORMIO_BUILD_DIR"),
+        ConfigPlatformioOption(name="libdeps_dir",
+                               sysenvvar="PLATFORMIO_LIBDEPS_DIR"),
+        ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"),
+        ConfigPlatformioOption(name="include_dir",
+                               sysenvvar="PLATFORMIO_INCLUDE_DIR"),
+        ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"),
+        ConfigPlatformioOption(name="test_dir",
+                               sysenvvar="PLATFORMIO_TEST_DIR"),
+        ConfigPlatformioOption(name="boards_dir",
+                               sysenvvar="PLATFORMIO_BOARDS_DIR"),
+        ConfigPlatformioOption(name="data_dir",
+                               sysenvvar="PLATFORMIO_DATA_DIR"),
+        ConfigPlatformioOption(name="shared_dir",
+                               sysenvvar="PLATFORMIO_SHARED_DIR"),
+
+        #
+        # [env]
+        #
+
+        # Generic
+        ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"),
+        ConfigEnvOption(name="platform_packages", multiple=True),
+        ConfigEnvOption(
+            name="framework", multiple=True, buildenvvar="PIOFRAMEWORK"),
+
+        # Board
+        ConfigEnvOption(name="board", buildenvvar="BOARD"),
+        ConfigEnvOption(name="board_build.mcu",
+                        oldnames=["board_mcu"],
+                        buildenvvar="BOARD_MCU"),
+        ConfigEnvOption(name="board_build.f_cpu",
+                        oldnames=["board_f_cpu"],
+                        buildenvvar="BOARD_F_CPU"),
+        ConfigEnvOption(name="board_build.f_flash",
+                        oldnames=["board_f_flash"],
+                        buildenvvar="BOARD_F_FLASH"),
+        ConfigEnvOption(name="board_build.flash_mode",
+                        oldnames=["board_flash_mode"],
+                        buildenvvar="BOARD_FLASH_MODE"),
+
+        # Build
+        ConfigEnvOption(name="build_type",
+                        type=click.Choice(["release", "debug"])),
+        ConfigEnvOption(name="build_flags",
+                        multiple=True,
+                        sysenvvar="PLATFORMIO_BUILD_FLAGS",
+                        buildenvvar="BUILD_FLAGS"),
+        ConfigEnvOption(name="src_build_flags",
+                        multiple=True,
+                        sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS",
+                        buildenvvar="SRC_BUILD_FLAGS"),
+        ConfigEnvOption(name="build_unflags",
+                        multiple=True,
+                        sysenvvar="PLATFORMIO_BUILD_UNFLAGS",
+                        buildenvvar="BUILD_UNFLAGS"),
+        ConfigEnvOption(name="src_filter",
+                        multiple=True,
+                        sysenvvar="PLATFORMIO_SRC_FILTER",
+                        buildenvvar="SRC_FILTER"),
+        ConfigEnvOption(name="targets", multiple=True),
+
+        # Upload
+        ConfigEnvOption(name="upload_port",
+                        sysenvvar="PLATFORMIO_UPLOAD_PORT",
+                        buildenvvar="UPLOAD_PORT"),
+        ConfigEnvOption(name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL"),
+        ConfigEnvOption(
+            name="upload_speed", type=click.INT, buildenvvar="UPLOAD_SPEED"),
+        ConfigEnvOption(name="upload_flags",
+                        multiple=True,
+                        sysenvvar="PLATFORMIO_UPLOAD_FLAGS",
+                        buildenvvar="UPLOAD_FLAGS"),
+        ConfigEnvOption(name="upload_resetmethod",
+                        buildenvvar="UPLOAD_RESETMETHOD"),
+        ConfigEnvOption(name="upload_command", buildenvvar="UPLOADCMD"),
+
+        # Monitor
+        ConfigEnvOption(name="monitor_port"),
+        ConfigEnvOption(name="monitor_speed", oldnames=["monitor_baud"]),
+        ConfigEnvOption(name="monitor_rts", type=click.IntRange(0, 1)),
+        ConfigEnvOption(name="monitor_dtr", type=click.IntRange(0, 1)),
+        ConfigEnvOption(name="monitor_flags", multiple=True),
+
+        # Library
+        ConfigEnvOption(name="lib_deps",
+                        oldnames=["lib_use", "lib_force", "lib_install"],
+                        multiple=True),
+        ConfigEnvOption(name="lib_ignore", multiple=True),
+        ConfigEnvOption(name="lib_extra_dirs",
+                        multiple=True,
+                        sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"),
+        ConfigEnvOption(name="lib_ldf_mode",
+                        type=click.Choice(
+                            ["off", "chain", "deep", "chain+", "deep+"])),
+        ConfigEnvOption(name="lib_compat_mode",
+                        type=click.Choice(["off", "soft", "strict"])),
+        ConfigEnvOption(name="lib_archive", type=click.BOOL),
+
+        # Test
+        ConfigEnvOption(name="test_filter", multiple=True),
+        ConfigEnvOption(name="test_ignore", multiple=True),
+        ConfigEnvOption(name="test_port"),
+        ConfigEnvOption(name="test_speed", type=click.INT),
+        ConfigEnvOption(name="test_transport"),
+        ConfigEnvOption(name="test_build_project_src", type=click.BOOL),
+
+        # Debug
+        ConfigEnvOption(name="debug_tool"),
+        ConfigEnvOption(name="debug_init_break"),
+        ConfigEnvOption(name="debug_init_cmds", multiple=True),
+        ConfigEnvOption(name="debug_extra_cmds", multiple=True),
+        ConfigEnvOption(name="debug_load_cmds",
+                        oldnames=["debug_load_cmd"],
+                        multiple=True),
+        ConfigEnvOption(name="debug_load_mode",
+                        type=click.Choice(["always", "modified", "manual"])),
+        ConfigEnvOption(name="debug_server", multiple=True),
+        ConfigEnvOption(name="debug_port"),
+        ConfigEnvOption(name="debug_svd_path",
+                        type=click.Path(
+                            exists=True, file_okay=True, dir_okay=False)),
+
+        # Other
+        ConfigEnvOption(name="extra_scripts",
+                        oldnames=["extra_script"],
+                        multiple=True,
+                        sysenvvar="PLATFORMIO_EXTRA_SCRIPTS")
+    ]
+])

platformio/projectconftpl.ini

@@ -1,9 +0,0 @@
-; PlatformIO Project Configuration File
-;
-;   Build options: build flags, source filter
-;   Upload options: custom upload port, speed and extra flags
-;   Library options: dependencies, extra library storages
-;   Advanced options: extra scripting
-;
-; Please visit documentation for the other options and examples
-; https://docs.platformio.org/page/projectconf.html

platformio/telemetry.py

@@ -14,7 +14,6 @@
 
 import atexit
 import platform
-import Queue
 import re
 import sys
 import threading
@@ -28,6 +27,14 @@ import click
 import requests
 
 from platformio import __version__, app, exception, util
+from platformio.commands import PlatformioCLI
+from platformio.compat import string_types
+from platformio.proc import is_ci, is_container
+
+try:
+    import queue
+except ImportError:
+    import Queue as queue
 
 
 class TelemetryBase(object):
@@ -78,12 +85,12 @@ class MeasurementProtocol(TelemetryBase):
     def __getitem__(self, name):
         if name in self.PARAMS_MAP:
             name = self.PARAMS_MAP[name]
-        return TelemetryBase.__getitem__(self, name)
+        return super(MeasurementProtocol, self).__getitem__(name)
 
     def __setitem__(self, name, value):
         if name in self.PARAMS_MAP:
             name = self.PARAMS_MAP[name]
-        TelemetryBase.__setitem__(self, name, value)
+        super(MeasurementProtocol, self).__setitem__(name, value)
 
     def _prefill_appinfo(self):
         self['av'] = __version__
@@ -117,7 +124,7 @@ class MeasurementProtocol(TelemetryBase):
                                         platform.platform())
         # self['cd3'] = " ".join(_filter_args(sys.argv[1:]))
         self['cd4'] = 1 if (not util.is_ci() and
-                            (caller_id or not util.is_container())) else 0
+                            (caller_id or not is_container())) else 0
         if caller_id:
             self['cd5'] = caller_id.lower()
 
@@ -129,12 +136,15 @@ class MeasurementProtocol(TelemetryBase):
                     return _arg
             return None
 
-        if not app.get_session_var("command_ctx"):
+        args = []
-            return
+        for arg in PlatformioCLI.leftover_args:
-        ctx_args = app.get_session_var("command_ctx").args
+            if not isinstance(arg, string_types):
-        args = [str(s).lower() for s in ctx_args if not str(s).startswith("-")]
+                arg = str(arg)
+            if not arg.startswith("-"):
+                args.append(arg.lower())
         if not args:
             return
+
         cmd_path = args[:1]
         if args[0] in ("platform", "platforms", "serialports", "device",
                        "settings", "account"):
@@ -182,7 +192,7 @@ class MPDataPusher(object):
     MAX_WORKERS = 5
 
     def __init__(self):
-        self._queue = Queue.LifoQueue()
+        self._queue = queue.LifoQueue()
         self._failedque = deque()
         self._http_session = requests.Session()
         self._http_offline = False
@@ -207,7 +217,7 @@ class MPDataPusher(object):
         try:
             while True:
                 items.append(self._queue.get_nowait())
-        except Queue.Empty:
+        except queue.Empty:
             pass
         return items
 
@@ -268,7 +278,7 @@ def on_command():
     mp = MeasurementProtocol()
     mp.send("screenview")
 
-    if util.is_ci():
+    if is_ci():
         measure_ci()
 
 
@@ -304,12 +314,15 @@ def measure_ci():
 
 
 def on_run_environment(options, targets):
-    opts = [
+    non_sensative_values = ["board", "platform", "framework"]
-        "%s=%s" % (opt, value.replace("\n", ", ") if "\n" in value else value)
+    safe_options = []
-        for opt, value in sorted(options.items())
+    for key, value in sorted(options.items()):
-    ]
+        if key in non_sensative_values:
+            safe_options.append("%s=%s" % (key, value))
+        else:
+            safe_options.append(key)
     targets = [t.title() for t in targets or ["run"]]
-    on_event("Env", " ".join(targets), "&".join(opts))
+    on_event("Env", " ".join(targets), "&".join(safe_options))
 
 
 def on_event(category, action, label=None, value=None, screen_name=None):
@@ -329,21 +342,17 @@ def on_exception(e):
 
     def _cleanup_description(text):
         text = text.replace("Traceback (most recent call last):", "")
-        text = re.sub(
+        text = re.sub(r'File "([^"]+)"',
-            r'File "([^"]+)"',
+                      lambda m: join(*m.group(1).split(sep)[-2:]),
-            lambda m: join(*m.group(1).split(sep)[-2:]),
+                      text,
-            text,
+                      flags=re.M)
-            flags=re.M)
         text = re.sub(r"\s+", " ", text, flags=re.M)
         return text.strip()
 
     skip_conditions = [
-        isinstance(e, cls)
+        isinstance(e, cls) for cls in (IOError, exception.ReturnErrorCode,
-        for cls in (IOError, exception.ReturnErrorCode,
+                                       exception.UserSideException,
-                    exception.AbortedByUser, exception.NotGlobalLibDir,
+                                       exception.PlatformIOProjectException)
-                    exception.InternetIsOffline,
-                    exception.NotPlatformIOProject,
-                    exception.UserSideException)
     ]
     try:
         skip_conditions.append("[API] Account: " in str(e))
@@ -388,7 +397,7 @@ def backup_reports(items):
 
     for params in items:
         # skip static options
-        for key in params.keys():
+        for key in list(params.keys()):
             if key in ("v", "tid", "cid", "cd1", "cd2", "sr", "an"):
                 del params[key]
 

platformio/unpacker.py

@@ -68,15 +68,14 @@ class ZIPArchive(ArchiveBase):
 
     @staticmethod
     def preserve_permissions(item, dest_dir):
-        attrs = item.external_attr >> 16L
+        attrs = item.external_attr >> 16
         if attrs:
             chmod(join(dest_dir, item.filename), attrs)
 
     @staticmethod
     def preserve_mtime(item, dest_dir):
-        util.change_filemtime(
+        util.change_filemtime(join(dest_dir, item.filename),
-            join(dest_dir, item.filename),
+                              mktime(tuple(item.date_time) + tuple([0, 0, 0])))
-            mktime(list(item.date_time) + [0] * 3))
 
     def get_items(self):
         return self._afo.infolist()

platformio/util.py

@@ -18,94 +18,21 @@ import platform
 import re
 import socket
 import stat
-import subprocess
 import sys
 import time
+from contextlib import contextmanager
 from functools import wraps
 from glob import glob
-from hashlib import sha1
+from os.path import abspath, basename, dirname, isfile, join
-from os.path import (abspath, basename, dirname, expanduser, isdir, isfile,
-                     join, normpath, splitdrive)
 from shutil import rmtree
-from threading import Thread
 
 import click
 import requests
 
 from platformio import __apiurl__, __version__, exception
-
+from platformio.commands import PlatformioCLI
-# pylint: disable=wrong-import-order, too-many-ancestors
+from platformio.compat import PY2, WINDOWS, get_file_contents
-
+from platformio.proc import exec_command, is_ci
-try:
-    import configparser as ConfigParser
-except ImportError:
-    import ConfigParser as ConfigParser
-
-
-class ProjectConfig(ConfigParser.ConfigParser):
-
-    VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}")
-
-    def items(self, section, **_):  # pylint: disable=arguments-differ
-        items = []
-        for option in ConfigParser.ConfigParser.options(self, section):
-            items.append((option, self.get(section, option)))
-        return items
-
-    def get(self, section, option, **kwargs):
-        try:
-            value = ConfigParser.ConfigParser.get(self, section, option,
-                                                  **kwargs)
-        except ConfigParser.Error as e:
-            raise exception.InvalidProjectConf(str(e))
-        if "${" not in value or "}" not in value:
-            return value
-        return self.VARTPL_RE.sub(self._re_sub_handler, value)
-
-    def _re_sub_handler(self, match):
-        section, option = match.group(1), match.group(2)
-        if section in ("env", "sysenv") and not self.has_section(section):
-            if section == "env":
-                click.secho(
-                    "Warning! Access to system environment variable via "
-                    "`${{env.{0}}}` is deprecated. Please use "
-                    "`${{sysenv.{0}}}` instead".format(option),
-                    fg="yellow")
-            return os.getenv(option)
-        return self.get(section, option)
-
-
-class AsyncPipe(Thread):
-
-    def __init__(self, outcallback=None):
-        super(AsyncPipe, self).__init__()
-        self.outcallback = outcallback
-
-        self._fd_read, self._fd_write = os.pipe()
-        self._pipe_reader = os.fdopen(self._fd_read)
-        self._buffer = []
-
-        self.start()
-
-    def get_buffer(self):
-        return self._buffer
-
-    def fileno(self):
-        return self._fd_write
-
-    def run(self):
-        for line in iter(self._pipe_reader.readline, ""):
-            line = line.strip()
-            self._buffer.append(line)
-            if self.outcallback:
-                self.outcallback(line)
-            else:
-                print(line)
-        self._pipe_reader.close()
-
-    def close(self):
-        os.close(self._fd_write)
-        self.join()
 
 
 class cd(object):
@@ -124,7 +51,12 @@ class cd(object):
 class memoized(object):
 
     def __init__(self, expire=0):
-        self.expire = expire / 1000  # milliseconds
+        expire = str(expire)
+        if expire.isdigit():
+            expire = "%ss" % int((int(expire) / 1000))
+        tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
+        assert expire.endswith(tuple(tdmap))
+        self.expire = int(tdmap[expire[-1]] * int(expire[:-1]))
         self.cache = {}
 
     def __call__(self, func):
@@ -142,7 +74,7 @@ class memoized(object):
         return wrapper
 
     def _reset(self):
-        self.cache = {}
+        self.cache.clear()
 
 
 class throttle(object):
@@ -176,8 +108,15 @@ def singleton(cls):
     return get_instance
 
 
-def path_to_unicode(path):
+@contextmanager
-    return path.decode(sys.getfilesystemencoding()).encode("utf-8")
+def capture_std_streams(stdout, stderr=None):
+    _stdout = sys.stdout
+    _stderr = sys.stderr
+    sys.stdout = stdout
+    sys.stderr = stderr or stdout
+    yield
+    sys.stdout = _stdout
+    sys.stderr = _stderr
 
 
 def load_json(file_path):
@@ -202,63 +141,6 @@ def pioversion_to_intstr():
     return [int(i) for i in vermatch.group(1).split(".")[:3]]
 
 
-def get_project_optional_dir(name, default=None):
-    paths = None
-    var_name = "PLATFORMIO_%s" % name.upper()
-    if var_name in os.environ:
-        paths = os.getenv(var_name)
-    else:
-        try:
-            config = load_project_config()
-            if (config.has_section("platformio")
-                    and config.has_option("platformio", name)):
-                paths = config.get("platformio", name)
-        except exception.NotPlatformIOProject:
-            pass
-
-    if not paths:
-        return default
-
-    items = []
-    for item in paths.split(", "):
-        if item.startswith("~"):
-            item = expanduser(item)
-        items.append(abspath(item))
-    paths = ", ".join(items)
-
-    while "$PROJECT_HASH" in paths:
-        paths = paths.replace("$PROJECT_HASH",
-                              sha1(get_project_dir()).hexdigest()[:10])
-
-    return paths
-
-
-def get_home_dir():
-    home_dir = get_project_optional_dir("home_dir",
-                                        join(expanduser("~"), ".platformio"))
-    win_home_dir = None
-    if "windows" in get_systype():
-        win_home_dir = splitdrive(home_dir)[0] + "\\.platformio"
-        if isdir(win_home_dir):
-            home_dir = win_home_dir
-
-    if not isdir(home_dir):
-        try:
-            os.makedirs(home_dir)
-        except:  # pylint: disable=bare-except
-            if win_home_dir:
-                os.makedirs(win_home_dir)
-                home_dir = win_home_dir
-
-    assert isdir(home_dir)
-    return home_dir
-
-
-def get_cache_dir():
-    return get_project_optional_dir("cache_dir", join(get_home_dir(),
-                                                      ".cache"))
-
-
 def get_source_dir():
     curpath = abspath(__file__)
     if not isfile(curpath):
@@ -269,174 +151,10 @@ def get_source_dir():
     return dirname(curpath)
 
 
-def get_project_dir():
-    return os.getcwd()
-
-
-def find_project_dir_above(path):
-    if isfile(path):
-        path = dirname(path)
-    if is_platformio_project(path):
-        return path
-    if isdir(dirname(path)):
-        return find_project_dir_above(dirname(path))
-    return None
-
-
-def is_platformio_project(project_dir=None):
-    if not project_dir:
-        project_dir = get_project_dir()
-    return isfile(join(project_dir, "platformio.ini"))
-
-
-def get_projectlib_dir():
-    return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib"))
-
-
-def get_projectlibdeps_dir():
-    return get_project_optional_dir("libdeps_dir",
-                                    join(get_project_dir(), ".piolibdeps"))
-
-
-def get_projectsrc_dir():
-    return get_project_optional_dir("src_dir", join(get_project_dir(), "src"))
-
-
-def get_projectinclude_dir():
-    return get_project_optional_dir("include_dir",
-                                    join(get_project_dir(), "include"))
-
-
-def get_projecttest_dir():
-    return get_project_optional_dir("test_dir", join(get_project_dir(),
-                                                     "test"))
-
-
-def get_projectboards_dir():
-    return get_project_optional_dir("boards_dir",
-                                    join(get_project_dir(), "boards"))
-
-
-def get_projectbuild_dir(force=False):
-    path = get_project_optional_dir("build_dir",
-                                    join(get_project_dir(), ".pioenvs"))
-    try:
-        if not isdir(path):
-            os.makedirs(path)
-        dontmod_path = join(path, "do-not-modify-files-here.url")
-        if not isfile(dontmod_path):
-            with open(dontmod_path, "w") as fp:
-                fp.write("""
-[InternetShortcut]
-URL=https://docs.platformio.org/page/projectconf/section_platformio.html#build-dir
-""")
-    except Exception as e:  # pylint: disable=broad-except
-        if not force:
-            raise Exception(e)
-    return path
-
-
-# compatibility with PIO Core+
-get_projectpioenvs_dir = get_projectbuild_dir
-
-
-def get_projectdata_dir():
-    return get_project_optional_dir("data_dir", join(get_project_dir(),
-                                                     "data"))
-
-
-def load_project_config(path=None):
-    if not path or isdir(path):
-        path = join(path or get_project_dir(), "platformio.ini")
-    if not isfile(path):
-        raise exception.NotPlatformIOProject(
-            dirname(path) if path.endswith("platformio.ini") else path)
-    cp = ProjectConfig()
-    try:
-        cp.read(path)
-    except ConfigParser.Error as e:
-        raise exception.InvalidProjectConf(str(e))
-    return cp
-
-
-def parse_conf_multi_values(items):
-    result = []
-    if not items:
-        return result
-    inline_comment_re = re.compile(r"\s+;.*$")
-    for item in items.split("\n" if "\n" in items else ", "):
-        item = item.strip()
-        # comment
-        if not item or item.startswith((";", "#")):
-            continue
-        if ";" in item:
-            item = inline_comment_re.sub("", item).strip()
-        result.append(item)
-    return result
-
-
 def change_filemtime(path, mtime):
     os.utime(path, (mtime, mtime))
 
 
-def is_ci():
-    return os.getenv("CI", "").lower() == "true"
-
-
-def is_container():
-    if not isfile("/proc/1/cgroup"):
-        return False
-    with open("/proc/1/cgroup") as fp:
-        for line in fp:
-            line = line.strip()
-            if ":" in line and not line.endswith(":/"):
-                return True
-    return False
-
-
-def exec_command(*args, **kwargs):
-    result = {"out": None, "err": None, "returncode": None}
-
-    default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    default.update(kwargs)
-    kwargs = default
-
-    p = subprocess.Popen(*args, **kwargs)
-    try:
-        result['out'], result['err'] = p.communicate()
-        result['returncode'] = p.returncode
-    except KeyboardInterrupt:
-        raise exception.AbortedByUser()
-    finally:
-        for s in ("stdout", "stderr"):
-            if isinstance(kwargs[s], AsyncPipe):
-                kwargs[s].close()
-
-    for s in ("stdout", "stderr"):
-        if isinstance(kwargs[s], AsyncPipe):
-            result[s[3:]] = "\n".join(kwargs[s].get_buffer())
-
-    for k, v in result.items():
-        if v and isinstance(v, basestring):
-            result[k].strip()
-
-    return result
-
-
-def copy_pythonpath_to_osenv():
-    _PYTHONPATH = []
-    if "PYTHONPATH" in os.environ:
-        _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep)
-    for p in os.sys.path:
-        conditions = [p not in _PYTHONPATH]
-        if "windows" not in get_systype():
-            conditions.append(
-                isdir(join(p, "click")) or isdir(join(p, "platformio")))
-        if all(conditions):
-            _PYTHONPATH.append(p)
-    os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH)
-
-
 def get_serial_ports(filter_hwid=False):
     try:
         from serial.tools.list_ports import comports
@@ -447,8 +165,9 @@ def get_serial_ports(filter_hwid=False):
     for p, d, h in comports():
         if not p:
             continue
-        if "windows" in get_systype():
+        if WINDOWS and PY2:
             try:
+                # pylint: disable=undefined-variable
                 d = unicode(d, errors="ignore")
             except TypeError:
                 pass
@@ -471,11 +190,11 @@ get_serialports = get_serial_ports
 
 def get_logical_devices():
     items = []
-    if "windows" in get_systype():
+    if WINDOWS:
         try:
             result = exec_command(
-                ["wmic", "logicaldisk", "get", "name,VolumeName"]).get(
+                ["wmic", "logicaldisk", "get",
-                    "out", "")
+                 "name,VolumeName"]).get("out", "")
             devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?")
             for line in result.split("\n"):
                 match = devicenamere.match(line.strip())
@@ -493,17 +212,17 @@ def get_logical_devices():
         for device in re.findall(r"[A-Z]:\\", result):
             items.append({"path": device, "name": None})
         return items
-    else:
+
-        result = exec_command(["df"]).get("out")
+    result = exec_command(["df"]).get("out")
-        devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I)
+    devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I)
-        for line in result.split("\n"):
+    for line in result.split("\n"):
-            match = devicenamere.match(line.strip())
+        match = devicenamere.match(line.strip())
-            if not match:
+        if not match:
-                continue
+            continue
-            items.append({
+        items.append({
-                "path": match.group(1),
+            "path": match.group(1),
-                "name": basename(match.group(1))
+            "name": basename(match.group(1))
-            })
+        })
     return items
 
 
@@ -560,19 +279,31 @@ def get_mdns_services():
         time.sleep(3)
         for service in mdns.get_services():
             properties = None
-            try:
+            if service.properties:
-                if service.properties:
+                try:
-                    json.dumps(service.properties)
+                    properties = {
-                properties = service.properties
+                        k.decode("utf8"):
-            except UnicodeDecodeError:
+                        v.decode("utf8") if isinstance(v, bytes) else v
-                pass
+                        for k, v in service.properties.items()
+                    }
+                    json.dumps(properties)
+                except UnicodeDecodeError:
+                    properties = None
 
             items.append({
-                "type": service.type,
+                "type":
-                "name": service.name,
+                service.type,
-                "ip": ".".join([str(ord(c)) for c in service.address]),
+                "name":
-                "port": service.port,
+                service.name,
-                "properties": properties
+                "ip":
+                ".".join([
+                    str(c if isinstance(c, int) else ord(c))
+                    for c in service.address
+                ]),
+                "port":
+                service.port,
+                "properties":
+                properties
             })
     return items
 
@@ -582,7 +313,7 @@ def get_request_defheaders():
     return {"User-Agent": "PlatformIO/%s CI/%d %s" % data}
 
 
-@memoized(expire=10000)
+@memoized(expire="60s")
 def _api_request_session():
     return requests.Session()
 
@@ -595,7 +326,7 @@ def _get_api_result(
         auth=None):
     from platformio.app import get_setting
 
-    result = None
+    result = {}
     r = None
     verify_ssl = sys.version_info >= (2, 7, 9)
 
@@ -607,33 +338,30 @@ def _get_api_result(
 
     try:
         if data:
-            r = _api_request_session().post(
+            r = _api_request_session().post(url,
-                url,
+                                            params=params,
-                params=params,
+                                            data=data,
-                data=data,
+                                            headers=headers,
-                headers=headers,
+                                            auth=auth,
-                auth=auth,
+                                            verify=verify_ssl)
-                verify=verify_ssl)
         else:
-            r = _api_request_session().get(
+            r = _api_request_session().get(url,
-                url,
+                                           params=params,
-                params=params,
+                                           headers=headers,
-                headers=headers,
+                                           auth=auth,
-                auth=auth,
+                                           verify=verify_ssl)
-                verify=verify_ssl)
         result = r.json()
         r.raise_for_status()
         return r.text
     except requests.exceptions.HTTPError as e:
         if result and "message" in result:
             raise exception.APIRequestError(result['message'])
-        elif result and "errors" in result:
+        if result and "errors" in result:
             raise exception.APIRequestError(result['errors'][0]['title'])
-        else:
+        raise exception.APIRequestError(e)
-            raise exception.APIRequestError(e)
     except ValueError:
-        raise exception.APIRequestError(
+        raise exception.APIRequestError("Invalid response: %s" %
-            "Invalid response: %s" % r.text.encode("utf-8"))
+                                        r.text.encode("utf-8"))
     finally:
         if r:
             r.close()
@@ -664,9 +392,8 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
             return json.loads(result)
         except (requests.exceptions.ConnectionError,
                 requests.exceptions.Timeout) as e:
-            from platformio.maintenance import in_silence
             total += 1
-            if not in_silence():
+            if not PlatformioCLI.in_silence():
                 click.secho(
                     "[API] ConnectionError: {0} (incremented retry: max={1}, "
                     "total={2})".format(e, max_retries, total),
@@ -684,18 +411,19 @@ PING_INTERNET_IPS = [
 ]
 
 
-@memoized(expire=5000)
+@memoized(expire="5s")
 def _internet_on():
     timeout = 2
     socket.setdefaulttimeout(timeout)
     for ip in PING_INTERNET_IPS:
         try:
             if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")):
-                requests.get(
+                requests.get("http://%s" % ip,
-                    "http://%s" % ip, allow_redirects=False, timeout=timeout)
+                             allow_redirects=False,
+                             timeout=timeout)
             else:
-                socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip,
+                socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect(
-                                                                           80))
+                    (ip, 80))
             return True
         except:  # pylint: disable=bare-except
             pass
@@ -709,35 +437,6 @@ def internet_on(raise_exception=False):
     return result
 
 
-def get_pythonexe_path():
-    return os.environ.get("PYTHONEXEPATH", normpath(sys.executable))
-
-
-def where_is_program(program, envpath=None):
-    env = os.environ
-    if envpath:
-        env['PATH'] = envpath
-
-    # try OS's built-in commands
-    try:
-        result = exec_command(
-            ["where" if "windows" in get_systype() else "which", program],
-            env=env)
-        if result['returncode'] == 0 and isfile(result['out'].strip()):
-            return result['out'].strip()
-    except OSError:
-        pass
-
-    # look up in $PATH
-    for bin_dir in env.get("PATH", "").split(os.pathsep):
-        if isfile(join(bin_dir, program)):
-            return join(bin_dir, program)
-        elif isfile(join(bin_dir, "%s.exe" % program)):
-            return join(bin_dir, "%s.exe" % program)
-
-    return program
-
-
 def pepver_to_semver(pepver):
     return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1)
 
@@ -791,15 +490,6 @@ def merge_dicts(d1, d2, path=None):
     return d1
 
 
-def get_file_contents(path):
-    try:
-        with open(path) as f:
-            return f.read()
-    except UnicodeDecodeError:
-        with open(path, encoding="latin-1") as f:
-            return f.read()
-
-
 def ensure_udev_rules():
 
     def _rules_to_set(rules_path):
@@ -831,6 +521,17 @@ def ensure_udev_rules():
     return True
 
 
+def get_original_version(version):
+    if version.count(".") != 2:
+        return None
+    _, raw = version.split(".")[:2]
+    if int(raw) <= 99:
+        return None
+    if int(raw) <= 9999:
+        return "%s.%s" % (raw[:-2], int(raw[-2:]))
+    return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:]))
+
+
 def rmtree_(path):
 
     def _onerror(_, name, __):
@@ -838,33 +539,9 @@ def rmtree_(path):
             os.chmod(name, stat.S_IWRITE)
             os.remove(name)
         except Exception as e:  # pylint: disable=broad-except
-            click.secho(
+            click.secho("%s \nPlease manually remove the file `%s`" %
-                "%s \nPlease manually remove the file `%s`" % (str(e), name),
+                        (str(e), name),
-                fg="red",
+                        fg="red",
-                err=True)
+                        err=True)
 
     return rmtree(path, onerror=_onerror)
-
-
-#
-# Glob.Escape from Python 3.4
-# https://github.com/python/cpython/blob/master/Lib/glob.py#L161
-#
-
-try:
-    from glob import escape as glob_escape  # pylint: disable=unused-import
-except ImportError:
-    magic_check = re.compile('([*?[])')
-    magic_check_bytes = re.compile(b'([*?[])')
-
-    def glob_escape(pathname):
-        """Escape all special characters."""
-        # Escaping is done by wrapping any of "*?[" between square brackets.
-        # Metacharacters do not work in the drive part and shouldn't be
-        # escaped.
-        drive, pathname = os.path.splitdrive(pathname)
-        if isinstance(pathname, bytes):
-            pathname = magic_check_bytes.sub(br'[\1]', pathname)
-        else:
-            pathname = magic_check.sub(r'[\1]', pathname)
-        return drive + pathname

platformio/vcsclient.py

@@ -16,10 +16,14 @@ import re
 from os.path import join
 from subprocess import CalledProcessError, check_call
 from sys import modules
-from urlparse import urlparse
 
-from platformio import util
 from platformio.exception import PlatformioException, UserSideException
+from platformio.proc import exec_command
+
+try:
+    from urllib.parse import urlparse
+except ImportError:
+    from urlparse import urlparse
 
 
 class VCSClientFactory(object):
@@ -38,10 +42,11 @@ class VCSClientFactory(object):
         if "#" in remote_url:
             remote_url, tag = remote_url.rsplit("#", 1)
         if not type_:
-            raise PlatformioException(
+            raise PlatformioException("VCS: Unknown repository type %s" %
-                "VCS: Unknown repository type %s" % remote_url)
+                                      remote_url)
-        obj = getattr(modules[__name__], "%sClient" % type_.title())(
+        obj = getattr(modules[__name__],
-            src_dir, remote_url, tag, silent)
+                      "%sClient" % type_.title())(src_dir, remote_url, tag,
+                                                  silent)
         assert isinstance(obj, VCSClientBase)
         return obj
 
@@ -98,14 +103,14 @@ class VCSClientBase(object):
             check_call(args, **kwargs)
             return True
         except CalledProcessError as e:
-            raise PlatformioException(
+            raise PlatformioException("VCS: Could not process command %s" %
-                "VCS: Could not process command %s" % e.cmd)
+                                      e.cmd)
 
     def get_cmd_output(self, args, **kwargs):
         args = [self.command] + args
         if "cwd" not in kwargs:
             kwargs['cwd'] = self.src_dir
-        result = util.exec_command(args, **kwargs)
+        result = exec_command(args, **kwargs)
         if result['returncode'] == 0:
             return result['out'].strip()
         raise PlatformioException(

scripts/99-platformio-udev.rules

@@ -63,10 +63,10 @@ SUBSYSTEMS=="usb", ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE="0666
 SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="05dc", MODE:="0666"
 
 # Teensy boards
-ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", ENV{ID_MM_DEVICE_IGNORE}="1"
+ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
-ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", ENV{MTP_NO_PROBE}="1"
+ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789A]?", ENV{MTP_NO_PROBE}="1"
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", MODE:="0666"
+SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", MODE:="0666"
-KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", MODE:="0666"
+KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", MODE:="0666"
 
 #TI Stellaris Launchpad
 SUBSYSTEMS=="usb", ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666"
@@ -84,176 +84,176 @@ SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic GDB Server"
 SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port"
 
 # opendous and estick
-ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="0666"
 
 # Original FT232/FT245 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="0666"
 
 # Original FT2232 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="0666"
 
 # Original FT4232 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="0666"
 
 # Original FT232H VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="0666"
 
 # DISTORTEC JTAG-lock-pick Tiny 2
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="0666"
 
 # TUMPA, TUMPA Lite
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="0666"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="0666"
 
 # XDS100v2
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="0666"
 
 # Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE)
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="0666"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="0666"
 
 # TI/Luminary Stellaris Evaluation Board FTDI (several)
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="0666"
 
 # TI/Luminary Stellaris In-Circuit Debug Interface FTDI (ICDI) Board
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="0666"
 
 # egnite Turtelizer 2
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="0666"
 
 # Section5 ICEbear
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="0666"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="0666"
 
 # Amontec JTAGkey and JTAGkey-tiny
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666"
 
 # TI ICDI
-ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="0666"
 
 # STLink v1
-ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="0666"
 
 # STLink v2
-ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="0666"
 
 # STLink v2-1
-ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="0666"
 
 # Hilscher NXHX Boards
-ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="0666"
 
 # Hitex STR9-comStick
-ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="0666"
 
 # Hitex STM32-PerformanceStick
-ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="0666"
 
 # Altera USB Blaster
-ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666"
 
 # Amontec JTAGkey-HiSpeed
-ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="0666"
 
 # SEGGER J-Link
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="0666"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="0666"
 
 # Raisonance RLink
-ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666"
 
 # Debug Board for Neo1973
-ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="0666"
 
 # Olimex ARM-USB-OCD
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="0666"
 
 # Olimex ARM-USB-OCD-TINY
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="0666"
 
 # Olimex ARM-JTAG-EW
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="0666"
 
 # Olimex ARM-USB-OCD-TINY-H
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="0666"
 
 # Olimex ARM-USB-OCD-H
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="0666"
 
 # USBprog with OpenOCD firmware
-ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="0666"
 
 # TI/Luminary Stellaris In-Circuit Debug Interface (ICDI) Board
-ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666"
 
 # Marvell Sheevaplug
-ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="0666"
 
 # Keil Software, Inc. ULink
-ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666"
 
 # CMSIS-DAP compatible adapters
-ATTRS{product}=="*CMSIS-DAP*", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{product}=="*CMSIS-DAP*", MODE="0666"
 
 #SEGGER J-LIK
-ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="0666"
-ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="0666"

scripts/docspregen.py

@@ -690,7 +690,7 @@ Uploading
 ---------
 %s supports the next uploading protocols:
 """ % board['name'])
-        for protocol in upload_protocols:
+        for protocol in sorted(upload_protocols):
             lines.append("* ``%s``" % protocol)
         lines.append("""
 Default protocol is ``%s``""" % variables['upload_protocol'])