Merge branch 'release/v3.6.1'

This reverts commit bfee896378.

.appveyor.yml

@@ -1,12 +1,18 @@
 build: off
-environment:
 
+platform:
+  - x86
+  - x64
+
+environment:
   matrix:
     - TOXENV: "py27"
 
 install:
   - cmd: git submodule update --init --recursive
-  - cmd: SET PATH=%PATH%;C:\Python27\Scripts;C:\MinGW\bin
+  - cmd: SET PATH=C:\MinGW\bin;%PATH%
+  - if %PLATFORM% == x64 SET PATH=C:\Python27-x64;C:\Python27-x64\Scripts;%PATH%
+  - if %PLATFORM% == x86 SET PATH=C:\Python27;C:\Python27\Scripts;%PATH%
   - cmd: pip install tox
 
 test_script:

.github/ISSUE_TEMPLATE.md

@@ -1,22 +1,28 @@
 What kind of issue is this?
 
-- [ ] Question. This issue tracker is not the place for questions. If you want to ask how to do
-      something, or to understand why something isn't working the way you expect it to, use
-      our Community Forums https://community.platformio.org
+- [ ] **Question**.
+      This issue tracker is not the place for questions. If you want to ask how to do something,
+      or to understand why something isn't working the way you expect it to, 
+      use [Community Forums](https://community.platformio.org) or [Premium Support](https://platformio.org/support)
 
-- [ ] PlatformIO IDE. All issues related to PlatformIO IDE should be reported to appropriate repository
-      https://github.com/platformio/platformio-atom-ide/issues
+- [ ] **PlatformIO IDE**.
+      All issues related to PlatformIO IDE should be reported to appropriate repository:
+      [PlatformIO IDE for Atom](https://github.com/platformio/platformio-atom-ide/issues) or
+      [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues)
 
-- [ ] Development Platform or Board. All issues related to Development Platforms or Embedded Boards
-      should be reported to appropriate repository. 
-      See full list with repositories and search for "platform-xxx" repository related to your hardware
-      https://github.com/platformio?query=platform-
+- [ ] **Development Platform or Board**.
+      All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms
+      should be reported to appropriate repository related to your hardware
+      https://github.com/topics/platformio-platform
 
-- [ ] Feature Request. Start by telling us what problem you’re trying to solve. Often a solution
+- [ ] **Feature Request**.
+      Start by telling us what problem you’re trying to solve. Often a solution
       already exists! Don’t send pull requests to implement new features without first getting our
       support. Sometimes we leave features out on purpose to keep the project small.
 
-- [ ] PlatformIO Core. If you’ve found a bug, please provide an information below.
+- [ ] **PlatformIO Core**.
+      If you’ve found a bug, please provide an information below.
+
 
 *You can erase any parts of this template not applicable to your Issue.*
 

.gitignore

@@ -9,3 +9,4 @@ build
 coverage.xml
 .coverage
 htmlcov
+.pytest_cache

.isort.cfg

@@ -1,3 +1,3 @@
 [settings]
 line_length=79
-known_third_party=bottle,click,lockfile,python-dateutil,pytest,requests,SCons,semantic_version,serial
+known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial

.pylintrc

@@ -20,4 +20,4 @@ confidence=
 # --disable=W"
 # disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
 
-disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import
+disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens

.travis.yml

@@ -21,7 +21,7 @@ matrix:
 install:
   - git submodule update --init --recursive
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl -fsSL https://bootstrap.pypa.io/get-pip.py | sudo python; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install tox; else pip install -U tox; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox==3.0.0"; else pip install -U tox; fi
 
   # ChipKIT issue: install 32-bit support for GCC PIC32
   - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi
@@ -29,10 +29,6 @@ install:
 script:
   - tox -e $TOX_ENV
 
-after_success:
-  - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then tox -e coverage; fi
-  - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then bash <(curl -s https://codecov.io/bash); fi
-
 notifications:
   email: false
 

README.rst

@@ -7,9 +7,6 @@ PlatformIO
 .. image:: https://ci.appveyor.com/api/projects/status/unnpw0n3c5k14btn/branch/develop?svg=true
     :target: https://ci.appveyor.com/project/ivankravets/platformio-core
     :alt: AppVeyor.CI Build Status
-.. image:: https://requires.io/github/platformio/platformio-core/requirements.svg?branch=develop
-    :target: https://requires.io/github/platformio/platformio-core/requirements/?branch=develop
-    :alt: Requirements Status
 .. image:: https://img.shields.io/pypi/v/platformio.svg
     :target: https://pypi.python.org/pypi/platformio/
     :alt: Latest Version
@@ -19,19 +16,20 @@ PlatformIO
 .. image:: https://img.shields.io/PlatformIO/Community.png
    :alt: Community Forums
    :target: https://community.platformio.org?utm_source=github&utm_medium=core
-.. image:: https://img.shields.io/PlatformIO/Plus.png?color=orange
-   :alt: PlatformIO Plus: Professional solutions for an awesome open source PlatformIO ecosystem
+.. image:: https://img.shields.io/PIO/Plus.png?color=orange
+   :alt: PIO Plus: Professional solutions for an awesome open source PlatformIO ecosystem
    :target: https://platformio.org/pricing?utm_source=github&utm_medium=core
 
-**Quick Links:** `Home Page <https://platformio.org?utm_source=github&utm_medium=core>`_ |
-`PlatformIO Plus <https://platformio.org/pricing?utm_source=github&utm_medium=core>`_ |
+**Quick Links:** `Web <https://platformio.org?utm_source=github&utm_medium=core>`_ |
+`PIO Plus <https://platformio.org/pricing?utm_source=github&utm_medium=core>`_ |
 `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_ |
 `Project Examples <https://github.com/platformio/platformio-examples/>`_ |
-`Docs <http://docs.platformio.org?utm_source=github&utm_medium=core>`_ |
+`Docs <https://docs.platformio.org?utm_source=github&utm_medium=core>`_ |
 `Donate <https://platformio.org/donate?utm_source=github&utm_medium=core>`_ |
 `Contact Us <https://platformio.org/contact?utm_source=github&utm_medium=core>`_
 
 **Social:** `Twitter <https://twitter.com/PlatformIO_Org>`_ |
+`LinkedIn <https://www.linkedin.com/company/platformio/>`_ |
 `Facebook <https://www.facebook.com/platformio>`_ |
 `Hackaday <https://hackaday.io/project/7980-platformio>`_ |
 `Bintray <https://bintray.com/platformio>`_ |
@@ -47,18 +45,27 @@ firmware updates.
 Get Started
 -----------
 
-* `What is PlatformIO? <http://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
+* `What is PlatformIO? <https://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
 
-Products
---------
+Open Source
+-----------
 
 * `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_
-* `PlatformIO Core (CLI) <http://docs.platformio.org/en/latest/core.html?utm_source=github&utm_medium=core>`_
-* `PIO Remote™ <http://docs.platformio.org/en/latest/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
-* `PIO Unified Debugger <http://docs.platformio.org/en/latest/plus/debugging.html?utm_source=github&utm_medium=core>`_
-* `PIO Unit Testing <http://docs.platformio.org/en/latest/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
-* `PIO Delivery™ <https://platformio.org/pricing?utm_source=github&utm_medium=core#solution-pio-delivery>`_
-* `Cloud Builder <https://platformio.org/pricing?utm_source=github&utm_medium=core#solution-cloud-builder>`_
+* `PlatformIO Core (CLI) <https://docs.platformio.org/en/latest/core.html?utm_source=github&utm_medium=core>`_
+* `Library Management <https://docs.platformio.org/page/librarymanager/index.html?utm_source=github&utm_medium=core>`_
+* `Project Examples <https://github.com/platformio/platformio-examples?utm_source=github&utm_medium=core>`_
+* `Desktop IDEs Integration <https://docs.platformio.org/page/ide.html?utm_source=github&utm_medium=core>`_
+* `Continuous Integration <https://docs.platformio.org/page/ci/index.html?utm_source=github&utm_medium=core>`_
+* `Advanced Scripting API <https://docs.platformio.org/page/projectconf/advanced_scripting.html?utm_source=github&utm_medium=core>`_
+
+PIO Plus
+--------
+
+* `PIO Remote <https://docs.platformio.org/page/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
+* `PIO Unified Debugger <https://docs.platformio.org/page/plus/debugging.html?utm_source=github&utm_medium=core>`_
+* `PIO Unit Testing <https://docs.platformio.org/en/latest/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
+* `Cloud IDEs Integration <https://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core#solution-pio-delivery>`_
+* `Integration Services <https://platformio.org/pricing?utm_source=github&utm_medium=core#enterprise-features>`_
 
 Registry
 --------
@@ -68,14 +75,6 @@ Registry
 * `Frameworks <https://platformio.org/frameworks?utm_source=github&utm_medium=core>`_
 * `Embedded Boards <https://platformio.org/boards?utm_source=github&utm_medium=core>`_
 
-Solutions
----------
-
-* `Library Manager <http://docs.platformio.org/en/latest/librarymanager/index.html?utm_source=github&utm_medium=core>`_
-* `Cloud IDEs Integration <https://platformio.org/pricing?utm_source=github&utm_medium=core#solution-cloud-ide>`_
-* `Standalone IDEs Integration <http://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core#other-ide>`_
-* `Continuous Integration <http://docs.platformio.org/en/latest/ci/index.html?utm_source=github&utm_medium=core>`_
-
 Development Platforms
 ---------------------
 
@@ -84,13 +83,17 @@ Development Platforms
 * `Espressif 32 <https://platformio.org/platforms/espressif32?utm_source=github&utm_medium=core>`_
 * `Espressif 8266 <https://platformio.org/platforms/espressif8266?utm_source=github&utm_medium=core>`_
 * `Freescale Kinetis <https://platformio.org/platforms/freescalekinetis?utm_source=github&utm_medium=core>`_
+* `Infineon XMC <https://platformio.org/platforms/infineonxmc?utm_source=github&utm_medium=core>`_
 * `Intel ARC32 <https://platformio.org/platforms/intel_arc32?utm_source=github&utm_medium=core>`_
+* `Intel MCS-51 (8051) <https://platformio.org/platforms/intel_mcs51?utm_source=github&utm_medium=core>`_
 * `Lattice iCE40 <https://platformio.org/platforms/lattice_ice40?utm_source=github&utm_medium=core>`_
 * `Maxim 32 <https://platformio.org/platforms/maxim32?utm_source=github&utm_medium=core>`_
 * `Microchip PIC32 <https://platformio.org/platforms/microchippic32?utm_source=github&utm_medium=core>`_
 * `Nordic nRF51 <https://platformio.org/platforms/nordicnrf51?utm_source=github&utm_medium=core>`_
 * `Nordic nRF52 <https://platformio.org/platforms/nordicnrf52?utm_source=github&utm_medium=core>`_
 * `NXP LPC <https://platformio.org/platforms/nxplpc?utm_source=github&utm_medium=core>`_
+* `RISC-V <https://platformio.org/platforms/riscv?utm_source=github&utm_medium=core>`_
+* `Samsung ARTIK <https://platformio.org/platforms/samsung_artik?utm_source=github&utm_medium=core>`_
 * `Silicon Labs EFM32 <https://platformio.org/platforms/siliconlabsefm32?utm_source=github&utm_medium=core>`_
 * `ST STM32 <https://platformio.org/platforms/ststm32?utm_source=github&utm_medium=core>`_
 * `Teensy <https://platformio.org/platforms/teensy?utm_source=github&utm_medium=core>`_

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (3, 5, 2)
+VERSION = (3, 6, 1)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -26,8 +26,8 @@ __description__ = (
     "FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3")
 __url__ = "https://platformio.org"
 
-__author__ = "Ivan Kravets"
-__email__ = "me@ikravets.com"
+__author__ = "PlatformIO"
+__email__ = "contact@platformio.org"
 
 __license__ = "Apache Software License"
 __copyright__ = "Copyright 2014-present PlatformIO"

platformio/__main__.py

@@ -107,8 +107,8 @@ def configure():
         try:
             click_echo_origin[origin](*args, **kwargs)
         except IOError:
-            (sys.stderr.write if kwargs.get("err") else
-             sys.stdout.write)("%s\n" % (args[0] if args else ""))
+            (sys.stderr.write if kwargs.get("err") else sys.stdout.write)(
+                "%s\n" % (args[0] if args else ""))
 
     click.echo = lambda *args, **kwargs: _safe_echo(0, *args, **kwargs)
     click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs)
@@ -135,7 +135,7 @@ An unexpected error occurred. Further steps:
   `pip install -U platformio` command
 
 * Try to find answer in FAQ Troubleshooting section
-  http://docs.platformio.org/page/faq.html
+  https://docs.platformio.org/page/faq.html
 
 * Report this problem to the developers
   https://github.com/platformio/platformio-core/issues

platformio/app.py

@@ -12,19 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import hashlib
 import json
 import os
 import uuid
 from copy import deepcopy
 from os import environ, getenv, listdir, remove
-from os.path import abspath, dirname, expanduser, getmtime, isdir, isfile, join
+from os.path import abspath, dirname, expanduser, isdir, isfile, join
 from time import time
 
 import requests
-from lockfile import LockFailed, LockFile
 
-from platformio import __version__, exception, util
+from platformio import exception, lockfile, util
 
 
 def projects_dir_validate(projects_dir):
@@ -63,7 +63,7 @@ DEFAULT_SETTINGS = {
     },
     "enable_telemetry": {
         "description":
-        ("Telemetry service <http://docs.platformio.org/page/"
+        ("Telemetry service <https://docs.platformio.org/page/"
          "userguide/cmd_settings.html?#enable-telemetry> (Yes/No)"),
         "value":
         True
@@ -106,11 +106,8 @@ class State(object):
     def __exit__(self, type_, value, traceback):
         if self._prev_state != self._state:
             try:
-                with open(self.path, "w") as fp:
-                    if "dev" in __version__:
-                        json.dump(self._state, fp, indent=4)
-                    else:
-                        json.dump(self._state, fp)
+                with codecs.open(self.path, "w", encoding="utf8") as fp:
+                    json.dump(self._state, fp)
             except IOError:
                 raise exception.HomeDirPermissionsError(util.get_home_dir())
         self._unlock_state_file()
@@ -118,21 +115,19 @@ class State(object):
     def _lock_state_file(self):
         if not self.lock:
             return
-        self._lockfile = LockFile(self.path)
-
-        if self._lockfile.is_locked() and \
-                (time() - getmtime(self._lockfile.lock_file)) > 10:
-            self._lockfile.break_lock()
-
+        self._lockfile = lockfile.LockFile(self.path)
         try:
             self._lockfile.acquire()
-        except LockFailed:
+        except IOError:
             raise exception.HomeDirPermissionsError(dirname(self.path))
 
     def _unlock_state_file(self):
         if self._lockfile:
             self._lockfile.release()
 
+    def __del__(self):
+        self._unlock_state_file()
+
 
 class ContentCache(object):
 
@@ -154,15 +149,10 @@ class ContentCache(object):
     def _lock_dbindex(self):
         if not self.cache_dir:
             os.makedirs(self.cache_dir)
-        self._lockfile = LockFile(self.cache_dir)
-        if self._lockfile.is_locked() and \
-                isfile(self._lockfile.lock_file) and \
-                (time() - getmtime(self._lockfile.lock_file)) > 10:
-            self._lockfile.break_lock()
-
+        self._lockfile = lockfile.LockFile(self.cache_dir)
         try:
             self._lockfile.acquire()
-        except LockFailed:
+        except:  # pylint: disable=bare-except
             return False
 
         return True
@@ -187,11 +177,8 @@ class ContentCache(object):
         cache_path = self.get_cache_path(key)
         if not isfile(cache_path):
             return None
-        with open(cache_path, "rb") as fp:
-            data = fp.read()
-            if data and data[0] in ("{", "["):
-                return json.loads(data)
-            return data
+        with codecs.open(cache_path, "rb", encoding="utf8") as fp:
+            return fp.read()
 
     def set(self, key, data, valid):
         if not get_setting("enable_cache"):
@@ -212,13 +199,17 @@ class ContentCache(object):
 
         if not isdir(dirname(cache_path)):
             os.makedirs(dirname(cache_path))
-        with open(cache_path, "wb") as fp:
-            if isinstance(data, (dict, list)):
-                json.dump(data, fp)
-            else:
-                fp.write(str(data))
-        with open(self._db_path, "a") as fp:
-            fp.write("%s=%s\n" % (str(expire_time), cache_path))
+        try:
+            with codecs.open(cache_path, "wb", encoding="utf8") as fp:
+                fp.write(data)
+            with open(self._db_path, "a") as fp:
+                fp.write("%s=%s\n" % (str(expire_time), cache_path))
+        except UnicodeError:
+            if isfile(cache_path):
+                try:
+                    remove(cache_path)
+                except OSError:
+                    pass
 
         return self._unlock_dbindex()
 
@@ -361,8 +352,8 @@ def get_cid():
                 pass
         cid = str(
             uuid.UUID(
-                bytes=hashlib.md5(str(_uid if _uid else uuid.getnode()))
-                .digest()))
+                bytes=hashlib.md5(str(_uid if _uid else uuid.getnode())).
+                digest()))
         if "windows" in util.get_systype() or os.getuid() > 0:
             set_state_item("cid", cid)
     return cid

platformio/builder/main.py

@@ -20,7 +20,7 @@ from os.path import expanduser, join
 from time import time
 
 from SCons.Script import (ARGUMENTS, COMMAND_LINE_TARGETS, DEFAULT_TARGETS,
-                          Action, AllowSubstExceptions, AlwaysBuild,
+                          AllowSubstExceptions, AlwaysBuild, Default,
                           DefaultEnvironment, Variables)
 
 from platformio import util
@@ -54,10 +54,12 @@ commonvars.AddVariables(
 
     # board options
     ("BOARD",),
+    # deprecated options, use board_{object.path} instead
     ("BOARD_MCU",),
     ("BOARD_F_CPU",),
     ("BOARD_F_FLASH",),
     ("BOARD_FLASH_MODE",),
+    # end of deprecated options
 
     # upload options
     ("UPLOAD_PORT",),
@@ -66,15 +68,19 @@ commonvars.AddVariables(
     ("UPLOAD_FLAGS",),
     ("UPLOAD_RESETMETHOD",),
 
+    # test options
+    ("TEST_BUILD_PROJECT_SRC",),
+
     # debug options
     ("DEBUG_TOOL",),
-
+    ("DEBUG_SVD_PATH",),
 
 )  # yapf: disable
 
 MULTILINE_VARS = [
     "EXTRA_SCRIPTS", "PIOFRAMEWORK", "BUILD_FLAGS", "SRC_BUILD_FLAGS",
-    "BUILD_UNFLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE", "LIB_EXTRA_DIRS"
+    "BUILD_UNFLAGS", "UPLOAD_FLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE",
+    "LIB_EXTRA_DIRS"
 ]
 
 DEFAULT_ENV_OPTIONS = dict(
@@ -99,6 +105,7 @@ DEFAULT_ENV_OPTIONS = dict(
     BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"),
     BUILDSRC_DIR=join("$BUILD_DIR", "src"),
     BUILDTEST_DIR=join("$BUILD_DIR", "test"),
+    LIBPATH=["$BUILD_DIR"],
     LIBSOURCE_DIRS=[
         util.get_projectlib_dir(),
         util.get_projectlibdeps_dir(),
@@ -128,7 +135,7 @@ if env.GetOption('clean'):
     env.PioClean(env.subst("$BUILD_DIR"))
     env.Exit(0)
 elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
-    print "Verbose mode can be enabled via `-v, --verbose` option"
+    print("Verbose mode can be enabled via `-v, --verbose` option")
 
 # Handle custom variables from system environment
 for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPTS",
@@ -156,37 +163,53 @@ env.LoadPioPlatform(commonvars)
 env.SConscriptChdir(0)
 env.SConsignFile(join("$PROJECTBUILD_DIR", ".sconsign.dblite"))
 
-for item in env.GetPreExtraScripts():
+for item in env.GetExtraScripts("pre"):
     env.SConscript(item, exports="env")
 
 env.SConscript("$BUILD_SCRIPT")
 
-AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS + ["size"]))
-AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS + ["size"]))
-
 if "UPLOAD_FLAGS" in env:
-    env.Append(UPLOADERFLAGS=["$UPLOAD_FLAGS"])
+    env.Prepend(UPLOADERFLAGS=["$UPLOAD_FLAGS"])
 
-for item in env.GetPostExtraScripts():
+for item in env.GetExtraScripts("post"):
     env.SConscript(item, exports="env")
 
+##############################################################################
+
+# Checking program size
+if env.get("SIZETOOL") and "nobuild" not in COMMAND_LINE_TARGETS:
+    env.Depends(["upload", "program"], "checkprogsize")
+    # Replace platform's "size" target with our
+    _new_targets = [t for t in DEFAULT_TARGETS if str(t) != "size"]
+    Default(None)
+    Default(_new_targets)
+    Default("checkprogsize")
+
+# Print configured protocols
+env.AddPreAction(
+    ["upload", "program"],
+    env.VerboseAction(lambda source, target, env: env.PrintUploadInfo(),
+                      "Configuring upload protocol..."))
+
+AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS))
+AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS))
+AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS))
+
+##############################################################################
+
 if "envdump" in COMMAND_LINE_TARGETS:
-    print env.Dump()
+    print(env.Dump())
     env.Exit(0)
 
 if "idedata" in COMMAND_LINE_TARGETS:
     try:
-        print "\n%s\n" % util.path_to_unicode(
-            json.dumps(env.DumpIDEData(), ensure_ascii=False))
+        print("\n%s\n" % util.path_to_unicode(
+            json.dumps(env.DumpIDEData(), ensure_ascii=False)))
         env.Exit(0)
     except UnicodeDecodeError:
         sys.stderr.write(
             "\nUnicodeDecodeError: Non-ASCII characters found in build "
             "environment\n"
             "See explanation in FAQ > Troubleshooting > Building\n"
-            "http://docs.platformio.org/page/faq.html\n\n")
+            "https://docs.platformio.org/page/faq.html\n\n")
         env.Exit(1)
-
-env.AddPreAction(["upload", "program"],
-                 Action(lambda source, target, env: env.PrintUploadInfo(),
-                        "Configuring upload protocol..."))

platformio/builder/tools/pioide.py

@@ -16,7 +16,7 @@ from __future__ import absolute_import
 
 from glob import glob
 from os import environ
-from os.path import join
+from os.path import abspath, isfile, join
 
 from SCons.Defaults import processDefines
 
@@ -25,7 +25,7 @@ from platformio.managers.core import get_core_package_dir
 
 
 def _dump_includes(env):
-    includes = []
+    includes = [env.subst("$PROJECTINCLUDE_DIR"), env.subst("$PROJECTSRC_DIR")]
 
     for item in env.get("CPPPATH", []):
         includes.append(env.subst(item))
@@ -53,11 +53,11 @@ def _dump_includes(env):
     if unity_dir:
         includes.append(unity_dir)
 
-    # remove dupicates
+    # remove duplicates
     result = []
     for item in includes:
         if item not in result:
-            result.append(item)
+            result.append(abspath(item))
 
     return result
 
@@ -97,16 +97,38 @@ def _dump_defines(env):
             board_mcu = env.BoardConfig().get("build.mcu")
         if board_mcu:
             defines.append(
-                str("__AVR_%s__" % board_mcu.upper()
-                    .replace("ATMEGA", "ATmega").replace("ATTINY", "ATtiny")))
+                str("__AVR_%s__" % board_mcu.upper().replace(
+                    "ATMEGA", "ATmega").replace("ATTINY", "ATtiny")))
 
     # built-in GCC marcos
-    if env.GetCompilerType() == "gcc":
-        defines.extend(_get_gcc_defines(env))
+    # if env.GetCompilerType() == "gcc":
+    #     defines.extend(_get_gcc_defines(env))
 
     return defines
 
 
+def _get_svd_path(env):
+    svd_path = env.subst("$DEBUG_SVD_PATH")
+    if svd_path:
+        return abspath(svd_path)
+
+    if "BOARD" not in env:
+        return None
+    try:
+        svd_path = env.BoardConfig().get("debug.svd_path")
+        assert svd_path
+    except (AssertionError, KeyError):
+        return None
+    # custom path to SVD file
+    if isfile(svd_path):
+        return svd_path
+    # default file from ./platform/misc/svd folder
+    p = env.PioPlatform()
+    if isfile(join(p.get_dir(), "misc", "svd", svd_path)):
+        return abspath(join(p.get_dir(), "misc", "svd", svd_path))
+    return None
+
+
 def DumpIDEData(env):
     LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS"
     LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS"
@@ -130,6 +152,12 @@ def DumpIDEData(env):
         util.where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
         "prog_path":
         env.subst("$PROG_PATH"),
+        "flash_extra_images": [{
+            "offset": item[0],
+            "path": env.subst(item[1])
+        } for item in env.get("FLASH_EXTRA_IMAGES", [])],
+        "svd_path":
+        _get_svd_path(env),
         "compiler_type":
         env.GetCompilerType()
     }

platformio/builder/tools/piolib.py

@@ -27,7 +27,7 @@ from os.path import (basename, commonprefix, dirname, isdir, isfile, join,
 import SCons.Scanner
 from SCons.Script import ARGUMENTS, COMMAND_LINE_TARGETS, DefaultEnvironment
 
-from platformio import util
+from platformio import exception, util
 from platformio.builder.tools import platformio as piotool
 from platformio.managers.lib import LibraryManager
 from platformio.managers.package import PackageManager
@@ -86,8 +86,8 @@ class LibBuilderBase(object):
     LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"]
     LDF_MODE_DEFAULT = "chain"
 
-    COMPAT_MODES = ["off", "light", "strict"]
-    COMPAT_MODE_DEFAULT = "light"
+    COMPAT_MODES = ["off", "soft", "strict"]
+    COMPAT_MODE_DEFAULT = "soft"
 
     CLASSIC_SCANNER = SCons.Scanner.C.CScanner()
     CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner()
@@ -279,9 +279,8 @@ class LibBuilderBase(object):
                 if (key in item and
                         not util.items_in_list(self.env[env_key], item[key])):
                     if self.verbose:
-                        sys.stderr.write(
-                            "Skip %s incompatible dependency %s\n" % (key[:-1],
-                                                                      item))
+                        sys.stderr.write("Skip %s incompatible dependency %s\n"
+                                         % (key[:-1], item))
                     skip = True
             if skip:
                 continue
@@ -374,7 +373,7 @@ class LibBuilderBase(object):
                         if isfile("%s.%s" % (_f_part, ext)):
                             incs.append(
                                 self.env.File("%s.%s" % (_f_part, ext)))
-            # print path, map(lambda n: n.get_abspath(), incs)
+            # print(path, map(lambda n: n.get_abspath(), incs))
             for inc in incs:
                 if inc not in result:
                     result.append(inc)
@@ -394,9 +393,9 @@ class LibBuilderBase(object):
         if self != lb:
             if _already_depends(lb):
                 if self.verbose:
-                    sys.stderr.write("Warning! Circular dependencies detected "
-                                     "between `%s` and `%s`\n" % (self.path,
-                                                                  lb.path))
+                    sys.stderr.write(
+                        "Warning! Circular dependencies detected "
+                        "between `%s` and `%s`\n" % (self.path, lb.path))
                 self._circular_deps.append(lb)
             elif lb not in self._depbuilders:
                 self._depbuilders.append(lb)
@@ -502,6 +501,28 @@ class ArduinoLibBuilder(LibBuilderBase):
     def is_frameworks_compatible(self, frameworks):
         return util.items_in_list(frameworks, ["arduino", "energia"])
 
+    def is_platforms_compatible(self, platforms):
+        platforms_map = {
+            "avr": "atmelavr",
+            "sam": "atmelsam",
+            "samd": "atmelsam",
+            "esp8266": "espressif8266",
+            "esp32": "espressif32",
+            "arc32": "intel_arc32",
+            "stm32": "ststm32"
+        }
+        items = []
+        for arch in self._manifest.get("architectures", "").split(","):
+            arch = arch.strip()
+            if arch == "*":
+                items = "*"
+                break
+            if arch in platforms_map:
+                items.append(platforms_map[arch])
+        if not items:
+            return LibBuilderBase.is_platforms_compatible(self, platforms)
+        return util.items_in_list(platforms, items)
+
 
 class MbedLibBuilder(LibBuilderBase):
 
@@ -565,15 +586,6 @@ class PlatformIOLibBuilder(LibBuilderBase):
     def _is_arduino_manifest(self):
         return isfile(join(self.path, "library.properties"))
 
-    @property
-    def src_dir(self):
-        if all([
-                "srcFilter" in self._manifest.get("build", {})
-                or self.env['SRC_FILTER'], not self._is_arduino_manifest()
-        ]):
-            return self.path
-        return LibBuilderBase.src_dir.fget(self)
-
     @property
     def src_filter(self):
         if "srcFilter" in self._manifest.get("build", {}):
@@ -758,7 +770,7 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
                 sys.stderr.write(
                     "Platform incompatible library %s\n" % lb.path)
             return False
-        if compat_mode == "light" and "PIOFRAMEWORK" in env and \
+        if compat_mode == "soft" and "PIOFRAMEWORK" in env and \
            not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])):
             if verbose:
                 sys.stderr.write(
@@ -777,7 +789,7 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             try:
                 lb = LibBuilderFactory.new(
                     env, join(libs_dir, item), verbose=verbose)
-            except ValueError:
+            except exception.InvalidJSONFile:
                 if verbose:
                     sys.stderr.write("Skip library with broken manifest: %s\n"
                                      % join(libs_dir, item))
@@ -796,14 +808,14 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
     if verbose and found_incompat:
         sys.stderr.write(
             "More details about \"Library Compatibility Mode\": "
-            "http://docs.platformio.org/page/librarymanager/ldf.html#"
+            "https://docs.platformio.org/page/librarymanager/ldf.html#"
             "ldf-compat-mode\n")
 
     DefaultEnvironment()['__PIO_LIB_BUILDERS'] = items
     return items
 
 
-def BuildProjectLibraries(env):
+def ConfigureProjectLibBuilder(env):
 
     def correct_found_libs(lib_builders):
         # build full dependency graph
@@ -822,7 +834,7 @@ def BuildProjectLibraries(env):
             title = "<%s>" % lb.name
             vcs_info = lb.vcs_info
             if lb.version:
-                title += " v%s" % lb.version
+                title += " %s" % lb.version
             if vcs_info and vcs_info.get("version"):
                 title += " #%s" % vcs_info.get("version")
             sys.stdout.write("%s|-- %s" % (margin, title))
@@ -837,29 +849,28 @@ def BuildProjectLibraries(env):
                 print_deps_tree(lb, level + 1)
 
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
-    project.env = env
     ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project)
 
-    print "Library Dependency Finder -> http://bit.ly/configure-pio-ldf"
-    print "LDF MODES: FINDER(%s) COMPATIBILITY(%s)" % (ldf_mode,
-                                                       project.lib_compat_mode)
+    print("Library Dependency Finder -> http://bit.ly/configure-pio-ldf")
+    print("LDF MODES: FINDER(%s) COMPATIBILITY(%s)" %
+          (ldf_mode, project.lib_compat_mode))
 
     lib_builders = env.GetLibBuilders()
-    print "Collected %d compatible libraries" % len(lib_builders)
+    print("Collected %d compatible libraries" % len(lib_builders))
 
-    print "Scanning dependencies..."
+    print("Scanning dependencies...")
     project.search_deps_recursive()
 
     if ldf_mode.startswith("chain") and project.depbuilders:
         correct_found_libs(lib_builders)
 
     if project.depbuilders:
-        print "Dependency Graph"
+        print("Dependency Graph")
         print_deps_tree(project)
     else:
-        print "No dependencies"
+        print("No dependencies")
 
-    return project.build()
+    return project
 
 
 def exists(_):
@@ -868,5 +879,5 @@ def exists(_):
 
 def generate(env):
     env.AddMethod(GetLibBuilders)
-    env.AddMethod(BuildProjectLibraries)
+    env.AddMethod(ConfigureProjectLibBuilder)
     return env

platformio/builder/tools/piomisc.py

@@ -18,7 +18,7 @@ import atexit
 import re
 import sys
 from os import environ, remove, walk
-from os.path import basename, isdir, isfile, join, relpath, sep
+from os.path import basename, isdir, isfile, join, realpath, relpath, sep
 from tempfile import mkstemp
 
 from SCons.Action import Action
@@ -30,9 +30,10 @@ from platformio.managers.core import get_core_package_dir
 
 class InoToCPPConverter(object):
 
-    PROTOTYPE_RE = re.compile(r"""^(
+    PROTOTYPE_RE = re.compile(
+        r"""^(
         (?:template\<.*\>\s*)?      # template
-        ([a-z_\d]+\*?\s+){1,2}      # return type
+        ([a-z_\d\&]+\*?\s+){1,2}      # return type
         ([a-z_\d]+\s*)              # name of prototype
         \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
         )\s*\{                      # must end with {
@@ -89,8 +90,8 @@ class InoToCPPConverter(object):
         self.env.Execute(
             self.env.VerboseAction(
                 '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
-                    out_file,
-                    tmp_path), "Converting " + basename(out_file[:-4])))
+                    out_file, tmp_path),
+                "Converting " + basename(out_file[:-4])))
         atexit.register(_delete_file, tmp_path)
         return isfile(out_file)
 
@@ -163,18 +164,17 @@ class InoToCPPConverter(object):
 
         prototype_names = set([m.group(3).strip() for m in prototypes])
         split_pos = prototypes[0].start()
-        match_ptrs = re.search(self.PROTOPTRS_TPLRE %
-                               ("|".join(prototype_names)),
-                               contents[:split_pos], re.M)
+        match_ptrs = re.search(
+            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
+            contents[:split_pos], re.M)
         if match_ptrs:
             split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
 
         result = []
         result.append(contents[:split_pos].strip())
         result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
-        result.append('#line %d "%s"' %
-                      (self._get_total_lines(contents[:split_pos]),
-                       self._main_ino.replace("\\", "/")))
+        result.append('#line %d "%s"' % (self._get_total_lines(
+            contents[:split_pos]), self._main_ino.replace("\\", "/")))
         result.append(contents[split_pos:].strip())
         return "\n".join(result)
 
@@ -199,7 +199,7 @@ def _delete_file(path):
         pass
 
 
-@util.memoized
+@util.memoized()
 def _get_compiler_type(env):
     try:
         sysenv = environ.copy()
@@ -231,14 +231,25 @@ def GetActualLDScript(env):
         return None
 
     script = None
+    script_in_next = False
     for f in env.get("LINKFLAGS", []):
-        if f.startswith("-Wl,-T"):
-            script = env.subst(f[6:].replace('"', "").strip())
-            if isfile(script):
-                return script
-            path = _lookup_in_ldpath(script)
-            if path:
-                return path
+        raw_script = None
+        if f == "-T":
+            script_in_next = True
+            continue
+        elif script_in_next:
+            script_in_next = False
+            raw_script = f
+        elif f.startswith("-Wl,-T"):
+            raw_script = f[6:]
+        else:
+            continue
+        script = env.subst(raw_script.replace('"', "").strip())
+        if isfile(script):
+            return script
+        path = _lookup_in_ldpath(script)
+        if path:
+            return path
 
     if script:
         sys.stderr.write(
@@ -263,13 +274,13 @@ def VerboseAction(_, act, actstr):
 
 def PioClean(env, clean_dir):
     if not isdir(clean_dir):
-        print "Build environment is clean"
+        print("Build environment is clean")
         env.Exit(0)
     for root, _, files in walk(clean_dir):
         for file_ in files:
             remove(join(root, file_))
-            print "Removed %s" % relpath(join(root, file_))
-    print "Done cleaning"
+            print("Removed %s" % relpath(join(root, file_)))
+    print("Done cleaning")
     util.rmtree_(clean_dir)
     env.Exit(0)
 
@@ -278,8 +289,13 @@ def ProcessDebug(env):
     if not env.subst("$PIODEBUGFLAGS"):
         env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"])
     env.Append(
-        BUILD_FLAGS=env.get("PIODEBUGFLAGS", []),
-        BUILD_UNFLAGS=["-Os", "-O0", "-O1", "-O2", "-O3"])
+        PIODEBUGFLAGS=["-D__PLATFORMIO_DEBUG__"],
+        BUILD_FLAGS=env.get("PIODEBUGFLAGS", []))
+    unflags = ["-Os"]
+    for level in [0, 1, 2]:
+        for flag in ("O", "g", "ggdb"):
+            unflags.append("-%s%d" % (flag, level))
+    env.Append(BUILD_UNFLAGS=unflags)
 
 
 def ProcessTest(env):
@@ -295,25 +311,18 @@ def ProcessTest(env):
         src_filter.append("+<%s%s>" % (env['PIOTEST'], sep))
     env.Replace(PIOTEST_SRC_FILTER=src_filter)
 
-    return env.CollectBuildFiles(
-        "$BUILDTEST_DIR",
-        "$PROJECTTEST_DIR",
-        "$PIOTEST_SRC_FILTER",
-        duplicate=False)
 
-
-def GetPreExtraScripts(env):
-    return [
-        item[4:] for item in env.get("EXTRA_SCRIPTS", [])
-        if item.startswith("pre:")
-    ]
-
-
-def GetPostExtraScripts(env):
-    return [
-        item[5:] if item.startswith("post:") else item
-        for item in env.get("EXTRA_SCRIPTS", []) if not item.startswith("pre:")
-    ]
+def GetExtraScripts(env, scope):
+    items = []
+    for item in env.get("EXTRA_SCRIPTS", []):
+        if scope == "post" and ":" not in item:
+            items.append(item)
+        elif item.startswith("%s:" % scope):
+            items.append(item[len(scope) + 1:])
+    if not items:
+        return items
+    with util.cd(env.subst("$PROJECT_DIR")):
+        return [realpath(item) for item in items]
 
 
 def exists(_):
@@ -328,6 +337,5 @@ def generate(env):
     env.AddMethod(PioClean)
     env.AddMethod(ProcessDebug)
     env.AddMethod(ProcessTest)
-    env.AddMethod(GetPreExtraScripts)
-    env.AddMethod(GetPostExtraScripts)
+    env.AddMethod(GetExtraScripts)
     return env

platformio/builder/tools/pioplatform.py

@@ -14,6 +14,7 @@
 
 from __future__ import absolute_import
 
+import base64
 import sys
 from os.path import isdir, isfile, join
 
@@ -22,8 +23,10 @@ from SCons.Script import COMMAND_LINE_TARGETS
 from platformio import exception, util
 from platformio.managers.platform import PlatformFactory
 
+# pylint: disable=too-many-branches, too-many-locals
 
-@util.memoized
+
+@util.memoized()
 def initPioPlatform(name):
     return PlatformFactory.newPlatform(name)
 
@@ -66,41 +69,63 @@ def LoadPioPlatform(env, variables):
     # Ensure real platform name
     env['PIOPLATFORM'] = p.name
 
-    # Add toolchains and uploaders to $PATH
+    # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH
+    systype = util.get_systype()
     for name in installed_packages:
         type_ = p.get_package_type(name)
-        if type_ not in ("toolchain", "uploader"):
+        if type_ not in ("toolchain", "uploader", "debugger"):
             continue
-        path = p.get_package_dir(name)
-        if isdir(join(path, "bin")):
-            path = join(path, "bin")
-        env.PrependENVPath("PATH", path)
+        pkg_dir = p.get_package_dir(name)
+        env.PrependENVPath(
+            "PATH",
+            join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir)
+        if ("windows" not in systype and isdir(join(pkg_dir, "lib"))
+                and type_ != "toolchain"):
+            env.PrependENVPath(
+                "DYLD_LIBRARY_PATH"
+                if "darwin" in systype else "LD_LIBRARY_PATH",
+                join(pkg_dir, "lib"))
 
     # Platform specific LD Scripts
     if isdir(join(p.get_dir(), "ldscripts")):
         env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")])
 
     if "BOARD" not in env:
+        # handle _MCU and _F_CPU variables for AVR native
+        for key, value in variables.UnknownVariables().items():
+            if not key.startswith("BOARD_"):
+                continue
+            env.Replace(
+                **{key.upper().replace("BUILD.", ""): base64.b64decode(value)})
         return
 
+    # update board manifest with a custom data
     board_config = env.BoardConfig()
-    for k in variables.keys():
-        if k in env or \
-                not any([k.startswith("BOARD_"), k.startswith("UPLOAD_")]):
+    for key, value in variables.UnknownVariables().items():
+        if not key.startswith("BOARD_"):
             continue
-        _opt, _val = k.lower().split("_", 1)
+        board_config.update(key.lower()[6:], base64.b64decode(value))
+
+    # update default environment variables
+    for key in variables.keys():
+        if key in env or \
+                not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]):
+            continue
+        _opt, _val = key.lower().split("_", 1)
         if _opt == "board":
             _opt = "build"
         if _val in board_config.get(_opt):
-            env.Replace(**{k: board_config.get("%s.%s" % (_opt, _val))})
+            env.Replace(**{key: board_config.get("%s.%s" % (_opt, _val))})
 
     if "build.ldscript" in board_config:
         env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript"))
 
 
-def PrintConfiguration(env):  # pylint: disable=too-many-branches
-    platform_data = ["PLATFORM: %s >" % env.PioPlatform().title]
+def PrintConfiguration(env):
+    platform = env.PioPlatform()
+    platform_data = ["PLATFORM: %s >" % platform.title]
     system_data = ["SYSTEM:"]
+    configuration_data = ["CONFIGURATION:"]
     mcu = env.subst("$BOARD_MCU")
     f_cpu = env.subst("$BOARD_F_CPU")
     if mcu:
@@ -119,11 +144,13 @@ def PrintConfiguration(env):  # pylint: disable=too-many-branches
         flash = board_config.get("upload", {}).get("maximum_size")
         system_data.append("%s RAM (%s Flash)" % (util.format_filesize(ram),
                                                   util.format_filesize(flash)))
+        configuration_data.append(
+            "https://docs.platformio.org/page/boards/%s/%s.html" %
+            (platform.name, board_config.id))
 
-    if platform_data:
-        print " ".join(platform_data)
-    if system_data:
-        print " ".join(system_data)
+    for data in (configuration_data, platform_data, system_data):
+        if len(data) > 1:
+            print(" ".join(data))
 
     # Debugging
     if not debug_tools:
@@ -145,7 +172,7 @@ def PrintConfiguration(env):  # pylint: disable=too-many-branches
     if external:
         data.append("EXTERNAL(%s)" % ", ".join(sorted(external)))
 
-    print "DEBUG: %s" % " ".join(data)
+    print("DEBUG: %s" % " ".join(data))
 
 
 def exists(_):

platformio/builder/tools/pioupload.py

@@ -14,6 +14,7 @@
 
 from __future__ import absolute_import
 
+import re
 import sys
 from fnmatch import fnmatch
 from os import environ
@@ -21,10 +22,12 @@ from os.path import isfile, join
 from shutil import copyfile
 from time import sleep
 
-from SCons.Node.Alias import Alias
+from SCons.Script import ARGUMENTS
 from serial import Serial, SerialException
 
-from platformio import util
+from platformio import exception, util
+
+# pylint: disable=unused-argument
 
 
 def FlushSerialBuffer(env, port):
@@ -40,18 +43,18 @@ def FlushSerialBuffer(env, port):
 
 def TouchSerialPort(env, port, baudrate):
     port = env.subst(port)
-    print "Forcing reset using %dbps open/close on port %s" % (baudrate, port)
+    print("Forcing reset using %dbps open/close on port %s" % (baudrate, port))
     try:
         s = Serial(port=port, baudrate=baudrate)
         s.setDTR(False)
         s.close()
-    except:  # pylint: disable=W0702
+    except:  # pylint: disable=bare-except
         pass
     sleep(0.4)  # DO NOT REMOVE THAT (required by SAM-BA based boards)
 
 
 def WaitForNewSerialPort(env, before):
-    print "Waiting for the new upload port..."
+    print("Waiting for the new upload port...")
     prev_port = env.subst("$UPLOAD_PORT")
     new_port = None
     elapsed = 0
@@ -88,7 +91,7 @@ def WaitForNewSerialPort(env, before):
     return new_port
 
 
-def AutodetectUploadPort(*args, **kwargs):  # pylint: disable=unused-argument
+def AutodetectUploadPort(*args, **kwargs):
     env = args[0]
 
     def _get_pattern():
@@ -130,10 +133,12 @@ def AutodetectUploadPort(*args, **kwargs):  # pylint: disable=unused-argument
             if not _is_match_pattern(item['port']):
                 continue
             port = item['port']
-            if upload_protocol.startswith("blackmagic") \
-                    and "GDB" in item['description']:
-                return ("\\\\.\\%s" % port if "windows" in util.get_systype()
-                        and port.startswith("COM") and len(port) > 4 else port)
+            if upload_protocol.startswith("blackmagic"):
+                if "windows" in util.get_systype() and \
+                        port.startswith("COM") and len(port) > 4:
+                    port = "\\\\.\\%s" % port
+                if "GDB" in item['description']:
+                    return port
             for hwid in board_hwids:
                 hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
                 if hwid_str in item['hwid']:
@@ -141,7 +146,7 @@ def AutodetectUploadPort(*args, **kwargs):  # pylint: disable=unused-argument
         return port
 
     if "UPLOAD_PORT" in env and not _get_pattern():
-        print env.subst("Use manually specified: $UPLOAD_PORT")
+        print(env.subst("Use manually specified: $UPLOAD_PORT"))
         return
 
     if (env.subst("$UPLOAD_PROTOCOL") == "mbed"
@@ -149,19 +154,14 @@ def AutodetectUploadPort(*args, **kwargs):  # pylint: disable=unused-argument
                 and not env.subst("$UPLOAD_PROTOCOL"))):
         env.Replace(UPLOAD_PORT=_look_for_mbed_disk())
     else:
-        if ("linux" in util.get_systype() and not any([
-                isfile("/etc/udev/rules.d/99-platformio-udev.rules"),
-                isfile("/lib/udev/rules.d/99-platformio-udev.rules")
-        ])):
-            sys.stderr.write(
-                "\nWarning! Please install `99-platformio-udev.rules` and "
-                "check that your board's PID and VID are listed in the rules."
-                "\n http://docs.platformio.org/en/latest/faq.html"
-                "#platformio-udev-rules\n")
+        try:
+            util.ensure_udev_rules()
+        except exception.InvalidUdevRules as e:
+            sys.stderr.write("\n%s\n\n" % e)
         env.Replace(UPLOAD_PORT=_look_for_serial_port())
 
     if env.subst("$UPLOAD_PORT"):
-        print env.subst("Auto-detected: $UPLOAD_PORT")
+        print(env.subst("Auto-detected: $UPLOAD_PORT"))
     else:
         sys.stderr.write(
             "Error: Please specify `upload_port` for environment or use "
@@ -171,7 +171,7 @@ def AutodetectUploadPort(*args, **kwargs):  # pylint: disable=unused-argument
         env.Exit(1)
 
 
-def UploadToDisk(_, target, source, env):  # pylint: disable=W0613,W0621
+def UploadToDisk(_, target, source, env):
     assert "UPLOAD_PORT" in env
     progname = env.subst("$PROGNAME")
     for ext in ("bin", "hex"):
@@ -180,36 +180,91 @@ def UploadToDisk(_, target, source, env):  # pylint: disable=W0613,W0621
             continue
         copyfile(fpath,
                  join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)))
-    print "Firmware has been successfully uploaded.\n"\
-          "(Some boards may require manual hard reset)"
+    print("Firmware has been successfully uploaded.\n"
+          "(Some boards may require manual hard reset)")
 
 
-def CheckUploadSize(_, target, source, env):  # pylint: disable=W0613,W0621
-    if "BOARD" not in env:
-        return
-    max_size = int(env.BoardConfig().get("upload.maximum_size", 0))
-    if max_size == 0 or "SIZETOOL" not in env:
-        return
-
-    sysenv = environ.copy()
-    sysenv['PATH'] = str(env['ENV']['PATH'])
-    cmd = [
-        env.subst("$SIZETOOL"), "-B",
-        str(source[0] if isinstance(target[0], Alias) else target[0])
+def CheckUploadSize(_, target, source, env):
+    check_conditions = [
+        env.get("BOARD"),
+        env.get("SIZETOOL") or env.get("SIZECHECKCMD")
     ]
-    result = util.exec_command(cmd, env=sysenv)
-    if result['returncode'] != 0:
+    if not all(check_conditions):
+        return
+    program_max_size = int(env.BoardConfig().get("upload.maximum_size", 0))
+    data_max_size = int(env.BoardConfig().get("upload.maximum_ram_size", 0))
+    if program_max_size == 0:
         return
-    print result['out'].strip()
 
-    line = result['out'].strip().splitlines()[1]
-    values = [v.strip() for v in line.split("\t")]
-    used_size = int(values[0]) + int(values[1])
+    def _configure_defaults():
+        env.Replace(
+            SIZECHECKCMD="$SIZETOOL -B -d $SOURCES",
+            SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s",
+            SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+")
 
-    if used_size > max_size:
+    def _get_size_output():
+        cmd = env.get("SIZECHECKCMD")
+        if not cmd:
+            return None
+        if not isinstance(cmd, list):
+            cmd = cmd.split()
+        cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
+        sysenv = environ.copy()
+        sysenv['PATH'] = str(env['ENV']['PATH'])
+        result = util.exec_command(env.subst(cmd), env=sysenv)
+        if result['returncode'] != 0:
+            return None
+        return result['out'].strip()
+
+    def _calculate_size(output, pattern):
+        if not output or not pattern:
+            return -1
+        size = 0
+        regexp = re.compile(pattern)
+        for line in output.split("\n"):
+            line = line.strip()
+            if not line:
+                continue
+            match = regexp.search(line)
+            if not match:
+                continue
+            size += sum(int(value) for value in match.groups())
+        return size
+
+    def _format_availale_bytes(value, total):
+        percent_raw = float(value) / float(total)
+        blocks_per_progress = 10
+        used_blocks = int(round(blocks_per_progress * percent_raw))
+        if used_blocks > blocks_per_progress:
+            used_blocks = blocks_per_progress
+        return "[{:{}}] {: 6.1%} (used {:d} bytes from {:d} bytes)".format(
+            "=" * used_blocks, blocks_per_progress, percent_raw, value, total)
+
+    if not env.get("SIZECHECKCMD") and not env.get("SIZEPROGREGEXP"):
+        _configure_defaults()
+    output = _get_size_output()
+    program_size = _calculate_size(output, env.get("SIZEPROGREGEXP"))
+    data_size = _calculate_size(output, env.get("SIZEDATAREGEXP"))
+
+    print("Memory Usage -> http://bit.ly/pio-memory-usage")
+    if data_max_size and data_size > -1:
+        print("DATA:    %s" % _format_availale_bytes(data_size, data_max_size))
+    if program_size > -1:
+        print("PROGRAM: %s" % _format_availale_bytes(program_size,
+                                                     program_max_size))
+    if int(ARGUMENTS.get("PIOVERBOSE", 0)):
+        print(output)
+
+    # raise error
+    # if data_max_size and data_size > data_max_size:
+    #     sys.stderr.write(
+    #         "Error: The data size (%d bytes) is greater "
+    #         "than maximum allowed (%s bytes)\n" % (data_size, data_max_size))
+    #     env.Exit(1)
+    if program_size > program_max_size:
         sys.stderr.write("Error: The program size (%d bytes) is greater "
-                         "than maximum allowed (%s bytes)\n" % (used_size,
-                                                                max_size))
+                         "than maximum allowed (%s bytes)\n" %
+                         (program_size, program_max_size))
         env.Exit(1)
 
 
@@ -220,9 +275,9 @@ def PrintUploadInfo(env):
         available.extend(env.BoardConfig().get("upload", {}).get(
             "protocols", []))
     if available:
-        print "AVAILABLE: %s" % ", ".join(sorted(available))
+        print("AVAILABLE: %s" % ", ".join(sorted(set(available))))
     if configured:
-        print "CURRENT: upload_protocol = %s" % configured
+        print("CURRENT: upload_protocol = %s" % configured)
 
 
 def exists(_):

platformio/builder/tools/platformio.py

@@ -20,9 +20,9 @@ from glob import glob
 from os import sep, walk
 from os.path import basename, dirname, isdir, join, realpath
 
-from SCons import Action, Builder, Util
+from SCons import Builder, Util
 from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild,
-                          DefaultEnvironment, SConscript)
+                          DefaultEnvironment, Export, SConscript)
 
 from platformio.util import glob_escape, pioversion_to_intstr
 
@@ -30,18 +30,54 @@ SRC_HEADER_EXT = ["h", "hpp"]
 SRC_C_EXT = ["c", "cc", "cpp"]
 SRC_BUILD_EXT = SRC_C_EXT + ["S", "spp", "SPP", "sx", "s", "asm", "ASM"]
 SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep]
+SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>")
 
 
 def scons_patched_match_splitext(path, suffixes=None):
-    """
-    Patch SCons Builder, append $OBJSUFFIX to the end of each target
-    """
+    """Patch SCons Builder, append $OBJSUFFIX to the end of each target"""
     tokens = Util.splitext(path)
     if suffixes and tokens[1] and tokens[1] in suffixes:
         return (path, tokens[1])
     return tokens
 
 
+def _build_project_deps(env):
+    project_lib_builder = env.ConfigureProjectLibBuilder()
+
+    # append project libs to the beginning of list
+    env.Prepend(LIBS=project_lib_builder.build())
+    # append extra linker related options from libs
+    env.AppendUnique(
+        **{
+            key: project_lib_builder.env.get(key)
+            for key in ("LIBS", "LIBPATH", "LINKFLAGS")
+            if project_lib_builder.env.get(key)
+        })
+
+    projenv = env.Clone()
+
+    # CPPPATH from dependencies
+    projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH"))
+    # extra build flags from `platformio.ini`
+    projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
+
+    is_test = "__test" in COMMAND_LINE_TARGETS
+    if is_test:
+        projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR",
+                             "$PIOTEST_SRC_FILTER")
+    if not is_test or env.get("TEST_BUILD_PROJECT_SRC") == "true":
+        projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR",
+                             env.get("SRC_FILTER"))
+
+    if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
+        sys.stderr.write(
+            "Error: Nothing to build. Please put your source code files "
+            "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR"))
+        env.Exit(1)
+
+    Export("projenv")
+
+
 def BuildProgram(env):
 
     def _append_pio_macros():
@@ -57,14 +93,13 @@ def BuildProgram(env):
     if not Util.case_sensitive_suffixes(".s", ".S"):
         env.Replace(AS="$CC", ASCOM="$ASPPCOM")
 
-    if "__debug" in COMMAND_LINE_TARGETS:
+    if set(["__debug", "debug"]) & set(COMMAND_LINE_TARGETS):
         env.ProcessDebug()
 
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
         env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
-    # remove base flags
-    env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
+
     # apply user flags
     env.ProcessFlags(env.get("BUILD_FLAGS"))
 
@@ -74,83 +109,84 @@ def BuildProgram(env):
     # restore PIO macros if it was deleted by framework
     _append_pio_macros()
 
-    # build dependent libs; place them before built-in libs
-    env.Prepend(LIBS=env.BuildProjectLibraries())
+    # remove specified flags
+    env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
-    # append specified LD_SCRIPT
-    if ("LDSCRIPT_PATH" in env
+    if "__test" in COMMAND_LINE_TARGETS:
+        env.ProcessTest()
+
+    # build project with dependencies
+    _build_project_deps(env)
+
+    # append into the beginning a main LD script
+    if (env.get("LDSCRIPT_PATH")
             and not any("-Wl,-T" in f for f in env['LINKFLAGS'])):
-        env.Append(LINKFLAGS=['-Wl,-T"$LDSCRIPT_PATH"'])
+        env.Prepend(LINKFLAGS=["-T", "$LDSCRIPT_PATH"])
 
     # enable "cyclic reference" for linker
     if env.get("LIBS") and env.GetCompilerType() == "gcc":
         env.Prepend(_LIBFLAGS="-Wl,--start-group ")
         env.Append(_LIBFLAGS=" -Wl,--end-group")
 
-    # Handle SRC_BUILD_FLAGS
-    env.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
-
-    env.Append(
-        LIBPATH=["$BUILD_DIR"],
-        PIOBUILDFILES=env.CollectBuildFiles(
-            "$BUILDSRC_DIR",
-            "$PROJECTSRC_DIR",
-            src_filter=env.get("SRC_FILTER"),
-            duplicate=False))
-
-    if "__test" in COMMAND_LINE_TARGETS:
-        env.Append(PIOBUILDFILES=env.ProcessTest())
-
-    if not env['PIOBUILDFILES'] and not COMMAND_LINE_TARGETS:
-        sys.stderr.write(
-            "Error: Nothing to build. Please put your source code files "
-            "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR"))
-        env.Exit(1)
-
     program = env.Program(
         join("$BUILD_DIR", env.subst("$PROGNAME")), env['PIOBUILDFILES'])
+    env.Replace(PIOMAINPROG=program)
 
-    checksize_action = Action.Action(env.CheckUploadSize,
-                                     "Checking program size")
-    AlwaysBuild(env.Alias("checkprogsize", program, checksize_action))
-    if set(["upload", "program"]) & set(COMMAND_LINE_TARGETS):
-        env.AddPostAction(program, checksize_action)
+    AlwaysBuild(
+        env.Alias(
+            "checkprogsize", program,
+            env.VerboseAction(env.CheckUploadSize,
+                              "Checking size $PIOMAINPROG")))
 
     return program
 
 
+def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
+    if not isinstance(flags, list):
+        flags = [flags]
+    result = {}
+    for raw in flags:
+        for key, value in env.ParseFlags(str(raw)).items():
+            if key not in result:
+                result[key] = []
+            result[key].extend(value)
+
+    cppdefines = []
+    for item in result['CPPDEFINES']:
+        if not Util.is_Sequence(item):
+            cppdefines.append(item)
+            continue
+        name, value = item[:2]
+        if '\"' in value:
+            value = value.replace('\"', '\\\"')
+        elif value.isdigit():
+            value = int(value)
+        elif value.replace(".", "", 1).isdigit():
+            value = float(value)
+        cppdefines.append((name, value))
+    result['CPPDEFINES'] = cppdefines
+
+    # fix relative CPPPATH & LIBPATH
+    for k in ("CPPPATH", "LIBPATH"):
+        for i, p in enumerate(result.get(k, [])):
+            if isdir(p):
+                result[k][i] = realpath(p)
+
+    # fix relative path for "-include"
+    for i, f in enumerate(result.get("CCFLAGS", [])):
+        if isinstance(f, tuple) and f[0] == "-include":
+            result['CCFLAGS'][i] = (f[0], env.File(realpath(f[1].get_path())))
+
+    return result
+
+
 def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
     if not flags:
         return
-    if isinstance(flags, list):
-        flags = " ".join(flags)
-    parsed_flags = env.ParseFlags(str(flags))
-    for flag in parsed_flags.pop("CPPDEFINES"):
-        if not Util.is_Sequence(flag):
-            env.Append(CPPDEFINES=flag)
-            continue
-        _key, _value = flag[:2]
-        if '\"' in _value:
-            _value = _value.replace('\"', '\\\"')
-        elif _value.isdigit():
-            _value = int(_value)
-        elif _value.replace(".", "", 1).isdigit():
-            _value = float(_value)
-        env.Append(CPPDEFINES=(_key, _value))
-    env.Append(**parsed_flags)
-
-    # fix relative CPPPATH & LIBPATH
-    for k in ("CPPPATH", "LIBPATH"):
-        for i, p in enumerate(env.get(k, [])):
-            if isdir(p):
-                env[k][i] = realpath(p)
-    # fix relative path for "-include"
-    for i, f in enumerate(env.get("CCFLAGS", [])):
-        if isinstance(f, tuple) and f[0] == "-include":
-            env['CCFLAGS'][i] = (f[0], env.File(realpath(f[1].get_path())))
+    env.Append(**env.ParseFlagsExtended(flags))
 
     # Cancel any previous definition of name, either built in or
-    # provided with a -D option // Issue #191
+    # provided with a -U option // Issue #191
     undefines = [
         u for u in env.get("CCFLAGS", [])
         if isinstance(u, basestring) and u.startswith("-U")
@@ -164,19 +200,27 @@ def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
 def ProcessUnFlags(env, flags):
     if not flags:
         return
-    if isinstance(flags, list):
-        flags = " ".join(flags)
-    parsed_flags = env.ParseFlags(str(flags))
-    all_flags = []
-    for items in parsed_flags.values():
-        all_flags.extend(items)
-    all_flags = set(all_flags)
+    parsed = env.ParseFlagsExtended(flags)
 
-    for key in parsed_flags:
-        cur_flags = set(env.Flatten(env.get(key, [])))
-        for item in cur_flags & all_flags:
-            while item in env[key]:
-                env[key].remove(item)
+    # get all flags and copy them to each "*FLAGS" variable
+    all_flags = []
+    for key, unflags in parsed.items():
+        if key.endswith("FLAGS"):
+            all_flags.extend(unflags)
+    for key, unflags in parsed.items():
+        if key.endswith("FLAGS"):
+            parsed[key].extend(all_flags)
+
+    for key, unflags in parsed.items():
+        for unflag in unflags:
+            for current in env.get(key, []):
+                conditions = [
+                    unflag == current,
+                    isinstance(current, (tuple, list))
+                    and unflag[0] == current[0]
+                ]
+                if any(conditions):
+                    env[key].remove(current)
 
 
 def IsFileWithExt(env, file_, ext):  # pylint: disable=W0613
@@ -190,8 +234,6 @@ def IsFileWithExt(env, file_, ext):  # pylint: disable=W0613
 
 def MatchSourceFiles(env, src_dir, src_filter=None):
 
-    SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>")
-
     def _append_build_item(items, item, src_dir):
         if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT):
             items.add(item.replace(src_dir + sep, ""))
@@ -270,10 +312,11 @@ def BuildFrameworks(env, frameworks):
         if f in ("arduino", "energia"):
             # Arduino IDE appends .o the end of filename
             Builder.match_splitext = scons_patched_match_splitext
-            env.ConvertInoToCpp()
+            if "nobuild" not in COMMAND_LINE_TARGETS:
+                env.ConvertInoToCpp()
 
         if f in board_frameworks:
-            SConscript(env.GetFrameworkScript(f))
+            SConscript(env.GetFrameworkScript(f), exports="env")
         else:
             sys.stderr.write(
                 "Error: This board doesn't support %s framework!\n" % f)
@@ -281,15 +324,16 @@ def BuildFrameworks(env, frameworks):
 
 
 def BuildLibrary(env, variant_dir, src_dir, src_filter=None):
-    lib = env.Clone()
-    return lib.StaticLibrary(
-        lib.subst(variant_dir),
-        lib.CollectBuildFiles(variant_dir, src_dir, src_filter))
+    env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
+    return env.StaticLibrary(
+        env.subst(variant_dir),
+        env.CollectBuildFiles(variant_dir, src_dir, src_filter))
 
 
 def BuildSources(env, variant_dir, src_dir, src_filter=None):
-    DefaultEnvironment().Append(PIOBUILDFILES=env.Clone().CollectBuildFiles(
-        variant_dir, src_dir, src_filter))
+    nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter)
+    DefaultEnvironment().Append(
+        PIOBUILDFILES=[env.Object(node) for node in nodes])
 
 
 def exists(_):
@@ -298,6 +342,7 @@ def exists(_):
 
 def generate(env):
     env.AddMethod(BuildProgram)
+    env.AddMethod(ParseFlagsExtended)
     env.AddMethod(ProcessFlags)
     env.AddMethod(ProcessUnFlags)
     env.AddMethod(IsFileWithExt)

platformio/commands/ci.py

@@ -86,7 +86,7 @@ def cli(  # pylint: disable=too-many-arguments
         app.set_session_var("force_option", True)
         _clean_dir(build_dir)
 
-        for dir_name, patterns in dict(lib=lib, src=src).iteritems():
+        for dir_name, patterns in dict(lib=lib, src=src).items():
             if not patterns:
                 continue
             contents = []

platformio/commands/device.py

@@ -55,7 +55,7 @@ def device_list(  # pylint: disable=too-many-branches
         "mdns": "Multicast DNS Services"
     }
 
-    for key, value in data.iteritems():
+    for key, value in data.items():
         if not single_key:
             click.secho(titles[key], bold=True)
             click.echo("=" * len(titles[key]))
@@ -85,7 +85,7 @@ def device_list(  # pylint: disable=too-many-branches
                 if item['properties']:
                     click.echo("Properties: %s" % ("; ".join([
                         "%s=%s" % (k, v)
-                        for k, v in item['properties'].iteritems()
+                        for k, v in item['properties'].items()
                     ])))
                 click.echo("")
 
@@ -165,8 +165,10 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
                                               kwargs['environment'])
         monitor_options = {k: v for k, v in project_options or []}
         if monitor_options:
-            for k in ("port", "baud", "rts", "dtr"):
+            for k in ("port", "baud", "speed", "rts", "dtr"):
                 k2 = "monitor_%s" % k
+                if k == "speed":
+                    k = "baud"
                 if kwargs[k] is None and k2 in monitor_options:
                     kwargs[k] = monitor_options[k2]
                     if k != "port":
@@ -180,7 +182,7 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
             kwargs['port'] = ports[0]['port']
 
     sys.argv = ["monitor"]
-    for k, v in kwargs.iteritems():
+    for k, v in kwargs.items():
         if k in ("port", "baud", "rts", "dtr", "environment", "project_dir"):
             continue
         k = "--" + k.replace("_", "-")

platformio/commands/home.py

@@ -15,7 +15,6 @@
 import sys
 
 import click
-import requests
 
 from platformio.managers.core import pioplus_call
 
@@ -30,13 +29,3 @@ from platformio.managers.core import pioplus_call
 @click.option("--no-open", is_flag=True)
 def cli(*args, **kwargs):  # pylint: disable=unused-argument
     pioplus_call(sys.argv[1:])
-
-
-def shutdown_servers():
-    port = 8010
-    while port < 9000:
-        try:
-            requests.get("http://127.0.0.1:%d?__shutdown__=1" % port)
-            port += 1
-        except:  # pylint: disable=bare-except
-            return

platformio/commands/init.py

@@ -73,22 +73,21 @@ def cli(
             click.secho(
                 "\nThe current working directory", fg="yellow", nl=False)
             click.secho(" %s " % project_dir, fg="cyan", nl=False)
-            click.secho(
-                "will be used for project.\n"
-                "You can specify another project directory via\n"
-                "`platformio init -d %PATH_TO_THE_PROJECT_DIR%` command.",
-                fg="yellow")
+            click.secho("will be used for the project.", fg="yellow")
             click.echo("")
 
         click.echo("The next files/directories have been created in %s" %
                    click.style(project_dir, fg="cyan"))
-        click.echo("%s - Project Configuration File" % click.style(
-            "platformio.ini", fg="cyan"))
-        click.echo(
-            "%s - Put your source files here" % click.style("src", fg="cyan"))
+        click.echo("%s - Put project header files here" % click.style(
+            "include", fg="cyan"))
         click.echo("%s - Put here project specific (private) libraries" %
                    click.style("lib", fg="cyan"))
+        click.echo("%s - Put project source files here" % click.style(
+            "src", fg="cyan"))
+        click.echo("%s - Project Configuration File" % click.style(
+            "platformio.ini", fg="cyan"))
 
+    is_new_project = util.is_platformio_project(project_dir)
     init_base_project(project_dir)
 
     if board:
@@ -102,16 +101,28 @@ def cli(
         pg = ProjectGenerator(project_dir, ide, env_name)
         pg.generate()
 
-    if not silent:
+    if is_new_project:
+        init_ci_conf(project_dir)
+        init_cvs_ignore(project_dir)
+
+    if silent:
+        return
+
+    if ide:
         click.secho(
-            "\nProject has been successfully initialized!\nUseful commands:\n"
-            "`platformio run` - process/build project from the current "
-            "directory\n"
-            "`platformio run --target upload` or `platformio run -t upload` "
-            "- upload firmware to embedded board\n"
-            "`platformio run --target clean` - clean project (remove compiled "
-            "files)\n"
-            "`platformio run --help` - additional information",
+            "\nProject has been successfully %s including configuration files "
+            "for `%s` IDE." % ("initialized" if is_new_project else "updated",
+                               ide),
+            fg="green")
+    else:
+        click.secho(
+            "\nProject has been successfully %s! Useful commands:\n"
+            "`pio run` - process/build project from the current directory\n"
+            "`pio run --target upload` or `pio run -t upload` "
+            "- upload firmware to a target\n"
+            "`pio run --target clean` - clean project (remove compiled files)"
+            "\n`pio run --help` - additional information" %
+            ("initialized" if is_new_project else "updated"),
             fg="green")
 
 
@@ -134,76 +145,147 @@ def get_best_envname(project_dir, boards=None):
 
 
 def init_base_project(project_dir):
-    if not util.is_platformio_project(project_dir):
-        copyfile(
-            join(util.get_source_dir(), "projectconftpl.ini"),
-            join(project_dir, "platformio.ini"))
+    if util.is_platformio_project(project_dir):
+        return
 
-    lib_dir = join(project_dir, "lib")
-    src_dir = join(project_dir, "src")
-    config = util.load_project_config(project_dir)
-    if config.has_option("platformio", "src_dir"):
-        src_dir = join(project_dir, config.get("platformio", "src_dir"))
+    copyfile(
+        join(util.get_source_dir(), "projectconftpl.ini"),
+        join(project_dir, "platformio.ini"))
 
-    for d in (src_dir, lib_dir):
-        if not isdir(d):
-            makedirs(d)
+    with util.cd(project_dir):
+        dir_to_readme = [
+            (util.get_projectsrc_dir(), None),
+            (util.get_projectinclude_dir(), init_include_readme),
+            (util.get_projectlib_dir(), init_lib_readme),
+            (util.get_projecttest_dir(), init_test_readme),
+        ]
+        for (path, cb) in dir_to_readme:
+            if isdir(path):
+                continue
+            makedirs(path)
+            if cb:
+                cb(path)
 
-    init_lib_readme(lib_dir)
-    init_ci_conf(project_dir)
-    init_cvs_ignore(project_dir)
+
+def init_include_readme(include_dir):
+    with open(join(include_dir, "README"), "w") as f:
+        f.write("""
+This directory is intended for project header files.
+
+A header file is a file containing C declarations and macro definitions
+to be shared between several project source files. You request the use of a
+header file in your project source file (C, C++, etc) located in `src` folder
+by including it, with the C preprocessing directive `#include'.
+
+```src/main.c
+
+#include "header.h"
+
+int main (void)
+{
+ ...
+}
+```
+
+Including a header file produces the same results as copying the header file
+into each source file that needs it. Such copying would be time-consuming
+and error-prone. With a header file, the related declarations appear
+in only one place. If they need to be changed, they can be changed in one
+place, and programs that include the header file will automatically use the
+new version when next recompiled. The header file eliminates the labor of
+finding and changing all the copies as well as the risk that a failure to
+find one copy will result in inconsistencies within a program.
+
+In C, the usual convention is to give header files names that end with `.h'.
+It is most portable to use only letters, digits, dashes, and underscores in
+header file names, and at most one dot.
+
+Read more about using header files in official GCC documentation:
+
+* Include Syntax
+* Include Operation
+* Once-Only Headers
+* Computed Includes
+
+https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html
+""")
 
 
 def init_lib_readme(lib_dir):
-    if isfile(join(lib_dir, "readme.txt")):
-        return
-    with open(join(lib_dir, "readme.txt"), "w") as f:
+    with open(join(lib_dir, "README"), "w") as f:
         f.write("""
-This directory is intended for the project specific (private) libraries.
-PlatformIO will compile them to static libraries and link to executable file.
+This directory is intended for project specific (private) libraries.
+PlatformIO will compile them to static libraries and link into executable file.
 
-The source code of each library should be placed in separate directory, like
-"lib/private_lib/[here are source files]".
+The source code of each library should be placed in a an own separate directory
+("lib/your_library_name/[here are source files]").
 
-For example, see how can be organized `Foo` and `Bar` libraries:
+For example, see a structure of the following two libraries `Foo` and `Bar`:
 
 |--lib
+|  |
 |  |--Bar
 |  |  |--docs
 |  |  |--examples
 |  |  |--src
 |  |     |- Bar.c
 |  |     |- Bar.h
+|  |  |- library.json (optional, custom build options, etc) https://docs.platformio.org/page/librarymanager/config.html
+|  |
 |  |--Foo
 |  |  |- Foo.c
 |  |  |- Foo.h
-|  |- readme.txt --> THIS FILE
+|  |
+|  |- README --> THIS FILE
+|
 |- platformio.ini
 |--src
    |- main.c
 
-Then in `src/main.c` you should use:
-
+and a contents of `src/main.c`:
+```
 #include <Foo.h>
 #include <Bar.h>
 
-// rest H/C/CPP code
+int main (void)
+{
+  ...
+}
 
-PlatformIO will find your libraries automatically, configure preprocessor's
-include paths and build them.
+```
+
+PlatformIO Library Dependency Finder will find automatically dependent
+libraries scanning project source files.
 
 More information about PlatformIO Library Dependency Finder
-- http://docs.platformio.org/page/librarymanager/ldf.html
+- https://docs.platformio.org/page/librarymanager/ldf.html
+""")
+
+
+def init_test_readme(test_dir):
+    with open(join(test_dir, "README"), "w") as f:
+        f.write("""
+This directory is intended for PIO Unit Testing and project tests.
+
+Unit Testing is a software testing method by which individual units of
+source code, sets of one or more MCU program modules together with associated
+control data, usage procedures, and operating procedures, are tested to
+determine whether they are fit for use. Unit testing finds problems early
+in the development cycle.
+
+More information about PIO Unit Testing:
+- https://docs.platformio.org/page/plus/unit-testing.html
 """)
 
 
 def init_ci_conf(project_dir):
-    if isfile(join(project_dir, ".travis.yml")):
+    conf_path = join(project_dir, ".travis.yml")
+    if isfile(conf_path):
         return
-    with open(join(project_dir, ".travis.yml"), "w") as f:
+    with open(conf_path, "w") as f:
         f.write("""# Continuous Integration (CI) is the practice, in software
 # engineering, of merging all developer working copies with a shared mainline
-# several times a day < http://docs.platformio.org/page/ci/index.html >
+# several times a day < https://docs.platformio.org/page/ci/index.html >
 #
 # Documentation:
 #
@@ -211,13 +293,13 @@ def init_ci_conf(project_dir):
 #   < https://docs.travis-ci.com/user/integration/platformio/ >
 #
 # * PlatformIO integration with Travis CI
-#   < http://docs.platformio.org/page/ci/travis.html >
+#   < https://docs.platformio.org/page/ci/travis.html >
 #
 # * User Guide for `platformio ci` command
-#   < http://docs.platformio.org/page/userguide/cmd_ci.html >
+#   < https://docs.platformio.org/page/userguide/cmd_ci.html >
 #
 #
-# Please choice one of the following templates (proposed below) and uncomment
+# Please choose one of the following templates (proposed below) and uncomment
 # it (remove "# " before each line) or use own configuration according to the
 # Travis CI documentation (see above).
 #
@@ -245,7 +327,7 @@ def init_ci_conf(project_dir):
 
 
 #
-# Template #2: The project is intended to by used as a library with examples
+# Template #2: The project is intended to be used as a library with examples.
 #
 
 # language: python
@@ -272,23 +354,11 @@ def init_ci_conf(project_dir):
 
 
 def init_cvs_ignore(project_dir):
-    ignore_path = join(project_dir, ".gitignore")
-    default = [".pioenvs\n", ".piolibdeps\n"]
-    current = []
-    modified = False
-    if isfile(ignore_path):
-        with open(ignore_path) as fp:
-            current = fp.readlines()
-        if current and not current[-1].endswith("\n"):
-            current[-1] += "\n"
-    for d in default:
-        if d not in current:
-            modified = True
-            current.append(d)
-    if not modified:
+    conf_path = join(project_dir, ".gitignore")
+    if isfile(conf_path):
         return
-    with open(ignore_path, "w") as fp:
-        fp.writelines(current)
+    with open(conf_path, "w") as fp:
+        fp.writelines([".pio\n", ".pioenvs\n", ".piolibdeps\n"])
 
 
 def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix,

platformio/commands/lib.py

@@ -203,7 +203,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
     if not isinstance(query, list):
         query = list(query)
 
-    for key, values in filters.iteritems():
+    for key, values in filters.items():
         for value in values:
             query.append('%s:"%s"' % (key, value))
 
@@ -228,7 +228,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         click.echo("For more examples and advanced search syntax, "
                    "please use documentation:")
         click.secho(
-            "http://docs.platformio.org/page/userguide/lib/cmd_search.html\n",
+            "https://docs.platformio.org/page/userguide/lib/cmd_search.html\n",
             fg="cyan")
         return
 
@@ -255,9 +255,10 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         elif not click.confirm("Show next libraries?"):
             break
         result = get_api_result(
-            "/v2/lib/search",
-            {"query": " ".join(query),
-             "page": int(result['page']) + 1},
+            "/v2/lib/search", {
+                "query": " ".join(query),
+                "page": int(result['page']) + 1
+            },
             cache_valid="1d")
 
 
@@ -306,13 +307,12 @@ def lib_builtin(storage, json_output):
 def lib_show(library, json_output):
     lm = LibraryManager()
     name, requirements, _ = lm.parse_pkg_uri(library)
-    lib_id = lm.search_lib_id(
-        {
-            "name": name,
-            "requirements": requirements
-        },
-        silent=json_output,
-        interactive=not json_output)
+    lib_id = lm.search_lib_id({
+        "name": name,
+        "requirements": requirements
+    },
+                              silent=json_output,
+                              interactive=not json_output)
     lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
     if json_output:
         return click.echo(json.dumps(lib))
@@ -422,24 +422,24 @@ def lib_stats(json_output):
         click.echo("-" * terminal_width)
 
     def _print_lib_item(item):
-        click.echo((printitemdate_tpl
-                    if "date" in item else printitem_tpl).format(
-                        name=click.style(item['name'], fg="cyan"),
-                        date=str(
-                            time.strftime("%c", util.parse_date(item['date']))
-                            if "date" in item else ""),
-                        url=click.style(
-                            "https://platformio.org/lib/show/%s/%s" %
-                            (item['id'], quote(item['name'])),
-                            fg="blue")))
+        date = str(
+            time.strftime("%c", util.parse_date(item['date'])) if "date" in
+            item else "")
+        url = click.style(
+            "https://platformio.org/lib/show/%s/%s" % (item['id'],
+                                                       quote(item['name'])),
+            fg="blue")
+        click.echo(
+            (printitemdate_tpl if "date" in item else printitem_tpl).format(
+                name=click.style(item['name'], fg="cyan"), date=date, url=url))
 
     def _print_tag_item(name):
         click.echo(
             printitem_tpl.format(
                 name=click.style(name, fg="cyan"),
                 url=click.style(
-                    "https://platformio.org/lib/search?query=" +
-                    quote("keyword:%s" % name),
+                    "https://platformio.org/lib/search?query=" + quote(
+                        "keyword:%s" % name),
                     fg="blue")))
 
     for key in ("updated", "added"):

platformio/commands/platform.py

@@ -85,6 +85,7 @@ def _get_installed_platform_data(platform,
         homepage=p.homepage,
         repository=p.repository_url,
         url=p.vendor_url,
+        docs=p.docs_url,
         license=p.license,
         forDesktop=not p.is_embedded(),
         frameworks=sorted(p.frameworks.keys() if p.frameworks else []),
@@ -364,8 +365,8 @@ def platform_update(platforms, only_packages, only_check, json_output):
             if not pkg_dir:
                 continue
             latest = pm.outdated(pkg_dir, requirements)
-            if (not latest and not PlatformFactory.newPlatform(pkg_dir)
-                    .are_outdated_packages()):
+            if (not latest and not PlatformFactory.newPlatform(pkg_dir).
+                    are_outdated_packages()):
                 continue
             data = _get_installed_platform_data(
                 pkg_dir, with_boards=False, expose_packages=False)

platformio/commands/run.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from datetime import datetime
 from hashlib import sha1
 from os import getcwd, makedirs, walk
 from os.path import getmtime, isdir, isfile, join
@@ -109,7 +108,9 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
             results.append(result)
             if result[1] and "monitor" in ep.get_build_targets() and \
                     "nobuild" not in ep.get_build_targets():
-                ctx.invoke(cmd_device_monitor)
+                ctx.invoke(
+                    cmd_device_monitor,
+                    environment=environment[0] if environment else None)
 
         found_error = any(status is False for (_, status) in results)
 
@@ -126,40 +127,45 @@ class EnvironmentProcessor(object):
 
     DEFAULT_DUMP_OPTIONS = ("platform", "framework", "board")
 
-    KNOWN_PLATFORMIO_OPTIONS = ("env_default", "home_dir", "lib_dir",
-                                "libdeps_dir", "include_dir", "src_dir",
-                                "build_dir", "data_dir", "test_dir",
-                                "boards_dir", "lib_extra_dirs")
+    KNOWN_PLATFORMIO_OPTIONS = [
+        "description", "env_default", "home_dir", "lib_dir", "libdeps_dir",
+        "include_dir", "src_dir", "build_dir", "data_dir", "test_dir",
+        "boards_dir", "lib_extra_dirs"
+    ]
 
-    KNOWN_ENV_OPTIONS = ("platform", "framework", "board", "board_mcu",
-                         "board_f_cpu", "board_f_flash", "board_flash_mode",
-                         "build_flags", "src_build_flags", "build_unflags",
-                         "src_filter", "extra_scripts", "targets",
-                         "upload_port", "upload_protocol", "upload_speed",
-                         "upload_flags", "upload_resetmethod", "lib_deps",
-                         "lib_ignore", "lib_extra_dirs", "lib_ldf_mode",
-                         "lib_compat_mode", "lib_archive", "piotest",
-                         "test_transport", "test_filter", "test_ignore",
-                         "test_port", "test_speed", "debug_tool", "debug_port",
-                         "debug_init_cmds", "debug_extra_cmds", "debug_server",
-                         "debug_init_break", "debug_load_cmd",
-                         "debug_load_mode", "monitor_port", "monitor_baud",
-                         "monitor_rts", "monitor_dtr")
+    KNOWN_ENV_OPTIONS = [
+        "platform", "framework", "board", "build_flags", "src_build_flags",
+        "build_unflags", "src_filter", "extra_scripts", "targets",
+        "upload_port", "upload_protocol", "upload_speed", "upload_flags",
+        "upload_resetmethod", "lib_deps", "lib_ignore", "lib_extra_dirs",
+        "lib_ldf_mode", "lib_compat_mode", "lib_archive", "piotest",
+        "test_transport", "test_filter", "test_ignore", "test_port",
+        "test_speed", "test_build_project_src", "debug_tool", "debug_port",
+        "debug_init_cmds", "debug_extra_cmds", "debug_server",
+        "debug_init_break", "debug_load_cmd", "debug_load_mode",
+        "debug_svd_path", "monitor_port", "monitor_speed", "monitor_rts",
+        "monitor_dtr"
+    ]
 
-    IGNORE_BUILD_OPTIONS = ("test_transport", "test_filter", "test_ignore",
-                            "test_port", "test_speed", "debug_port",
-                            "debug_init_cmds", "debug_extra_cmds",
-                            "debug_server", "debug_init_break",
-                            "debug_load_cmd", "debug_load_mode",
-                            "monitor_port", "monitor_baud", "monitor_rts",
-                            "monitor_dtr")
+    IGNORE_BUILD_OPTIONS = [
+        "test_transport", "test_filter", "test_ignore", "test_port",
+        "test_speed", "debug_port", "debug_init_cmds", "debug_extra_cmds",
+        "debug_server", "debug_init_break", "debug_load_cmd",
+        "debug_load_mode", "monitor_port", "monitor_speed", "monitor_rts",
+        "monitor_dtr"
+    ]
 
     REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"}
 
     RENAMED_OPTIONS = {
         "lib_use": "lib_deps",
         "lib_force": "lib_deps",
-        "extra_script": "extra_scripts"
+        "extra_script": "extra_scripts",
+        "monitor_baud": "monitor_speed",
+        "board_mcu": "board_build.mcu",
+        "board_f_cpu": "board_build.f_cpu",
+        "board_f_flash": "board_build.f_flash",
+        "board_flash_mode": "board_build.flash_mode"
     }
 
     RENAMED_PLATFORMS = {"espressif": "espressif8266"}
@@ -193,10 +199,8 @@ class EnvironmentProcessor(object):
                     "%s: %s" % (k, ", ".join(util.parse_conf_multi_values(v))))
 
         if not self.silent:
-            click.echo("[%s] Processing %s (%s)" %
-                       (datetime.now().strftime("%c"),
-                        click.style(self.name, fg="cyan", bold=True),
-                        "; ".join(env_dump)))
+            click.echo("Processing %s (%s)" % (click.style(
+                self.name, fg="cyan", bold=True), "; ".join(env_dump)))
             click.secho("-" * terminal_width, bold=True)
 
         self.options = self._validate_options(self.options)
@@ -208,10 +212,10 @@ class EnvironmentProcessor(object):
 
         if is_error or "piotest_processor" not in self.cmd_ctx.meta:
             print_header(
-                "[%s] Took %.2f seconds" %
-                ((click.style("ERROR", fg="red", bold=True)
-                  if is_error else click.style(
-                      "SUCCESS", fg="green", bold=True)), time() - start_time),
+                "[%s] Took %.2f seconds" % (
+                    (click.style("ERROR", fg="red", bold=True) if is_error else
+                     click.style("SUCCESS", fg="green", bold=True)),
+                    time() - start_time),
                 is_error=is_error)
 
         return not is_error
@@ -237,7 +241,11 @@ class EnvironmentProcessor(object):
                 v = self.RENAMED_PLATFORMS[v]
 
             # warn about unknown options
-            if k not in self.KNOWN_ENV_OPTIONS and not k.startswith("custom_"):
+            unknown_conditions = [
+                k not in self.KNOWN_ENV_OPTIONS, not k.startswith("custom_"),
+                not k.startswith("board_")
+            ]
+            if all(unknown_conditions):
                 click.secho(
                     "Detected non-PlatformIO `%s` option in `[env:%s]` section"
                     % (k, self.name),
@@ -288,10 +296,10 @@ class EnvironmentProcessor(object):
                     if d.strip()
                 ], self.verbose)
             if "lib_deps" in self.options:
-                _autoinstall_libdeps(self.cmd_ctx,
-                                     util.parse_conf_multi_values(
-                                         self.options['lib_deps']),
-                                     self.verbose)
+                _autoinstall_libdeps(
+                    self.cmd_ctx,
+                    util.parse_conf_multi_values(self.options['lib_deps']),
+                    self.verbose)
 
         try:
             p = PlatformFactory.newPlatform(self.options['platform'])
@@ -377,10 +385,9 @@ def print_summary(results, start_time):
             err=status is False)
 
     print_header(
-        "[%s] Took %.2f seconds" %
-        ((click.style("SUCCESS", fg="green", bold=True)
-          if successed else click.style("ERROR", fg="red", bold=True)),
-         time() - start_time),
+        "[%s] Took %.2f seconds" % (
+            (click.style("SUCCESS", fg="green", bold=True) if successed else
+             click.style("ERROR", fg="red", bold=True)), time() - start_time),
         is_error=not successed)
 
 
@@ -411,7 +418,7 @@ def check_project_envs(config, environments=None):
 
 def calculate_project_hash():
     check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
-    structure = [__version__]
+    chunks = [__version__]
     for d in (util.get_projectsrc_dir(), util.get_projectlib_dir()):
         if not isdir(d):
             continue
@@ -419,5 +426,10 @@ def calculate_project_hash():
             for f in files:
                 path = join(root, f)
                 if path.endswith(check_suffixes):
-                    structure.append(path)
-    return sha1(",".join(sorted(structure))).hexdigest()
+                    chunks.append(path)
+    chunks_to_str = ",".join(sorted(chunks))
+    if "windows" in util.get_systype():
+        # Fix issue with useless project rebuilding for case insensitive FS.
+        # A case of disk drive can differ...
+        chunks_to_str = chunks_to_str.lower()
+    return sha1(chunks_to_str).hexdigest()

platformio/commands/settings.py

@@ -32,8 +32,8 @@ def settings_get(name):
     click.echo(
         list_tpl.format(
             name=click.style("Name", fg="cyan"),
-            value=(click.style("Value", fg="green") +
-                   click.style(" [Default]", fg="yellow")),
+            value=(click.style("Value", fg="green") + click.style(
+                " [Default]", fg="yellow")),
             description="Description"))
     click.echo("-" * terminal_width)
 

platformio/commands/update.py

@@ -32,14 +32,14 @@ from platformio.managers.lib import LibraryManager
     help="Do not update, only check for new version")
 @click.pass_context
 def cli(ctx, core_packages, only_check):
+    # cleanup lib search results, cached board and platform lists
+    app.clean_cache()
+
     update_core_packages(only_check)
 
     if core_packages:
         return
 
-    # cleanup lib search results, cached board and platform lists
-    app.clean_cache()
-
     click.echo()
     click.echo("Platform Manager")
     click.echo("================")

platformio/commands/upgrade.py

@@ -20,7 +20,7 @@ import click
 import requests
 
 from platformio import VERSION, __version__, exception, util
-from platformio.commands.home import shutdown_servers
+from platformio.managers.core import shutdown_piohome_servers
 
 
 @click.command(
@@ -36,7 +36,7 @@ def cli(dev):
     click.secho("Please wait while upgrading PlatformIO ...", fg="yellow")
 
     # kill all PIO Home servers, they block `pioplus` binary
-    shutdown_servers()
+    shutdown_piohome_servers()
 
     to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".")
     cmds = (["pip", "install", "--upgrade",
@@ -63,7 +63,7 @@ def cli(dev):
             fg="green")
         click.echo("Release notes: ", nl=False)
         click.secho(
-            "http://docs.platformio.org/en/latest/history.html", fg="cyan")
+            "https://docs.platformio.org/en/latest/history.html", fg="cyan")
     except Exception as e:  # pylint: disable=broad-except
         if not r:
             raise exception.UpgradeError("\n".join([str(cmd), str(e)]))
@@ -102,8 +102,9 @@ def get_pip_package(to_develop):
     pkg_name = os.path.join(cache_dir, "piocoredevelop.zip")
     try:
         with open(pkg_name, "w") as fp:
-            r = util.exec_command(
-                ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True)
+            r = util.exec_command(["curl", "-fsSL", dl_url],
+                                  stdout=fp,
+                                  universal_newlines=True)
             assert r['returncode'] == 0
         # check ZIP structure
         with ZipFile(pkg_name) as zp:
@@ -150,7 +151,7 @@ def get_develop_latest_version():
 
 def get_pypi_latest_version():
     r = requests.get(
-        "https://pypi.python.org/pypi/platformio/json",
+        "https://pypi.org/pypi/platformio/json",
         headers=util.get_request_defheaders())
     r.raise_for_status()
     return r.json()['info']['version']

platformio/downloader.py

@@ -21,7 +21,7 @@ from time import mktime
 import click
 import requests
 
-from platformio import app, util
+from platformio import util
 from platformio.exception import (FDSHASumMismatch, FDSizeMismatch,
                                   FDUnrecognizedStatusCode)
 
@@ -43,14 +43,12 @@ class FileDownloader(object):
 
         disposition = self._request.headers.get("content-disposition")
         if disposition and "filename=" in disposition:
-            self._fname = disposition[
-                disposition.index("filename=") + 9:].replace('"', "").replace(
-                    "'", "")
+            self._fname = disposition[disposition.index("filename=") +
+                                      9:].replace('"', "").replace("'", "")
             self._fname = self._fname.encode("utf8")
         else:
             self._fname = [p for p in url.split("/") if p][-1]
 
-        self._progressbar = None
         self._destination = self._fname
         if dest_dir:
             self.set_destination(
@@ -70,12 +68,12 @@ class FileDownloader(object):
             return -1
         return int(self._request.headers['content-length'])
 
-    def start(self):
+    def start(self, with_progress=True):
         label = "Downloading"
         itercontent = self._request.iter_content(chunk_size=self.CHUNK_SIZE)
         f = open(self._destination, "wb")
         try:
-            if app.is_disabled_progressbar() or self.get_size() == -1:
+            if not with_progress or self.get_size() == -1:
                 click.echo("%s..." % label)
                 for chunk in itercontent:
                     if chunk:
@@ -85,12 +83,6 @@ class FileDownloader(object):
                 with click.progressbar(length=chunks, label=label) as pb:
                     for _ in pb:
                         f.write(next(itercontent))
-        except IOError as e:
-            click.secho(
-                "Error: Please read http://bit.ly/package-manager-ioerror",
-                fg="red",
-                err=True)
-            raise e
         finally:
             f.close()
             self._request.close()
@@ -98,6 +90,8 @@ class FileDownloader(object):
         if self.get_lmtime():
             self._preserve_filemtime(self.get_lmtime())
 
+        return True
+
     def verify(self, sha1=None):
         _dlsize = getsize(self._destination)
         if self.get_size() != -1 and _dlsize != self.get_size():

platformio/exception.py

@@ -28,6 +28,10 @@ class ReturnErrorCode(PlatformioException):
     MESSAGE = "{0}"
 
 
+class LockFileTimeoutError(PlatformioException):
+    pass
+
+
 class MinitermException(PlatformioException):
     pass
 
@@ -102,6 +106,13 @@ class PackageInstallError(PlatformioException):
                "Please try this solution -> http://bit.ly/faq-package-manager")
 
 
+class ExtractArchiveItemError(PlatformioException):
+
+    MESSAGE = (
+        "Could not extract `{0}` to `{1}`. Try to disable antivirus "
+        "tool or check this solution -> http://bit.ly/faq-package-manager")
+
+
 class FDUnrecognizedStatusCode(PlatformioException):
 
     MESSAGE = "Got an unrecognized status code '{0}' when downloaded {1}"
@@ -167,7 +178,7 @@ class InternetIsOffline(PlatformioException):
     MESSAGE = (
         "You are not connected to the Internet.\n"
         "If you build a project first time, we need Internet connection "
-        "to install all dependencies and toolchain.")
+        "to install all dependencies and toolchains.")
 
 
 class LibNotFound(PlatformioException):
@@ -192,6 +203,11 @@ class InvalidLibConfURL(PlatformioException):
     MESSAGE = "Invalid library config URL '{0}'"
 
 
+class InvalidProjectConf(PlatformioException):
+
+    MESSAGE = "Invalid `platformio.ini`, project configuration file: '{0}'"
+
+
 class BuildScriptNotFound(PlatformioException):
 
     MESSAGE = "Invalid path '{0}' to build script"
@@ -207,6 +223,11 @@ class InvalidSettingValue(PlatformioException):
     MESSAGE = "Invalid value '{0}' for the setting '{1}'"
 
 
+class InvalidJSONFile(PlatformioException):
+
+    MESSAGE = "Could not load broken JSON: {0}"
+
+
 class CIBuildEnvsEmpty(PlatformioException):
 
     MESSAGE = ("Can't find PlatformIO build environments.\n"
@@ -214,13 +235,32 @@ class CIBuildEnvsEmpty(PlatformioException):
                "predefined environments using `--project-conf` option")
 
 
+class InvalidUdevRules(PlatformioException):
+    pass
+
+
+class MissedUdevRules(InvalidUdevRules):
+
+    MESSAGE = (
+        "Warning! Please install `99-platformio-udev.rules`. \nMode details: "
+        "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules")
+
+
+class OutdatedUdevRules(InvalidUdevRules):
+
+    MESSAGE = (
+        "Warning! Your `{0}` are outdated. Please update or reinstall them."
+        "\n Mode details: https://docs.platformio.org"
+        "/en/latest/faq.html#platformio-udev-rules")
+
+
 class UpgradeError(PlatformioException):
 
     MESSAGE = """{0}
 
 * Upgrade using `pip install -U platformio`
 * Try different installation/upgrading steps:
-  http://docs.platformio.org/page/installation.html
+  https://docs.platformio.org/page/installation.html
 """
 
 
@@ -243,8 +283,8 @@ class CygwinEnvDetected(PlatformioException):
 class DebugSupportError(PlatformioException):
 
     MESSAGE = ("Currently, PlatformIO does not support debugging for `{0}`.\n"
-               "Please mail contact@pioplus.com or visit "
-               "< http://docs.platformio.org/page/plus/debugging.html >")
+               "Please contact support@pioplus.com or visit "
+               "< https://docs.platformio.org/page/plus/debugging.html >")
 
 
 class DebugInvalidOptions(PlatformioException):

platformio/ide/projectgenerator.py

@@ -40,7 +40,7 @@ class ProjectGenerator(object):
         return sorted(
             [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))])
 
-    @util.memoized
+    @util.memoized()
     def get_project_env(self):
         data = {}
         config = util.load_project_config(self.project_dir)
@@ -54,7 +54,6 @@ class ProjectGenerator(object):
                 data[k] = v
         return data
 
-    @util.memoized
     def get_project_build_data(self):
         data = {
             "defines": [],
@@ -128,22 +127,8 @@ class ProjectGenerator(object):
 
     @staticmethod
     def _merge_contents(dst_path, contents):
-        file_name = basename(dst_path)
-
-        # merge .gitignore
-        if file_name == ".gitignore" and isfile(dst_path):
-            modified = False
-            default = [l.strip() for l in contents.split("\n")]
-            with open(dst_path) as fp:
-                current = [l.strip() for l in fp.readlines()]
-            for d in default:
-                if d and d not in current:
-                    modified = True
-                    current.append(d)
-            if not modified:
-                return
-            contents = "\n".join(current) + "\n"
-
+        if basename(dst_path) == ".gitignore" and isfile(dst_path):
+            return
         with open(dst_path, "w") as f:
             f.write(contents)
 

platformio/ide/tpls/atom/.clang_complete.tpl

@@ -3,4 +3,4 @@
 % end
 % for define in defines:
 -D{{!define}}
-% end
+% end

platformio/ide/tpls/atom/.gitignore.tpl

@@ -1,3 +1,4 @@
+.pio
 .pioenvs
 .piolibdeps
 .clang_complete

platformio/ide/tpls/clion/.gitignore.tpl

@@ -1,3 +1,4 @@
+.pio
 .pioenvs
 .piolibdeps
 CMakeListsPrivate.txt

platformio/ide/tpls/clion/.idea/misc.xml.tpl

@@ -9,6 +9,9 @@
       <file path="$PROJECT_DIR$/lib" />
       <file path="$PROJECT_DIR$/.piolibdeps" />
     </libraryRoots>
+    <excludeRoots>
+      <file path="$PROJECT_DIR$/.pio" />
+    </excludeRoots>
     <excludeRoots>
       <file path="$PROJECT_DIR$/.pioenvs" />
     </excludeRoots>

platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl

@@ -17,8 +17,8 @@
 <stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_ID" value="gdb"/>
 <stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_REGISTER_GROUPS" value=""/>
 <stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_START_MODE" value="run"/>
-<booleanAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN" value="true"/>
-<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN_SYMBOL" value="main"/>
+<booleanAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN" value="false"/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN_SYMBOL" value=""/>
 <stringAttribute key="org.eclipse.cdt.launch.PROGRAM_NAME" value="{{prog_path}}"/>
 <stringAttribute key="org.eclipse.cdt.launch.PROJECT_ATTR" value="{{project_name}}"/>
 <booleanAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_AUTO_ATTR" value="false"/>

platformio/ide/tpls/emacs/.clang_complete.tpl

@@ -1,6 +1,6 @@
 % for include in includes:
--I{{include}}
+-I"{{include}}"
 % end
 % for define in defines:
 -D{{!define}}
-% end
+% end

platformio/ide/tpls/emacs/.gitignore.tpl

@@ -1,3 +1,4 @@
+.pio
 .pioenvs
 .piolibdeps
 .clang_complete

platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl

@@ -11,7 +11,7 @@
       <itemPath>nbproject/private/launcher.properties</itemPath>
     </logicalFolder>
   </logicalFolder>
-  <sourceFolderFilter>^(nbproject|.pioenvs)$</sourceFolderFilter>
+  <sourceFolderFilter>^(nbproject|.pio|.pioenvs)$</sourceFolderFilter>
   <sourceRootList>
     <Elem>.</Elem>
   </sourceRootList>

platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl

@@ -97,7 +97,7 @@
     "settings":
     {
          "sublimegdb_workingdir": "{{project_dir}}",
-         "sublimegdb_exec_cmd": "-exec-continue",
+         "sublimegdb_exec_cmd": "",
          "sublimegdb_commandline": "{{platformio_path}} -f -c sublimetext debug --interface=gdb --interpreter=mi -x .pioinit"
 
     }

platformio/ide/tpls/vim/.clang_complete.tpl

@@ -1,6 +1,6 @@
 % for include in includes:
--I{{include}}
+-I"{{include}}"
 % end
 % for define in defines:
 -D{{!define}}
-% end
+% end

platformio/ide/tpls/vim/.gcc-flags.json.tpl

@@ -3,6 +3,6 @@
   "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }}",
   "gccDefaultCppFlags": "-fsyntax-only {{! cxx_flags.replace(' -MMD ', ' ').replace('"', '\\"') }}",
   "gccErrorLimit": 15,
-  "gccIncludePaths": "{{ ','.join(includes).replace("\\", "/") }}",
+  "gccIncludePaths": "{{! ','.join("'{}'".format(w.replace("\\", '/')) for w in includes)}}",
   "gccSuppressWarnings": false
 }

platformio/ide/tpls/vim/.gitignore.tpl

@@ -1,3 +1,4 @@
+.pio
 .pioenvs
 .piolibdeps
 .clang_complete

platformio/ide/tpls/vscode/.gitignore.tpl

@@ -1,3 +1,4 @@
+.pio
 .pioenvs
 .piolibdeps
 .vscode/.browse.c_cpp.db*

platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl

@@ -1,17 +1,29 @@
 {
+    "!!! WARNING !!!": "PLEASE DO NOT MODIFY THIS FILE! USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags",
     "configurations": [
         {
 % import platform
+% from os.path import commonprefix, dirname
+%
 % systype = platform.system().lower()
+%
+% cleaned_includes = []
+% for include in includes:
+%   if "toolchain-" not in dirname(commonprefix([include, cc_path])):
+%       cleaned_includes.append(include)
+%   end
+% end
+%
 % if systype == "windows":
             "name": "Win32",
 % elif systype == "darwin":
             "name": "Mac",
+            "macFrameworkPath": [],
 % else:
             "name": "Linux",
 % end
             "includePath": [
-% for include in includes:
+% for include in cleaned_includes:
                 "{{include.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
 % end
                 ""
@@ -20,7 +32,7 @@
                 "limitSymbolsToIncludedHeaders": true,
                 "databaseFilename": "${workspaceRoot}/.vscode/.browse.c_cpp.db",
                 "path": [
-% for include in includes:
+% for include in cleaned_includes:
                     "{{include.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
 % end
                     ""
@@ -32,7 +44,19 @@
 % end
                 ""
             ],
-            "intelliSenseMode": "clang-x64"
+            "intelliSenseMode": "clang-x64",
+% import re
+% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
+% cc_stds = STD_RE.findall(cc_flags)
+% cxx_stds = STD_RE.findall(cxx_flags)
+%
+% if cc_stds:
+            "cStandard": "c{{ cc_stds[-1] }}",
+% end
+% if cxx_stds:
+            "cppStandard": "c++{{ cxx_stds[-1] }}",
+% end
+            "compilerPath": "{{ cc_path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"') }}"
         }
     ]
 }

platformio/ide/tpls/vscode/.vscode/extensions.json.tpl

@@ -0,0 +1,7 @@
+{
+	// See http://go.microsoft.com/fwlink/?LinkId=827846
+	// for the documentation about the extensions.json format
+	"recommendations": [
+		"platformio.platformio-ide"
+	]
+}

platformio/ide/tpls/vscode/.vscode/launch.json.tpl

@@ -1,16 +1,43 @@
+// AUTOMATICALLY GENERATED FILE. PLEASE DO NOT MODIFY IT MANUALLY
+
+// PIO Unified Debugger
+//
+// Documentation: https://docs.platformio.org/page/plus/debugging.html
+// Configuration: https://docs.platformio.org/page/projectconf/section_env_debug.html
+
 % from os.path import dirname, join
+%
+% def _escape_path(path):
+%   return path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')
+% end
+%
 {
     "version": "0.2.0",
     "configurations": [
         {
-            "type": "gdb",
+            "type": "platformio-debug",
             "request": "launch",
-            "cwd": "${workspaceRoot}",
-            "name": "PlatformIO Debugger",
-            "target": "{{prog_path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
-            "gdbpath": "{{join(dirname(platformio_path), "piodebuggdb").replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
-            "autorun": [ "source .pioinit" ],
-            "preLaunchTask": "PlatformIO: Pre-Debug",
+            "name": "PIO Debug",
+            "executable": "{{ _escape_path(prog_path) }}",
+            "toolchainBinDir": "{{ _escape_path(dirname(gdb_path)) }}",
+% if svd_path:
+            "svdPath": "{{ _escape_path(svd_path) }}",
+% end
+            "preLaunchTask": {
+                "type": "PlatformIO",
+                "task": "Pre-Debug"
+            },
+            "internalConsoleOptions": "openOnSessionStart"
+        },
+        {
+            "type": "platformio-debug",
+            "request": "launch",
+            "name": "PIO Debug (skip Pre-Debug)",
+            "executable": "{{ _escape_path(prog_path) }}",
+            "toolchainBinDir": "{{ _escape_path(dirname(gdb_path)) }}",
+% if svd_path:
+            "svdPath": "{{ _escape_path(svd_path) }}",
+% end
             "internalConsoleOptions": "openOnSessionStart"
         }
     ]

platformio/lockfile.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from os import remove
+from os.path import abspath, exists, getmtime
+from time import sleep, time
+
+from platformio import exception
+
+LOCKFILE_TIMEOUT = 3600  # in seconds, 1 hour
+LOCKFILE_DELAY = 0.2
+
+LOCKFILE_INTERFACE_FCNTL = 1
+LOCKFILE_INTERFACE_MSVCRT = 2
+
+try:
+    import fcntl
+    LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_FCNTL
+except ImportError:
+    try:
+        import msvcrt
+        LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_MSVCRT
+    except ImportError:
+        LOCKFILE_CURRENT_INTERFACE = None
+
+
+class LockFileExists(Exception):
+    pass
+
+
+class LockFile(object):
+
+    def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY):
+        self.timeout = timeout
+        self.delay = delay
+        self._lock_path = abspath(path) + ".lock"
+        self._fp = None
+
+    def _lock(self):
+        if not LOCKFILE_CURRENT_INTERFACE and exists(self._lock_path):
+            # remove stale lock
+            if time() - getmtime(self._lock_path) > 10:
+                try:
+                    remove(self._lock_path)
+                except:  # pylint: disable=bare-except
+                    pass
+            else:
+                raise LockFileExists
+
+        self._fp = open(self._lock_path, "w")
+        try:
+            if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL:
+                fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+            elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT:
+                msvcrt.locking(self._fp.fileno(), msvcrt.LK_NBLCK, 1)
+        except IOError:
+            self._fp = None
+            raise LockFileExists
+        return True
+
+    def _unlock(self):
+        if not self._fp:
+            return
+        if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL:
+            fcntl.flock(self._fp.fileno(), fcntl.LOCK_UN)
+        elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT:
+            msvcrt.locking(self._fp.fileno(), msvcrt.LK_UNLCK, 1)
+        self._fp.close()
+        self._fp = None
+
+    def acquire(self):
+        elapsed = 0
+        while elapsed < self.timeout:
+            try:
+                return self._lock()
+            except LockFileExists:
+                sleep(self.delay)
+                elapsed += self.delay
+
+        raise exception.LockFileTimeoutError()
+
+    def release(self):
+        self._unlock()
+        if exists(self._lock_path):
+            try:
+                remove(self._lock_path)
+            except:  # pylint: disable=bare-except
+                pass
+
+    def __enter__(self):
+        self.acquire()
+
+    def __exit__(self, type_, value, traceback):
+        self.release()
+
+    def __del__(self):
+        self.release()

platformio/maintenance.py

@@ -179,7 +179,7 @@ def after_upgrade(ctx):
         click.secho(
             "Please remove multiple PIO Cores from a system:", fg="yellow")
         click.secho(
-            "http://docs.platformio.org/page/faq.html"
+            "https://docs.platformio.org/page/faq.html"
             "#multiple-pio-cores-in-a-system",
             fg="cyan")
         click.secho("*" * terminal_width, fg="yellow")
@@ -208,8 +208,8 @@ def after_upgrade(ctx):
 
     # PlatformIO banner
     click.echo("*" * terminal_width)
-    click.echo("If you like %s, please:" %
-               (click.style("PlatformIO", fg="cyan")))
+    click.echo(
+        "If you like %s, please:" % (click.style("PlatformIO", fg="cyan")))
     click.echo("- %s us on Twitter to stay up-to-date "
                "on the latest project news > %s" %
                (click.style("follow", fg="cyan"),
@@ -224,9 +224,9 @@ def after_upgrade(ctx):
             (click.style("try", fg="cyan"),
              click.style("https://platformio.org/platformio-ide", fg="cyan")))
     if not util.is_ci():
-        click.echo("- %s us with PlatformIO Plus > %s" %
-                   (click.style("support", fg="cyan"),
-                    click.style("https://pioplus.com", fg="cyan")))
+        click.echo("- %s us with PlatformIO Plus > %s" % (click.style(
+            "support", fg="cyan"), click.style(
+                "https://pioplus.com", fg="cyan")))
 
     click.echo("*" * terminal_width)
     click.echo("")
@@ -274,7 +274,8 @@ def check_platformio_upgrade():
         click.secho("pip install -U platformio", fg="cyan", nl=False)
         click.secho("` command.", fg="yellow")
     click.secho("Changes: ", fg="yellow", nl=False)
-    click.secho("http://docs.platformio.org/en/latest/history.html", fg="cyan")
+    click.secho(
+        "https://docs.platformio.org/en/latest/history.html", fg="cyan")
     click.echo("*" * terminal_width)
     click.echo("")
 
@@ -296,8 +297,8 @@ def check_internal_updates(ctx, what):
         if manifest['name'] in outdated_items:
             continue
         conds = [
-            pm.outdated(manifest['__pkg_dir']),
-            what == "platforms" and PlatformFactory.newPlatform(
+            pm.outdated(manifest['__pkg_dir']), what == "platforms"
+            and PlatformFactory.newPlatform(
                 manifest['__pkg_dir']).are_outdated_packages()
         ]
         if any(conds):

platformio/managers/core.py

@@ -16,15 +16,18 @@ import os
 import subprocess
 import sys
 from os.path import dirname, join
+from time import sleep
+
+import requests
 
 from platformio import __version__, exception, util
 from platformio.managers.package import PackageManager
 
 CORE_PACKAGES = {
-    "contrib-piohome": ">=0.7.1,<2",
-    "contrib-pysite": ">=0.1.5,<2",
-    "tool-pioplus": ">=0.14.5,<2",
-    "tool-unity": "~1.20302.1",
+    "contrib-piohome": "^1.0.2",
+    "contrib-pysite": ">=0.3.2,<2",
+    "tool-pioplus": "^1.5.0",
+    "tool-unity": "~1.20403.0",
     "tool-scons": "~2.20501.4"
 }
 
@@ -36,11 +39,12 @@ PIOPLUS_AUTO_UPDATES_MAX = 100
 class CorePackageManager(PackageManager):
 
     def __init__(self):
-        PackageManager.__init__(self, join(util.get_home_dir(), "packages"), [
-            "https://dl.bintray.com/platformio/dl-packages/manifest.json",
-            "http%s://dl.platformio.org/packages/manifest.json" %
-            ("" if sys.version_info < (2, 7, 9) else "s")
-        ])
+        super(CorePackageManager, self).__init__(
+            join(util.get_home_dir(), "packages"), [
+                "https://dl.bintray.com/platformio/dl-packages/manifest.json",
+                "http%s://dl.platformio.org/packages/manifest.json" %
+                ("" if sys.version_info < (2, 7, 9) else "s")
+            ])
 
     def install(  # pylint: disable=keyword-arg-before-vararg
             self,
@@ -69,7 +73,7 @@ class CorePackageManager(PackageManager):
             if manifest['name'] not in best_pkg_versions:
                 continue
             if manifest['version'] != best_pkg_versions[manifest['name']]:
-                self.uninstall(manifest['__pkg_dir'], trigger_event=False)
+                self.uninstall(manifest['__pkg_dir'], after_update=True)
         self.cache_reset()
         return True
 
@@ -92,10 +96,24 @@ def update_core_packages(only_check=False, silent=False):
         if not pkg_dir:
             continue
         if not silent or pm.outdated(pkg_dir, requirements):
+            if name == "tool-pioplus" and not only_check:
+                shutdown_piohome_servers()
+                if "windows" in util.get_systype():
+                    sleep(1)
             pm.update(name, requirements, only_check=only_check)
     return True
 
 
+def shutdown_piohome_servers():
+    port = 8010
+    while port < 9000:
+        try:
+            requests.get("http://127.0.0.1:%d?__shutdown__=1" % port)
+            port += 1
+        except:  # pylint: disable=bare-except
+            return
+
+
 def pioplus_call(args, **kwargs):
     if "windows" in util.get_systype() and sys.version_info < (2, 7, 6):
         raise exception.PlatformioException(
@@ -107,6 +125,7 @@ def pioplus_call(args, **kwargs):
     pythonexe_path = util.get_pythonexe_path()
     os.environ['PYTHONEXEPATH'] = pythonexe_path
     os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite")
+    os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "")
     os.environ['PATH'] = (os.pathsep).join(
         [dirname(pythonexe_path), os.environ['PATH']])
     util.copy_pythonpath_to_osenv()

platformio/managers/lib.py

@@ -32,7 +32,7 @@ class LibraryManager(BasePkgManager):
     def __init__(self, package_dir=None):
         if not package_dir:
             package_dir = join(util.get_home_dir(), "lib")
-        BasePkgManager.__init__(self, package_dir)
+        super(LibraryManager, self).__init__(package_dir)
 
     @property
     def manifest_names(self):
@@ -208,9 +208,9 @@ class LibraryManager(BasePkgManager):
             cache_valid="30d")
         assert dl_data
 
-        return self._install_from_url(name, dl_data['url'].replace(
-            "http://", "https://") if app.get_setting("enable_ssl") else
-                                      dl_data['url'], requirements)
+        return self._install_from_url(
+            name, dl_data['url'].replace("http://", "https://")
+            if app.get_setting("enable_ssl") else dl_data['url'], requirements)
 
     def search_lib_id(  # pylint: disable=too-many-branches
             self,
@@ -237,9 +237,8 @@ class LibraryManager(BasePkgManager):
             if not isinstance(values, list):
                 values = [v.strip() for v in values.split(",") if v]
             for value in values:
-                query.append('%s:"%s"' % (key[:-1]
-                                          if key.endswith("s") else key,
-                                          value))
+                query.append('%s:"%s"' %
+                             (key[:-1] if key.endswith("s") else key, value))
 
         lib_info = None
         result = util.get_api_result(
@@ -290,9 +289,9 @@ class LibraryManager(BasePkgManager):
     def _get_lib_id_from_installed(self, filters):
         if filters['name'].startswith("id="):
             return int(filters['name'][3:])
-        package_dir = self.get_package_dir(filters['name'],
-                                           filters.get("requirements",
-                                                       filters.get("version")))
+        package_dir = self.get_package_dir(
+            filters['name'], filters.get("requirements",
+                                         filters.get("version")))
         if not package_dir:
             return None
         manifest = self.load_manifest(package_dir)
@@ -332,25 +331,24 @@ class LibraryManager(BasePkgManager):
             name,
             requirements=None,
             silent=False,
-            trigger_event=True,
+            after_update=False,
             interactive=False,
             force=False):
         _name, _requirements, _url = self.parse_pkg_uri(name, requirements)
         if not _url:
-            name = "id=%d" % self.search_lib_id(
-                {
-                    "name": _name,
-                    "requirements": _requirements
-                },
-                silent=silent,
-                interactive=interactive)
+            name = "id=%d" % self.search_lib_id({
+                "name": _name,
+                "requirements": _requirements
+            },
+                                                silent=silent,
+                                                interactive=interactive)
             requirements = _requirements
         pkg_dir = BasePkgManager.install(
             self,
             name,
             requirements,
             silent=silent,
-            trigger_event=trigger_event,
+            after_update=after_update,
             force=force)
 
         if not pkg_dir:
@@ -365,11 +363,20 @@ class LibraryManager(BasePkgManager):
 
         for filters in self.normalize_dependencies(manifest['dependencies']):
             assert "name" in filters
+
+            # avoid circle dependencies
+            if not self.INSTALL_HISTORY:
+                self.INSTALL_HISTORY = []
+            history_key = str(filters)
+            if history_key in self.INSTALL_HISTORY:
+                continue
+            self.INSTALL_HISTORY.append(history_key)
+
             if any(s in filters.get("version", "") for s in ("\\", "/")):
                 self.install(
                     "{name}={version}".format(**filters),
                     silent=silent,
-                    trigger_event=trigger_event,
+                    after_update=after_update,
                     interactive=interactive,
                     force=force)
             else:
@@ -385,20 +392,20 @@ class LibraryManager(BasePkgManager):
                         lib_id,
                         filters.get("version"),
                         silent=silent,
-                        trigger_event=trigger_event,
+                        after_update=after_update,
                         interactive=interactive,
                         force=force)
                 else:
                     self.install(
                         lib_id,
                         silent=silent,
-                        trigger_event=trigger_event,
+                        after_update=after_update,
                         interactive=interactive,
                         force=force)
         return pkg_dir
 
 
-@util.memoized
+@util.memoized()
 def get_builtin_libs(storage_names=None):
     items = []
     storage_names = storage_names or []
@@ -417,7 +424,7 @@ def get_builtin_libs(storage_names=None):
     return items
 
 
-@util.memoized
+@util.memoized()
 def is_builtin_lib(name):
     for storage in get_builtin_libs():
         if any(l.get("name") == name for l in storage['items']):

platformio/managers/package.py

@@ -18,7 +18,7 @@ import json
 import os
 import re
 import shutil
-from os.path import basename, getsize, isdir, isfile, islink, join
+from os.path import abspath, basename, getsize, isdir, isfile, islink, join
 from tempfile import mkdtemp
 
 import click
@@ -27,6 +27,7 @@ import semantic_version
 
 from platformio import __version__, app, exception, telemetry, util
 from platformio.downloader import FileDownloader
+from platformio.lockfile import LockFile
 from platformio.unpacker import FileUnpacker
 from platformio.vcsclient import VCSClientFactory
 
@@ -90,7 +91,8 @@ class PkgRepoMixin(object):
         reqspec = None
         if requirements:
             try:
-                reqspec = semantic_version.Spec(requirements)
+                reqspec = self.parse_semver_spec(
+                    requirements, raise_exception=True)
             except ValueError:
                 pass
 
@@ -98,8 +100,8 @@ class PkgRepoMixin(object):
             if not self.is_system_compatible(v.get("system")):
                 continue
             if "platformio" in v.get("engines", {}):
-                if PkgRepoMixin.PIO_VERSION not in semantic_version.Spec(
-                        v['engines']['platformio']):
+                if PkgRepoMixin.PIO_VERSION not in self.parse_semver_spec(
+                        v['engines']['platformio'], raise_exception=True):
                     continue
             specver = semantic_version.Version(v['version'])
             if reqspec and specver not in reqspec:
@@ -177,8 +179,25 @@ class PkgInstallerMixin(object):
                     shutil.copy(cache_path, dst_path)
                     return dst_path
 
-        fd = FileDownloader(url, dest_dir)
-        fd.start()
+        with_progress = not app.is_disabled_progressbar()
+        try:
+            fd = FileDownloader(url, dest_dir)
+            fd.start(with_progress=with_progress)
+        except IOError as e:
+            raise_error = not with_progress
+            if with_progress:
+                try:
+                    fd = FileDownloader(url, dest_dir)
+                    fd.start(with_progress=False)
+                except IOError:
+                    raise_error = True
+            if raise_error:
+                click.secho(
+                    "Error: Please read http://bit.ly/package-manager-ioerror",
+                    fg="red",
+                    err=True)
+                raise e
+
         if sha1:
             fd.verify(sha1)
         dst_path = fd.get_filepath()
@@ -194,13 +213,33 @@ class PkgInstallerMixin(object):
 
     @staticmethod
     def unpack(source_path, dest_dir):
-        with FileUnpacker(source_path) as fu:
-            return fu.unpack(dest_dir)
+        with_progress = not app.is_disabled_progressbar()
+        try:
+            with FileUnpacker(source_path) as fu:
+                return fu.unpack(dest_dir, with_progress=with_progress)
+        except IOError as e:
+            if not with_progress:
+                raise e
+            with FileUnpacker(source_path) as fu:
+                return fu.unpack(dest_dir, with_progress=False)
 
     @staticmethod
     def parse_semver_spec(value, raise_exception=False):
         try:
-            return semantic_version.Spec(value)
+            # Workaround for ^ issue and pre-releases
+            # https://github.com/rbarrois/python-semanticversion/issues/61
+            requirements = []
+            for item in str(value).split(","):
+                item = item.strip()
+                if not item:
+                    continue
+                if item.startswith("^"):
+                    major = semantic_version.Version.coerce(item[1:]).major
+                    requirements.append(">=%s" % major)
+                    requirements.append("<%s" % (int(major) + 1))
+                else:
+                    requirements.append(item)
+            return semantic_version.Spec(*requirements)
         except ValueError as e:
             if raise_exception:
                 raise e
@@ -220,6 +259,69 @@ class PkgInstallerMixin(object):
                 raise e
         return None
 
+    @staticmethod
+    def parse_pkg_uri(  # pylint: disable=too-many-branches
+            text, requirements=None):
+        text = str(text)
+        name, url = None, None
+
+        # Parse requirements
+        req_conditions = [
+            "@" in text, not requirements, ":" not in text
+            or text.rfind("/") < text.rfind("@")
+        ]
+        if all(req_conditions):
+            text, requirements = text.rsplit("@", 1)
+
+        # Handle PIO Library Registry ID
+        if text.isdigit():
+            text = "id=" + text
+        # Parse custom name
+        elif "=" in text and not text.startswith("id="):
+            name, text = text.split("=", 1)
+
+        # Parse URL
+        # if valid URL with scheme vcs+protocol://
+        if "+" in text and text.find("+") < text.find("://"):
+            url = text
+        elif "/" in text or "\\" in text:
+            git_conditions = [
+                # Handle GitHub URL (https://github.com/user/package)
+                text.startswith("https://github.com/") and not text.endswith(
+                    (".zip", ".tar.gz")),
+                (text.split("#", 1)[0]
+                 if "#" in text else text).endswith(".git")
+            ]
+            hg_conditions = [
+                # Handle Developer Mbed URL
+                # (https://developer.mbed.org/users/user/code/package/)
+                # (https://os.mbed.com/users/user/code/package/)
+                text.startswith("https://developer.mbed.org"),
+                text.startswith("https://os.mbed.com")
+            ]
+            if any(git_conditions):
+                url = "git+" + text
+            elif any(hg_conditions):
+                url = "hg+" + text
+            elif "://" not in text and (isfile(text) or isdir(text)):
+                url = "file://" + text
+            elif "://" in text:
+                url = text
+            # Handle short version of GitHub URL
+            elif text.count("/") == 1:
+                url = "git+https://github.com/" + text
+
+        # Parse name from URL
+        if url and not name:
+            _url = url.split("#", 1)[0] if "#" in url else url
+            if _url.endswith(("\\", "/")):
+                _url = _url[:-1]
+            name = basename(_url)
+            if "." in name and not name.startswith("."):
+                name = name.rsplit(".", 1)[0]
+
+        return (name or text, requirements, url)
+
     @staticmethod
     def get_install_dirname(manifest):
         name = re.sub(r"[^\da-z\_\-\. ]", "_", manifest['name'], flags=re.I)
@@ -277,11 +379,13 @@ class PkgInstallerMixin(object):
                     manifest[key.strip()] = value.strip()
 
         if src_manifest:
-            if "name" not in manifest:
-                manifest['name'] = src_manifest['name']
             if "version" in src_manifest:
                 manifest['version'] = src_manifest['version']
             manifest['__src_url'] = src_manifest['url']
+            # handle a custom package name
+            autogen_name = self.parse_pkg_uri(manifest['__src_url'])[0]
+            if "name" not in manifest or autogen_name != src_manifest['name']:
+                manifest['name'] = src_manifest['name']
 
         if "name" not in manifest:
             manifest['name'] = basename(pkg_dir)
@@ -343,6 +447,12 @@ class PkgInstallerMixin(object):
         return manifest.get("__pkg_dir") if manifest and isdir(
             manifest.get("__pkg_dir")) else None
 
+    def get_package_by_dir(self, pkg_dir):
+        for manifest in self.get_installed():
+            if manifest['__pkg_dir'] == util.path_to_unicode(abspath(pkg_dir)):
+                return manifest
+        return None
+
     def find_pkg_root(self, src_dir):
         if self.manifest_exists(src_dir):
             return src_dir
@@ -369,6 +479,7 @@ class PkgInstallerMixin(object):
                 click.secho("Looking for another mirror...", fg="yellow")
 
         if versions is None:
+            util.internet_on(raise_exception=True)
             raise exception.UnknownPackage(name)
         elif not pkgdata:
             raise exception.UndefinedPackageVersion(requirements or "latest",
@@ -450,8 +561,8 @@ class PkgInstallerMixin(object):
                 "Package version %s doesn't satisfy requirements %s" %
                 (tmp_manifest['version'], requirements))
             try:
-                assert tmp_semver and tmp_semver in semantic_version.Spec(
-                    requirements), mismatch_error
+                assert tmp_semver and tmp_semver in self.parse_semver_spec(
+                    requirements, raise_exception=True), mismatch_error
             except (AssertionError, ValueError):
                 assert tmp_manifest['version'] == requirements, mismatch_error
 
@@ -476,23 +587,23 @@ class PkgInstallerMixin(object):
                                             cur_manifest['version'])
                 if "__src_url" in cur_manifest:
                     target_dirname = "%s@src-%s" % (
-                        pkg_dirname,
-                        hashlib.md5(cur_manifest['__src_url']).hexdigest())
-                os.rename(pkg_dir, join(self.package_dir, target_dirname))
+                        pkg_dirname, hashlib.md5(
+                            cur_manifest['__src_url']).hexdigest())
+                shutil.move(pkg_dir, join(self.package_dir, target_dirname))
             # fix to a version
             elif action == 2:
                 target_dirname = "%s@%s" % (pkg_dirname,
                                             tmp_manifest['version'])
                 if "__src_url" in tmp_manifest:
                     target_dirname = "%s@src-%s" % (
-                        pkg_dirname,
-                        hashlib.md5(tmp_manifest['__src_url']).hexdigest())
+                        pkg_dirname, hashlib.md5(
+                            tmp_manifest['__src_url']).hexdigest())
                 pkg_dir = join(self.package_dir, target_dirname)
 
         # remove previous/not-satisfied package
         if isdir(pkg_dir):
             util.rmtree_(pkg_dir)
-        os.rename(tmp_dir, pkg_dir)
+        shutil.move(tmp_dir, pkg_dir)
         assert isdir(pkg_dir)
         self.cache_reset()
         return pkg_dir
@@ -517,73 +628,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
     def print_message(self, message, nl=True):
         click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl)
 
-    @staticmethod
-    def parse_pkg_uri(  # pylint: disable=too-many-branches
-            text, requirements=None):
-        text = str(text)
-        name, url = None, None
-
-        # Parse requirements
-        req_conditions = [
-            "@" in text, not requirements, ":" not in text
-            or text.rfind("/") < text.rfind("@")
-        ]
-        if all(req_conditions):
-            text, requirements = text.rsplit("@", 1)
-
-        # Handle PIO Library Registry ID
-        if text.isdigit():
-            text = "id=" + text
-        # Parse custom name
-        elif "=" in text and not text.startswith("id="):
-            name, text = text.split("=", 1)
-
-        # Parse URL
-        # if valid URL with scheme vcs+protocol://
-        if "+" in text and text.find("+") < text.find("://"):
-            url = text
-        elif "/" in text or "\\" in text:
-            git_conditions = [
-                # Handle GitHub URL (https://github.com/user/package)
-                text.startswith("https://github.com/") and not text.endswith(
-                    (".zip", ".tar.gz")),
-                (text.split("#", 1)[0]
-                 if "#" in text else text).endswith(".git")
-            ]
-            hg_conditions = [
-                # Handle Developer Mbed URL
-                # (https://developer.mbed.org/users/user/code/package/)
-                # (https://os.mbed.com/users/user/code/package/)
-                text.startswith("https://developer.mbed.org"),
-                text.startswith("https://os.mbed.com")
-            ]
-            if any(git_conditions):
-                url = "git+" + text
-            elif any(hg_conditions):
-                url = "hg+" + text
-            elif "://" not in text and (isfile(text) or isdir(text)):
-                url = "file://" + text
-            elif "://" in text:
-                url = text
-            # Handle short version of GitHub URL
-            elif text.count("/") == 1:
-                url = "git+https://github.com/" + text
-
-        # Parse name from URL
-        if url and not name:
-            _url = url.split("#", 1)[0] if "#" in url else url
-            if _url.endswith(("\\", "/")):
-                _url = _url[:-1]
-            name = basename(_url)
-            if "." in name and not name.startswith("."):
-                name = name.rsplit(".", 1)[0]
-
-        return (name or text, requirements, url)
-
     def outdated(self, pkg_dir, requirements=None):
         """
         Has 3 different results:
-        `None` - unknown package, VCS is fixed to commit
+        `None` - unknown package, VCS is detached to commit
         `False` - package is up-to-date
         `String` - a found latest version
         """
@@ -591,7 +639,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             return None
         latest = None
         manifest = self.load_manifest(pkg_dir)
-        # skip fixed package to a specific version
+        # skip detached package to a specific version
         if "@" in pkg_dir and "__src_url" not in manifest and not requirements:
             return None
 
@@ -633,112 +681,123 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 name,
                 requirements=None,
                 silent=False,
-                trigger_event=True,
+                after_update=False,
                 force=False):
-        name, requirements, url = self.parse_pkg_uri(name, requirements)
-        package_dir = self.get_package_dir(name, requirements, url)
+        pkg_dir = None
+        # interprocess lock
+        with LockFile(self.package_dir):
+            self.cache_reset()
 
-        # avoid circle dependencies
-        if not self.INSTALL_HISTORY:
-            self.INSTALL_HISTORY = []
-        history_key = "%s-%s-%s" % (name, requirements or "", url or "")
-        if history_key in self.INSTALL_HISTORY:
-            return package_dir
-        self.INSTALL_HISTORY.append(history_key)
+            name, requirements, url = self.parse_pkg_uri(name, requirements)
+            package_dir = self.get_package_dir(name, requirements, url)
 
-        if package_dir and force:
-            self.uninstall(package_dir)
-            package_dir = None
+            # avoid circle dependencies
+            if not self.INSTALL_HISTORY:
+                self.INSTALL_HISTORY = []
+            history_key = "%s-%s-%s" % (name, requirements or "", url or "")
+            if history_key in self.INSTALL_HISTORY:
+                return package_dir
+            self.INSTALL_HISTORY.append(history_key)
+
+            if package_dir and force:
+                self.uninstall(package_dir)
+                package_dir = None
+
+            if not package_dir or not silent:
+                msg = "Installing " + click.style(name, fg="cyan")
+                if requirements:
+                    msg += " @ " + requirements
+                self.print_message(msg)
+            if package_dir:
+                if not silent:
+                    click.secho(
+                        "{name} @ {version} is already installed".format(
+                            **self.load_manifest(package_dir)),
+                        fg="yellow")
+                return package_dir
+
+            if url:
+                pkg_dir = self._install_from_url(
+                    name, url, requirements, track=True)
+            else:
+                pkg_dir = self._install_from_piorepo(name, requirements)
+
+            if not pkg_dir or not self.manifest_exists(pkg_dir):
+                raise exception.PackageInstallError(name, requirements or "*",
+                                                    util.get_systype())
+
+            manifest = self.load_manifest(pkg_dir)
+            assert manifest
+
+            if not after_update:
+                telemetry.on_event(
+                    category=self.__class__.__name__,
+                    action="Install",
+                    label=manifest['name'])
 
-        if not package_dir or not silent:
-            msg = "Installing " + click.style(name, fg="cyan")
-            if requirements:
-                msg += " @ " + requirements
-            self.print_message(msg)
-        if package_dir:
             if not silent:
                 click.secho(
-                    "{name} @ {version} is already installed".format(
-                        **self.load_manifest(package_dir)),
-                    fg="yellow")
-            return package_dir
-
-        if url:
-            pkg_dir = self._install_from_url(
-                name, url, requirements, track=True)
-        else:
-            pkg_dir = self._install_from_piorepo(name, requirements)
-
-        if not pkg_dir or not self.manifest_exists(pkg_dir):
-            raise exception.PackageInstallError(name, requirements or "*",
-                                                util.get_systype())
-
-        manifest = self.load_manifest(pkg_dir)
-        assert manifest
-
-        if trigger_event:
-            telemetry.on_event(
-                category=self.__class__.__name__,
-                action="Install",
-                label=manifest['name'])
-
-        if not silent:
-            click.secho(
-                "{name} @ {version} has been successfully installed!".format(
-                    **manifest),
-                fg="green")
+                    "{name} @ {version} has been successfully installed!".
+                    format(**manifest),
+                    fg="green")
 
         return pkg_dir
 
-    def uninstall(self, package, requirements=None, trigger_event=True):
-        if isdir(package):
-            pkg_dir = package
-        else:
-            name, requirements, url = self.parse_pkg_uri(package, requirements)
-            pkg_dir = self.get_package_dir(name, requirements, url)
-
-        if not pkg_dir:
-            raise exception.UnknownPackage("%s @ %s" % (package,
-                                                        requirements or "*"))
-
-        manifest = self.load_manifest(pkg_dir)
-        click.echo(
-            "Uninstalling %s @ %s: \t" %
-            (click.style(manifest['name'], fg="cyan"), manifest['version']),
-            nl=False)
-
-        if islink(pkg_dir):
-            os.unlink(pkg_dir)
-        else:
-            util.rmtree_(pkg_dir)
-        self.cache_reset()
-
-        # unfix package with the same name
-        pkg_dir = self.get_package_dir(manifest['name'])
-        if pkg_dir and "@" in pkg_dir:
-            os.rename(pkg_dir,
-                      join(self.package_dir,
-                           self.get_install_dirname(manifest)))
+    def uninstall(self, package, requirements=None, after_update=False):
+        # interprocess lock
+        with LockFile(self.package_dir):
             self.cache_reset()
 
-        click.echo("[%s]" % click.style("OK", fg="green"))
+            if isdir(package) and self.get_package_by_dir(package):
+                pkg_dir = package
+            else:
+                name, requirements, url = self.parse_pkg_uri(
+                    package, requirements)
+                pkg_dir = self.get_package_dir(name, requirements, url)
+
+            if not pkg_dir:
+                raise exception.UnknownPackage(
+                    "%s @ %s" % (package, requirements or "*"))
+
+            manifest = self.load_manifest(pkg_dir)
+            click.echo(
+                "Uninstalling %s @ %s: \t" % (click.style(
+                    manifest['name'], fg="cyan"), manifest['version']),
+                nl=False)
+
+            if islink(pkg_dir):
+                os.unlink(pkg_dir)
+            else:
+                util.rmtree_(pkg_dir)
+            self.cache_reset()
+
+            # unfix package with the same name
+            pkg_dir = self.get_package_dir(manifest['name'])
+            if pkg_dir and "@" in pkg_dir:
+                shutil.move(
+                    pkg_dir,
+                    join(self.package_dir, self.get_install_dirname(manifest)))
+                self.cache_reset()
+
+            click.echo("[%s]" % click.style("OK", fg="green"))
+
+            if not after_update:
+                telemetry.on_event(
+                    category=self.__class__.__name__,
+                    action="Uninstall",
+                    label=manifest['name'])
 
-        if trigger_event:
-            telemetry.on_event(
-                category=self.__class__.__name__,
-                action="Uninstall",
-                label=manifest['name'])
         return True
 
     def update(self, package, requirements=None, only_check=False):
-        if isdir(package):
+        if isdir(package) and self.get_package_by_dir(package):
             pkg_dir = package
         else:
             pkg_dir = self.get_package_dir(*self.parse_pkg_uri(package))
 
         if not pkg_dir:
-            raise exception.UnknownPackage("%s @ %s" % (package,
-                                                        requirements or "*"))
+            raise exception.UnknownPackage(
+                "%s @ %s" % (package, requirements or "*"))
 
         manifest = self.load_manifest(pkg_dir)
         name = manifest['name']
@@ -758,7 +817,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         elif latest is False:
             click.echo("[%s]" % (click.style("Up-to-date", fg="green")))
         else:
-            click.echo("[%s]" % (click.style("Fixed", fg="yellow")))
+            click.echo("[%s]" % (click.style("Detached", fg="yellow")))
 
         if only_check or not latest:
             return True
@@ -769,8 +828,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             self._update_src_manifest(
                 dict(version=vcs.get_current_revision()), vcs.storage_dir)
         else:
-            self.uninstall(pkg_dir, trigger_event=False)
-            self.install(name, latest, trigger_event=False)
+            self.uninstall(pkg_dir, after_update=True)
+            self.install(name, latest, after_update=True)
 
         telemetry.on_event(
             category=self.__class__.__name__,

platformio/managers/platform.py

@@ -30,7 +30,7 @@ from platformio.managers.package import BasePkgManager, PackageManager
 
 class PlatformManager(BasePkgManager):
 
-    FILE_CACHE_VALID = None  # disable platform caching
+    FILE_CACHE_VALID = None  # disable platform download caching
 
     def __init__(self, package_dir=None, repositories=None):
         if not repositories:
@@ -39,9 +39,9 @@ class PlatformManager(BasePkgManager):
                 "{0}://dl.platformio.org/platforms/manifest.json".format(
                     "https" if app.get_setting("enable_ssl") else "http")
             ]
-        BasePkgManager.__init__(self, package_dir
-                                or join(util.get_home_dir(), "platforms"),
-                                repositories)
+        BasePkgManager.__init__(
+            self, package_dir or join(util.get_home_dir(), "platforms"),
+            repositories)
 
     @property
     def manifest_names(self):
@@ -62,7 +62,7 @@ class PlatformManager(BasePkgManager):
                 with_packages=None,
                 without_packages=None,
                 skip_default_package=False,
-                trigger_event=True,
+                after_update=False,
                 silent=False,
                 force=False,
                 **_):  # pylint: disable=too-many-arguments, arguments-differ
@@ -70,20 +70,20 @@ class PlatformManager(BasePkgManager):
             self, name, requirements, silent=silent, force=force)
         p = PlatformFactory.newPlatform(platform_dir)
 
-        # @Hook: when 'update' operation (trigger_event is False),
-        # don't cleanup packages or install them
-        if not trigger_event:
+        # don't cleanup packages or install them after update
+        # we check packages for updates in def update()
+        if after_update:
             return True
+
         p.install_packages(
             with_packages,
             without_packages,
             skip_default_package,
             silent=silent,
             force=force)
-        self.cleanup_packages(p.packages.keys())
-        return True
+        return self.cleanup_packages(p.packages.keys())
 
-    def uninstall(self, package, requirements=None, trigger_event=True):
+    def uninstall(self, package, requirements=None, after_update=False):
         if isdir(package):
             pkg_dir = package
         else:
@@ -96,13 +96,12 @@ class PlatformManager(BasePkgManager):
         p = PlatformFactory.newPlatform(pkg_dir)
         BasePkgManager.uninstall(self, pkg_dir, requirements)
 
-        # @Hook: when 'update' operation (trigger_event is False),
-        # don't cleanup packages or install them
-        if not trigger_event:
+        # don't cleanup packages or install them after update
+        # we check packages for updates in def update()
+        if after_update:
             return True
 
-        self.cleanup_packages(p.packages.keys())
-        return True
+        return self.cleanup_packages(p.packages.keys())
 
     def update(  # pylint: disable=arguments-differ
             self,
@@ -154,11 +153,15 @@ class PlatformManager(BasePkgManager):
                 continue
             if (manifest['name'] not in deppkgs
                     or manifest['version'] not in deppkgs[manifest['name']]):
-                pm.uninstall(manifest['__pkg_dir'], trigger_event=False)
+                try:
+                    pm.uninstall(manifest['__pkg_dir'], after_update=True)
+                except exception.UnknownPackage:
+                    pass
 
         self.cache_reset()
         return True
 
+    @util.memoized(expire=5000)
     def get_installed_boards(self):
         boards = []
         for manifest in self.get_installed():
@@ -170,7 +173,7 @@ class PlatformManager(BasePkgManager):
         return boards
 
     @staticmethod
-    @util.memoized
+    @util.memoized()
     def get_registered_boards():
         return util.get_api_result("/boards", cache_valid="7d")
 
@@ -232,8 +235,8 @@ class PlatformFactory(object):
                 name = pm.load_manifest(platform_dir)['name']
 
         if not platform_dir:
-            raise exception.UnknownPlatform(name if not requirements else
-                                            "%s@%s" % (name, requirements))
+            raise exception.UnknownPlatform(
+                name if not requirements else "%s@%s" % (name, requirements))
 
         platform_cls = None
         if isfile(join(platform_dir, "platform.py")):
@@ -280,21 +283,25 @@ class PlatformPackagesMixin(object):
 
         return True
 
-    def find_pkg_names(self, items):
+    def find_pkg_names(self, candidates):
         result = []
-        for item in items:
-            candidate = item
+        for candidate in candidates:
+            found = False
 
             # lookup by package types
             for _name, _opts in self.packages.items():
-                if _opts.get("type") == item:
-                    candidate = _name
+                if _opts.get("type") == candidate:
+                    result.append(_name)
+                    found = True
 
-            if (self.frameworks and item.startswith("framework-")
-                    and item[10:] in self.frameworks):
-                candidate = self.frameworks[item[10:]]['package']
+            if (self.frameworks and candidate.startswith("framework-")
+                    and candidate[10:] in self.frameworks):
+                result.append(self.frameworks[candidate[10:]]['package'])
+                found = True
+
+            if not found:
+                result.append(candidate)
 
-            result.append(candidate)
         return result
 
     def update_packages(self, only_check=False):
@@ -324,8 +331,8 @@ class PlatformPackagesMixin(object):
     def get_package_dir(self, name):
         version = self.packages[name].get("version", "")
         if ":" in version:
-            return self.pm.get_package_dir(*self.pm.parse_pkg_uri(
-                "%s=%s" % (name, version)))
+            return self.pm.get_package_dir(
+                *self.pm.parse_pkg_uri("%s=%s" % (name, version)))
         return self.pm.get_package_dir(name, version)
 
     def get_package_version(self, name):
@@ -489,6 +496,10 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
     def vendor_url(self):
         return self._manifest.get("url")
 
+    @property
+    def docs_url(self):
+        return self._manifest.get("docs")
+
     @property
     def repository_url(self):
         return self._manifest.get("repository", {}).get("url")
@@ -593,12 +604,13 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
 
         # enable upload tools for upload targets
         if any(["upload" in t for t in targets] + ["program" in targets]):
-            for _name, _opts in self.packages.iteritems():
-                if _opts.get("type") == "uploader":
-                    self.packages[_name]['optional'] = False
-                elif "nobuild" in targets:
-                    # skip all packages, allow only upload tools
-                    self.packages[_name]['optional'] = True
+            for name, opts in self.packages.items():
+                if opts.get("type") == "uploader":
+                    self.packages[name]['optional'] = False
+                # skip all packages in "nobuild" mode
+                # allow only upload tools and frameworks
+                elif "nobuild" in targets and opts.get("type") != "framework":
+                    self.packages[name]['optional'] = True
 
     def get_lib_storages(self):
         storages = []
@@ -654,6 +666,15 @@ class PlatformBoardConfig(object):
             else:
                 raise KeyError("Invalid board option '%s'" % path)
 
+    def update(self, path, value):
+        newdict = None
+        for key in path.split(".")[::-1]:
+            if newdict is None:
+                newdict = {key: value}
+            else:
+                newdict = {key: newdict}
+        util.merge_dicts(self._manifest, newdict)
+
     def __contains__(self, key):
         try:
             self.get(key)

platformio/projectconftpl.ini

@@ -6,4 +6,4 @@
 ;   Advanced options: extra scripting
 ;
 ; Please visit documentation for the other options and examples
-; http://docs.platformio.org/page/projectconf.html
+; https://docs.platformio.org/page/projectconf.html

platformio/telemetry.py

@@ -16,6 +16,7 @@ import atexit
 import platform
 import Queue
 import re
+import sys
 import threading
 from collections import deque
 from os import getenv, sep
@@ -60,7 +61,7 @@ class MeasurementProtocol(TelemetryBase):
     }
 
     def __init__(self):
-        TelemetryBase.__init__(self)
+        super(MeasurementProtocol, self).__init__()
         self['v'] = 1
         self['tid'] = self.TID
         self['cid'] = app.get_cid()
@@ -111,8 +112,8 @@ class MeasurementProtocol(TelemetryBase):
         self['cd2'] = "Python/%s %s" % (platform.python_version(),
                                         platform.platform())
         # self['cd3'] = " ".join(_filter_args(sys.argv[1:]))
-        self['cd4'] = 1 if (not util.is_ci()
-                            and (caller_id or not util.is_container())) else 0
+        self['cd4'] = 1 if (not util.is_ci() and
+                            (caller_id or not util.is_container())) else 0
         if caller_id:
             self['cd5'] = caller_id.lower()
 
@@ -152,16 +153,22 @@ class MeasurementProtocol(TelemetryBase):
                         cmd_path.append(sub_cmd)
         self['screen_name'] = " ".join([p.title() for p in cmd_path])
 
-    def send(self, hittype):
+    @staticmethod
+    def _ignore_hit():
         if not app.get_setting("enable_telemetry"):
+            return True
+        if app.get_session_var("caller_id") and \
+                all(c in sys.argv for c in ("run", "idedata")):
+            return True
+        return False
+
+    def send(self, hittype):
+        if self._ignore_hit():
             return
-
         self['t'] = hittype
-
         # correct queue time
         if "qt" in self._params and isinstance(self['qt'], float):
             self['qt'] = int((time() - self['qt']) * 1000)
-
         MPDataPusher().push(self._params)
 
 
@@ -284,7 +291,7 @@ def measure_ci():
         }
     }
 
-    for key, value in envmap.iteritems():
+    for key, value in envmap.items():
         if getenv(key, "").lower() != "true":
             continue
         event.update({"action": key, "label": value['label']})

platformio/unpacker.py

@@ -13,15 +13,14 @@
 # limitations under the License.
 
 from os import chmod
-from os.path import join
+from os.path import exists, islink, join
 from tarfile import open as tarfile_open
 from time import mktime
 from zipfile import ZipFile
 
 import click
 
-from platformio import app, util
-from platformio.exception import UnsupportedArchiveType
+from platformio import exception, util
 
 
 class ArchiveBase(object):
@@ -32,6 +31,9 @@ class ArchiveBase(object):
     def get_items(self):
         raise NotImplementedError()
 
+    def get_item_filename(self, item):
+        raise NotImplementedError()
+
     def extract_item(self, item, dest_dir):
         self._afo.extract(item, dest_dir)
         self.after_extract(item, dest_dir)
@@ -46,16 +48,19 @@ class ArchiveBase(object):
 class TARArchive(ArchiveBase):
 
     def __init__(self, archpath):
-        ArchiveBase.__init__(self, tarfile_open(archpath))
+        super(TARArchive, self).__init__(tarfile_open(archpath))
 
     def get_items(self):
         return self._afo.getmembers()
 
+    def get_item_filename(self, item):
+        return item.name
+
 
 class ZIPArchive(ArchiveBase):
 
     def __init__(self, archpath):
-        ArchiveBase.__init__(self, ZipFile(archpath))
+        super(ZIPArchive, self).__init__(ZipFile(archpath))
 
     @staticmethod
     def preserve_permissions(item, dest_dir):
@@ -72,6 +77,9 @@ class ZIPArchive(ArchiveBase):
     def get_items(self):
         return self._afo.infolist()
 
+    def get_item_filename(self, item):
+        return item.filename
+
     def after_extract(self, item, dest_dir):
         self.preserve_permissions(item, dest_dir)
         self.preserve_mtime(item, dest_dir)
@@ -89,16 +97,16 @@ class FileUnpacker(object):
         elif self.archpath.lower().endswith(".zip"):
             self._unpacker = ZIPArchive(self.archpath)
         if not self._unpacker:
-            raise UnsupportedArchiveType(self.archpath)
+            raise exception.UnsupportedArchiveType(self.archpath)
         return self
 
     def __exit__(self, *args):
         if self._unpacker:
             self._unpacker.close()
 
-    def unpack(self, dest_dir="."):
+    def unpack(self, dest_dir=".", with_progress=True):
         assert self._unpacker
-        if app.is_disabled_progressbar():
+        if not with_progress:
             click.echo("Unpacking...")
             for item in self._unpacker.get_items():
                 self._unpacker.extract_item(item, dest_dir)
@@ -107,4 +115,12 @@ class FileUnpacker(object):
             with click.progressbar(items, label="Unpacking") as pb:
                 for item in pb:
                     self._unpacker.extract_item(item, dest_dir)
+
+        # check on disk
+        for item in self._unpacker.get_items():
+            filename = self._unpacker.get_item_filename(item)
+            item_path = join(dest_dir, filename)
+            if not islink(item_path) and not exists(item_path):
+                raise exception.ExtractArchiveItemError(filename, dest_dir)
+
         return True

platformio/util.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import collections
-import functools
 import json
 import os
 import platform
@@ -25,6 +23,7 @@ import sys
 import time
 from functools import wraps
 from glob import glob
+from hashlib import sha1
 from os.path import (abspath, basename, dirname, expanduser, isdir, isfile,
                      join, normpath, splitdrive)
 from shutil import rmtree
@@ -38,30 +37,40 @@ from platformio import __apiurl__, __version__, exception
 # pylint: disable=wrong-import-order, too-many-ancestors
 
 try:
-    from configparser import ConfigParser
+    import configparser as ConfigParser
 except ImportError:
-    from ConfigParser import ConfigParser
+    import ConfigParser as ConfigParser
 
 
-class ProjectConfig(ConfigParser):
+class ProjectConfig(ConfigParser.ConfigParser):
 
     VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}")
 
     def items(self, section, **_):  # pylint: disable=arguments-differ
         items = []
-        for option in ConfigParser.options(self, section):
+        for option in ConfigParser.ConfigParser.options(self, section):
             items.append((option, self.get(section, option)))
         return items
 
     def get(self, section, option, **kwargs):
-        value = ConfigParser.get(self, section, option, **kwargs)
+        try:
+            value = ConfigParser.ConfigParser.get(self, section, option,
+                                                  **kwargs)
+        except ConfigParser.Error as e:
+            raise exception.InvalidProjectConf(str(e))
         if "${" not in value or "}" not in value:
             return value
         return self.VARTPL_RE.sub(self._re_sub_handler, value)
 
     def _re_sub_handler(self, match):
         section, option = match.group(1), match.group(2)
-        if section == "env" and not self.has_section(section):
+        if section in ("env", "sysenv") and not self.has_section(section):
+            if section == "env":
+                click.secho(
+                    "Warning! Access to system environment variable via "
+                    "`${{env.{0}}}` is deprecated. Please use "
+                    "`${{sysenv.{0}}}` instead".format(option),
+                    fg="yellow")
             return os.getenv(option)
         return self.get(section, option)
 
@@ -69,7 +78,7 @@ class ProjectConfig(ConfigParser):
 class AsyncPipe(Thread):
 
     def __init__(self, outcallback=None):
-        Thread.__init__(self)
+        super(AsyncPipe, self).__init__()
         self.outcallback = outcallback
 
         self._fd_read, self._fd_write = os.pipe()
@@ -91,7 +100,7 @@ class AsyncPipe(Thread):
             if self.outcallback:
                 self.outcallback(line)
             else:
-                print line
+                print(line)
         self._pipe_reader.close()
 
     def close(self):
@@ -113,37 +122,24 @@ class cd(object):
 
 
 class memoized(object):
-    '''
-    Decorator. Caches a function's return value each time it is called.
-    If called later with the same arguments, the cached value is returned
-    (not reevaluated).
-    https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize
-    '''
 
-    def __init__(self, func):
-        self.func = func
+    def __init__(self, expire=0):
+        self.expire = expire / 1000  # milliseconds
         self.cache = {}
 
-    def __call__(self, *args):
-        if not isinstance(args, collections.Hashable):
-            # uncacheable. a list, for instance.
-            # better to not cache than blow up.
-            return self.func(*args)
-        if args in self.cache:
-            return self.cache[args]
-        value = self.func(*args)
-        self.cache[args] = value
-        return value
+    def __call__(self, func):
 
-    def __repr__(self):
-        '''Return the function's docstring.'''
-        return self.func.__doc__
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            key = str(args) + str(kwargs)
+            if (key not in self.cache
+                    or (self.expire > 0
+                        and self.cache[key][0] < time.time() - self.expire)):
+                self.cache[key] = (time.time(), func(*args, **kwargs))
+            return self.cache[key][1]
 
-    def __get__(self, obj, objtype):
-        '''Support instance methods.'''
-        fn = functools.partial(self.__call__, obj)
-        fn.reset = self._reset
-        return fn
+        wrapper.reset = self._reset
+        return wrapper
 
     def _reset(self):
         self.cache = {}
@@ -155,15 +151,15 @@ class throttle(object):
         self.threshhold = threshhold  # milliseconds
         self.last = 0
 
-    def __call__(self, fn):
+    def __call__(self, func):
 
-        @wraps(fn)
+        @wraps(func)
         def wrapper(*args, **kwargs):
             diff = int(round((time.time() - self.last) * 1000))
             if diff < self.threshhold:
                 time.sleep((self.threshhold - diff) * 0.001)
             self.last = time.time()
-            return fn(*args, **kwargs)
+            return func(*args, **kwargs)
 
         return wrapper
 
@@ -189,8 +185,7 @@ def load_json(file_path):
         with open(file_path, "r") as f:
             return json.load(f)
     except ValueError:
-        raise exception.PlatformioException(
-            "Could not load broken JSON: %s" % file_path)
+        raise exception.InvalidJSONFile(file_path)
 
 
 def get_systype():
@@ -319,6 +314,9 @@ def get_projectboards_dir():
 def get_projectbuild_dir(force=False):
     path = get_project_optional_dir("build_dir",
                                     join(get_project_dir(), ".pioenvs"))
+    if "$PROJECT_HASH" in path:
+        path = path.replace("$PROJECT_HASH",
+                            sha1(get_project_dir()).hexdigest()[:10])
     try:
         if not isdir(path):
             os.makedirs(path)
@@ -327,7 +325,7 @@ def get_projectbuild_dir(force=False):
             with open(dontmod_path, "w") as fp:
                 fp.write("""
 [InternetShortcut]
-URL=http://docs.platformio.org/page/projectconf/section_platformio.html#build-dir
+URL=https://docs.platformio.org/page/projectconf/section_platformio.html#build-dir
 """)
     except Exception as e:  # pylint: disable=broad-except
         if not force:
@@ -351,17 +349,27 @@ def load_project_config(path=None):
         raise exception.NotPlatformIOProject(
             dirname(path) if path.endswith("platformio.ini") else path)
     cp = ProjectConfig()
-    cp.read(path)
+    try:
+        cp.read(path)
+    except ConfigParser.Error as e:
+        raise exception.InvalidProjectConf(str(e))
     return cp
 
 
 def parse_conf_multi_values(items):
+    result = []
     if not items:
-        return []
-    return [
-        item.strip() for item in items.split("\n" if "\n" in items else ", ")
-        if item.strip()
-    ]
+        return result
+    inline_comment_re = re.compile(r"\s+;.*$")
+    for item in items.split("\n" if "\n" in items else ", "):
+        item = item.strip()
+        # comment
+        if not item or item.startswith((";", "#")):
+            continue
+        if ";" in item:
+            item = inline_comment_re.sub("", item).strip()
+        result.append(item)
+    return result
 
 
 def change_filemtime(path, mtime):
@@ -405,7 +413,7 @@ def exec_command(*args, **kwargs):
         if isinstance(kwargs[s], AsyncPipe):
             result[s[3:]] = "\n".join(kwargs[s].get_buffer())
 
-    for k, v in result.iteritems():
+    for k, v in result.items():
         if v and isinstance(v, basestring):
             result[k].strip()
 
@@ -548,6 +556,14 @@ def get_mdns_services():
     with mDNSListener() as mdns:
         time.sleep(3)
         for service in mdns.get_services():
+            properties = None
+            try:
+                if service.properties:
+                    json.dumps(service.properties)
+                properties = service.properties
+            except UnicodeDecodeError:
+                pass
+
             items.append({
                 "type":
                 service.type,
@@ -558,7 +574,7 @@ def get_mdns_services():
                 "port":
                 service.port,
                 "properties":
-                service.properties
+                properties
             })
     return items
 
@@ -568,7 +584,7 @@ def get_request_defheaders():
     return {"User-Agent": "PlatformIO/%s CI/%d %s" % data}
 
 
-@memoized
+@memoized(expire=10000)
 def _api_request_session():
     return requests.Session()
 
@@ -609,6 +625,7 @@ def _get_api_result(
                 verify=verify_ssl)
         result = r.json()
         r.raise_for_status()
+        return r.text
     except requests.exceptions.HTTPError as e:
         if result and "message" in result:
             raise exception.APIRequestError(result['message'])
@@ -622,7 +639,7 @@ def _get_api_result(
     finally:
         if r:
             r.close()
-    return result
+    return None
 
 
 def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
@@ -637,7 +654,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
                 if cache_key:
                     result = cc.get(cache_key)
                     if result is not None:
-                        return result
+                        return json.loads(result)
 
             # check internet before and resolve issue with 60 seconds timeout
             internet_on(raise_exception=True)
@@ -646,7 +663,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
             if cache_valid:
                 with ContentCache() as cc:
                     cc.set(cache_key, result, cache_valid)
-            return result
+            return json.loads(result)
         except (requests.exceptions.ConnectionError,
                 requests.exceptions.Timeout) as e:
             from platformio.maintenance import in_silence
@@ -670,7 +687,7 @@ PING_INTERNET_IPS = [
 ]
 
 
-@memoized
+@memoized(expire=5000)
 def _internet_on():
     timeout = 2
     socket.setdefaulttimeout(timeout)
@@ -765,6 +782,55 @@ def format_filesize(filesize):
     return "%d%sB" % ((base * filesize / unit), suffix)
 
 
+def merge_dicts(d1, d2, path=None):
+    if path is None:
+        path = []
+    for key in d2:
+        if (key in d1 and isinstance(d1[key], dict)
+                and isinstance(d2[key], dict)):
+            merge_dicts(d1[key], d2[key], path + [str(key)])
+        else:
+            d1[key] = d2[key]
+    return d1
+
+
+def ensure_udev_rules():
+
+    def _rules_to_set(rules_path):
+        result = set([])
+        with open(rules_path, "rb") as fp:
+            for line in fp.readlines():
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                result.add(line)
+        return result
+
+    if "linux" not in get_systype():
+        return None
+    installed_rules = [
+        "/etc/udev/rules.d/99-platformio-udev.rules",
+        "/lib/udev/rules.d/99-platformio-udev.rules"
+    ]
+    if not any(isfile(p) for p in installed_rules):
+        raise exception.MissedUdevRules
+
+    origin_path = abspath(
+        join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules"))
+    if not isfile(origin_path):
+        return None
+
+    origin_rules = _rules_to_set(origin_path)
+    for rules_path in installed_rules:
+        if not isfile(rules_path):
+            continue
+        current_rules = _rules_to_set(rules_path)
+        if not origin_rules <= current_rules:
+            raise exception.OutdatedUdevRules(rules_path)
+
+    return True
+
+
 def rmtree_(path):
 
     def _onerror(_, name, __):
@@ -773,8 +839,9 @@ def rmtree_(path):
             os.remove(name)
         except Exception as e:  # pylint: disable=broad-except
             click.secho(
-                "Please manually remove file `%s`" % name, fg="red", err=True)
-            raise e
+                "%s \nPlease manually remove the file `%s`" % (str(e), name),
+                fg="red",
+                err=True)
 
     return rmtree(path, onerror=_onerror)
 
@@ -791,8 +858,7 @@ except ImportError:
     magic_check_bytes = re.compile(b'([*?[])')
 
     def glob_escape(pathname):
-        """Escape all special characters.
-        """
+        """Escape all special characters."""
         # Escaping is done by wrapping any of "*?[" between square brackets.
         # Metacharacters do not work in the drive part and shouldn't be
         # escaped.

scripts/99-platformio-udev.rules

@@ -16,7 +16,7 @@
 #
 # INSTALLATION
 #
-# Please visit > http://docs.platformio.org/en/latest/faq.html#platformio-udev-rules
+# Please visit > https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules
 #
 #####################################################################################
 
@@ -53,6 +53,9 @@ KERNEL=="ttyACM*", ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="066
 # STM32 discovery boards, with onboard st/linkv2
 SUBSYSTEMS=="usb", ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374?", MODE:="0666"
 
+# Maple with DFU
+SUBSYSTEMS=="usb", ATTRS{idVendor}=="1eaf", ATTRS{idProduct}=="000[34]", MODE:="0666"
+
 # USBtiny
 SUBSYSTEMS=="usb", ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE="0666"
 
@@ -205,3 +208,52 @@ ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="660", GROUP="plugdev",
 
 # CMSIS-DAP compatible adapters
 ATTRS{product}=="*CMSIS-DAP*", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+#SEGGER J-LIK
+ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="666"
+ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="666"

scripts/install_devplatforms.py

@@ -15,6 +15,7 @@
 import json
 import subprocess
 import sys
+from platformio import util
 
 
 def main():
@@ -24,6 +25,14 @@ def main():
     for platform in platforms:
         if platform['forDesktop']:
             continue
+        # RISC-V GAP does not support Windows 86
+        if (util.get_systype() == "windows_x86"
+                and platform['name'] == "riscv_gap"):
+            continue
+        # unknown issue on Linux
+        if ("linux" in util.get_systype()
+                and platform['name'] == "aceinna_imu"):
+            continue
         subprocess.check_call(
             ["platformio", "platform", "install", platform['repository']])
 

setup.py

@@ -21,7 +21,6 @@ install_requires = [
     "bottle<0.13",
     "click>=5,<6",
     "colorama",
-    "lockfile>=0.9.1,<0.13",
     "pyserial>=3,<4,!=3.3",
     "requests>=2.4.0,<3",
     "semantic_version>=2.5.0,<3"
@@ -38,7 +37,7 @@ setup(
     license=__license__,
     python_requires='>=2.7, <3',
     install_requires=install_requires,
-    packages=find_packages(),
+    packages=find_packages() + ["scripts"],
     package_data={
         "platformio": [
             "projectconftpl.ini",
@@ -46,6 +45,9 @@ setup(
             "ide/tpls/*/*.tpl",
             "ide/tpls/*/*/*.tpl",
             "ide/tpls/*/.*/*.tpl"
+        ],
+        "scripts": [
+            "99-platformio-udev.rules"
         ]
     },
     entry_points={

tests/commands/test_lib.py

@@ -61,8 +61,8 @@ def test_global_install_archive(clirunner, validate_cliresult,
         "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip",
         "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip",
         "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2",
-        "http://dl.platformio.org/libraries/archives/0/9540.tar.gz",
-        "https://github.com/adafruit/Adafruit-ST7735-Library/archive/master.zip"
+        "SomeLib=http://dl.platformio.org/libraries/archives/0/9540.tar.gz",
+        "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip"
     ])
     validate_cliresult(result)
 
@@ -75,8 +75,8 @@ def test_global_install_archive(clirunner, validate_cliresult,
 
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
     items2 = [
-        "RadioHead-1.62", "ArduinoJson", "DallasTemperature_ID54",
-        "OneWire_ID1", "Adafruit ST7735 Library"
+        "RadioHead-1.62", "ArduinoJson", "SomeLib_ID54",
+        "OneWire_ID1", "ESP32WebServer"
     ]
     assert set(items1) >= set(items2)
 
@@ -142,7 +142,7 @@ def test_global_lib_list(clirunner, validate_cliresult):
     validate_cliresult(result)
     assert all([
         n in result.output for n in
-        ("Source: https://github.com/adafruit/Adafruit-ST7735-Library/archive/master.zip",
+        ("Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
          "Version: 5.10.1",
          "Source: git+https://github.com/gioblu/PJON.git#3.0",
          "Version: 1fb26fd", "RadioHead-1.62")
@@ -157,8 +157,8 @@ def test_global_lib_list(clirunner, validate_cliresult):
     ])
     items1 = [i['name'] for i in json.loads(result.output)]
     items2 = [
-        "Adafruit ST7735 Library", "ArduinoJson", "ArduinoJson", "ArduinoJson",
-        "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "DallasTemperature",
+        "ESP32WebServer", "ArduinoJson", "ArduinoJson", "ArduinoJson",
+        "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "SomeLib",
         "ESPAsyncTCP", "NeoPixelBus", "OneWire", "PJON", "PJON",
         "PubSubClient", "RFcontrol", "RadioHead-1.62", "platformio-libmirror",
         "rs485-nodeproto"
@@ -166,14 +166,12 @@ def test_global_lib_list(clirunner, validate_cliresult):
     assert sorted(items1) == sorted(items2)
 
     versions1 = [
-        "{name}@{version}".format(**item)
-        for item in json.loads(result.output)
+        "{name}@{version}".format(**item) for item in json.loads(result.output)
     ]
     versions2 = [
         'ArduinoJson@5.8.2', 'ArduinoJson@5.10.1', 'AsyncMqttClient@0.8.2',
-        'AsyncTCP@1.0.1', 'ESPAsyncTCP@1.1.3', 'NeoPixelBus@2.2.4',
-        'PJON@07fe9aa', 'PJON@1fb26fd', 'PubSubClient@bef5814',
-        'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0'
+        'AsyncTCP@1.0.1', 'NeoPixelBus@2.2.4', 'PJON@07fe9aa', 'PJON@1fb26fd',
+        'PubSubClient@bef5814', 'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0'
     ]
     assert set(versions1) >= set(versions2)
 
@@ -204,7 +202,7 @@ def test_global_lib_update(clirunner, validate_cliresult):
     # update rest libraries
     result = clirunner.invoke(cmd_lib, ["-g", "update"])
     validate_cliresult(result)
-    assert result.output.count("[Fixed]") == 6
+    assert result.output.count("[Detached]") == 6
     assert result.output.count("[Up-to-date]") == 11
     assert "Uninstalling RFcontrol @ 77d4eb3f8a" in result.output
 
@@ -221,9 +219,9 @@ def test_global_lib_uninstall(clirunner, validate_cliresult,
     validate_cliresult(result)
     items = json.loads(result.output)
     result = clirunner.invoke(cmd_lib,
-                              ["-g", "uninstall", items[0]['__pkg_dir']])
+                              ["-g", "uninstall", items[5]['__pkg_dir']])
     validate_cliresult(result)
-    assert "Uninstalling Adafruit ST7735 Library" in result.output
+    assert "Uninstalling AsyncTCP" in result.output
 
     # uninstall the rest libraries
     result = clirunner.invoke(cmd_lib, [
@@ -236,9 +234,9 @@ def test_global_lib_uninstall(clirunner, validate_cliresult,
     items2 = [
         "RadioHead-1.62", "rs485-nodeproto", "platformio-libmirror",
         "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81",
-        "ESPAsyncTCP_ID305", "DallasTemperature_ID54", "NeoPixelBus_ID547",
+        "ESPAsyncTCP_ID305", "SomeLib_ID54", "NeoPixelBus_ID547",
         "PJON", "AsyncMqttClient_ID346", "ArduinoJson_ID64",
-        "PJON@src-79de467ebe19de18287becff0a1fb42d", "AsyncTCP_ID1826"
+        "PJON@src-79de467ebe19de18287becff0a1fb42d", "ESP32WebServer"
     ]
     assert set(items1) == set(items2)
 

tests/commands/test_platform.py

@@ -61,13 +61,14 @@ def test_install_known_version(clirunner, validate_cliresult,
     assert len(isolated_pio_home.join("packages").listdir()) == 1
 
 
-def test_install_from_vcs(clirunner, validate_cliresult):
+def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home):
     result = clirunner.invoke(cli_platform.platform_install, [
         "https://github.com/platformio/"
         "platform-espressif8266.git#feature/stage", "--skip-default-package"
     ])
     validate_cliresult(result)
     assert "espressif8266" in result.output
+    assert len(isolated_pio_home.join("packages").listdir()) == 1
 
 
 def test_list_json_output(clirunner, validate_cliresult):

tests/test_builder.py

@@ -0,0 +1,103 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.run import cli as cmd_run
+
+
+def test_build_flags(clirunner, validate_cliresult, tmpdir):
+    build_flags = [("-D TEST_INT=13", "-DTEST_INT=13"),
+                   ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"),
+                   ('-DTEST_STR_SPACE="Andrew Smith"',
+                    '"-DTEST_STR_SPACE=Andrew Smith"')]
+
+    tmpdir.join("platformio.ini").write("""
+[env:native]
+platform = native
+extra_scripts = extra.py
+build_flags =
+    ; -DCOMMENTED_MACRO
+    %s ; inline comment
+    """ % " ".join([f[0] for f in build_flags]))
+
+    tmpdir.join("extra.py").write("""
+Import("projenv")
+
+projenv.Append(CPPDEFINES="POST_SCRIPT_MACRO")
+    """)
+
+    tmpdir.mkdir("src").join("main.cpp").write("""
+#if !defined(TEST_INT) || TEST_INT != 13
+#error "TEST_INT"
+#endif
+
+#ifndef TEST_STR_SPACE
+#error "TEST_STR_SPACE"
+#endif
+
+#ifndef POST_SCRIPT_MACRO
+#error "POST_SCRIPT_MACRO"
+#endif
+
+#ifdef COMMENTED_MACRO
+#error "COMMENTED_MACRO"
+#endif
+
+int main() {
+}
+""")
+
+    result = clirunner.invoke(
+        cmd_run, ["--project-dir", str(tmpdir), "--verbose"])
+    validate_cliresult(result)
+    build_output = result.output[result.output.find(
+        "Scanning dependencies..."):]
+    for flag in build_flags:
+        assert flag[1] in build_output, flag
+
+
+def test_build_unflags(clirunner, validate_cliresult, tmpdir):
+    tmpdir.join("platformio.ini").write("""
+[env:native]
+platform = native
+build_unflags = -DTMP_MACRO1=45 -I. -DNON_EXISTING_MACRO -lunknownLib -Os
+extra_scripts = pre:extra.py
+""")
+
+    tmpdir.join("extra.py").write("""
+Import("env")
+env.Append(CPPPATH="%s")
+env.Append(CPPDEFINES="TMP_MACRO1")
+env.Append(CPPDEFINES=["TMP_MACRO2"])
+env.Append(CPPDEFINES=("TMP_MACRO3", 13))
+env.Append(CCFLAGS=["-Os"])
+env.Append(LIBS=["unknownLib"])
+    """ % str(tmpdir))
+
+    tmpdir.mkdir("src").join("main.c").write("""
+#ifdef TMP_MACRO1
+#error "TMP_MACRO1 should be removed"
+#endif
+
+int main() {
+}
+""")
+
+    result = clirunner.invoke(
+        cmd_run, ["--project-dir", str(tmpdir), "--verbose"])
+    validate_cliresult(result)
+    build_output = result.output[result.output.find(
+        "Scanning dependencies..."):]
+    assert "-DTMP_MACRO1" not in build_output
+    assert "-Os" not in build_output
+    assert str(tmpdir) not in build_output

tests/test_examples.py

@@ -36,16 +36,24 @@ def pytest_generate_tests(metafunc):
         p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
         if not p.is_embedded():
             continue
+        # issue with "version `CXXABI_1.3.9' not found (required by sdcc)"
+        if "linux" in util.get_systype() and p.name == "intel_mcs51":
+            continue
         examples_dir = join(p.get_dir(), "examples")
         assert isdir(examples_dir)
         examples_dirs.append(examples_dir)
 
     project_dirs = []
     for examples_dir in examples_dirs:
+        platform_examples = []
         for root, _, files in walk(examples_dir):
             if "platformio.ini" not in files or ".skiptest" in files:
                 continue
-            project_dirs.append(root)
+            platform_examples.append(root)
+
+        # test random 3 examples
+        random.shuffle(platform_examples)
+        project_dirs.extend(platform_examples[:3])
     project_dirs.sort()
     metafunc.parametrize("pioproject_dir", project_dirs)
 

tests/test_pkgmanifest.py

@@ -16,11 +16,11 @@ import pytest
 import requests
 
 
-def validate_response(req):
-    assert req.status_code == 200
-    assert int(req.headers['Content-Length']) > 0
-    assert req.headers['Content-Type'] in ("application/gzip",
-                                           "application/octet-stream")
+def validate_response(r):
+    assert r.status_code == 200, r.url
+    assert int(r.headers['Content-Length']) > 0, r.url
+    assert r.headers['Content-Type'] in ("application/gzip",
+                                         "application/octet-stream")
 
 
 def test_packages():