Merge branch 'release/v3.4.1'

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

.coveragerc

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

.github/ISSUE_TEMPLATE.md

@@ -7,7 +7,9 @@ What kind of issue is this?
 - [ ] PlatformIO IDE. All issues related to PlatformIO IDE should be reported to appropriate repository
       https://github.com/platformio/platformio-atom-ide/issues
 
-- [ ] Development Platform. All issues related to Development Platform should be reported to appropriate repository. Search it using link below
+- [ ] Development Platform or Board. All issues related to Development Platforms or Embedded Boards
+      should be reported to appropriate repository. 
+      See full list with repositories and search for "platform-xxx" repository related to your hardware
       https://github.com/platformio?query=platform-
 
 - [ ] Feature Request. Start by telling us what problem you’re trying to solve. Often a solution

.isort.cfg

@@ -1,3 +1,3 @@
 [settings]
 line_length=79
-known_third_party=bottle,click,lockfile,pytest,requests,semantic_version,serial,SCons
+known_third_party=arrow,bottle,click,lockfile,pytest,requests,SCons,semantic_version,serial

.travis.yml

@@ -16,7 +16,7 @@ matrix:
       env: TOX_ENV=py27
     - os: osx
       language: generic
-      env: TOX_ENV=py27
+      env: TOX_ENV=skipexamples
 
 install:
   - git submodule update --init --recursive

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+  "python.pythonPath": "${workspaceRoot}/.tox/develop/bin/python",
+  "python.formatting.provider": "yapf",
+  "files.exclude": {
+    "**/*.pyc": true,
+    "*.egg-info": true,
+    ".cache": true,
+    "build": true,
+    "dist": true
+  },
+  "editor.rulers": [79],
+  "restructuredtext.builtDocumentationPath": "${workspaceRoot}/docs/_build/html",
+  "restructuredtext.confPath": "${workspaceRoot}/docs",
+  "restructuredtext.linter.executablePath": "${workspaceRoot}/.tox/docs/bin/restructuredtext-lint"
+}

CONTRIBUTING.md

@@ -1,7 +1,7 @@
 Contributing
 ------------
 
-To get started, <a href="https://www.clahub.com/agreements/platformio/platformio">sign the Contributor License Agreement</a>.
+To get started, <a href="https://www.clahub.com/agreements/platformio/platformio-core">sign the Contributor License Agreement</a>.
 
 1. Fork the repository on GitHub.
 2. Make a branch off of ``develop``

README.rst

@@ -7,8 +7,8 @@ PlatformIO
 .. image:: https://ci.appveyor.com/api/projects/status/unnpw0n3c5k14btn/branch/develop?svg=true
     :target: https://ci.appveyor.com/project/ivankravets/platformio-core
     :alt: AppVeyor.CI Build Status
-.. image:: https://requires.io/github/platformio/platformio/requirements.svg?branch=develop
-    :target: https://requires.io/github/platformio/platformio/requirements/?branch=develop
+.. image:: https://requires.io/github/platformio/platformio-core/requirements.svg?branch=develop
+    :target: https://requires.io/github/platformio/platformio-core/requirements/?branch=develop
     :alt: Requirements Status
 .. image:: https://img.shields.io/pypi/v/platformio.svg
     :target: https://pypi.python.org/pypi/platformio/
@@ -26,7 +26,7 @@ PlatformIO
 **Quick Links:** `Home Page <http://platformio.org>`_ |
 `PlatformIO Plus <https://pioplus.com>`_ |
 `PlatformIO IDE <http://platformio.org/platformio-ide>`_ |
-`Project Examples <https://github.com/platformio/platformio-examples/tree/develop>`_ |
+`Project Examples <https://github.com/platformio/platformio-examples/>`_ |
 `Docs <http://docs.platformio.org>`_ |
 `Donate <http://platformio.org/donate>`_ |
 `Contact Us <https://pioplus.com/contact.html>`_
@@ -37,153 +37,83 @@ PlatformIO
 `Bintray <https://bintray.com/platformio>`_ |
 `Community <https://community.platformio.org>`_
 
-.. image:: http://docs.platformio.org/en/stable/_static/platformio-logo.png
+.. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png
     :target: http://platformio.org
 
 `PlatformIO <http://platformio.org>`_ is an open source ecosystem for IoT
-development. Cross-platform build system and library manager. Continuous and
-IDE integration. Arduino, ESP8266 and ARM mbed compatible
+development. Cross-platform IDE and unified debugger. Remote unit testing and
+firmware updates.
 
-* **PlatformIO IDE** - The next-generation integrated development environment for IoT.
-  C/C++ Intelligent Code Completion and Smart Code Linter for the super-fast coding.
-  Multi-projects workflow with Multiple Panes. Themes Support with dark and light colors.
-  Built-in Terminal with PlatformIO Core tool and support for the powerful Serial Port Monitor.
-  All advanced instruments without leaving your favourite development environment.
-* **Development Platforms** - Embedded and Desktop development platforms with
-  pre-built toolchains, debuggers, uploaders and frameworks which work under
-  popular host OS: Mac, Windows, Linux (+ARM)
-* **Embedded Boards** - Rapid Embedded Programming, IDE and Continuous
-  Integration in a few steps with PlatformIO thanks to built-in project
-  generator for the most popular embedded boards and IDE
-* **Library Manager** - Hundreds Popular Libraries are organized into single
-  Web 2.0 platform: list by categories, keywords, authors, compatible
-  platforms and frameworks; learn via examples; be up-to-date with the latest
-  version.
+Get Started
+-----------
 
-*Atmel AVR & SAM, Espressif, Freescale Kinetis, Intel ARC32, Lattice iCE40,
-Microchip PIC32, Nordic nRF51, NXP LPC, Silicon Labs EFM32, ST STM32,
-TI MSP430 & Tiva, Teensy, Arduino, mbed, libOpenCM3, etc.*
+* `What is PlatformIO? <http://docs.platformio.org/page/what-is-platformio.html>`_
 
-.. image:: http://docs.platformio.org/en/stable/_static/platformio-demo-wiring.gif
-    :target: http://platformio.org
+Products
+--------
 
-* `PlatformIO Plus and professional solutions <https://pioplus.com>`_
 * `PlatformIO IDE <http://platformio.org/platformio-ide>`_
-* `Get Started <http://platformio.org/get-started>`_
-* `Library Search and Registry <http://platformio.org/lib>`_
+* `PlatformIO Core <http://docs.platformio.org/page/core.html>`_
+* `PIO Remote™ <http://docs.platformio.org/page/plus/pio-remote.html>`_
+* `PIO Unified Debugger <http://docs.platformio.org/page/plus/debugging.html>`_
+* `PIO Unit Testing <http://docs.platformio.org/page/plus/unit-testing.html>`_
+* `PIO Delivery™ <http://platformio.org/pricing#solution-pio-delivery>`_
+* `Cloud Builder <http://platformio.org/pricing#solution-cloud-builder>`_
+
+Registry
+--------
+
+* `Libraries <http://platformio.org/lib>`_
 * `Development Platforms <http://platformio.org/platforms>`_
 * `Frameworks <http://platformio.org/frameworks>`_
-* `Embedded Boards Explorer <http://platformio.org/boards>`_
-* `Library Manager <http://docs.platformio.org/en/stable/librarymanager/index.html>`_
-* `User Guide <http://docs.platformio.org/en/stable/userguide/index.html>`_
-* `Continuous Integration <http://docs.platformio.org/en/stable/ci/index.html>`_
-* `IDE Integration <http://docs.platformio.org/en/stable/ide.html>`_
-* `Articles about us <http://docs.platformio.org/en/stable/articles.html>`_
-* `FAQ <http://docs.platformio.org/en/stable/faq.html>`_
-* `Release Notes <http://docs.platformio.org/en/stable/history.html>`_
+* `Embedded Boards <http://platformio.org/boards>`_
 
-Use whenever. *Run everywhere.*
--------------------------------
-*PlatformIO* is written in pure *Python* and **doesn't depend** on any
-additional libraries/tools from an operating system. It allows you to use
-*PlatformIO* beginning from *PC (Mac, Linux, Win)* and ending with credit-card
-sized computers (`Raspberry Pi <http://www.raspberrypi.org>`_,
-`BeagleBone <http://beagleboard.org>`_,
-`CubieBoard <http://cubieboard.org>`_).
+Solutions
+---------
 
-Embedded Development. *Easier Than Ever.*
------------------------------------------
-*PlatformIO* is well suited for embedded development and has pre-configured
-settings for most popular `Embedded Boards <http://platformio.org/boards>`_.
+* `Library Manager <http://docs.platformio.org/page/librarymanager/index.html>`_
+* `Cloud IDEs Integration <http://platformio.org/pricing#solution-cloud-ide>`_
+* `Standalone IDEs Integration <http://docs.platformio.org/page/ide.html#other-ide>`_
+* `Continuous Integration <http://docs.platformio.org/page/ci/index.html>`_
 
-* Colourful `command-line output <https://raw.githubusercontent.com/platformio/platformio/develop/examples/platformio-examples.png>`_
-* `IDE Integration <http://docs.platformio.org/en/stable/ide.html>`_ with
-  *Arduino, Atom, Eclipse, Emacs, Energia, Qt Creator, Sublime Text, Vim, Visual Studio*
-* Cloud compiling and `Continuous Integration <http://docs.platformio.org/en/stable/ci/index.html>`_
-  with *AppVeyor, Circle CI, Drone, Shippable, Travis CI*
-* Built-in `Serial Port Monitor <http://docs.platformio.org/en/stable/userguide/cmd_serialports.html#platformio-serialports-monitor>`_ and configurable
-  `build -flags/-options <http://docs.platformio.org/en/stable/projectconf.html#build-flags>`_
-* Automatic **firmware uploading**
-* Pre-built tool chains, frameworks for the popular `Hardware Platforms <http://platformio.org/platforms>`_
-
-.. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-embedded-development.png
-    :target: http://platformio.org
-    :alt:  PlatformIO Embedded Development Process
-
-The Missing Library Manager. *It's here!*
------------------------------------------
-*PlatformIO Library Manager* is the missing library manager for development
-platforms which allows you to organize and have up-to-date external libraries.
-
-* Friendly `Command-Line Interface <http://docs.platformio.org/en/stable/librarymanager/index.html>`_
-* Modern `Web 2.0 Library Search <http://platformio.org/lib>`_
-* Open Source `Library Registry API <https://github.com/platformio/platformio-api>`_
-* Library Crawler based on `library.json <http://docs.platformio.org/en/stable/librarymanager/config.html>`_
-  specification
-* Library **dependency management**
-* Automatic library updating
-
-.. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-library-manager.png
-    :target: http://platformio.org
-    :alt:  PlatformIO Library Manager Architecture
-
-Smart Build System. *Fast and Reliable.*
-----------------------------------------
-*PlatformIO Code Builder* is built-on a next-generation software construction
-tool named `SCons <http://www.scons.org/>`_. Think of *SCons* as an improved,
-cross-platform substitute for the classic *Make* utility.
-
-* Reliable, automatic *dependency analysis*
-* Reliable detection of *build changes*
-* Improved support for *parallel builds*
-* Ability to share *built files in a cache*
-* Lookup for external libraries which are installed via `Library Manager <http://docs.platformio.org/en/stable/librarymanager/index.html>`_
-
-.. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-scons-builder.png
-    :target: http://platformio.org
-    :alt:  PlatformIO Build System Architecture
-
-Single source code. *Multiple platforms.*
------------------------------------------
-*PlatformIO* allows the developer to compile the same code with different
-development platforms using only *One Command*
-`platformio run <http://docs.platformio.org/en/stable/userguide/cmd_run.html>`_.
-This happens due to
-`Project Configuration File (platformio.ini) <http://docs.platformio.org/en/stable/projectconf.html>`_
-where you can setup different environments with specific options (platform
-type, firmware uploading settings, pre-built framework, build flags and many
-more).
-
-It has support for the most popular embedded platforms:
+Development Platforms
+---------------------
 
 * `Atmel AVR <http://platformio.org/platforms/atmelavr>`_
 * `Atmel SAM <http://platformio.org/platforms/atmelsam>`_
-* `Espressif <http://platformio.org/platforms/espressif>`_
+* `Espressif 32 <http://platformio.org/platforms/espressif32>`_
+* `Espressif 8266 <http://platformio.org/platforms/espressif8266>`_
 * `Freescale Kinetis <http://platformio.org/platforms/freescalekinetis>`_
 * `Intel ARC32 <http://platformio.org/platforms/intel_arc32>`_
 * `Lattice iCE40 <http://platformio.org/platforms/lattice_ice40>`_
+* `Maxim 32 <http://platformio.org/platforms/maxim32>`_
 * `Microchip PIC32 <http://platformio.org/platforms/microchippic32>`_
 * `Nordic nRF51 <http://platformio.org/platforms/nordicnrf51>`_
+* `Nordic nRF52 <http://platformio.org/platforms/nordicnrf52>`_
 * `NXP LPC <http://platformio.org/platforms/nxplpc>`_
-* `ST STM32 <http://platformio.org/platforms/ststm32>`_
 * `Silicon Labs EFM32 <http://platformio.org/platforms/siliconlabsefm32>`_
+* `ST STM32 <http://platformio.org/platforms/ststm32>`_
 * `Teensy <http://platformio.org/platforms/teensy>`_
 * `TI MSP430 <http://platformio.org/platforms/timsp430>`_
-* `TI TIVA C <http://platformio.org/platforms/titiva>`_
+* `TI Tiva <http://platformio.org/platforms/titiva>`_
+* `WIZNet W7500 <http://platformio.org/platforms/wiznet7500>`_
 
-Frameworks:
+Frameworks
+----------
 
 * `Arduino <http://platformio.org/frameworks/arduino>`_
+* `ARTIK SDK <http://platformio.org/frameworks/artik-sdk>`_
 * `CMSIS <http://platformio.org/frameworks/cmsis>`_
 * `Energia <http://platformio.org/frameworks/energia>`_
+* `ESP-IDF <http://platformio.org/frameworks/espidf>`_
 * `libOpenCM3 <http://platformio.org/frameworks/libopencm3>`_
 * `mbed <http://platformio.org/frameworks/mbed>`_
+* `Pumbaa <http://platformio.org/frameworks/pumbaa>`_
 * `Simba <http://platformio.org/frameworks/simba>`_
 * `SPL <http://platformio.org/frameworks/spl>`_
+* `STM32Cube <http://platformio.org/frameworks/stm32cube>`_
 * `WiringPi <http://platformio.org/frameworks/wiringpi>`_
 
-For further details, please refer to `What is PlatformIO? <http://docs.platformio.org/en/stable/faq.html#what-is-platformio>`_
-
 Contributing
 ------------
 
@@ -192,7 +122,7 @@ See `contributing guidelines <https://github.com/platformio/platformio/blob/deve
 License
 -------
 
-Copyright 2014-present PlatformIO <contact@platformio.org>
+Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 
 The PlatformIO is licensed under the permissive Apache 2.0 license,
 so you can use it in both commercial and personal projects with confidence.

platformio/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (3, 2, 0)
+VERSION = (3, 4, 1)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -33,7 +33,8 @@ __copyright__ = "Copyright 2014-present PlatformIO"
 __apiurl__ = "https://api.platformio.org"
 
 if sys.version_info < (2, 7, 0) or sys.version_info >= (3, 0, 0):
-    msg = ("PlatformIO version %s does not run under Python version %s.\n"
+    msg = ("PlatformIO Core v%s does not run under Python version %s.\n"
+           "Minimum supported version is 2.7, please upgrade Python.\n"
            "Python 3 is not yet supported.\n")
     sys.stderr.write(msg % (__version__, sys.version.split()[0]))
     sys.exit(1)

platformio/__main__.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,14 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from os import getenv, listdir
+import os
+import sys
 from os.path import join
 from platform import system
-from sys import exit as sys_exit
 from traceback import format_exc
 
 import click
-import requests
 
 from platformio import __version__, exception, maintenance
 from platformio.util import get_source_dir
@@ -29,7 +28,7 @@ class PlatformioCLI(click.MultiCommand):  # pylint: disable=R0904
 
     def list_commands(self, ctx):
         cmds = []
-        for filename in listdir(join(get_source_dir(), "commands")):
+        for filename in os.listdir(join(get_source_dir(), "commands")):
             if filename.startswith("__init__"):
                 continue
             if filename.endswith(".py"):
@@ -37,16 +36,16 @@ class PlatformioCLI(click.MultiCommand):  # pylint: disable=R0904
         cmds.sort()
         return cmds
 
-    def get_command(self, ctx, name):
+    def get_command(self, ctx, cmd_name):
         mod = None
         try:
-            mod = __import__("platformio.commands." + name, None, None,
+            mod = __import__("platformio.commands." + cmd_name, None, None,
                              ["cli"])
         except ImportError:
             try:
-                return self._handle_obsolate_command(name)
+                return self._handle_obsolate_command(cmd_name)
             except AttributeError:
-                raise click.UsageError('No such command "%s"' % name, ctx)
+                raise click.UsageError('No such command "%s"' % cmd_name, ctx)
         return mod.cli
 
     @staticmethod
@@ -89,14 +88,13 @@ def main():
         # https://urllib3.readthedocs.org
         # /en/latest/security.html#insecureplatformwarning
         try:
-            requests.packages.urllib3.disable_warnings()
-        except AttributeError:
-            raise exception.PlatformioException(
-                "Invalid installation of Python `requests` package`. See "
-                "< https://github.com/platformio/platformio-core/issues/252 >")
+            import urllib3
+            urllib3.disable_warnings()
+        except (AttributeError, ImportError):
+            pass
 
         # handle PLATFORMIO_FORCE_COLOR
-        if str(getenv("PLATFORMIO_FORCE_COLOR", "")).lower() == "true":
+        if str(os.getenv("PLATFORMIO_FORCE_COLOR", "")).lower() == "true":
             try:
                 # pylint: disable=protected-access
                 click._compat.isatty = lambda stream: True
@@ -133,5 +131,10 @@ An unexpected error occurred. Further steps:
     return 0
 
 
+def debug_gdb_main():
+    sys.argv = [sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]
+    return main()
+
+
 if __name__ == "__main__":
-    sys_exit(main())
+    sys.exit(main())

platformio/app.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -64,7 +64,8 @@ DEFAULT_SETTINGS = {
         "description":
         ("Telemetry service <http://docs.platformio.org/page/"
          "userguide/cmd_settings.html?#enable-telemetry> (Yes/No)"),
-        "value": True
+        "value":
+        True
     }
 }
 
@@ -87,7 +88,7 @@ class State(object):
             self._lock_state_file()
             if isfile(self.path):
                 self._state = util.load_json(self.path)
-        except ValueError:
+        except exception.PlatformioException:
             self._state = {}
         self._prev_state = deepcopy(self._state)
         return self._state
@@ -137,41 +138,21 @@ class ContentCache(object):
             return
 
         self.cache_dir = cache_dir or join(util.get_home_dir(), ".cache")
-        if not self.cache_dir:
-            os.makedirs(self.cache_dir)
         self._db_path = join(self.cache_dir, "db.data")
 
     def __enter__(self):
         if not self._db_path or not isfile(self._db_path):
             return self
-        found = False
-        newlines = []
-        with open(self._db_path) as fp:
-            for line in fp.readlines():
-                if "=" not in line:
-                    continue
-                line = line.strip()
-                expire, path = line.split("=")
-                if time() < int(expire):
-                    newlines.append(line)
-                    continue
-                found = True
-                if isfile(path):
-                    remove(path)
-                    if not len(listdir(dirname(path))):
-                        util.rmtree_(dirname(path))
-
-        if found and self._lock_dbindex():
-            with open(self._db_path, "w") as fp:
-                fp.write("\n".join(newlines) + "\n")
-            self._unlock_dbindex()
 
+        self.delete()
         return self
 
     def __exit__(self, type_, value, traceback):
         pass
 
     def _lock_dbindex(self):
+        if not self.cache_dir:
+            os.makedirs(self.cache_dir)
         self._lockfile = LockFile(self.cache_dir)
         if self._lockfile.is_locked() and \
                 (time() - getmtime(self._lockfile.lock_file)) > 10:
@@ -187,6 +168,7 @@ class ContentCache(object):
     def _unlock_dbindex(self):
         if self._lockfile:
             self._lockfile.release()
+        return True
 
     def get_cache_path(self, key):
         assert len(key) > 3
@@ -200,46 +182,86 @@ class ContentCache(object):
         return h.hexdigest()
 
     def get(self, key):
-        if not self.cache_dir:
-            return None
         cache_path = self.get_cache_path(key)
         if not isfile(cache_path):
             return None
         with open(cache_path, "rb") as fp:
             data = fp.read()
-            if data[0] in ("{", "["):
+            if data and data[0] in ("{", "["):
                 return json.loads(data)
             return data
 
     def set(self, key, data, valid):
-        if not self.cache_dir or not data:
+        cache_path = self.get_cache_path(key)
+        if isfile(cache_path):
+            self.delete(key)
+        if not data:
             return
         if not isdir(self.cache_dir):
             os.makedirs(self.cache_dir)
         tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
         assert valid.endswith(tuple(tdmap.keys()))
-        cache_path = self.get_cache_path(key)
         expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1]))
 
         if not self._lock_dbindex():
             return False
-        with open(self._db_path, "a") as fp:
-            fp.write("%s=%s\n" % (str(expire_time), cache_path))
-        self._unlock_dbindex()
 
         if not isdir(dirname(cache_path)):
             os.makedirs(dirname(cache_path))
         with open(cache_path, "wb") as fp:
-            if isinstance(data, dict) or isinstance(data, list):
+            if isinstance(data, (dict, list)):
                 json.dump(data, fp)
             else:
                 fp.write(str(data))
+        with open(self._db_path, "a") as fp:
+            fp.write("%s=%s\n" % (str(expire_time), cache_path))
+
+        return self._unlock_dbindex()
+
+    def delete(self, keys=None):
+        """ Keys=None, delete expired items """
+        if not keys:
+            keys = []
+        if not isinstance(keys, list):
+            keys = [keys]
+        paths_for_delete = [self.get_cache_path(k) for k in keys]
+        found = False
+        newlines = []
+        with open(self._db_path) as fp:
+            for line in fp.readlines():
+                if "=" not in line:
+                    continue
+                line = line.strip()
+                expire, path = line.split("=")
+                if time() < int(expire) and isfile(path) and \
+                        path not in paths_for_delete:
+                    newlines.append(line)
+                    continue
+                found = True
+                if isfile(path):
+                    try:
+                        remove(path)
+                        if not listdir(dirname(path)):
+                            util.rmtree_(dirname(path))
+                    except OSError:
+                        pass
+
+        if found and self._lock_dbindex():
+            with open(self._db_path, "w") as fp:
+                fp.write("\n".join(newlines) + "\n")
+            self._unlock_dbindex()
 
         return True
 
     def clean(self):
-        if self.cache_dir and isdir(self.cache_dir):
-            util.rmtree_(self.cache_dir)
+        if not self.cache_dir or not isdir(self.cache_dir):
+            return
+        util.rmtree_(self.cache_dir)
+
+
+def clean_cache():
+    with ContentCache() as cc:
+        cc.clean()
 
 
 def sanitize_setting(name, value):
@@ -270,6 +292,12 @@ def set_state_item(name, value):
         data[name] = value
 
 
+def delete_state_item(name):
+    with State(lock=True) as data:
+        if name in data:
+            del data[name]
+
+
 def get_setting(name):
     _env_name = "PLATFORMIO_SETTING_%s" % name.upper()
     if _env_name in environ:
@@ -306,7 +334,8 @@ def set_session_var(name, value):
 
 def is_disabled_progressbar():
     return any([
-        get_session_var("force_option"), util.is_ci(),
+        get_session_var("force_option"),
+        util.is_ci(),
         getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true"
     ])
 
@@ -325,7 +354,7 @@ def get_cid():
             except:  # pylint: disable=bare-except
                 pass
         cid = str(
-            uuid.UUID(bytes=hashlib.md5(
-                str(_uid if _uid else uuid.getnode())).digest()))
+            uuid.UUID(bytes=hashlib.md5(str(_uid if _uid else uuid.getnode()))
+                      .digest()))
         set_state_item("cid", cid)
     return cid

platformio/builder/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

platformio/builder/main.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 
 import base64
 import json
+import sys
 from os import environ
 from os.path import join
 from time import time
@@ -31,7 +32,7 @@ commonvars = Variables(None)
 commonvars.AddVariables(
     ("PLATFORM_MANIFEST",),
     ("BUILD_SCRIPT",),
-    ("EXTRA_SCRIPT",),
+    ("EXTRA_SCRIPTS",),
     ("PIOENV",),
     ("PIOTEST",),
     ("PIOPLATFORM",),
@@ -49,6 +50,7 @@ commonvars.AddVariables(
     ("LIB_DEPS",),
     ("LIB_IGNORE",),
     ("LIB_EXTRA_DIRS",),
+    ("LIB_ARCHIVE",),
 
     # board options
     ("BOARD",),
@@ -63,12 +65,18 @@ commonvars.AddVariables(
     ("UPLOAD_SPEED",),
     ("UPLOAD_FLAGS",),
     ("UPLOAD_RESETMETHOD",)
+
 )  # yapf: disable
 
+MULTILINE_VARS = [
+    "EXTRA_SCRIPTS", "PIOFRAMEWORK", "BUILD_FLAGS", "SRC_BUILD_FLAGS",
+    "BUILD_UNFLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE", "LIB_EXTRA_DIRS"
+]
+
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
         "ar", "as", "gcc", "g++", "gnulink", "platformio", "pioplatform",
-        "piowinhooks", "piolib", "piotest", "pioupload", "piomisc"
+        "piowinhooks", "piolib", "pioupload", "piomisc", "pioide"
     ],  # yapf: disable
     toolpath=[join(util.get_source_dir(), "builder", "tools")],
     variables=commonvars,
@@ -77,7 +85,6 @@ DEFAULT_ENV_OPTIONS = dict(
     PIOVARIABLES=commonvars.keys(),
     ENV=environ,
     UNIX_TIME=int(time()),
-    PROGNAME="program",
     PIOHOME_DIR=util.get_home_dir(),
     PROJECT_DIR=util.get_project_dir(),
     PROJECTSRC_DIR=util.get_projectsrc_dir(),
@@ -88,9 +95,12 @@ DEFAULT_ENV_OPTIONS = dict(
     BUILDSRC_DIR=join("$BUILD_DIR", "src"),
     BUILDTEST_DIR=join("$BUILD_DIR", "test"),
     LIBSOURCE_DIRS=[
-        util.get_projectlib_dir(), util.get_projectlibdeps_dir(),
+        util.get_projectlib_dir(),
+        util.get_projectlibdeps_dir(),
         join("$PIOHOME_DIR", "lib")
     ],
+    PROGNAME="program",
+    PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
     PYTHONEXE=util.get_pythonexe_path())
 
 if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
@@ -106,6 +116,8 @@ env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
 for k in commonvars.keys():
     if k in env:
         env[k] = base64.b64decode(env[k])
+        if k in MULTILINE_VARS:
+            env[k] = util.parse_conf_multi_values(env[k])
 
 if env.GetOption('clean'):
     env.PioClean(env.subst("$BUILD_DIR"))
@@ -114,45 +126,53 @@ elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     print "Verbose mode can be enabled via `-v, --verbose` option"
 
 # Handle custom variables from system environment
-for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPT",
+for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPTS",
             "UPLOAD_PORT", "UPLOAD_FLAGS", "LIB_EXTRA_DIRS"):
     k = "PLATFORMIO_%s" % var
-    if environ.get(k):
-        env[var] = environ.get(k)
-
-# Parse comma separated items
-for opt in ("PIOFRAMEWORK", "LIB_DEPS", "LIB_IGNORE", "LIB_EXTRA_DIRS"):
-    if opt not in env:
+    if k not in environ:
         continue
-    env[opt] = [l.strip() for l in env[opt].split(", ") if l.strip()]
+    if var in ("UPLOAD_PORT", ):
+        env[var] = environ.get(k)
+        continue
+    env.Append(**{var: util.parse_conf_multi_values(environ.get(k))})
 
 # Configure extra library source directories for LDF
 if util.get_project_optional_dir("lib_extra_dirs"):
-    env.Prepend(LIBSOURCE_DIRS=[
-        l.strip()
-        for l in util.get_project_optional_dir("lib_extra_dirs").split(", ")
-        if l.strip()
-    ])
+    env.Prepend(LIBSOURCE_DIRS=util.parse_conf_multi_values(
+        util.get_project_optional_dir("lib_extra_dirs")))
 env.Prepend(LIBSOURCE_DIRS=env.get("LIB_EXTRA_DIRS", []))
 
 env.LoadPioPlatform(commonvars)
 
 env.SConscriptChdir(0)
 env.SConsignFile(join("$PROJECTPIOENVS_DIR", ".sconsign.dblite"))
+
+for item in env.GetPreExtraScripts():
+    env.SConscript(item, exports="env")
+
 env.SConscript("$BUILD_SCRIPT")
 
+AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS + ["size"]))
 AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS + ["size"]))
 
 if "UPLOAD_FLAGS" in env:
     env.Append(UPLOADERFLAGS=["$UPLOAD_FLAGS"])
 
-if env.get("EXTRA_SCRIPT"):
-    env.SConscript(env.get("EXTRA_SCRIPT"), exports="env")
+for item in env.GetPostExtraScripts():
+    env.SConscript(item, exports="env")
 
 if "envdump" in COMMAND_LINE_TARGETS:
     print env.Dump()
     env.Exit(0)
 
 if "idedata" in COMMAND_LINE_TARGETS:
-    print "\n%s\n" % json.dumps(env.DumpIDEData())
-    env.Exit(0)
+    try:
+        print "\n%s\n" % json.dumps(env.DumpIDEData())
+        env.Exit(0)
+    except UnicodeDecodeError:
+        sys.stderr.write(
+            "\nUnicodeDecodeError: Non-ASCII characters found in build "
+            "environment\n"
+            "See explanation in FAQ > Troubleshooting > Building\n"
+            "http://docs.platformio.org/page/faq.html\n\n")
+        env.Exit(1)

platformio/builder/tools/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

platformio/builder/tools/pioide.py

@@ -0,0 +1,120 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from glob import glob
+from os.path import join
+
+from SCons.Defaults import processDefines
+
+from platformio import util
+from platformio.managers.core import get_core_package_dir
+
+
+def dump_includes(env):
+    includes = []
+
+    for item in env.get("CPPPATH", []):
+        includes.append(env.subst(item))
+
+    # installed libs
+    for lb in env.GetLibBuilders():
+        includes.extend(lb.get_inc_dirs())
+
+    # includes from toolchains
+    p = env.PioPlatform()
+    for name in p.get_installed_packages():
+        if p.get_package_type(name) != "toolchain":
+            continue
+        toolchain_dir = util.glob_escape(p.get_package_dir(name))
+        toolchain_incglobs = [
+            join(toolchain_dir, "*", "include*"),
+            join(toolchain_dir, "lib", "gcc", "*", "*", "include*")
+        ]
+        for g in toolchain_incglobs:
+            includes.extend(glob(g))
+
+    unity_dir = get_core_package_dir("tool-unity")
+    if unity_dir:
+        includes.append(unity_dir)
+
+    return includes
+
+
+def dump_defines(env):
+    defines = []
+    # global symbols
+    for item in processDefines(env.get("CPPDEFINES", [])):
+        defines.append(env.subst(item).replace('\\', ''))
+
+    # special symbol for Atmel AVR MCU
+    if env['PIOPLATFORM'] == "atmelavr":
+        defines.append(
+            "__AVR_%s__" % env.BoardConfig().get("build.mcu").upper()
+            .replace("ATMEGA", "ATmega").replace("ATTINY", "ATtiny"))
+    return defines
+
+
+def DumpIDEData(env):
+    LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS"
+    LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS"
+
+    data = {
+        "libsource_dirs":
+        [env.subst(l) for l in env.get("LIBSOURCE_DIRS", [])],
+        "defines":
+        dump_defines(env),
+        "includes":
+        dump_includes(env),
+        "cc_flags":
+        env.subst(LINTCCOM),
+        "cxx_flags":
+        env.subst(LINTCXXCOM),
+        "cc_path":
+        util.where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
+        "cxx_path":
+        util.where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
+        "gdb_path":
+        util.where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
+        "prog_path":
+        env.subst("$PROG_PATH")
+    }
+
+    env_ = env.Clone()
+    # https://github.com/platformio/platformio-atom-ide/issues/34
+    _new_defines = []
+    for item in processDefines(env_.get("CPPDEFINES", [])):
+        item = item.replace('\\"', '"')
+        if " " in item:
+            _new_defines.append(item.replace(" ", "\\\\ "))
+        else:
+            _new_defines.append(item)
+    env_.Replace(CPPDEFINES=_new_defines)
+
+    data.update({
+        "cc_flags": env_.subst(LINTCCOM),
+        "cxx_flags": env_.subst(LINTCXXCOM)
+    })
+
+    return data
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    env.AddMethod(DumpIDEData)
+    return env

platformio/builder/tools/piolib.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -39,16 +39,15 @@ class LibBuilderFactory(object):
             clsname = "PlatformIOLibBuilder"
         else:
             used_frameworks = LibBuilderFactory.get_used_frameworks(env, path)
-            common_frameworks = (set(env.get("PIOFRAMEWORK", [])) &
-                                 set(used_frameworks))
+            common_frameworks = (
+                set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks))
             if common_frameworks:
                 clsname = "%sLibBuilder" % list(common_frameworks)[0].title()
             elif used_frameworks:
                 clsname = "%sLibBuilder" % used_frameworks[0].title()
 
-        obj = getattr(sys.modules[__name__], clsname)(env,
-                                                      path,
-                                                      verbose=verbose)
+        obj = getattr(sys.modules[__name__], clsname)(
+            env, path, verbose=verbose)
         assert isinstance(obj, LibBuilderBase)
         return obj
 
@@ -66,7 +65,8 @@ class LibBuilderFactory(object):
         # check source files
         for root, _, files in os.walk(path, followlinks=True):
             for fname in files:
-                if not env.IsFileWithExt(fname, ("c", "cpp", "h", "hpp")):
+                if not env.IsFileWithExt(
+                        fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT):
                     continue
                 with open(join(root, fname)) as f:
                     content = f.read()
@@ -80,7 +80,7 @@ class LibBuilderFactory(object):
 class LibBuilderBase(object):
 
     LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"]
-    LDF_MODE_DEFAULT = "chain+"
+    LDF_MODE_DEFAULT = "chain"
 
     CLASSIC_SCANNER = SCons.Scanner.C.CScanner()
     ADVANCED_SCANNER = SCons.Scanner.C.CScanner(advanced=True)
@@ -101,6 +101,9 @@ class LibBuilderBase(object):
         self._circular_deps = list()
         self._scanned_paths = list()
 
+        # reset source filter, could be overridden with extra script
+        self.env['SRC_FILTER'] = ""
+
         # process extra options and append to build environment
         self.process_extra_options()
 
@@ -131,8 +134,10 @@ class LibBuilderBase(object):
     @property
     def src_filter(self):
         return piotool.SRC_FILTER_DEFAULT + [
-            "-<example%s>" % os.sep, "-<examples%s>" % os.sep, "-<test%s>" %
-            os.sep, "-<tests%s>" % os.sep
+            "-<example%s>" % os.sep,
+            "-<examples%s>" % os.sep,
+            "-<test%s>" % os.sep,
+            "-<tests%s>" % os.sep
         ]
 
     @property
@@ -145,7 +150,10 @@ class LibBuilderBase(object):
         return join("$BUILD_DIR", "lib", basename(self.path))
 
     def get_inc_dirs(self):
-        return [self.src_dir]
+        items = [self.src_dir]
+        if all([isdir(join(self.path, d)) for d in ("inc", "src")]):
+            items.append(join(self.path, "inc"))
+        return items
 
     @property
     def build_flags(self):
@@ -161,7 +169,7 @@ class LibBuilderBase(object):
 
     @property
     def lib_archive(self):
-        return True
+        return self.env.get("LIB_ARCHIVE", "") != "false"
 
     @staticmethod
     def validate_ldf_mode(mode):
@@ -226,6 +234,7 @@ class LibBuilderBase(object):
             self.env.ProcessUnFlags(self.build_unflags)
             self.env.ProcessFlags(self.build_flags)
             if self.extra_script:
+                self.env.SConscriptChdir(1)
                 self.env.SConscript(
                     realpath(self.extra_script),
                     exports={"env": self.env,
@@ -243,8 +252,9 @@ class LibBuilderBase(object):
                 if (key in item and
                         not self.items_in_list(self.env[env_key], item[key])):
                     if self.verbose:
-                        sys.stderr.write("Skip %s incompatible dependency %s\n"
-                                         % (key[:-1], item))
+                        sys.stderr.write(
+                            "Skip %s incompatible dependency %s\n" % (key[:-1],
+                                                                      item))
                     skip = True
             if skip:
                 continue
@@ -333,8 +343,8 @@ class LibBuilderBase(object):
             if _already_depends(lb):
                 if self.verbose:
                     sys.stderr.write("Warning! Circular dependencies detected "
-                                     "between `%s` and `%s`\n" %
-                                     (self.path, lb.path))
+                                     "between `%s` and `%s`\n" % (self.path,
+                                                                  lb.path))
                 self._circular_deps.append(lb)
             elif lb not in self._depbuilders:
                 self._depbuilders.append(lb)
@@ -376,24 +386,26 @@ class LibBuilderBase(object):
         for lb in self._circular_deps:
             self.env.AppendUnique(CPPPATH=lb.get_inc_dirs())
 
-        if not self._is_built:
-            self.env.AppendUnique(CPPPATH=self.get_inc_dirs())
+        if self._is_built:
+            return libs
+        self._is_built = True
 
-            if self.lib_ldf_mode == "off":
-                for lb in self.envorigin.GetLibBuilders():
-                    if self == lb or not lb.is_built:
-                        continue
-                    for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
-                        self.env.AppendUnique(**{key: lb.env.get(key)})
+        self.env.AppendUnique(CPPPATH=self.get_inc_dirs())
 
-            if self.lib_archive:
-                libs.append(
-                    self.env.BuildLibrary(self.build_dir, self.src_dir,
-                                          self.src_filter))
-            else:
-                self.env.BuildSources(self.build_dir, self.src_dir,
-                                      self.src_filter)
-            self._is_built = True
+        if self.lib_ldf_mode == "off":
+            for lb in self.envorigin.GetLibBuilders():
+                if self == lb or not lb.is_built:
+                    continue
+                for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
+                    self.env.AppendUnique(**{key: lb.env.get(key)})
+
+        if self.lib_archive:
+            libs.append(
+                self.env.BuildLibrary(self.build_dir, self.src_dir,
+                                      self.src_filter))
+        else:
+            self.env.BuildSources(self.build_dir, self.src_dir,
+                                  self.src_filter)
         return libs
 
 
@@ -401,41 +413,6 @@ class UnknownLibBuilder(LibBuilderBase):
     pass
 
 
-class ProjectAsLibBuilder(LibBuilderBase):
-
-    def __init__(self, *args, **kwargs):
-        LibBuilderBase.__init__(self, *args, **kwargs)
-        self._is_built = True
-
-    @property
-    def lib_ldf_mode(self):
-        mode = LibBuilderBase.lib_ldf_mode.fget(self)
-        if not mode.startswith("chain"):
-            return mode
-        # parse all project files
-        return "deep+" if "+" in mode else "deep"
-
-    @property
-    def src_filter(self):
-        return self.env.get("SRC_FILTER", LibBuilderBase.src_filter.fget(self))
-
-    def process_extra_options(self):
-        # skip for project, options are already processed
-        pass
-
-    def search_deps_recursive(self, search_paths=None):
-        for dep in self.env.get("LIB_DEPS", []):
-            for token in ("@", "="):
-                if token in dep:
-                    dep, _ = dep.split(token, 1)
-            for lb in self.envorigin.GetLibBuilders():
-                if lb.name == dep:
-                    if lb not in self.depbuilders:
-                        self.depend_recursive(lb)
-                    break
-        return LibBuilderBase.search_deps_recursive(self, search_paths)
-
-
 class ArduinoLibBuilder(LibBuilderBase):
 
     def load_manifest(self):
@@ -510,10 +487,21 @@ class PlatformIOLibBuilder(LibBuilderBase):
     def _is_arduino_manifest(self):
         return isfile(join(self.path, "library.properties"))
 
+    @property
+    def src_dir(self):
+        if all([
+                "srcFilter" in self._manifest.get("build", {})
+                or self.env['SRC_FILTER'], not self._is_arduino_manifest()
+        ]):
+            return self.path
+        return LibBuilderBase.src_dir.fget(self)
+
     @property
     def src_filter(self):
         if "srcFilter" in self._manifest.get("build", {}):
             return self._manifest.get("build").get("srcFilter")
+        elif self.env['SRC_FILTER']:
+            return self.env['SRC_FILTER']
         elif self._is_arduino_manifest():
             return ArduinoLibBuilder.src_filter.fget(self)
         return LibBuilderBase.src_filter.fget(self)
@@ -565,17 +553,57 @@ class PlatformIOLibBuilder(LibBuilderBase):
         inc_dirs = LibBuilderBase.get_inc_dirs(self)
 
         #  backwards compatibility with PlatformIO 2.0
-        if ("build" not in self._manifest and self._is_arduino_manifest() and
-                not isdir(join(self.path, "src")) and
-                isdir(join(self.path, "utility"))):
+        if ("build" not in self._manifest and self._is_arduino_manifest()
+                and not isdir(join(self.path, "src"))
+                and isdir(join(self.path, "utility"))):
             inc_dirs.append(join(self.path, "utility"))
 
         for path in self.env.get("CPPPATH", []):
-            if path not in self.envorigin['CPPPATH']:
+            if path not in self.envorigin.get("CPPPATH", []):
                 inc_dirs.append(self.env.subst(path))
         return inc_dirs
 
 
+class ProjectAsLibBuilder(LibBuilderBase):
+
+    @property
+    def src_dir(self):
+        return self.env.subst("$PROJECTSRC_DIR")
+
+    @property
+    def lib_ldf_mode(self):
+        mode = LibBuilderBase.lib_ldf_mode.fget(self)
+        if not mode.startswith("chain"):
+            return mode
+        # parse all project files
+        return "deep+" if "+" in mode else "deep"
+
+    @property
+    def src_filter(self):
+        return self.env.get("SRC_FILTER", LibBuilderBase.src_filter.fget(self))
+
+    def process_extra_options(self):
+        # skip for project, options are already processed
+        pass
+
+    def search_deps_recursive(self, search_paths=None):
+        for dep in self.env.get("LIB_DEPS", []):
+            for token in ("@", "="):
+                if token in dep:
+                    dep, _ = dep.split(token, 1)
+            for lb in self.envorigin.GetLibBuilders():
+                if lb.name == dep:
+                    if lb not in self.depbuilders:
+                        self.depend_recursive(lb)
+                    break
+        return LibBuilderBase.search_deps_recursive(self, search_paths)
+
+    def build(self):
+        self._is_built = True  # do not build Project now
+        self.env.AppendUnique(CPPPATH=self.get_inc_dirs())
+        return LibBuilderBase.build(self)
+
+
 def GetLibBuilders(env):  # pylint: disable=too-many-branches
 
     if "__PIO_LIB_BUILDERS" in DefaultEnvironment():
@@ -583,25 +611,25 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
 
     items = []
     compat_mode = int(env.get("LIB_COMPAT_MODE", 1))
-    verbose = (int(ARGUMENTS.get("PIOVERBOSE", 0)) and
-               not env.GetOption('clean'))
+    verbose = (int(ARGUMENTS.get("PIOVERBOSE", 0))
+               and not env.GetOption('clean'))
 
     def _check_lib_builder(lb):
         if lb.name in env.get("LIB_IGNORE", []):
             if verbose:
                 sys.stderr.write("Ignored library %s\n" % lb.path)
             return
-        if compat_mode > 1 and not lb.is_platforms_compatible(env[
-                'PIOPLATFORM']):
+        if compat_mode > 1 and not lb.is_platforms_compatible(
+                env['PIOPLATFORM']):
             if verbose:
-                sys.stderr.write("Platform incompatible library %s\n" %
-                                 lb.path)
+                sys.stderr.write(
+                    "Platform incompatible library %s\n" % lb.path)
             return False
         if compat_mode > 0 and "PIOFRAMEWORK" in env and \
            not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])):
             if verbose:
-                sys.stderr.write("Framework incompatible library %s\n" %
-                                 lb.path)
+                sys.stderr.write(
+                    "Framework incompatible library %s\n" % lb.path)
             return False
         return True
 
@@ -614,9 +642,8 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             if item == "__cores__" or not isdir(join(libs_dir, item)):
                 continue
             try:
-                lb = LibBuilderFactory.new(env,
-                                           join(libs_dir, item),
-                                           verbose=verbose)
+                lb = LibBuilderFactory.new(
+                    env, join(libs_dir, item), verbose=verbose)
             except ValueError:
                 if verbose:
                     sys.stderr.write("Skip library with broken manifest: %s\n"
@@ -643,7 +670,7 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
     return items
 
 
-def BuildDependentLibraries(env, src_dir):
+def BuildProjectLibraries(env):
     lib_builders = env.GetLibBuilders()
 
     def correct_found_libs():
@@ -672,7 +699,7 @@ def BuildDependentLibraries(env, src_dir):
     print "Collected %d compatible libraries" % len(lib_builders)
     print "Looking for dependencies..."
 
-    project = ProjectAsLibBuilder(env, src_dir)
+    project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
     project.env = env
     project.search_deps_recursive()
 
@@ -696,5 +723,5 @@ def exists(_):
 
 def generate(env):
     env.AddMethod(GetLibBuilders)
-    env.AddMethod(BuildDependentLibraries)
+    env.AddMethod(BuildProjectLibraries)
     return env

platformio/builder/tools/piomisc.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,21 +17,21 @@ from __future__ import absolute_import
 import atexit
 import re
 import sys
-from glob import glob
 from os import environ, remove, walk
-from os.path import basename, isdir, isfile, join, relpath
+from os.path import basename, isdir, isfile, join, relpath, sep
 from tempfile import mkstemp
 
 from SCons.Action import Action
-from SCons.Defaults import processDefines
 from SCons.Script import ARGUMENTS
 
 from platformio import util
+from platformio.managers.core import get_core_package_dir
 
 
 class InoToCPPConverter(object):
 
     PROTOTYPE_RE = re.compile(r"""^(
+        (?:template\<.*\>\s*)?      # template
         ([a-z_\d]+\*?\s+){1,2}      # return type
         ([a-z_\d]+\s*)              # name of prototype
         \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
@@ -89,8 +89,8 @@ class InoToCPPConverter(object):
         self.env.Execute(
             self.env.VerboseAction(
                 '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
-                    out_file, tmp_path), "Converting " + basename(
-                        out_file[:-4])))
+                    out_file,
+                    tmp_path), "Converting " + basename(out_file[:-4])))
         atexit.register(_delete_file, tmp_path)
         return isfile(out_file)
 
@@ -115,7 +115,7 @@ class InoToCPPConverter(object):
                 elif stropen:
                     newlines[len(newlines) - 1] += line[:-1]
                     continue
-            elif stropen and line.endswith('";'):
+            elif stropen and line.endswith(('",', '";')):
                 newlines[len(newlines) - 1] += line
                 stropen = False
                 newlines.append('#line %d "%s"' %
@@ -139,8 +139,8 @@ class InoToCPPConverter(object):
         prototypes = []
         reserved_keywords = set(["if", "else", "while"])
         for match in self.PROTOTYPE_RE.finditer(contents):
-            if (set([match.group(2).strip(), match.group(3).strip()]) &
-                    reserved_keywords):
+            if (set([match.group(2).strip(),
+                     match.group(3).strip()]) & reserved_keywords):
                 continue
             prototypes.append(match)
         return prototypes
@@ -180,8 +180,9 @@ class InoToCPPConverter(object):
 
 
 def ConvertInoToCpp(env):
-    ino_nodes = (env.Glob(join("$PROJECTSRC_DIR", "*.ino")) +
-                 env.Glob(join("$PROJECTSRC_DIR", "*.pde")))
+    src_dir = util.glob_escape(env.subst("$PROJECTSRC_DIR"))
+    ino_nodes = (
+        env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde")))
     if not ino_nodes:
         return
     c = InoToCPPConverter(env)
@@ -198,81 +199,6 @@ def _delete_file(path):
         pass
 
 
-def DumpIDEData(env):
-
-    def get_includes(env_):
-        includes = []
-
-        for item in env_.get("CPPPATH", []):
-            includes.append(env_.subst(item))
-
-        # installed libs
-        for lb in env.GetLibBuilders():
-            includes.extend(lb.get_inc_dirs())
-
-        # includes from toolchains
-        p = env.PioPlatform()
-        for name in p.get_installed_packages():
-            if p.get_package_type(name) != "toolchain":
-                continue
-            toolchain_dir = p.get_package_dir(name)
-            toolchain_incglobs = [
-                join(toolchain_dir, "*", "include*"),
-                join(toolchain_dir, "lib", "gcc", "*", "*", "include*")
-            ]
-            for g in toolchain_incglobs:
-                includes.extend(glob(g))
-
-        return includes
-
-    def get_defines(env_):
-        defines = []
-        # global symbols
-        for item in processDefines(env_.get("CPPDEFINES", [])):
-            defines.append(env_.subst(item).replace('\\', ''))
-
-        # special symbol for Atmel AVR MCU
-        if env['PIOPLATFORM'] == "atmelavr":
-            defines.append(
-                "__AVR_%s__" % env.BoardConfig().get("build.mcu").upper()
-                .replace("ATMEGA", "ATmega").replace("ATTINY", "ATtiny"))
-        return defines
-
-    LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS"
-    LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS"
-    env_ = env.Clone()
-
-    data = {
-        "libsource_dirs":
-        [env_.subst(l) for l in env_.get("LIBSOURCE_DIRS", [])],
-        "defines": get_defines(env_),
-        "includes": get_includes(env_),
-        "cc_flags": env_.subst(LINTCCOM),
-        "cxx_flags": env_.subst(LINTCXXCOM),
-        "cc_path": util.where_is_program(
-            env_.subst("$CC"), env_.subst("${ENV['PATH']}")),
-        "cxx_path": util.where_is_program(
-            env_.subst("$CXX"), env_.subst("${ENV['PATH']}"))
-    }
-
-    # https://github.com/platformio/platformio-atom-ide/issues/34
-    _new_defines = []
-    for item in processDefines(env_.get("CPPDEFINES", [])):
-        item = item.replace('\\"', '"')
-        if " " in item:
-            _new_defines.append(item.replace(" ", "\\\\ "))
-        else:
-            _new_defines.append(item)
-    env_.Replace(CPPDEFINES=_new_defines)
-
-    data.update({
-        "cc_flags": env_.subst(LINTCCOM),
-        "cxx_flags": env_.subst(LINTCXXCOM)
-    })
-
-    return data
-
-
 def GetCompilerType(env):
     try:
         sysenv = environ.copy()
@@ -327,8 +253,7 @@ def GetActualLDScript(env):
 def VerboseAction(_, act, actstr):
     if int(ARGUMENTS.get("PIOVERBOSE", 0)):
         return act
-    else:
-        return Action(act, actstr)
+    return Action(act, actstr)
 
 
 def PioClean(env, clean_dir):
@@ -344,15 +269,59 @@ def PioClean(env, clean_dir):
     env.Exit(0)
 
 
+def ProcessDebug(env):
+    if not env.subst("$PIODEBUGFLAGS"):
+        env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb"])
+    env.Append(
+        BUILD_FLAGS=env.get("PIODEBUGFLAGS", []),
+        BUILD_UNFLAGS=["-Os", "-O0", "-O1", "-O2", "-O3"])
+
+
+def ProcessTest(env):
+    env.Append(
+        CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
+        CPPPATH=[join("$BUILD_DIR", "UnityTestLib")])
+    unitylib = env.BuildLibrary(
+        join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity"))
+    env.Prepend(LIBS=[unitylib])
+
+    src_filter = ["+<*.cpp>", "+<*.c>"]
+    if "PIOTEST" in env:
+        src_filter.append("+<%s%s>" % (env['PIOTEST'], sep))
+
+    return env.CollectBuildFiles(
+        "$BUILDTEST_DIR",
+        "$PROJECTTEST_DIR",
+        src_filter=src_filter,
+        duplicate=False)
+
+
+def GetPreExtraScripts(env):
+    return [
+        item[4:] for item in env.get("EXTRA_SCRIPTS", [])
+        if item.startswith("pre:")
+    ]
+
+
+def GetPostExtraScripts(env):
+    return [
+        item[5:] if item.startswith("post:") else item
+        for item in env.get("EXTRA_SCRIPTS", []) if not item.startswith("pre:")
+    ]
+
+
 def exists(_):
     return True
 
 
 def generate(env):
     env.AddMethod(ConvertInoToCpp)
-    env.AddMethod(DumpIDEData)
     env.AddMethod(GetCompilerType)
     env.AddMethod(GetActualLDScript)
     env.AddMethod(VerboseAction)
     env.AddMethod(PioClean)
+    env.AddMethod(ProcessDebug)
+    env.AddMethod(ProcessTest)
+    env.AddMethod(GetPreExtraScripts)
+    env.AddMethod(GetPostExtraScripts)
     return env

platformio/builder/tools/pioplatform.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -80,8 +80,9 @@ def LoadPioPlatform(env, variables):
 
     board_config = env.BoardConfig()
     for k in variables.keys():
-        if (k in env or
-                not any([k.startswith("BOARD_"), k.startswith("UPLOAD_")])):
+        if (k in env
+                or not any([k.startswith("BOARD_"),
+                            k.startswith("UPLOAD_")])):
             continue
         _opt, _val = k.lower().split("_", 1)
         if _opt == "board":

platformio/builder/tools/piotest.py

@@ -1,47 +0,0 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-
-from os.path import join, sep
-
-
-def ProcessTest(env):
-    env.Append(
-        CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
-        CPPPATH=[join("$BUILD_DIR", "UnityTestLib")])
-    unitylib = env.BuildLibrary(
-        join("$BUILD_DIR", "UnityTestLib"),
-        env.PioPlatform().get_package_dir("tool-unity"))
-    env.Prepend(LIBS=[unitylib])
-
-    src_filter = None
-    if "PIOTEST" in env:
-        src_filter = "+<output_export.cpp>"
-        src_filter += " +<%s%s>" % (env['PIOTEST'], sep)
-
-    return env.CollectBuildFiles(
-        "$BUILDTEST_DIR",
-        "$PROJECTTEST_DIR",
-        src_filter=src_filter,
-        duplicate=False)
-
-
-def exists(_):
-    return True
-
-
-def generate(env):
-    env.AddMethod(ProcessTest)
-    return env

platformio/builder/tools/pioupload.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@ from shutil import copyfile
 from time import sleep
 
 from SCons.Node.Alias import Alias
-from serial import Serial
+from serial import Serial, SerialException
 
 from platformio import util
 
@@ -48,7 +48,7 @@ def TouchSerialPort(env, port, baudrate):
         s.close()
     except:  # pylint: disable=W0702
         pass
-    sleep(0.4)
+    sleep(0.4)  # DO NOT REMOVE THAT (required by SAM-BA based boards)
 
 
 def WaitForNewSerialPort(env, before):
@@ -56,12 +56,12 @@ def WaitForNewSerialPort(env, before):
     prev_port = env.subst("$UPLOAD_PORT")
     new_port = None
     elapsed = 0
-    sleep(1)
+    before = [p['port'] for p in before]
     while elapsed < 5 and new_port is None:
-        now = util.get_serialports()
+        now = [p['port'] for p in util.get_serialports()]
         for p in now:
             if p not in before:
-                new_port = p['port']
+                new_port = p
                 break
         before = now
         sleep(0.25)
@@ -69,10 +69,16 @@ def WaitForNewSerialPort(env, before):
 
     if not new_port:
         for p in now:
-            if prev_port == p['port']:
-                new_port = p['port']
+            if prev_port == p:
+                new_port = p
                 break
 
+    try:
+        s = Serial(new_port)
+        s.close()
+    except SerialException:
+        sleep(1)
+
     if not new_port:
         sys.stderr.write("Error: Couldn't find a board on the selected port. "
                          "Check that you have the correct port selected. "
@@ -102,12 +108,16 @@ def AutodetectUploadPort(*args, **kwargs):  # pylint: disable=unused-argument
     def _look_for_mbed_disk():
         msdlabels = ("mbed", "nucleo", "frdm", "microbit")
         for item in util.get_logicaldisks():
-            if not _is_match_pattern(item['disk']):
+            if item['disk'].startswith(
+                    "/net") or not _is_match_pattern(item['disk']):
                 continue
-            if (item['name'] and
-                    any([l in item['name'].lower() for l in msdlabels])):
+            mbed_pages = [
+                join(item['disk'], n) for n in ("mbed.htm", "mbed.html")
+            ]
+            if any([isfile(p) for p in mbed_pages]):
                 return item['disk']
-            if isfile(join(item['disk'], "mbed.html")):
+            if (item['name']
+                    and any([l in item['name'].lower() for l in msdlabels])):
                 return item['disk']
         return None
 
@@ -192,8 +202,8 @@ def CheckUploadSize(_, target, source, env):  # pylint: disable=W0613,W0621
 
     if used_size > max_size:
         sys.stderr.write("Error: The program size (%d bytes) is greater "
-                         "than maximum allowed (%s bytes)\n" %
-                         (used_size, max_size))
+                         "than maximum allowed (%s bytes)\n" % (used_size,
+                                                                max_size))
         env.Exit(1)
 
 

platformio/builder/tools/piowinhooks.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

platformio/builder/tools/platformio.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,9 +25,9 @@ from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild,
                           DefaultEnvironment, SConscript)
 from SCons.Util import case_sensitive_suffixes, is_Sequence
 
-from platformio.util import pioversion_to_intstr
+from platformio.util import glob_escape, pioversion_to_intstr
 
-SRC_BUILD_EXT = ["c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"]
+SRC_BUILD_EXT = ["c", "cc", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"]
 SRC_HEADER_EXT = ["h", "hpp"]
 SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep]
 
@@ -35,9 +35,10 @@ SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep]
 def BuildProgram(env):
 
     def _append_pio_macros():
-        env.AppendUnique(CPPDEFINES=[(
-            "PLATFORMIO",
-            int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())))])
+        env.AppendUnique(CPPDEFINES=[
+            ("PLATFORMIO",
+             int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())))
+        ])
 
     _append_pio_macros()
 
@@ -45,6 +46,9 @@ def BuildProgram(env):
     if not case_sensitive_suffixes(".s", ".S"):
         env.Replace(AS="$CC", ASCOM="$ASPPCOM")
 
+    if "__debug" in COMMAND_LINE_TARGETS:
+        env.ProcessDebug()
+
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
         env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
@@ -59,11 +63,11 @@ def BuildProgram(env):
     _append_pio_macros()
 
     # build dependent libs
-    deplibs = env.BuildDependentLibraries("$PROJECTSRC_DIR")
+    deplibs = env.BuildProjectLibraries()
 
     # append specified LD_SCRIPT
-    if ("LDSCRIPT_PATH" in env and
-            not any(["-Wl,-T" in f for f in env['LINKFLAGS']])):
+    if ("LDSCRIPT_PATH" in env
+            and not any(["-Wl,-T" in f for f in env['LINKFLAGS']])):
         env.Append(LINKFLAGS=['-Wl,-T"$LDSCRIPT_PATH"'])
 
     # enable "cyclic reference" for linker
@@ -75,7 +79,6 @@ def BuildProgram(env):
     env.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
 
     env.Append(
-        CPPPATH=["$PROJECTSRC_DIR"],
         LIBS=deplibs,
         LIBPATH=["$BUILD_DIR"],
         PIOBUILDFILES=env.CollectBuildFiles(
@@ -183,7 +186,7 @@ def MatchSourceFiles(env, src_dir, src_filter=None):
 
     src_dir = env.subst(src_dir)
     src_filter = src_filter or SRC_FILTER_DEFAULT
-    if isinstance(src_filter, list) or isinstance(src_filter, tuple):
+    if isinstance(src_filter, (list, tuple)):
         src_filter = " ".join(src_filter)
 
     matches = set()
@@ -191,7 +194,7 @@ def MatchSourceFiles(env, src_dir, src_filter=None):
     src_filter = src_filter.replace("/", sep).replace("\\", sep)
     for (action, pattern) in SRC_FILTER_PATTERNS_RE.findall(src_filter):
         items = set()
-        for item in glob(join(src_dir, pattern)):
+        for item in glob(join(glob_escape(src_dir), pattern)):
             if isdir(item):
                 for root, _, files in walk(item, followlinks=True):
                     for f in files:
@@ -266,8 +269,7 @@ def BuildLibrary(env, variant_dir, src_dir, src_filter=None):
     lib = env.Clone()
     return lib.StaticLibrary(
         lib.subst(variant_dir),
-        lib.CollectBuildFiles(
-            variant_dir, src_dir, src_filter=src_filter))
+        lib.CollectBuildFiles(variant_dir, src_dir, src_filter=src_filter))
 
 
 def BuildSources(env, variant_dir, src_dir, src_filter=None):

platformio/commands/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

platformio/commands/account.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@ import sys
 
 import click
 
-from platformio.pioplus import pioplus_call
+from platformio.managers.core import pioplus_call
 
 
 @click.group("account", short_help="Manage PIO Account")
@@ -45,12 +45,16 @@ def account_logout():
 
 
 @cli.command("password", short_help="Change password")
-def account_password():
+@click.option("--old-password")
+@click.option("--new-password")
+def account_password(**kwargs):
     pioplus_call(sys.argv[1:])
 
 
 @cli.command("token", short_help="Get or regenerate Authentication Token")
+@click.option("-p", "--password")
 @click.option("--regenerate", is_flag=True)
+@click.option("--json-output", is_flag=True)
 def account_token(**kwargs):
     pioplus_call(sys.argv[1:])
 
@@ -61,7 +65,8 @@ def account_forgot(**kwargs):
     pioplus_call(sys.argv[1:])
 
 
-@cli.command("show", short_help="PIO Account information: groups, permissions")
+@cli.command("show", short_help="PIO Account information")
+@click.option("--offline", is_flag=True)
 @click.option("--json-output", is_flag=True)
 def account_show(**kwargs):
     pioplus_call(sys.argv[1:])

platformio/commands/boards.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,72 +20,63 @@ from platformio.exception import APIRequestError, InternetIsOffline
 from platformio.managers.platform import PlatformManager
 
 
-@click.command("boards", short_help="Pre-configured Embedded Boards")
+@click.command("boards", short_help="Embedded Board Explorer")
 @click.argument("query", required=False)
 @click.option("--installed", is_flag=True)
 @click.option("--json-output", is_flag=True)
 def cli(query, installed, json_output):  # pylint: disable=R0912
     if json_output:
-        return _ouput_boards_json(query, installed)
-
-    BOARDLIST_TPL = ("{type:<30} {mcu:<14} {frequency:<8} "
-                     " {flash:<7} {ram:<6} {name}")
-    terminal_width, _ = click.get_terminal_size()
+        return _print_boards_json(query, installed)
 
     grpboards = {}
     for board in _get_boards(installed):
+        if query and query.lower() not in json.dumps(board).lower():
+            continue
         if board['platform'] not in grpboards:
             grpboards[board['platform']] = []
         grpboards[board['platform']].append(board)
 
-    for (platform, pboards) in sorted(grpboards.items()):
-        if query:
-            search_data = json.dumps(pboards).lower()
-            if query.lower() not in search_data.lower():
-                continue
-
+    terminal_width, _ = click.get_terminal_size()
+    for (platform, boards) in sorted(grpboards.items()):
         click.echo("")
         click.echo("Platform: ", nl=False)
         click.secho(platform, bold=True)
         click.echo("-" * terminal_width)
+        print_boards(boards)
+
+
+def print_boards(boards):
+    terminal_width, _ = click.get_terminal_size()
+    BOARDLIST_TPL = ("{type:<30} {mcu:<14} {frequency:<8} "
+                     " {flash:<7} {ram:<6} {name}")
+    click.echo(
+        BOARDLIST_TPL.format(
+            type=click.style("ID", fg="cyan"),
+            mcu="MCU",
+            frequency="Frequency",
+            flash="Flash",
+            ram="RAM",
+            name="Name"))
+    click.echo("-" * terminal_width)
+
+    for board in boards:
+        ram_size = board['ram']
+        if ram_size >= 1024:
+            if ram_size % 1024:
+                ram_size = "%.1fkB" % (ram_size / 1024.0)
+            else:
+                ram_size = "%dkB" % (ram_size / 1024)
+        else:
+            ram_size = "%dB" % ram_size
+
         click.echo(
             BOARDLIST_TPL.format(
-                type=click.style(
-                    "ID", fg="cyan"),
-                mcu="MCU",
-                frequency="Frequency",
-                flash="Flash",
-                ram="RAM",
-                name="Name"))
-        click.echo("-" * terminal_width)
-
-        for board in sorted(pboards, key=lambda b: b['id']):
-            if query:
-                search_data = "%s %s" % (board['id'],
-                                         json.dumps(board).lower())
-                if query.lower() not in search_data.lower():
-                    continue
-
-            flash_size = "%dkB" % (board['rom'] / 1024)
-
-            ram_size = board['ram']
-            if ram_size >= 1024:
-                if ram_size % 1024:
-                    ram_size = "%.1fkB" % (ram_size / 1024.0)
-                else:
-                    ram_size = "%dkB" % (ram_size / 1024)
-            else:
-                ram_size = "%dB" % ram_size
-
-            click.echo(
-                BOARDLIST_TPL.format(
-                    type=click.style(
-                        board['id'], fg="cyan"),
-                    mcu=board['mcu'],
-                    frequency="%dMhz" % (board['fcpu'] / 1000000),
-                    flash=flash_size,
-                    ram=ram_size,
-                    name=board['name']))
+                type=click.style(board['id'], fg="cyan"),
+                mcu=board['mcu'],
+                frequency="%dMhz" % (board['fcpu'] / 1000000),
+                flash="%dkB" % (board['rom'] / 1024),
+                ram=ram_size,
+                name=board['name']))
 
 
 def _get_boards(installed=False):
@@ -99,10 +90,10 @@ def _get_boards(installed=False):
                     boards.append(board)
         except InternetIsOffline:
             pass
-    return boards
+    return sorted(boards, key=lambda b: b['name'])
 
 
-def _ouput_boards_json(query, installed=False):
+def _print_boards_json(query, installed=False):
     result = []
     try:
         boards = _get_boards(installed)

platformio/commands/ci.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -152,7 +152,7 @@ def _copy_contents(dst_dir, contents):
 def _exclude_contents(dst_dir, patterns):
     contents = []
     for p in patterns:
-        contents += glob(join(dst_dir, p))
+        contents += glob(join(util.glob_escape(dst_dir), p))
     for path in contents:
         path = abspath(path)
         if isdir(path):

platformio/commands/debug.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from os import getcwd
+
+import click
+
+from platformio.managers.core import pioplus_call
+
+
+@click.command(
+    "debug",
+    context_settings=dict(ignore_unknown_options=True),
+    short_help="PIO Unified Debugger")
+@click.option(
+    "-d",
+    "--project-dir",
+    default=getcwd,
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        dir_okay=True,
+        writable=True,
+        resolve_path=True))
+@click.option("--environment", "-e", metavar="<environment>")
+@click.option("--verbose", "-v", is_flag=True)
+@click.option("--interface", type=click.Choice(["gdb"]))
+@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
+def cli(*args, **kwargs):  # pylint: disable=unused-argument
+    pioplus_call(sys.argv[1:])

platformio/commands/device.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,12 +14,12 @@
 
 import json
 import sys
+from os import getcwd
 
 import click
 from serial.tools import miniterm
 
-from platformio.exception import MinitermException
-from platformio.util import get_serialports
+from platformio import exception, util
 
 
 @click.group(short_help="Monitor device or list existing")
@@ -32,10 +32,10 @@ def cli():
 def device_list(json_output):
 
     if json_output:
-        click.echo(json.dumps(get_serialports()))
+        click.echo(json.dumps(util.get_serialports()))
         return
 
-    for item in get_serialports():
+    for item in util.get_serialports():
         click.secho(item['port'], fg="cyan")
         click.echo("-" * len(item['port']))
         click.echo("Hardware ID: %s" % item['hwid'])
@@ -45,8 +45,7 @@ def device_list(json_output):
 
 @cli.command("monitor", short_help="Monitor device (Serial)")
 @click.option("--port", "-p", help="Port, a number or a device name")
-@click.option(
-    "--baud", "-b", type=int, default=9600, help="Set baud rate, default=9600")
+@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
 @click.option(
     "--parity",
     default="N",
@@ -61,12 +60,12 @@ def device_list(json_output):
 @click.option(
     "--rts",
     default=None,
-    type=click.Choice(["0", "1"]),
+    type=click.IntRange(0, 1),
     help="Set initial RTS line state")
 @click.option(
     "--dtr",
     default=None,
-    type=click.Choice(["0", "1"]),
+    type=click.IntRange(0, 1),
     help="Set initial DTR line state")
 @click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
 @click.option(
@@ -98,15 +97,39 @@ def device_list(json_output):
     "--quiet",
     is_flag=True,
     help="Diagnostics: suppress non-error messages, default=Off")
-def device_monitor(**kwargs):
+@click.option(
+    "-d",
+    "--project-dir",
+    default=getcwd,
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, resolve_path=True))
+@click.option(
+    "-e",
+    "--environment",
+    help="Load configuration from `platformio.ini` and specified environment")
+def device_monitor(**kwargs):  # pylint: disable=too-many-branches
+    try:
+        project_options = get_project_options(kwargs['project_dir'],
+                                              kwargs['environment'])
+        monitor_options = {k: v for k, v in project_options or []}
+        if monitor_options:
+            for k in ("port", "baud", "rts", "dtr"):
+                k2 = "monitor_%s" % k
+                if kwargs[k] is None and k2 in monitor_options:
+                    kwargs[k] = monitor_options[k2]
+                    if k != "port":
+                        kwargs[k] = int(kwargs[k])
+    except exception.NotPlatformIOProject:
+        pass
+
     if not kwargs['port']:
-        ports = get_serialports(filter_hwid=True)
+        ports = util.get_serialports(filter_hwid=True)
         if len(ports) == 1:
             kwargs['port'] = ports[0]['port']
 
     sys.argv = ["monitor"]
     for k, v in kwargs.iteritems():
-        if k in ("port", "baud", "rts", "dtr"):
+        if k in ("port", "baud", "rts", "dtr", "environment", "project_dir"):
             continue
         k = "--" + k.replace("_", "-")
         if isinstance(v, bool):
@@ -121,8 +144,31 @@ def device_monitor(**kwargs):
     try:
         miniterm.main(
             default_port=kwargs['port'],
-            default_baudrate=kwargs['baud'],
+            default_baudrate=kwargs['baud'] or 9600,
             default_rts=kwargs['rts'],
             default_dtr=kwargs['dtr'])
     except Exception as e:
-        raise MinitermException(e)
+        raise exception.MinitermException(e)
+
+
+def get_project_options(project_dir, environment):
+    config = util.load_project_config(project_dir)
+    if not config.sections():
+        return
+
+    known_envs = [s[4:] for s in config.sections() if s.startswith("env:")]
+    if environment:
+        if environment in known_envs:
+            return config.items("env:%s" % environment)
+        raise exception.UnknownEnvNames(environment, ", ".join(known_envs))
+
+    if not known_envs:
+        return
+
+    if config.has_option("platformio", "env_default"):
+        env_default = config.get("platformio",
+                                 "env_default").split(", ")[0].strip()
+        if env_default and env_default in known_envs:
+            return config.items("env:%s" % env_default)
+
+    return config.items("env:%s" % known_envs[0])

platformio/commands/init.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -57,6 +57,7 @@ def validate_boards(ctx, param, value):  # pylint: disable=W0613
     "--ide", type=click.Choice(ProjectGenerator.get_supported_ides()))
 @click.option("-O", "--project-option", multiple=True)
 @click.option("--env-prefix", default="")
+@click.option("-s", "--silent", is_flag=True)
 @click.pass_context
 def cli(
         ctx,  # pylint: disable=R0913
@@ -64,28 +65,29 @@ def cli(
         board,
         ide,
         project_option,
-        env_prefix):
+        env_prefix,
+        silent):
 
-    if project_dir == getcwd():
-        click.secho("\nThe current working directory", fg="yellow", nl=False)
-        click.secho(" %s " % project_dir, fg="cyan", nl=False)
-        click.secho(
-            "will be used for project.\n"
-            "You can specify another project directory via\n"
-            "`platformio init -d %PATH_TO_THE_PROJECT_DIR%` command.",
-            fg="yellow")
-        click.echo("")
+    if not silent:
+        if project_dir == getcwd():
+            click.secho(
+                "\nThe current working directory", fg="yellow", nl=False)
+            click.secho(" %s " % project_dir, fg="cyan", nl=False)
+            click.secho(
+                "will be used for project.\n"
+                "You can specify another project directory via\n"
+                "`platformio init -d %PATH_TO_THE_PROJECT_DIR%` command.",
+                fg="yellow")
+            click.echo("")
 
-    click.echo("The next files/directories have been created in %s" %
-               click.style(
-                   project_dir, fg="cyan"))
-    click.echo("%s - Project Configuration File" % click.style(
-        "platformio.ini", fg="cyan"))
-    click.echo("%s - Put your source files here" % click.style(
-        "src", fg="cyan"))
-    click.echo("%s - Put here project specific (private) libraries" %
-               click.style(
-                   "lib", fg="cyan"))
+        click.echo("The next files/directories have been created in %s" %
+                   click.style(project_dir, fg="cyan"))
+        click.echo("%s - Project Configuration File" % click.style(
+            "platformio.ini", fg="cyan"))
+        click.echo(
+            "%s - Put your source files here" % click.style("src", fg="cyan"))
+        click.echo("%s - Put here project specific (private) libraries" %
+                   click.style("lib", fg="cyan"))
 
     init_base_project(project_dir)
 
@@ -94,43 +96,41 @@ def cli(
                           ide is not None)
 
     if ide:
-        if not board:
-            board = get_first_board(project_dir)
-            if board:
-                board = [board]
-        if not board:
+        env_name = get_best_envname(project_dir, board)
+        if not env_name:
             raise exception.BoardNotDefined()
-        if len(board) > 1:
-            click.secho(
-                "Warning! You have initialised project with more than 1 board"
-                " for the specified IDE.\n"
-                "However, the IDE features (code autocompletion, syntax "
-                "linter) have been configured for the first board '%s' from "
-                "your list '%s'." % (board[0], ", ".join(board)),
-                fg="yellow")
-        pg = ProjectGenerator(project_dir, ide, board[0])
+        pg = ProjectGenerator(project_dir, ide, env_name)
         pg.generate()
 
-    click.secho(
-        "\nProject has been successfully initialized!\nUseful commands:\n"
-        "`platformio run` - process/build project from the current "
-        "directory\n"
-        "`platformio run --target upload` or `platformio run -t upload` "
-        "- upload firmware to embedded board\n"
-        "`platformio run --target clean` - clean project (remove compiled "
-        "files)\n"
-        "`platformio run --help` - additional information",
-        fg="green")
+    if not silent:
+        click.secho(
+            "\nProject has been successfully initialized!\nUseful commands:\n"
+            "`platformio run` - process/build project from the current "
+            "directory\n"
+            "`platformio run --target upload` or `platformio run -t upload` "
+            "- upload firmware to embedded board\n"
+            "`platformio run --target clean` - clean project (remove compiled "
+            "files)\n"
+            "`platformio run --help` - additional information",
+            fg="green")
 
 
-def get_first_board(project_dir):
+def get_best_envname(project_dir, boards=None):
     config = util.load_project_config(project_dir)
+    env_default = None
+    if config.has_option("platformio", "env_default"):
+        env_default = config.get("platformio",
+                                 "env_default").split(", ")[0].strip()
+    if env_default:
+        return env_default
+    section = None
     for section in config.sections():
         if not section.startswith("env:"):
             continue
-        elif config.has_option(section, "board"):
-            return config.get(section, "board")
-    return None
+        elif config.has_option(section, "board") and (not boards or config.get(
+                section, "board") in boards):
+            break
+    return section[4:] if section else None
 
 
 def init_base_project(project_dir):
@@ -298,7 +298,8 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix,
     config = util.load_project_config(project_dir)
     for section in config.sections():
         cond = [
-            section.startswith("env:"), config.has_option(section, "board")
+            section.startswith("env:"),
+            config.has_option(section, "board")
         ]
         if all(cond):
             used_boards.append(config.get(section, "board"))

platformio/commands/lib.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,14 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
-from os.path import join
-from time import sleep
+# pylint: disable=too-many-branches, too-many-locals
 
+import json
+from os.path import isdir, join
+from time import sleep
+from urllib import quote
+
+import arrow
 import click
 
 from platformio import exception, util
 from platformio.managers.lib import LibraryManager
+from platformio.managers.platform import PlatformFactory, PlatformManager
 from platformio.util import get_api_result
 
 
@@ -43,8 +48,9 @@ from platformio.util import get_api_result
     help="Manage custom library storage")
 @click.pass_context
 def cli(ctx, **options):
+    non_storage_cmds = ("search", "show", "register", "stats", "builtin")
     # skip commands that don't need storage folder
-    if ctx.invoked_subcommand in ("search", "register") or \
+    if ctx.invoked_subcommand in non_storage_cmds or \
             (len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")):
         return
     storage_dir = options['storage_dir']
@@ -60,6 +66,9 @@ def cli(ctx, **options):
                 "Please use `platformio lib --global %s` command to remove "
                 "this warning." % ctx.invoked_subcommand,
                 fg="yellow")
+    elif util.is_platformio_project(storage_dir):
+        with util.cd(storage_dir):
+            storage_dir = util.get_projectlibdeps_dir()
 
     if not storage_dir and not util.is_platformio_project():
         raise exception.NotGlobalLibDir(util.get_project_dir(),
@@ -106,57 +115,69 @@ def lib_uninstall(lm, libraries):
     "--only-check",
     is_flag=True,
     help="Do not update, only check for new version")
+@click.option("--json-output", is_flag=True)
 @click.pass_obj
-def lib_update(lm, libraries, only_check):
+def lib_update(lm, libraries, only_check, json_output):
     if not libraries:
-        libraries = [str(m.get("id", m['name'])) for m in lm.get_installed()]
-    for library in libraries:
-        lm.update(library, only_check=only_check)
+        libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()]
+
+    if only_check and json_output:
+        result = []
+        for library in libraries:
+            pkg_dir = library if isdir(library) else None
+            requirements = None
+            url = None
+            if not pkg_dir:
+                name, requirements, url = lm.parse_pkg_input(library)
+                pkg_dir = lm.get_package_dir(name, requirements, url)
+            if not pkg_dir:
+                continue
+            latest = lm.outdated(pkg_dir, requirements)
+            if not latest:
+                continue
+            manifest = lm.load_manifest(pkg_dir)
+            manifest['versionLatest'] = latest
+            result.append(manifest)
+        return click.echo(json.dumps(result))
+    else:
+        for library in libraries:
+            lm.update(library, only_check=only_check)
 
 
-#######
+def print_lib_item(item):
+    click.secho(item['name'], fg="cyan")
+    click.echo("=" * len(item['name']))
+    if "id" in item:
+        click.secho("#ID: %d" % item['id'], bold=True)
+    if "description" in item or "url" in item:
+        click.echo(item.get("description", item.get("url", "")))
+    click.echo()
 
-LIBLIST_TPL = ("[{id:^14}] {name:<25} {compatibility:<30} "
-               "\"{authornames}\": {description}")
+    for key in ("version", "homepage", "license", "keywords"):
+        if key not in item or not item[key]:
+            continue
+        if isinstance(item[key], list):
+            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
+        else:
+            click.echo("%s: %s" % (key.title(), item[key]))
+
+    for key in ("frameworks", "platforms"):
+        if key not in item:
+            continue
+        click.echo("Compatible %s: %s" % (key, ", ".join(
+            [i['title'] if isinstance(i, dict) else i for i in item[key]])))
+
+    if "authors" in item or "authornames" in item:
+        click.echo("Authors: %s" % ", ".join(
+            item.get("authornames",
+                     [a.get("name", "") for a in item.get("authors", [])])))
+
+    if "__src_url" in item:
+        click.secho("Source: %s" % item['__src_url'])
+    click.echo()
 
 
-def echo_liblist_header():
-    click.echo(
-        LIBLIST_TPL.format(
-            id=click.style(
-                "ID", fg="green"),
-            name=click.style(
-                "Name", fg="cyan"),
-            compatibility=click.style(
-                "Compatibility", fg="yellow"),
-            authornames="Authors",
-            description="Description"))
-
-    terminal_width, _ = click.get_terminal_size()
-    click.echo("-" * terminal_width)
-
-
-def echo_liblist_item(item):
-    description = item.get("description", item.get("url", "")).encode("utf-8")
-    if "version" in item:
-        description += " | @" + click.style(item['version'], fg="yellow")
-
-    click.echo(
-        LIBLIST_TPL.format(
-            id=click.style(
-                str(item.get("id", "-")), fg="green"),
-            name=click.style(
-                item['name'], fg="cyan"),
-            compatibility=click.style(
-                ", ".join(
-                    item.get("frameworks", ["-"]) + item.get("platforms", [])),
-                fg="yellow"),
-            authornames=", ".join(item.get("authornames", ["Unknown"])).encode(
-                "utf-8"),
-            description=description))
-
-
-@cli.command("search", short_help="Search for library")
+@cli.command("search", short_help="Search for a library")
 @click.argument("query", required=False, nargs=-1)
 @click.option("--json-output", is_flag=True)
 @click.option("--page", type=click.INT, default=1)
@@ -181,9 +202,8 @@ def lib_search(query, json_output, page, noninteractive, **filters):
             query.append('%s:"%s"' % (key, value))
 
     result = get_api_result(
-        "/lib/search",
-        dict(
-            query=" ".join(query), page=page),
+        "/v2/lib/search",
+        dict(query=" ".join(query), page=page),
         cache_valid="3d")
 
     if json_output:
@@ -210,12 +230,9 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         "Found %d libraries:\n" % result['total'],
         fg="green" if result['total'] else "yellow")
 
-    if result['total']:
-        echo_liblist_header()
-
     while True:
         for item in result['items']:
-            echo_liblist_item(item)
+            print_lib_item(item)
 
         if (int(result['page']) * int(result['perpage']) >=
                 int(result['total'])):
@@ -232,9 +249,9 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         elif not click.confirm("Show next libraries?"):
             break
         result = get_api_result(
-            "/lib/search",
-            dict(
-                query=" ".join(query), page=int(result['page']) + 1),
+            "/v2/lib/search",
+            {"query": " ".join(query),
+             "page": int(result['page']) + 1},
             cache_valid="3d")
 
 
@@ -245,41 +262,87 @@ def lib_list(lm, json_output):
     items = lm.get_installed()
 
     if json_output:
-        click.echo(json.dumps(items))
-        return
+        return click.echo(json.dumps(items))
 
     if not items:
         return
 
-    echo_liblist_header()
     for item in sorted(items, key=lambda i: i['name']):
-        if "authors" in item:
-            item['authornames'] = [i['name'] for i in item['authors']]
-        echo_liblist_item(item)
+        print_lib_item(item)
 
 
-@cli.command("show", short_help="Show details about installed library")
-@click.pass_obj
+@util.memoized
+def get_builtin_libs(storage_names=None):
+    items = []
+    storage_names = storage_names or []
+    pm = PlatformManager()
+    for manifest in pm.get_installed():
+        p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
+        for storage in p.get_lib_storages():
+            if storage_names and storage['name'] not in storage_names:
+                continue
+            lm = LibraryManager(storage['path'])
+            items.append({
+                "name": storage['name'],
+                "path": storage['path'],
+                "items": lm.get_installed()
+            })
+    return items
+
+
+@cli.command("builtin", short_help="List built-in libraries")
+@click.option("--storage", multiple=True)
+@click.option("--json-output", is_flag=True)
+def lib_builtin(storage, json_output):
+    items = get_builtin_libs(storage)
+    if json_output:
+        return click.echo(json.dumps(items))
+
+    for storage_ in items:
+        if not storage_['items']:
+            continue
+        click.secho(storage_['name'], fg="green")
+        click.echo("*" * len(storage_['name']))
+        click.echo()
+
+        for item in sorted(storage_['items'], key=lambda i: i['name']):
+            print_lib_item(item)
+
+
+@cli.command("show", short_help="Show detailed info about a library")
 @click.argument("library", metavar="[LIBRARY]")
-def lib_show(lm, library):  # pylint: disable=too-many-branches
-    name, requirements, url = lm.parse_pkg_name(library)
-    package_dir = lm.get_package_dir(name, requirements, url)
-    if not package_dir:
-        click.secho(
-            "%s @ %s is not installed" % (name, requirements or "*"),
-            fg="yellow")
-        return
+@click.option("--json-output", is_flag=True)
+def lib_show(library, json_output):
+    lm = LibraryManager()
+    name, requirements, _ = lm.parse_pkg_input(library)
+    lib_id = lm.get_pkg_id_by_name(
+        name, requirements, silent=json_output, interactive=not json_output)
+    lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
+    if json_output:
+        return click.echo(json.dumps(lib))
 
-    manifest = lm.load_manifest(package_dir)
-
-    click.secho(manifest['name'], fg="cyan")
-    click.echo("=" * len(manifest['name']))
-    if "description" in manifest:
-        click.echo(manifest['description'])
+    click.secho(lib['name'], fg="cyan")
+    click.echo("=" * len(lib['name']))
+    click.secho("#ID: %d" % lib['id'], bold=True)
+    click.echo(lib['description'])
     click.echo()
 
+    click.echo("Version: %s, released %s" %
+               (lib['version']['name'],
+                arrow.get(lib['version']['released']).humanize()))
+    click.echo("Manifest: %s" % lib['confurl'])
+    for key in ("homepage", "repository", "license"):
+        if key not in lib or not lib[key]:
+            continue
+        if isinstance(lib[key], list):
+            click.echo("%s: %s" % (key.title(), ", ".join(lib[key])))
+        else:
+            click.echo("%s: %s" % (key.title(), lib[key]))
+
+    blocks = []
+
     _authors = []
-    for author in manifest.get("authors", []):
+    for author in lib.get("authors", []):
         _data = []
         for key in ("name", "email", "url", "maintainer"):
             if not author[key]:
@@ -292,23 +355,38 @@ def lib_show(lm, library):  # pylint: disable=too-many-branches
                 _data.append(author[key])
         _authors.append(" ".join(_data))
     if _authors:
-        click.echo("Authors: %s" % ", ".join(_authors))
+        blocks.append(("Authors", _authors))
 
-    for key in ("keywords", "frameworks", "platforms", "license", "url",
-                "version"):
-        if key not in manifest:
+    blocks.append(("Keywords", lib['keywords']))
+    for key in ("frameworks", "platforms"):
+        if key not in lib or not lib[key]:
             continue
-        if isinstance(manifest[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(manifest[key])))
-        else:
-            click.echo("%s: %s" % (key.title(), manifest[key]))
+        blocks.append(("Compatible %s" % key, [i['title'] for i in lib[key]]))
+    blocks.append(("Headers", lib['headers']))
+    blocks.append(("Examples", lib['examples']))
+    blocks.append(("Versions", [
+        "%s, released %s" % (v['name'], arrow.get(v['released']).humanize())
+        for v in lib['versions']
+    ]))
+    blocks.append(("Unique Downloads", [
+        "Today: %s" % lib['dlstats']['day'],
+        "Week: %s" % lib['dlstats']['week'],
+        "Month: %s" % lib['dlstats']['month']
+    ]))
+
+    for (title, rows) in blocks:
+        click.echo()
+        click.secho(title, bold=True)
+        click.echo("-" * len(title))
+        for row in rows:
+            click.echo(row)
 
 
-@cli.command("register", short_help="Register new library")
+@cli.command("register", short_help="Register a new library")
 @click.argument("config_url")
 def lib_register(config_url):
-    if (not config_url.startswith("http://") and
-            not config_url.startswith("https://")):
+    if (not config_url.startswith("http://")
+            and not config_url.startswith("https://")):
         raise exception.InvalidLibConfURL(config_url)
 
     result = get_api_result("/lib/register", data=dict(config_url=config_url))
@@ -317,3 +395,76 @@ def lib_register(config_url):
             result['message'],
             fg="green"
             if "successed" in result and result['successed'] else "red")
+
+
+@cli.command("stats", short_help="Library Registry Statistics")
+@click.option("--json-output", is_flag=True)
+def lib_stats(json_output):
+    result = get_api_result("/lib/stats", cache_valid="1h")
+
+    if json_output:
+        return click.echo(json.dumps(result))
+
+    printitem_tpl = "{name:<33} {url}"
+    printitemdate_tpl = "{name:<33} {date:23} {url}"
+
+    def _print_title(title):
+        click.secho(title.upper(), bold=True)
+        click.echo("*" * len(title))
+
+    def _print_header(with_date=False):
+        click.echo((printitemdate_tpl if with_date else printitem_tpl).format(
+            name=click.style("Name", fg="cyan"),
+            date="Date",
+            url=click.style("Url", fg="blue")))
+
+        terminal_width, _ = click.get_terminal_size()
+        click.echo("-" * terminal_width)
+
+    def _print_lib_item(item):
+        click.echo((printitemdate_tpl
+                    if "date" in item else printitem_tpl).format(
+                        name=click.style(item['name'], fg="cyan"),
+                        date=str(
+                            arrow.get(item['date']).humanize()
+                            if "date" in item else ""),
+                        url=click.style(
+                            "http://platformio.org/lib/show/%s/%s" %
+                            (item['id'], quote(item['name'])),
+                            fg="blue")))
+
+    def _print_tag_item(name):
+        click.echo(
+            printitem_tpl.format(
+                name=click.style(name, fg="cyan"),
+                url=click.style(
+                    "http://platformio.org/lib/search?query=" + quote(
+                        "keyword:%s" % name),
+                    fg="blue")))
+
+    for key in ("updated", "added"):
+        _print_title("Recently " + key)
+        _print_header(with_date=True)
+        for item in result.get(key, []):
+            _print_lib_item(item)
+        click.echo()
+
+    _print_title("Recent keywords")
+    _print_header(with_date=False)
+    for item in result.get("lastkeywords"):
+        _print_tag_item(item)
+    click.echo()
+
+    _print_title("Popular keywords")
+    _print_header(with_date=False)
+    for item in result.get("topkeywords"):
+        _print_tag_item(item)
+    click.echo()
+
+    for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth",
+                                                                "Month")):
+        _print_title("Featured: " + title)
+        _print_header(with_date=False)
+        for item in result.get(key, []):
+            _print_lib_item(item)
+        click.echo()

platformio/commands/platform.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,10 +13,12 @@
 # limitations under the License.
 
 import json
+from os.path import dirname, isdir
 
 import click
 
-from platformio import exception, util
+from platformio import app, exception, util
+from platformio.commands.boards import print_boards
 from platformio.managers.platform import PlatformFactory, PlatformManager
 
 
@@ -28,40 +30,152 @@ def cli():
 def _print_platforms(platforms):
     for platform in platforms:
         click.echo("{name} ~ {title}".format(
-            name=click.style(
-                platform['name'], fg="cyan"),
+            name=click.style(platform['name'], fg="cyan"),
             title=platform['title']))
         click.echo("=" * (3 + len(platform['name'] + platform['title'])))
         click.echo(platform['description'])
         click.echo()
-        click.echo("Home: %s" % "http://platformio.org/platforms/" + platform[
-            'name'])
-        if platform['packages']:
+        if "homepage" in platform:
+            click.echo("Home: %s" % platform['homepage'])
+        if "frameworks" in platform and platform['frameworks']:
+            click.echo("Frameworks: %s" % ", ".join(platform['frameworks']))
+        if "packages" in platform:
             click.echo("Packages: %s" % ", ".join(platform['packages']))
         if "version" in platform:
             click.echo("Version: " + platform['version'])
         click.echo()
 
 
+def _get_registry_platforms():
+    platforms = util.get_api_result("/platforms", cache_valid="30d")
+    pm = PlatformManager()
+    for platform in platforms or []:
+        platform['versions'] = pm.get_all_repo_versions(platform['name'])
+    return platforms
+
+
+def _original_version(version):
+    if version.count(".") != 2:
+        return None
+    _, y = version.split(".")[:2]
+    if int(y) < 100:
+        return None
+    if len(y) % 2 != 0:
+        y = "0" + y
+    parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(len(y) / 2)]
+    return ".".join(parts)
+
+
+def _get_platform_data(*args, **kwargs):
+    try:
+        return _get_installed_platform_data(*args, **kwargs)
+    except exception.UnknownPlatform:
+        return _get_registry_platform_data(*args, **kwargs)
+
+
+def _get_installed_platform_data(platform,
+                                 with_boards=True,
+                                 expose_packages=True):
+    p = PlatformFactory.newPlatform(platform)
+    data = dict(
+        name=p.name,
+        title=p.title,
+        description=p.description,
+        version=p.version,
+        homepage=p.homepage,
+        repository=p.repository_url,
+        url=p.vendor_url,
+        license=p.license,
+        forDesktop=not p.is_embedded(),
+        frameworks=sorted(p.frameworks.keys() if p.frameworks else []),
+        packages=p.packages.keys() if p.packages else [])
+
+    # if dump to API
+    # del data['version']
+    # return data
+
+    # overwrite VCS version and add extra fields
+    manifest = PlatformManager().load_manifest(dirname(p.manifest_path))
+    assert manifest
+    for key in manifest:
+        if key == "version" or key.startswith("__"):
+            data[key] = manifest[key]
+
+    if with_boards:
+        data['boards'] = [c.get_brief_data() for c in p.get_boards().values()]
+
+    if not data['packages'] or not expose_packages:
+        return data
+
+    data['packages'] = []
+    installed_pkgs = p.get_installed_packages()
+    for name, opts in p.packages.items():
+        item = dict(
+            name=name,
+            type=p.get_package_type(name),
+            requirements=opts.get("version"),
+            optional=opts.get("optional") is True)
+        if name in installed_pkgs:
+            for key, value in installed_pkgs[name].items():
+                if key not in ("url", "version", "description"):
+                    continue
+                item[key] = value
+                if key == "version":
+                    item["originalVersion"] = _original_version(value)
+        data['packages'].append(item)
+
+    return data
+
+
+def _get_registry_platform_data(  # pylint: disable=unused-argument
+        platform,
+        with_boards=True,
+        expose_packages=True):
+    _data = None
+    for p in _get_registry_platforms():
+        if p['name'] == platform:
+            _data = p
+            break
+
+    if not _data:
+        return None
+
+    data = dict(
+        name=_data['name'],
+        title=_data['title'],
+        description=_data['description'],
+        homepage=_data['homepage'],
+        repository=_data['repository'],
+        url=_data['url'],
+        license=_data['license'],
+        forDesktop=_data['forDesktop'],
+        frameworks=_data['frameworks'],
+        packages=_data['packages'],
+        versions=_data['versions'])
+
+    if with_boards:
+        data['boards'] = [
+            board for board in PlatformManager().get_registered_boards()
+            if board['platform'] == _data['name']
+        ]
+
+    return data
+
+
 @cli.command("search", short_help="Search for development platform")
 @click.argument("query", required=False)
 @click.option("--json-output", is_flag=True)
 def platform_search(query, json_output):
     platforms = []
-    for platform in util.get_api_result("/platforms", cache_valid="365d"):
+    for platform in _get_registry_platforms():
         if query == "all":
             query = ""
-
         search_data = json.dumps(platform)
         if query and query.lower() not in search_data.lower():
             continue
-
-        platforms.append({
-            "name": platform['name'],
-            "title": platform['title'],
-            "description": platform['description'],
-            "packages": platform['packages']
-        })
+        platforms.append(
+            _get_registry_platform_data(
+                platform['name'], with_boards=False, expose_packages=False))
 
     if json_output:
         click.echo(json.dumps(platforms))
@@ -69,6 +183,111 @@ def platform_search(query, json_output):
         _print_platforms(platforms)
 
 
+@cli.command("frameworks", short_help="List supported frameworks, SDKs")
+@click.argument("query", required=False)
+@click.option("--json-output", is_flag=True)
+def platform_frameworks(query, json_output):
+    frameworks = []
+    for framework in util.get_api_result("/frameworks", cache_valid="30d"):
+        if query == "all":
+            query = ""
+        search_data = json.dumps(framework)
+        if query and query.lower() not in search_data.lower():
+            continue
+        framework['homepage'] = (
+            "http://platformio.org/frameworks/" + framework['name'])
+        framework['platforms'] = [
+            platform['name'] for platform in _get_registry_platforms()
+            if framework['name'] in platform['frameworks']
+        ]
+        frameworks.append(framework)
+
+    frameworks = sorted(frameworks, key=lambda manifest: manifest['name'])
+    if json_output:
+        click.echo(json.dumps(frameworks))
+    else:
+        _print_platforms(frameworks)
+
+
+@cli.command("list", short_help="List installed development platforms")
+@click.option("--json-output", is_flag=True)
+def platform_list(json_output):
+    platforms = []
+    pm = PlatformManager()
+    for manifest in pm.get_installed():
+        platforms.append(
+            _get_installed_platform_data(
+                manifest['__pkg_dir'],
+                with_boards=False,
+                expose_packages=False))
+
+    platforms = sorted(platforms, key=lambda manifest: manifest['name'])
+    if json_output:
+        click.echo(json.dumps(platforms))
+    else:
+        _print_platforms(platforms)
+
+
+@cli.command("show", short_help="Show details about development platform")
+@click.argument("platform")
+@click.option("--json-output", is_flag=True)
+def platform_show(platform, json_output):  # pylint: disable=too-many-branches
+    data = _get_platform_data(platform)
+    if not data:
+        raise exception.UnknownPlatform(platform)
+    if json_output:
+        return click.echo(json.dumps(data))
+
+    click.echo("{name} ~ {title}".format(
+        name=click.style(data['name'], fg="cyan"), title=data['title']))
+    click.echo("=" * (3 + len(data['name'] + data['title'])))
+    click.echo(data['description'])
+    click.echo()
+    if "version" in data:
+        click.echo("Version: %s" % data['version'])
+    if data['homepage']:
+        click.echo("Home: %s" % data['homepage'])
+    if data['repository']:
+        click.echo("Repository: %s" % data['repository'])
+    if data['url']:
+        click.echo("Vendor: %s" % data['url'])
+    if data['license']:
+        click.echo("License: %s" % data['license'])
+    if data['frameworks']:
+        click.echo("Frameworks: %s" % ", ".join(data['frameworks']))
+
+    if not data['packages']:
+        return
+
+    if not isinstance(data['packages'][0], dict):
+        click.echo("Packages: %s" % ", ".join(data['packages']))
+    else:
+        click.echo()
+        click.secho("Packages", bold=True)
+        click.echo("--------")
+        for item in data['packages']:
+            click.echo()
+            click.echo("Package %s" % click.style(item['name'], fg="yellow"))
+            click.echo("-" * (8 + len(item['name'])))
+            if item['type']:
+                click.echo("Type: %s" % item['type'])
+            click.echo("Requirements: %s" % item['requirements'])
+            click.echo("Installed: %s" %
+                       ("Yes" if item.get("version") else "No (optional)"))
+            if "version" in item:
+                click.echo("Version: %s" % item['version'])
+            if "originalVersion" in item:
+                click.echo("Original version: %s" % item['originalVersion'])
+            if "description" in item:
+                click.echo("Description: %s" % item['description'])
+
+    if data['boards']:
+        click.echo()
+        click.secho("Boards", bold=True)
+        click.echo("------")
+        print_boards(data['boards'])
+
+
 @cli.command("install", short_help="Install new development platform")
 @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]")
 @click.option("--with-package", multiple=True)
@@ -108,99 +327,51 @@ def platform_uninstall(platforms):
     "-p",
     "--only-packages",
     is_flag=True,
-    help="Update only platform packages")
+    help="Update only the platform packages")
 @click.option(
     "-c",
     "--only-check",
     is_flag=True,
-    help="Do not update, only check for new version")
-def platform_update(platforms, only_packages, only_check):
-    pm = PlatformManager()
-    if not platforms:
-        platforms = set([m['name'] for m in pm.get_installed()])
-    for platform in platforms:
-        click.echo("Platform %s" % click.style(platform, fg="cyan"))
-        click.echo("--------")
-        pm.update(platform, only_packages=only_packages, only_check=only_check)
-        click.echo()
-
-
-@cli.command("list", short_help="List installed development platforms")
+    help="Do not update, only check for a new version")
 @click.option("--json-output", is_flag=True)
-def platform_list(json_output):
-    platforms = []
+def platform_update(platforms, only_packages, only_check, json_output):
     pm = PlatformManager()
-    for manifest in pm.get_installed():
-        p = PlatformFactory.newPlatform(
-            pm.get_manifest_path(manifest['__pkg_dir']))
-        platforms.append({
-            "name": p.name,
-            "title": p.title,
-            "description": p.description,
-            "version": p.version,
-            "url": p.vendor_url,
-            "packages": p.get_installed_packages().keys(),
-            'forDesktop': any([
-                p.name.startswith(n) for n in ("native", "linux", "windows")
-            ])
-        })
+    pkg_dir_to_name = {}
+    if not platforms:
+        platforms = []
+        for manifest in pm.get_installed():
+            platforms.append(manifest['__pkg_dir'])
+            pkg_dir_to_name[manifest['__pkg_dir']] = manifest.get(
+                "title", manifest['name'])
 
-    if json_output:
-        click.echo(json.dumps(platforms))
+    if only_check and json_output:
+        result = []
+        for platform in platforms:
+            pkg_dir = platform if isdir(platform) else None
+            requirements = None
+            url = None
+            if not pkg_dir:
+                name, requirements, url = pm.parse_pkg_input(platform)
+                pkg_dir = pm.get_package_dir(name, requirements, url)
+            if not pkg_dir:
+                continue
+            latest = pm.outdated(pkg_dir, requirements)
+            if (not latest and not PlatformFactory.newPlatform(pkg_dir)
+                    .are_outdated_packages()):
+                continue
+            data = _get_installed_platform_data(
+                pkg_dir, with_boards=False, expose_packages=False)
+            if latest:
+                data['versionLatest'] = latest
+            result.append(data)
+        return click.echo(json.dumps(result))
     else:
-        _print_platforms(platforms)
-
-
-@cli.command("show", short_help="Show details about installed platform")
-@click.argument("platform")
-def platform_show(platform):
-
-    def _detail_version(version):
-        if version.count(".") != 2:
-            return version
-        _, y = version.split(".")[:2]
-        if int(y) < 100:
-            return version
-        if len(y) % 2 != 0:
-            y = "0" + y
-        parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(len(y) / 2)]
-        return "%s (%s)" % (version, ".".join(parts))
-
-    try:
-        p = PlatformFactory.newPlatform(platform)
-    except exception.UnknownPlatform:
-        raise exception.PlatformNotInstalledYet(platform)
-
-    click.echo("{name} ~ {title}".format(
-        name=click.style(
-            p.name, fg="cyan"), title=p.title))
-    click.echo("=" * (3 + len(p.name + p.title)))
-    click.echo(p.description)
-    click.echo()
-    click.echo("Version: %s" % p.version)
-    if p.homepage:
-        click.echo("Home: %s" % p.homepage)
-    if p.license:
-        click.echo("License: %s" % p.license)
-    if p.frameworks:
-        click.echo("Frameworks: %s" % ", ".join(p.frameworks.keys()))
-
-    if not p.packages:
-        return
-
-    installed_pkgs = p.get_installed_packages()
-    for name, opts in p.packages.items():
-        click.echo()
-        click.echo("Package %s" % click.style(name, fg="yellow"))
-        click.echo("-" * (8 + len(name)))
-        if p.get_package_type(name):
-            click.echo("Type: %s" % p.get_package_type(name))
-        click.echo("Requirements: %s" % opts.get("version"))
-        click.echo("Installed: %s" % ("Yes" if name in installed_pkgs else
-                                      "No (optional)"))
-        if name in installed_pkgs:
-            for key, value in installed_pkgs[name].items():
-                if key in ("url", "version", "description"):
-                    if key == "version":
-                        value = _detail_version(value)
-                    click.echo("%s: %s" % (key.title(), value))
+        # cleanup cached board and platform lists
+        app.clean_cache()
+        for platform in platforms:
+            click.echo("Platform %s" % click.style(
+                pkg_dir_to_name.get(platform, platform), fg="cyan"))
+            click.echo("--------")
+            pm.update(
+                platform, only_packages=only_packages, only_check=only_check)
+            click.echo()

platformio/commands/remote.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@ import click
 
 from platformio import exception, util
 from platformio.commands.device import device_monitor as cmd_device_monitor
-from platformio.pioplus import pioplus_call
+from platformio.managers.core import pioplus_call
 
 # pylint: disable=unused-argument
 
@@ -147,12 +147,12 @@ def device_list(json_output):
 @click.option(
     "--rts",
     default=None,
-    type=click.Choice(["0", "1"]),
+    type=click.IntRange(0, 1),
     help="Set initial RTS line state")
 @click.option(
     "--dtr",
     default=None,
-    type=click.Choice(["0", "1"]),
+    type=click.IntRange(0, 1),
     help="Set initial DTR line state")
 @click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
 @click.option(

platformio/commands/run.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,9 @@ from time import time
 import click
 
 from platformio import __version__, exception, telemetry, util
+from platformio.commands.device import device_monitor as cmd_device_monitor
 from platformio.commands.lib import lib_install as cmd_lib_install
+from platformio.commands.lib import get_builtin_libs
 from platformio.commands.platform import \
     platform_install as cmd_platform_install
 from platformio.managers.lib import LibraryManager
@@ -75,10 +77,8 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
 
         env_default = None
         if config.has_option("platformio", "env_default"):
-            env_default = [
-                e.strip()
-                for e in config.get("platformio", "env_default").split(",")
-            ]
+            env_default = util.parse_conf_multi_values(
+                config.get("platformio", "env_default"))
 
         results = []
         start_time = time()
@@ -88,14 +88,14 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
 
             envname = section[4:]
             skipenv = any([
-                environment and envname not in environment, not environment and
-                env_default and envname not in env_default
+                environment and envname not in environment, not environment
+                and env_default and envname not in env_default
             ])
             if skipenv:
                 results.append((envname, None))
                 continue
 
-            if results:
+            if not silent and results:
                 click.echo()
 
             options = {}
@@ -106,31 +106,52 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
 
             ep = EnvironmentProcessor(ctx, envname, options, target,
                                       upload_port, silent, verbose)
-            results.append((envname, ep.process()))
+            result = (envname, ep.process())
+            results.append(result)
+            if result[1] and "monitor" in ep.get_build_targets() and \
+                    "nobuild" not in ep.get_build_targets():
+                ctx.invoke(cmd_device_monitor)
 
-        if len(results) > 1:
+        found_error = any([status is False for (_, status) in results])
+
+        if (found_error or not silent) and len(results) > 1:
             click.echo()
             print_summary(results, start_time)
 
-        if any([status is False for (_, status) in results]):
+        if found_error:
             raise exception.ReturnErrorCode(1)
         return True
 
 
 class EnvironmentProcessor(object):
 
-    KNOWN_OPTIONS = (
-        "platform", "framework", "board", "board_mcu", "board_f_cpu",
-        "board_f_flash", "board_flash_mode", "build_flags", "src_build_flags",
-        "build_unflags", "src_filter", "extra_script", "targets",
-        "upload_port", "upload_protocol", "upload_speed", "upload_flags",
-        "upload_resetmethod", "lib_install", "lib_deps", "lib_force",
-        "lib_ignore", "lib_extra_dirs", "lib_ldf_mode", "lib_compat_mode",
-        "test_ignore", "test_port", "piotest")
+    KNOWN_OPTIONS = ("platform", "framework", "board", "board_mcu",
+                     "board_f_cpu", "board_f_flash", "board_flash_mode",
+                     "build_flags", "src_build_flags", "build_unflags",
+                     "src_filter", "extra_scripts", "targets", "upload_port",
+                     "upload_protocol", "upload_speed", "upload_flags",
+                     "upload_resetmethod", "lib_deps", "lib_ignore",
+                     "lib_extra_dirs", "lib_ldf_mode", "lib_compat_mode",
+                     "lib_archive", "piotest", "test_transport", "test_ignore",
+                     "test_port", "debug_tool", "debug_port",
+                     "debug_init_cmds", "debug_extra_cmds", "debug_server",
+                     "debug_init_break", "debug_load_cmd", "monitor_port",
+                     "monitor_baud", "monitor_rts", "monitor_dtr")
+
+    IGNORE_BUILD_OPTIONS = ("test_transport", "test_filter", "test_ignore",
+                            "test_port", "debug_tool", "debug_port",
+                            "debug_init_cmds", "debug_extra_cmds",
+                            "debug_server", "debug_init_break",
+                            "debug_load_cmd", "monitor_port", "monitor_baud",
+                            "monitor_rts", "monitor_dtr")
 
     REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"}
 
-    RENAMED_OPTIONS = {"lib_use": "lib_deps", "lib_force": "lib_deps"}
+    RENAMED_OPTIONS = {
+        "lib_use": "lib_deps",
+        "lib_force": "lib_deps",
+        "extra_script": "extra_scripts"
+    }
 
     RENAMED_PLATFORMS = {"espressif": "espressif8266"}
 
@@ -155,29 +176,32 @@ class EnvironmentProcessor(object):
         terminal_width, _ = click.get_terminal_size()
         start_time = time()
 
-        # multi-line values to one line
         for k, v in self.options.items():
-            if "\n" in v:
-                self.options[k] = self.options[k].strip().replace("\n", ", ")
+            self.options[k] = self.options[k].strip()
 
-        click.echo("[%s] Processing %s (%s)" % (
-            datetime.now().strftime("%c"), click.style(
-                self.name, fg="cyan", bold=True),
-            ", ".join(["%s: %s" % (k, v) for k, v in self.options.items()])))
-        click.secho("-" * terminal_width, bold=True)
-        if self.silent:
-            click.echo("Please wait...")
+        if not self.silent:
+            click.echo(
+                "[%s] Processing %s (%s)" %
+                (datetime.now().strftime("%c"),
+                 click.style(self.name, fg="cyan", bold=True), "; ".join([
+                     "%s: %s" % (k, ", ".join(util.parse_conf_multi_values(v)))
+                     for k, v in self.options.items()
+                 ])))
+            click.secho("-" * terminal_width, bold=True)
 
         self.options = self._validate_options(self.options)
         result = self._run()
-
         is_error = result['returncode'] != 0
+
+        if self.silent and not is_error:
+            return True
+
         if is_error or "piotest_processor" not in self.cmd_ctx.meta:
             print_header(
-                "[%s] Took %.2f seconds" % ((click.style(
-                    "ERROR", fg="red", bold=True) if is_error else click.style(
-                        "SUCCESS", fg="green", bold=True)),
-                                            time() - start_time),
+                "[%s] Took %.2f seconds" %
+                ((click.style("ERROR", fg="red", bold=True)
+                  if is_error else click.style(
+                      "SUCCESS", fg="green", bold=True)), time() - start_time),
                 is_error=is_error)
 
         return not is_error
@@ -205,41 +229,46 @@ class EnvironmentProcessor(object):
             # warn about unknown options
             if k not in self.KNOWN_OPTIONS:
                 click.secho(
-                    "Detected non-PlatformIO `%s` option in `[env:]` section" %
-                    k,
+                    "Detected non-PlatformIO `%s` option in `[env:%s]` section"
+                    % (k, self.name),
                     fg="yellow")
             result[k] = v
         return result
 
-    def _get_build_variables(self):
+    def get_build_variables(self):
         variables = {"pioenv": self.name}
         if self.upload_port:
             variables['upload_port'] = self.upload_port
         for k, v in self.options.items():
             if k in self.REMAPED_OPTIONS:
                 k = self.REMAPED_OPTIONS[k]
+            if k in self.IGNORE_BUILD_OPTIONS:
+                continue
             if k == "targets" or (k == "upload_port" and self.upload_port):
                 continue
             variables[k] = v
         return variables
 
-    def _get_build_targets(self):
+    def get_build_targets(self):
         targets = []
         if self.targets:
             targets = [t for t in self.targets]
         elif "targets" in self.options:
-            targets = self.options['targets'].split()
+            targets = self.options['targets'].split(", ")
         return targets
 
     def _run(self):
         if "platform" not in self.options:
             raise exception.UndefinedEnvPlatform(self.name)
 
-        build_vars = self._get_build_variables()
-        build_targets = self._get_build_targets()
+        build_vars = self.get_build_variables()
+        build_targets = self.get_build_targets()
 
         telemetry.on_run_environment(self.options, build_targets)
 
+        # skip monitor target, we call it above
+        if "monitor" in build_targets:
+            build_targets.remove("monitor")
         if "nobuild" not in build_targets:
             # install dependent libraries
             if "lib_install" in self.options:
@@ -249,10 +278,10 @@ class EnvironmentProcessor(object):
                     if d.strip()
                 ], self.verbose)
             if "lib_deps" in self.options:
-                _autoinstall_libdeps(self.cmd_ctx, [
-                    d.strip() for d in self.options['lib_deps'].split(", ")
-                    if d.strip()
-                ], self.verbose)
+                _autoinstall_libdeps(
+                    self.cmd_ctx,
+                    util.parse_conf_multi_values(self.options['lib_deps']),
+                    self.verbose)
 
         try:
             p = PlatformFactory.newPlatform(self.options['platform'])
@@ -267,6 +296,8 @@ class EnvironmentProcessor(object):
 
 
 def _autoinstall_libdeps(ctx, libraries, verbose=False):
+    if not libraries:
+        return
     storage_dir = util.get_projectlibdeps_dir()
     ctx.obj = LibraryManager(storage_dir)
     if verbose:
@@ -275,7 +306,15 @@ def _autoinstall_libdeps(ctx, libraries, verbose=False):
         try:
             ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose)
         except exception.LibNotFound as e:
-            click.secho("Warning! %s" % e, fg="yellow")
+            if not _is_builtin_lib(lib):
+                click.secho("Warning! %s" % e, fg="yellow")
+
+
+def _is_builtin_lib(lib_name):
+    for storage in get_builtin_libs():
+        if any([l.get("name") == lib_name for l in storage['items']]):
+            return True
+    return False
 
 
 def _clean_pioenvs_dir(pioenvs_dir):
@@ -283,8 +322,8 @@ def _clean_pioenvs_dir(pioenvs_dir):
     proj_hash = calculate_project_hash()
 
     # if project's config is modified
-    if (isdir(pioenvs_dir) and
-            getmtime(join(util.get_project_dir(), "platformio.ini")) >
+    if (isdir(pioenvs_dir)
+            and getmtime(join(util.get_project_dir(), "platformio.ini")) >
             getmtime(pioenvs_dir)):
         util.rmtree_(pioenvs_dir)
 
@@ -329,23 +368,23 @@ def print_summary(results, start_time):
         format_str = (
             "Environment {0:<" + str(envname_max_len + 9) + "}\t[{1}]")
         click.echo(
-            format_str.format(
-                click.style(
-                    envname, fg="cyan"), status_str),
+            format_str.format(click.style(envname, fg="cyan"), status_str),
             err=status is False)
 
     print_header(
-        "[%s] Took %.2f seconds" % ((click.style(
-            "SUCCESS", fg="green", bold=True) if successed else click.style(
-                "ERROR", fg="red", bold=True)), time() - start_time),
+        "[%s] Took %.2f seconds" %
+        ((click.style("SUCCESS", fg="green", bold=True)
+          if successed else click.style("ERROR", fg="red", bold=True)),
+         time() - start_time),
         is_error=not successed)
 
 
 def check_project_defopts(config):
     if not config.has_section("platformio"):
         return True
-    known = ("home_dir", "lib_dir", "libdeps_dir", "src_dir", "envs_dir",
-             "data_dir", "test_dir", "env_default", "lib_extra_dirs")
+    known = ("env_default", "home_dir", "lib_dir", "libdeps_dir", "src_dir",
+             "envs_dir", "data_dir", "test_dir", "boards_dir",
+             "lib_extra_dirs")
     unknown = set([k for k, _ in config.items("platformio")]) - set(known)
     if not unknown:
         return True

platformio/commands/settings.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,11 +31,9 @@ def settings_get(name):
 
     click.echo(
         list_tpl.format(
-            name=click.style(
-                "Name", fg="cyan"),
-            value=(click.style(
-                "Value", fg="green") + click.style(
-                    " [Default]", fg="yellow")),
+            name=click.style("Name", fg="cyan"),
+            value=(click.style("Value", fg="green") + click.style(
+                " [Default]", fg="yellow")),
             description="Description"))
     click.echo("-" * terminal_width)
 
@@ -59,8 +57,7 @@ def settings_get(name):
 
         click.echo(
             list_tpl.format(
-                name=click.style(
-                    _name, fg="cyan"),
+                name=click.style(_name, fg="cyan"),
                 value=_value_str,
                 description=_data['description']))
 

platformio/commands/test.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,12 +17,23 @@ from os import getcwd
 
 import click
 
-from platformio.pioplus import pioplus_call
+from platformio.managers.core import pioplus_call
 
 
 @click.command("test", short_help="Local Unit Testing")
 @click.option("--environment", "-e", multiple=True, metavar="<environment>")
-@click.option("--ignore", "-i", multiple=True, metavar="<pattern>")
+@click.option(
+    "--filter",
+    "-f",
+    multiple=True,
+    metavar="<pattern>",
+    help="Filter tests by a pattern")
+@click.option(
+    "--ignore",
+    "-i",
+    multiple=True,
+    metavar="<pattern>",
+    help="Ignore tests by a pattern")
 @click.option("--upload-port")
 @click.option("--test-port")
 @click.option(
@@ -37,6 +48,20 @@ from platformio.pioplus import pioplus_call
         resolve_path=True))
 @click.option("--without-building", is_flag=True)
 @click.option("--without-uploading", is_flag=True)
+@click.option(
+    "--no-reset",
+    is_flag=True,
+    help="Disable software reset via Serial.DTR/RST")
+@click.option(
+    "--monitor-rts",
+    default=None,
+    type=click.IntRange(0, 1),
+    help="Set initial RTS line state for Serial Monitor")
+@click.option(
+    "--monitor-dtr",
+    default=None,
+    type=click.IntRange(0, 1),
+    help="Set initial DTR line state for Serial Monitor")
 @click.option("--verbose", "-v", is_flag=True)
 def cli(*args, **kwargs):  # pylint: disable=unused-argument
     pioplus_call(sys.argv[1:])

platformio/commands/update.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,25 +14,36 @@
 
 import click
 
+from platformio import app
 from platformio.commands.lib import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
+from platformio.managers.core import update_core_packages
 from platformio.managers.lib import LibraryManager
-from platformio.pioplus import pioplus_update
 
 
 @click.command(
-    "update", short_help="Update installed Platforms, Packages and Libraries")
+    "update", short_help="Update installed platforms, packages and libraries")
+@click.option(
+    "--core-packages", is_flag=True, help="Update only the core packages")
 @click.option(
     "-c",
     "--only-check",
     is_flag=True,
     help="Do not update, only check for new version")
 @click.pass_context
-def cli(ctx, only_check):
+def cli(ctx, core_packages, only_check):
+    update_core_packages(only_check)
+
+    if core_packages:
+        return
+
+    # cleanup lib search results, cached board and platform lists
+    app.clean_cache()
+
+    click.echo()
     click.echo("Platform Manager")
     click.echo("================")
     ctx.invoke(cmd_platform_update, only_check=only_check)
-    pioplus_update()
 
     click.echo()
     click.echo("Library Manager")

platformio/commands/upgrade.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,11 +18,15 @@ import click
 import requests
 
 from platformio import VERSION, __version__, exception, util
+from platformio.managers.core import update_core_packages
 
 
 @click.command(
     "upgrade", short_help="Upgrade PlatformIO to the latest version")
 def cli():
+    # Update PlatformIO's Core packages
+    update_core_packages(silent=True)
+
     latest = get_latest_version()
     if __version__ == latest:
         return click.secho(
@@ -66,8 +70,8 @@ def cli():
             if not r:
                 raise exception.UpgradeError("\n".join([str(cmd), str(e)]))
             permission_errors = ("permission denied", "not permitted")
-            if (any([m in r['err'].lower() for m in permission_errors]) and
-                    "windows" not in util.get_systype()):
+            if (any([m in r['err'].lower() for m in permission_errors])
+                    and "windows" not in util.get_systype()):
                 click.secho(
                     """
 -----------------
@@ -83,8 +87,8 @@ WARNING! Don't use `sudo` for the rest PlatformIO commands.
                     err=True)
                 raise exception.ReturnErrorCode(1)
             else:
-                raise exception.UpgradeError("\n".join(
-                    [str(cmd), r['out'], r['err']]))
+                raise exception.UpgradeError(
+                    "\n".join([str(cmd), r['out'], r['err']]))
 
 
 def get_latest_version():
@@ -101,9 +105,10 @@ def get_latest_version():
 
 def get_develop_latest_version():
     version = None
-    r = requests.get("https://raw.githubusercontent.com/platformio/platformio"
-                     "/develop/platformio/__init__.py",
-                     headers=util.get_request_defheaders())
+    r = requests.get(
+        "https://raw.githubusercontent.com/platformio/platformio"
+        "/develop/platformio/__init__.py",
+        headers=util.get_request_defheaders())
     r.raise_for_status()
     for line in r.text.split("\n"):
         line = line.strip()
@@ -121,7 +126,8 @@ def get_develop_latest_version():
 
 
 def get_pypi_latest_version():
-    r = requests.get("https://pypi.python.org/pypi/platformio/json",
-                     headers=util.get_request_defheaders())
+    r = requests.get(
+        "https://pypi.python.org/pypi/platformio/json",
+        headers=util.get_request_defheaders())
     r.raise_for_status()
     return r.json()['info']['version']

platformio/downloader.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -31,16 +31,16 @@ class FileDownloader(object):
 
     def __init__(self, url, dest_dir=None):
         # make connection
-        self._request = requests.get(url,
-                                     stream=True,
-                                     headers=util.get_request_defheaders())
+        self._request = requests.get(
+            url, stream=True, headers=util.get_request_defheaders())
         if self._request.status_code != 200:
             raise FDUnrecognizedStatusCode(self._request.status_code, url)
 
         disposition = self._request.headers.get("content-disposition")
         if disposition and "filename=" in disposition:
-            self._fname = disposition[disposition.index("filename=") +
-                                      9:].replace('"', "").replace("'", "")
+            self._fname = disposition[
+                disposition.index("filename=") + 9:].replace('"', "").replace(
+                    "'", "")
             self._fname = self._fname.encode("utf8")
         else:
             self._fname = url.split("/")[-1]

platformio/exception.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,8 +20,7 @@ class PlatformioException(Exception):
     def __str__(self):  # pragma: no cover
         if self.MESSAGE:
             return self.MESSAGE.format(*self.args)
-        else:
-            return Exception.__str__(self)
+        return Exception.__str__(self)
 
 
 class ReturnErrorCode(PlatformioException):
@@ -33,6 +32,10 @@ class MinitermException(PlatformioException):
     pass
 
 
+class UserSideException(PlatformioException):
+    pass
+
+
 class AbortedByUser(PlatformioException):
 
     MESSAGE = "Aborted by user"
@@ -40,7 +43,12 @@ class AbortedByUser(PlatformioException):
 
 class UnknownPlatform(PlatformioException):
 
-    MESSAGE = "Unknown platform '{0}'"
+    MESSAGE = "Unknown development platform '{0}'"
+
+
+class IncompatiblePlatform(PlatformioException):
+
+    MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}"
 
 
 class PlatformNotInstalledYet(PlatformioException):
@@ -53,7 +61,7 @@ class BoardNotDefined(PlatformioException):
 
     MESSAGE = "You need to specify board ID using `-b` or `--board` "\
         "option. Supported boards list is available via "\
-        " `platformio boards` command"
+        "`platformio boards` command"
 
 
 class UnknownBoard(PlatformioException):
@@ -78,7 +86,7 @@ class UnknownPackage(PlatformioException):
 
 class MissingPackageManifest(PlatformioException):
 
-    MESSAGE = "Could not find '{0}' manifest file in the package"
+    MESSAGE = "Could not find one of '{0}' manifest files in the package"
 
 
 class UndefinedPackageVersion(PlatformioException):
@@ -89,8 +97,10 @@ class UndefinedPackageVersion(PlatformioException):
 
 class PackageInstallError(PlatformioException):
 
-    MESSAGE = "Can not install '{0}' with version requirements '{1}' "\
-              "for your system '{2}'"
+    MESSAGE = "Could not install '{0}' with version requirements '{1}' "\
+              "for your system '{2}'.\n"\
+              "If you use Antivirus, it can block PlatformIO Package "\
+              "Manager. Try to disable it for a while."
 
 
 class FDUnrecognizedStatusCode(PlatformioException):

platformio/ide/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

platformio/ide/projectgenerator.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,19 +15,22 @@
 import json
 import os
 import re
+from cStringIO import StringIO
 from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath
 
 import bottle
+import click
 
-from platformio import app, exception, util
+from platformio import exception, util
+from platformio.commands.run import cli as cmd_run
 
 
 class ProjectGenerator(object):
 
-    def __init__(self, project_dir, ide, board):
+    def __init__(self, project_dir, ide, env_name):
         self.project_dir = project_dir
         self.ide = ide
-        self.board = board
+        self.env_name = env_name
         self._tplvars = {}
 
         with util.cd(self.project_dir):
@@ -43,36 +46,38 @@ class ProjectGenerator(object):
 
     @util.memoized
     def get_project_env(self):
-        data = {"env_name": "PlatformIO"}
+        data = {}
         config = util.load_project_config(self.project_dir)
         for section in config.sections():
             if not section.startswith("env:"):
                 continue
+            if self.env_name != section[4:]:
+                continue
             data = {"env_name": section[4:]}
             for k, v in config.items(section):
                 data[k] = v
-            if self.board == data.get("board"):
-                break
         return data
 
     @util.memoized
     def get_project_build_data(self):
         data = {"defines": [], "includes": [], "cxx_path": None}
         envdata = self.get_project_env()
-        if "env_name" not in envdata:
+        if not envdata:
             return data
-        cmd = [util.get_pythonexe_path(), "-m", "platformio", "-f"]
-        if app.get_session_var("caller_id"):
-            cmd.extend(["-c", app.get_session_var("caller_id")])
-        cmd.extend(["run", "-t", "idedata", "-e", envdata['env_name']])
-        cmd.extend(["-d", self.project_dir])
-        result = util.exec_command(cmd)
 
-        if result['returncode'] != 0 or '"includes":' not in result['out']:
-            raise exception.PlatformioException("\n".join(
-                [result['out'], result['err']]))
+        out = StringIO()
+        with util.capture_stdout(out):
+            click.get_current_context().invoke(
+                cmd_run,
+                project_dir=self.project_dir,
+                environment=[envdata['env_name']],
+                target=["idedata"])
+        result = out.getvalue()
 
-        for line in result['out'].split("\n"):
+        if '"includes":' not in result:
+            raise exception.PlatformioException(result)
+
+        for line in result.split("\n"):
             line = line.strip()
             if line.startswith('{"') and line.endswith("}"):
                 data = json.loads(line)
@@ -146,16 +151,24 @@ class ProjectGenerator(object):
         self._tplvars.update(self.get_project_env())
         self._tplvars.update(self.get_project_build_data())
         self._tplvars.update({
-            "project_name": self.get_project_name(),
-            "src_files": self.get_src_files(),
-            "user_home_dir": abspath(expanduser("~")),
-            "project_dir": self.project_dir,
-            "project_src_dir": self.project_src_dir,
-            "systype": util.get_systype(),
+            "project_name":
+            self.get_project_name(),
+            "src_files":
+            self.get_src_files(),
+            "user_home_dir":
+            abspath(expanduser("~")),
+            "project_dir":
+            self.project_dir,
+            "project_src_dir":
+            self.project_src_dir,
+            "systype":
+            util.get_systype(),
             "platformio_path":
             self._fix_os_path(util.where_is_program("platformio")),
-            "env_pathsep": os.pathsep,
-            "env_path": self._fix_os_path(os.getenv("PATH"))
+            "env_pathsep":
+            os.pathsep,
+            "env_path":
+            self._fix_os_path(os.getenv("PATH"))
         })
 
     @staticmethod

platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -5,6 +5,7 @@ SET(CMAKE_C_COMPILER "{{cc_path.replace("\\", "/")}}")
 SET(CMAKE_CXX_COMPILER "{{cxx_path.replace("\\", "/")}}")
 SET(CMAKE_CXX_FLAGS_DISTRIBUTION "{{cxx_flags}}")
 SET(CMAKE_C_FLAGS_DISTRIBUTION "{{cc_flags}}")
+set(CMAKE_CXX_STANDARD 11)
 
 % for define in defines:
 add_definitions(-D{{!define}})

platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.cdt.launch.applicationLaunchType">
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.AUTO_SOLIB" value="true"/>
+<listAttribute key="org.eclipse.cdt.dsf.gdb.AUTO_SOLIB_LIST"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.DEBUG_NAME" value="platformio -c eclipse debug -d ${project_loc} --interface=gdb"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.DEBUG_ON_FORK" value="false"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.GDB_INIT" value=".pioinit"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.NON_STOP" value="false"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.REVERSE" value="false"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.REVERSE_MODE" value="UseSoftTrace"/>
+<listAttribute key="org.eclipse.cdt.dsf.gdb.SOLIB_PATH"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.TRACEPOINT_MODE" value="TP_NORMAL_ONLY"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.UPDATE_THREADLIST_ON_SUSPEND" value="false"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.internal.ui.launching.LocalApplicationCDebuggerTab.DEFAULTS_SET" value="true"/>
+<intAttribute key="org.eclipse.cdt.launch.ATTR_BUILD_BEFORE_LAUNCH_ATTR" value="0"/>
+<stringAttribute key="org.eclipse.cdt.launch.COREFILE_PATH" value=""/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_ID" value="gdb"/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_REGISTER_GROUPS" value=""/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_START_MODE" value="run"/>
+<booleanAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN" value="true"/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN_SYMBOL" value="main"/>
+<stringAttribute key="org.eclipse.cdt.launch.PROGRAM_NAME" value=".pioenvs/{{env_name}}/firmware.elf"/>
+<stringAttribute key="org.eclipse.cdt.launch.PROJECT_ATTR" value="{{project_name}}"/>
+<booleanAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_AUTO_ATTR" value="false"/>
+<stringAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_ID_ATTR" value=""/>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+<listEntry value="/{{project_name}}"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+<listEntry value="4"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.ui.favoriteGroups">
+<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>
+</listAttribute>
+<stringAttribute key="org.eclipse.dsf.launch.MEMORY_BLOCKS" value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&#10;&lt;memoryBlockExpressionList context=&quot;reserved-for-future-use&quot;/&gt;&#10;"/>
+<stringAttribute key="process_factory_id" value="org.eclipse.cdt.dsf.gdb.GdbProcessFactory"/>
+<stringAttribute key="saved_expressions&lt;seperator&gt;Unknown" value="0x55f4"/>
+</launchConfiguration>

platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl

@@ -93,5 +93,12 @@
 		{
 			"path": "."
 		}
-	]
+	],
+    "settings":
+    {
+         "sublimegdb_workingdir": "{{project_dir}}",
+         "sublimegdb_exec_cmd": "-exec-continue",
+         "sublimegdb_commandline": "{{platformio_path}} -f -c sublimetext debug --interface=gdb --interpreter=mi -x .pioinit"
+
+    }
 }

platformio/ide/tpls/vim/.clang_complete.tpl

@@ -0,0 +1,6 @@
+% for include in includes:
+-I{{include}}
+% end
+% for define in defines:
+-D{{!define}}
+% end

platformio/ide/tpls/vim/.gcc-flags.json.tpl

@@ -0,0 +1,8 @@
+{
+  "execPath": "{{ cxx_path.replace("\\", "/") }}",
+  "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }}",
+  "gccDefaultCppFlags": "-fsyntax-only {{! cxx_flags.replace(' -MMD ', ' ').replace('"', '\\"') }}",
+  "gccErrorLimit": 15,
+  "gccIncludePaths": "{{ ','.join(includes).replace("\\", "/") }}",
+  "gccSuppressWarnings": false
+}

platformio/ide/tpls/vim/.gitignore.tpl

@@ -0,0 +1,4 @@
+.pioenvs
+.piolibdeps
+.clang_complete
+.gcc-flags.json

platformio/ide/tpls/vscode/.gitignore.tpl

@@ -0,0 +1,3 @@
+.pioenvs
+.piolibdeps
+.vscode

platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl

@@ -0,0 +1,37 @@
+{
+    "configurations": [
+        {
+% import platform
+% systype = platform.system().lower()
+% if systype == "windows":
+            "name": "Win32",
+% elif systype == "darwin":
+            "name": "Mac",
+% else:
+            "name": "Linux",
+% end
+            "includePath": [
+% for include in includes:
+                "{{include.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
+% end
+                ""
+            ],
+            "browse": {
+                "limitSymbolsToIncludedHeaders": true,
+                "databaseFilename": "",
+                "path": [
+% for include in includes:
+                    "{{include.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
+% end
+                    ""
+                ]
+            },
+            "defines": [
+% for define in defines:
+                "{{!define.replace('"', '\\"')}}",
+% end
+                ""
+            ]
+        }
+    ]
+}

platformio/ide/tpls/vscode/.vscode/launch.json.tpl

@@ -0,0 +1,15 @@
+% from os.path import dirname, join
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "gdb",
+            "request": "launch",
+            "cwd": "${workspaceRoot}",
+            "name": "PlatformIO Debugger",
+            "target": "{{prog_path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
+            "gdbpath": "{{join(dirname(platformio_path), "piodebuggdb").replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
+            "autorun": [ "source .pioinit" ]
+        }
+    ]
+}

platformio/maintenance.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -29,28 +29,27 @@ from platformio.commands.platform import \
     platform_uninstall as cmd_platform_uninstall
 from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.commands.upgrade import get_latest_version
+from platformio.managers.core import update_core_packages
 from platformio.managers.lib import LibraryManager
-from platformio.managers.platform import PlatformManager
-from platformio.pioplus import pioplus_update
+from platformio.managers.platform import PlatformFactory, PlatformManager
 
 
 def in_silence(ctx=None):
     ctx = ctx or app.get_session_var("command_ctx")
     assert ctx
     ctx_args = ctx.args or []
-    return (ctx_args and
-            (ctx.args[0] == "upgrade" or "--json-output" in ctx_args))
-
-
-def clean_cache():
-    with app.ContentCache() as cc:
-        cc.clean()
+    return ctx_args and any([
+        ctx.args[0] == "upgrade", "--json-output" in ctx_args,
+        "--version" in ctx_args
+    ])
 
 
 def on_platformio_start(ctx, force, caller):
     if not caller:
         if getenv("PLATFORMIO_CALLER"):
             caller = getenv("PLATFORMIO_CALLER")
+        elif getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"):
+            caller = "vscode"
         elif util.is_container():
             if getenv("C9_UID"):
                 caller = "C9"
@@ -64,8 +63,6 @@ def on_platformio_start(ctx, force, caller):
     app.set_session_var("caller_id", caller)
     telemetry.on_command()
 
-    if ctx.args and (ctx.args[0] == "upgrade" or "update" in ctx.args):
-        clean_cache()
     if not in_silence(ctx):
         after_upgrade(ctx)
 
@@ -97,20 +94,22 @@ class Upgrader(object):
         self.to_version = semantic_version.Version.coerce(
             util.pepver_to_semver(to_version))
 
-        self._upgraders = [
-            (semantic_version.Version("3.0.0-a1"), self._upgrade_to_3_0_0),
-            (semantic_version.Version("3.0.0-b11"), self._upgrade_to_3_0_0)
-        ]
+        self._upgraders = [(semantic_version.Version("3.0.0-a.1"),
+                            self._upgrade_to_3_0_0),
+                           (semantic_version.Version("3.0.0-b.11"),
+                            self._upgrade_to_3_0_0b11),
+                           (semantic_version.Version("3.4.0-a.9"),
+                            self._update_dev_platforms)]
 
     def run(self, ctx):
         if self.from_version > self.to_version:
             return True
 
         result = [True]
-        for item in self._upgraders:
-            if self.from_version >= item[0] or self.to_version < item[0]:
+        for version, callback in self._upgraders:
+            if self.from_version >= version or self.to_version < version:
                 continue
-            result.append(item[1](ctx))
+            result.append(callback(ctx))
 
         return all(result)
 
@@ -146,39 +145,55 @@ class Upgrader(object):
             m['name'] for m in PlatformManager().get_installed()
         ]
         if "espressif" not in current_platforms:
-            return
+            return True
         ctx.invoke(cmd_platform_install, platforms=["espressif8266"])
         ctx.invoke(cmd_platform_uninstall, platforms=["espressif"])
+        return True
+
+    @staticmethod
+    def _update_dev_platforms(ctx):
+        ctx.invoke(cmd_platform_update)
+        return True
 
 
 def after_upgrade(ctx):
+    terminal_width, _ = click.get_terminal_size()
     last_version = app.get_state_item("last_version", "0.0.0")
     if last_version == __version__:
         return
 
     if last_version == "0.0.0":
         app.set_state_item("last_version", __version__)
+    elif semantic_version.Version.coerce(util.pepver_to_semver(
+            last_version)) > semantic_version.Version.coerce(
+                util.pepver_to_semver(__version__)):
+        click.secho("*" * terminal_width, fg="yellow")
+        click.secho(
+            "Obsolete PIO Core v%s is used (previous was %s)" % (__version__,
+                                                                 last_version),
+            fg="yellow")
+        click.secho(
+            "Please remove multiple PIO Cores from a system:", fg="yellow")
+        click.secho(
+            "http://docs.platformio.org/page/faq.html"
+            "#multiple-pio-cores-in-a-system",
+            fg="cyan")
+        click.secho("*" * terminal_width, fg="yellow")
+        return
     else:
-        click.secho("Please wait while upgrading PlatformIO ...", fg="yellow")
-        clean_cache()
+        click.secho("Please wait while upgrading PlatformIO...", fg="yellow")
+        app.clean_cache()
+
+        # Update PlatformIO's Core packages
+        update_core_packages(silent=True)
+
         u = Upgrader(last_version, __version__)
         if u.run(ctx):
             app.set_state_item("last_version", __version__)
-
-            # update development platforms
-            pm = PlatformManager()
-            for manifest in pm.get_installed():
-                # pm.update(manifest['name'], "^" + manifest['version'])
-                pm.update(manifest['name'])
-
-            # update PlatformIO Plus tool if installed
-            pioplus_update()
-
             click.secho(
                 "PlatformIO has been successfully upgraded to %s!\n" %
                 __version__,
                 fg="green")
-
             telemetry.on_event(
                 category="Auto",
                 action="Upgrade",
@@ -188,26 +203,26 @@ def after_upgrade(ctx):
         click.echo("")
 
     # PlatformIO banner
-    terminal_width, _ = click.get_terminal_size()
     click.echo("*" * terminal_width)
-    click.echo("If you like %s, please:" % (click.style(
-        "PlatformIO", fg="cyan")))
+    click.echo("If you like %s, please:" %
+               (click.style("PlatformIO", fg="cyan")))
     click.echo("- %s us on Twitter to stay up-to-date "
-               "on the latest project news > %s" % (click.style(
-                   "follow", fg="cyan"), click.style(
-                       "https://twitter.com/PlatformIO_Org", fg="cyan")))
-    click.echo("- %s it on GitHub > %s" % (click.style(
-        "star", fg="cyan"), click.style(
-            "https://github.com/platformio/platformio", fg="cyan")))
+               "on the latest project news > %s" %
+               (click.style("follow", fg="cyan"),
+                click.style("https://twitter.com/PlatformIO_Org", fg="cyan")))
+    click.echo(
+        "- %s it on GitHub > %s" %
+        (click.style("star", fg="cyan"),
+         click.style("https://github.com/platformio/platformio", fg="cyan")))
     if not getenv("PLATFORMIO_IDE"):
-        click.echo("- %s PlatformIO IDE for IoT development > %s" %
-                   (click.style(
-                       "try", fg="cyan"), click.style(
-                           "http://platformio.org/platformio-ide", fg="cyan")))
+        click.echo(
+            "- %s PlatformIO IDE for IoT development > %s" %
+            (click.style("try", fg="cyan"),
+             click.style("http://platformio.org/platformio-ide", fg="cyan")))
     if not util.is_ci():
-        click.echo("- %s us with PlatformIO Plus > %s" % (click.style(
-            "support", fg="cyan"), click.style(
-                "https://pioplus.com", fg="cyan")))
+        click.echo("- %s us with PlatformIO Plus > %s" %
+                   (click.style("support", fg="cyan"),
+                    click.style("https://pioplus.com", fg="cyan")))
 
     click.echo("*" * terminal_width)
     click.echo("")
@@ -219,6 +234,9 @@ def check_platformio_upgrade():
     if (time() - interval) < last_check.get("platformio_upgrade", 0):
         return
 
+    # Update PlatformIO's Core packages
+    update_core_packages(silent=True)
+
     last_check['platformio_upgrade'] = int(time())
     app.set_state_item("last_check", last_check)
 
@@ -267,8 +285,14 @@ def check_internal_updates(ctx, what):
     pm = PlatformManager() if what == "platforms" else LibraryManager()
     outdated_items = []
     for manifest in pm.get_installed():
-        if manifest['name'] not in outdated_items and \
-                pm.is_outdated(manifest['name']):
+        if manifest['name'] in outdated_items:
+            continue
+        conds = [
+            pm.outdated(manifest['__pkg_dir']),
+            what == "platforms" and PlatformFactory.newPlatform(
+                manifest['__pkg_dir']).are_outdated_packages()
+        ]
+        if any(conds):
             outdated_items.append(manifest['name'])
 
     if not outdated_items:
@@ -279,8 +303,8 @@ def check_internal_updates(ctx, what):
     click.echo("")
     click.echo("*" * terminal_width)
     click.secho(
-        "There are the new updates for %s (%s)" %
-        (what, ", ".join(outdated_items)),
+        "There are the new updates for %s (%s)" % (what,
+                                                   ", ".join(outdated_items)),
         fg="yellow")
 
     if not app.get_setting("auto_update_" + what):

platformio/managers/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

platformio/managers/core.py

@@ -0,0 +1,126 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+from os.path import join
+
+from platformio import __version__, exception, util
+from platformio.managers.package import PackageManager
+
+CORE_PACKAGES = {
+    "pysite-pioplus": ">=0.3.0,<2",
+    "tool-pioplus": ">=0.9.1,<2",
+    "tool-unity": "~1.20302.1",
+    "tool-scons": "~3.20501.2"
+}
+
+PIOPLUS_AUTO_UPDATES_MAX = 100
+
+# pylint: disable=arguments-differ
+
+
+class CorePackageManager(PackageManager):
+
+    def __init__(self):
+        PackageManager.__init__(
+            self,
+            join(util.get_home_dir(), "packages"), [
+                "https://dl.bintray.com/platformio/dl-packages/manifest.json",
+                "http%s://dl.platformio.org/packages/manifest.json" %
+                ("" if sys.version_info < (2, 7, 9) else "s")
+            ])
+
+    def install(self, name, requirements=None, *args, **kwargs):
+        PackageManager.install(self, name, requirements, *args, **kwargs)
+        self.cleanup_packages()
+        return self.get_package_dir(name, requirements)
+
+    def update(self, *args, **kwargs):
+        result = PackageManager.update(self, *args, **kwargs)
+        self.cleanup_packages()
+        return result
+
+    def cleanup_packages(self):
+        self.cache_reset()
+        best_pkg_versions = {}
+        for name, requirements in CORE_PACKAGES.items():
+            pkg_dir = self.get_package_dir(name, requirements)
+            if not pkg_dir:
+                continue
+            best_pkg_versions[name] = self.load_manifest(pkg_dir)['version']
+        for manifest in self.get_installed():
+            if manifest['name'] not in best_pkg_versions:
+                continue
+            if manifest['version'] != best_pkg_versions[manifest['name']]:
+                self.uninstall(manifest['__pkg_dir'], trigger_event=False)
+        self.cache_reset()
+        return True
+
+
+def get_core_package_dir(name):
+    assert name in CORE_PACKAGES
+    requirements = CORE_PACKAGES[name]
+    pm = CorePackageManager()
+    pkg_dir = pm.get_package_dir(name, requirements)
+    if pkg_dir:
+        return pkg_dir
+    return pm.install(name, requirements)
+
+
+def update_core_packages(only_check=False, silent=False):
+    pm = CorePackageManager()
+    for name, requirements in CORE_PACKAGES.items():
+        pkg_dir = pm.get_package_dir(name)
+        if not pkg_dir:
+            continue
+        if not silent or pm.outdated(pkg_dir, requirements):
+            pm.update(name, requirements, only_check=only_check)
+
+
+def pioplus_call(args, **kwargs):
+    if "windows" in util.get_systype() and sys.version_info < (2, 7, 6):
+        raise exception.PlatformioException(
+            "PlatformIO Core Plus v%s does not run under Python version %s.\n"
+            "Minimum supported version is 2.7.6, please upgrade Python.\n"
+            "Python 3 is not yet supported.\n" % (__version__,
+                                                  sys.version.split()[0]))
+
+    pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus")
+    os.environ['PYTHONEXEPATH'] = util.get_pythonexe_path()
+    os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("pysite-pioplus")
+    util.copy_pythonpath_to_osenv()
+    code = subprocess.call([pioplus_path] + args, **kwargs)
+
+    # handle remote update request
+    if code == 13:
+        count_attr = "_update_count"
+        try:
+            count_value = getattr(pioplus_call, count_attr)
+        except AttributeError:
+            count_value = 0
+            setattr(pioplus_call, count_attr, 1)
+        count_value += 1
+        setattr(pioplus_call, count_attr, count_value)
+        if count_value < PIOPLUS_AUTO_UPDATES_MAX:
+            update_core_packages()
+            return pioplus_call(args, **kwargs)
+
+    # handle reload request
+    elif code == 14:
+        return pioplus_call(args, **kwargs)
+
+    if code != 0:
+        raise exception.ReturnErrorCode(1)

platformio/managers/lib.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,10 +15,11 @@
 # pylint: disable=too-many-arguments, too-many-locals, too-many-branches
 
 import json
-import os
-from hashlib import md5
-from os.path import dirname, join
+import re
+from glob import glob
+from os.path import isdir, join
 
+import arrow
 import click
 import semantic_version
 
@@ -34,70 +35,93 @@ class LibraryManager(BasePkgManager):
         BasePkgManager.__init__(self, package_dir)
 
     @property
-    def manifest_name(self):
-        return ".library.json"
+    def manifest_names(self):
+        return [
+            ".library.json", "library.json", "library.properties",
+            "module.json"
+        ]
 
-    def check_pkg_structure(self, pkg_dir):
-        try:
-            return BasePkgManager.check_pkg_structure(self, pkg_dir)
-        except exception.MissingPackageManifest:
-            # we will generate manifest automatically
-            pass
+    def get_manifest_path(self, pkg_dir):
+        path = BasePkgManager.get_manifest_path(self, pkg_dir)
+        if path:
+            return path
 
-        manifest = {
-            "name": "Library_" + md5(pkg_dir).hexdigest()[:5],
-            "version": "0.0.0"
-        }
-        manifest_path = self._find_any_manifest(pkg_dir)
-        if manifest_path:
-            _manifest = self._parse_manifest(manifest_path)
-            pkg_dir = dirname(manifest_path)
-            for key in ("name", "version"):
-                if key not in _manifest:
-                    _manifest[key] = manifest[key]
-            manifest = _manifest
-        else:
-            for root, dirs, files in os.walk(pkg_dir):
-                if len(dirs) == 1 and not files:
-                    manifest['name'] = dirs[0]
-                    continue
-                if dirs or files:
-                    pkg_dir = root
-                    break
+        # if library without manifest, returns first source file
+        src_dir = join(util.glob_escape(pkg_dir))
+        if isdir(join(pkg_dir, "src")):
+            src_dir = join(src_dir, "src")
+        chs_files = glob(join(src_dir, "*.[chS]"))
+        if chs_files:
+            return chs_files[0]
+        cpp_files = glob(join(src_dir, "*.cpp"))
+        if cpp_files:
+            return cpp_files[0]
 
-        with open(join(pkg_dir, self.manifest_name), "w") as fp:
-            json.dump(manifest, fp)
-
-        return pkg_dir
-
-    @staticmethod
-    def _find_any_manifest(pkg_dir):
-        manifests = ("library.json", "library.properties", "module.json")
-        for root, _, files in os.walk(pkg_dir):
-            for manifest in manifests:
-                if manifest in files:
-                    return join(root, manifest)
         return None
 
-    @staticmethod
-    def _parse_manifest(path):
-        manifest = {}
-        if path.endswith(".json"):
-            return util.load_json(path)
-        elif path.endswith("library.properties"):
-            with open(path) as fp:
-                for line in fp.readlines():
-                    if "=" not in line:
-                        continue
-                    key, value = line.split("=", 1)
-                    manifest[key.strip()] = value.strip()
+    def load_manifest(self, pkg_dir):
+        manifest = BasePkgManager.load_manifest(self, pkg_dir)
+        if not manifest:
+            return manifest
+
+        # if Arduino library.properties
+        if "sentence" in manifest:
             manifest['frameworks'] = ["arduino"]
-            if "author" in manifest:
-                manifest['authors'] = [{"name": manifest['author']}]
-                del manifest['author']
-            if "sentence" in manifest:
-                manifest['description'] = manifest['sentence']
-                del manifest['sentence']
+            manifest['description'] = manifest['sentence']
+            del manifest['sentence']
+
+        if "author" in manifest:
+            manifest['authors'] = [{"name": manifest['author']}]
+            del manifest['author']
+
+        if "authors" in manifest and not isinstance(manifest['authors'], list):
+            manifest['authors'] = [manifest['authors']]
+
+        if "keywords" not in manifest:
+            keywords = []
+            for keyword in re.split(r"[\s/]+",
+                                    manifest.get("category", "Uncategorized")):
+                keyword = keyword.strip()
+                if not keyword:
+                    continue
+                keywords.append(keyword.lower())
+            manifest['keywords'] = keywords
+            if "category" in manifest:
+                del manifest['category']
+
+        # don't replace VCS URL
+        if "url" in manifest and "description" in manifest:
+            manifest['homepage'] = manifest['url']
+            del manifest['url']
+
+        if "architectures" in manifest:
+            platforms = []
+            platforms_map = {
+                "avr": "atmelavr",
+                "sam": "atmelsam",
+                "samd": "atmelsam",
+                "esp8266": "espressif8266",
+                "arc32": "intel_arc32"
+            }
+            for arch in manifest['architectures'].split(","):
+                arch = arch.strip()
+                if arch == "*":
+                    platforms = "*"
+                    break
+                if arch in platforms_map:
+                    platforms.append(platforms_map[arch])
+            manifest['platforms'] = platforms
+            del manifest['architectures']
+
+        # convert listed items via comma to array
+        for key in ("keywords", "frameworks", "platforms"):
+            if key not in manifest or \
+                    not isinstance(manifest[key], basestring):
+                continue
+            manifest[key] = [
+                i.strip() for i in manifest[key].split(",") if i.strip()
+            ]
+
         return manifest
 
     @staticmethod
@@ -129,13 +153,8 @@ class LibraryManager(BasePkgManager):
     def max_satisfying_repo_version(versions, requirements=None):
 
         def _cmp_dates(datestr1, datestr2):
-            from datetime import datetime
-            assert "T" in datestr1 and "T" in datestr2
-            dateformat = "%Y-%m-%d %H:%M:%S"
-            date1 = datetime.strptime(datestr1[:-1].replace("T", " "),
-                                      dateformat)
-            date2 = datetime.strptime(datestr2[:-1].replace("T", " "),
-                                      dateformat)
+            date1 = arrow.get(datestr1)
+            date2 = arrow.get(datestr2)
             if date1 == date2:
                 return 0
             return -1 if date1 < date2 else 1
@@ -150,7 +169,7 @@ class LibraryManager(BasePkgManager):
         for v in versions:
             specver = None
             try:
-                specver = semantic_version.Version(v['version'], partial=True)
+                specver = semantic_version.Version(v['name'], partial=True)
             except ValueError:
                 pass
 
@@ -158,30 +177,30 @@ class LibraryManager(BasePkgManager):
                 if not specver or specver not in reqspec:
                     continue
                 if not item or semantic_version.Version(
-                        item['version'], partial=True) < specver:
+                        item['name'], partial=True) < specver:
                     item = v
             elif requirements:
-                if requirements == v['version']:
+                if requirements == v['name']:
                     return v
             else:
-                if not item or _cmp_dates(item['date'], v['date']) == -1:
+                if not item or _cmp_dates(item['released'],
+                                          v['released']) == -1:
                     item = v
         return item
 
-    def get_latest_repo_version(self, name, requirements):
+    def get_latest_repo_version(self, name, requirements, silent=False):
         item = self.max_satisfying_repo_version(
             util.get_api_result(
-                "/lib/versions/%d" % self._get_pkg_id_by_name(name,
-                                                              requirements),
-                cache_valid="1h"),
-            requirements)
-        return item['version'] if item else None
+                "/lib/info/%d" % self.get_pkg_id_by_name(
+                    name, requirements, silent=silent),
+                cache_valid="1d")['versions'], requirements)
+        return item['name'] if item else None
 
-    def _get_pkg_id_by_name(self,
-                            name,
-                            requirements,
-                            silent=False,
-                            interactive=False):
+    def get_pkg_id_by_name(self,
+                           name,
+                           requirements,
+                           silent=False,
+                           interactive=False):
         if name.startswith("id="):
             return int(name[3:])
         # try to find ID from installed packages
@@ -196,7 +215,7 @@ class LibraryManager(BasePkgManager):
             }, silent, interactive)['id'])
 
     def _install_from_piorepo(self, name, requirements):
-        assert name.startswith("id=")
+        assert name.startswith("id="), name
         version = self.get_latest_repo_version(name, requirements)
         if not version:
             raise exception.UndefinedPackageVersion(requirements or "latest",
@@ -207,36 +226,36 @@ class LibraryManager(BasePkgManager):
             cache_valid="30d")
         assert dl_data
 
-        return self._install_from_url(
-            name, dl_data['url'].replace("http://", "https://")
-            if app.get_setting("enable_ssl") else dl_data['url'], requirements)
-
-    def install(self,
-                name,
-                requirements=None,
-                silent=False,
-                trigger_event=True,
-                interactive=False):
-        already_installed = False
-        _name, _requirements, _url = self.parse_pkg_name(name, requirements)
+        return self._install_from_url(name, dl_data['url'].replace(
+            "http://", "https://") if app.get_setting("enable_ssl") else
+                                      dl_data['url'], requirements)
 
+    def install(  # pylint: disable=arguments-differ
+            self,
+            name,
+            requirements=None,
+            silent=False,
+            trigger_event=True,
+            interactive=False):
+        pkg_dir = None
         try:
+            _name, _requirements, _url = self.parse_pkg_input(
+                name, requirements)
             if not _url:
-                _name = "id=%d" % self._get_pkg_id_by_name(
+                name = "id=%d" % self.get_pkg_id_by_name(
                     _name,
                     _requirements,
                     silent=silent,
                     interactive=interactive)
-            already_installed = self.get_package(_name, _requirements, _url)
-            pkg_dir = BasePkgManager.install(
-                self, _name
-                if not _url else name, _requirements, silent, trigger_event)
+                requirements = _requirements
+            pkg_dir = BasePkgManager.install(self, name, requirements, silent,
+                                             trigger_event)
         except exception.InternetIsOffline as e:
             if not silent:
                 click.secho(str(e), fg="yellow")
             return
 
-        if already_installed:
+        if not pkg_dir:
             return
 
         manifest = self.load_manifest(pkg_dir)
@@ -290,39 +309,43 @@ class LibraryManager(BasePkgManager):
             if not isinstance(values, list):
                 values = [v.strip() for v in values.split(",") if v]
             for value in values:
-                query.append('%s:"%s"' % (key[:-1] if key.endswith("s") else
-                                          key, value))
+                query.append('%s:"%s"' % (key[:-1]
+                                          if key.endswith("s") else key,
+                                          value))
 
         lib_info = None
         result = util.get_api_result(
-            "/lib/search", dict(query=" ".join(query)), cache_valid="3d")
+            "/v2/lib/search", dict(query=" ".join(query)), cache_valid="3d")
         if result['total'] == 1:
             lib_info = result['items'][0]
         elif result['total'] > 1:
-            click.secho(
-                "Conflict: More than one library has been found "
-                "by request %s:" % json.dumps(filters),
-                fg="red",
-                err=True)
-            commands.lib.echo_liblist_header()
-            for item in result['items']:
-                commands.lib.echo_liblist_item(item)
-
-            if not interactive:
-                click.secho(
-                    "Automatically chose the first available library "
-                    "(use `--interactive` option to make a choice)",
-                    fg="yellow",
-                    err=True)
+            if silent and not interactive:
                 lib_info = result['items'][0]
             else:
-                deplib_id = click.prompt(
-                    "Please choose library ID",
-                    type=click.Choice([str(i['id']) for i in result['items']]))
+                click.secho(
+                    "Conflict: More than one library has been found "
+                    "by request %s:" % json.dumps(filters),
+                    fg="yellow",
+                    err=True)
                 for item in result['items']:
-                    if item['id'] == int(deplib_id):
-                        lib_info = item
-                        break
+                    commands.lib.print_lib_item(item)
+
+                if not interactive:
+                    click.secho(
+                        "Automatically chose the first available library "
+                        "(use `--interactive` option to make a choice)",
+                        fg="yellow",
+                        err=True)
+                    lib_info = result['items'][0]
+                else:
+                    deplib_id = click.prompt(
+                        "Please choose library ID",
+                        type=click.Choice(
+                            [str(i['id']) for i in result['items']]))
+                    for item in result['items']:
+                        if item['id'] == int(deplib_id):
+                            lib_info = item
+                            break
 
         if not lib_info:
             if filters.keys() == ["name"]:

platformio/managers/package.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,17 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
+import hashlib
 import json
 import os
+import re
 import shutil
-from os.path import basename, dirname, getsize, isdir, isfile, islink, join
+from os.path import basename, getsize, isdir, isfile, islink, join
 from tempfile import mkdtemp
 
 import click
 import requests
 import semantic_version
 
-from platformio import app, exception, telemetry, util
+from platformio import __version__, app, exception, telemetry, util
 from platformio.downloader import FileDownloader
 from platformio.unpacker import FileUnpacker
 from platformio.vcsclient import VCSClientFactory
@@ -67,12 +70,13 @@ class PackageRepoIterator(object):
 
         if self.package in manifest:
             return manifest[self.package]
-        else:
-            return self.next()
+        return self.next()
 
 
 class PkgRepoMixin(object):
 
+    PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__))
+
     @staticmethod
     def max_satisfying_repo_version(versions, requirements=None):
         item = None
@@ -85,9 +89,13 @@ class PkgRepoMixin(object):
                 pass
 
         for v in versions:
-            if ("system" in v and v['system'] not in ("all", "*") and
-                    systype not in v['system']):
+            if "system" in v and v['system'] not in ("all", "*") and \
+                    systype not in v['system']:
                 continue
+            if "platformio" in v.get("engines", {}):
+                if PkgRepoMixin.PIO_VERSION not in semantic_version.Spec(
+                        v['engines']['platformio']):
+                    continue
             specver = semantic_version.Version(v['version'])
             if reqspec and specver not in reqspec:
                 continue
@@ -95,7 +103,11 @@ class PkgRepoMixin(object):
                 item = v
         return item
 
-    def get_latest_repo_version(self, name, requirements):
+    def get_latest_repo_version(  # pylint: disable=unused-argument
+            self,
+            name,
+            requirements,
+            silent=False):
         version = None
         for versions in PackageRepoIterator(name, self.repositories):
             pkgdata = self.max_satisfying_repo_version(versions, requirements)
@@ -106,54 +118,204 @@ class PkgRepoMixin(object):
                 version = pkgdata['version']
         return version
 
+    def get_all_repo_versions(self, name):
+        result = []
+        for versions in PackageRepoIterator(name, self.repositories):
+            result.extend([v['version'] for v in versions])
+        return sorted(set(result))
+
 
 class PkgInstallerMixin(object):
 
-    VCS_MANIFEST_NAME = ".piopkgmanager.json"
+    SRC_MANIFEST_NAME = ".piopkgmanager.json"
 
-    def get_vcs_manifest_path(self, pkg_dir):
+    FILE_CACHE_VALID = "1m"  # 1 month
+    FILE_CACHE_MAX_SIZE = 1024 * 1024
+
+    MEMORY_CACHE = {}
+
+    @staticmethod
+    def cache_get(key, default=None):
+        return PkgInstallerMixin.MEMORY_CACHE.get(key, default)
+
+    @staticmethod
+    def cache_set(key, value):
+        PkgInstallerMixin.MEMORY_CACHE[key] = value
+
+    @staticmethod
+    def cache_reset():
+        PkgInstallerMixin.MEMORY_CACHE = {}
+
+    def read_dirs(self, src_dir):
+        cache_key = "read_dirs-%s" % src_dir
+        result = self.cache_get(cache_key)
+        if result:
+            return result
+        result = [
+            join(src_dir, name) for name in sorted(os.listdir(src_dir))
+            if isdir(join(src_dir, name))
+        ]
+        self.cache_set(cache_key, result)
+        return result
+
+    def download(self, url, dest_dir, sha1=None):
+        cache_key_fname = app.ContentCache.key_from_args(url, "fname")
+        cache_key_data = app.ContentCache.key_from_args(url, "data")
+        if self.FILE_CACHE_VALID:
+            with app.ContentCache() as cc:
+                fname = cc.get(cache_key_fname)
+                cache_path = cc.get_cache_path(cache_key_data)
+                if fname and isfile(cache_path):
+                    dst_path = join(dest_dir, fname)
+                    shutil.copy(cache_path, dst_path)
+                    return dst_path
+
+        fd = FileDownloader(url, dest_dir)
+        fd.start()
+        if sha1:
+            fd.verify(sha1)
+        dst_path = fd.get_filepath()
+        if not self.FILE_CACHE_VALID or getsize(
+                dst_path) > PkgInstallerMixin.FILE_CACHE_MAX_SIZE:
+            return dst_path
+
+        with app.ContentCache() as cc:
+            cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID)
+            cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID)
+            shutil.copy(dst_path, cc.get_cache_path(cache_key_data))
+        return dst_path
+
+    @staticmethod
+    def unpack(source_path, dest_dir):
+        fu = FileUnpacker(source_path, dest_dir)
+        return fu.start()
+
+    @staticmethod
+    def get_install_dirname(manifest):
+        name = re.sub(r"[^\da-z\_\-\. ]", "_", manifest['name'], flags=re.I)
+        if "id" in manifest:
+            name += "_ID%d" % manifest['id']
+        return name
+
+    def get_src_manifest_path(self, pkg_dir):
+        if not isdir(pkg_dir):
+            return None
         for item in os.listdir(pkg_dir):
             if not isdir(join(pkg_dir, item)):
                 continue
-            if isfile(join(pkg_dir, item, self.VCS_MANIFEST_NAME)):
-                return join(pkg_dir, item, self.VCS_MANIFEST_NAME)
+            if isfile(join(pkg_dir, item, self.SRC_MANIFEST_NAME)):
+                return join(pkg_dir, item, self.SRC_MANIFEST_NAME)
         return None
 
     def get_manifest_path(self, pkg_dir):
         if not isdir(pkg_dir):
             return None
-        manifest_path = join(pkg_dir, self.manifest_name)
-        if isfile(manifest_path):
-            return manifest_path
-        return self.get_vcs_manifest_path(pkg_dir)
-
-    def manifest_exists(self, pkg_dir):
-        return self.get_manifest_path(pkg_dir) is not None
-
-    def load_manifest(self, path):
-        assert path
-        pkg_dir = path
-        if isdir(path):
-            path = self.get_manifest_path(path)
-        else:
-            pkg_dir = dirname(pkg_dir)
-        if path:
-            if isfile(path) and path.endswith(self.VCS_MANIFEST_NAME):
-                pkg_dir = dirname(dirname(path))
-            manifest = util.load_json(path)
-            manifest['__pkg_dir'] = pkg_dir
-            return manifest
+        for name in self.manifest_names:
+            manifest_path = join(pkg_dir, name)
+            if isfile(manifest_path):
+                return manifest_path
         return None
 
-    def check_pkg_structure(self, pkg_dir):
-        if self.manifest_exists(pkg_dir):
-            return pkg_dir
+    def manifest_exists(self, pkg_dir):
+        return self.get_manifest_path(pkg_dir) or \
+            self.get_src_manifest_path(pkg_dir)
 
-        for root, _, _ in os.walk(pkg_dir):
+    def load_manifest(self, pkg_dir):
+        cache_key = "load_manifest-%s" % pkg_dir
+        result = self.cache_get(cache_key)
+        if result:
+            return result
+
+        manifest = {}
+        src_manifest = None
+        manifest_path = self.get_manifest_path(pkg_dir)
+        src_manifest_path = self.get_src_manifest_path(pkg_dir)
+        if src_manifest_path:
+            src_manifest = util.load_json(src_manifest_path)
+
+        if not manifest_path and not src_manifest_path:
+            return None
+
+        if manifest_path and manifest_path.endswith(".json"):
+            manifest = util.load_json(manifest_path)
+        elif manifest_path and manifest_path.endswith(".properties"):
+            with codecs.open(manifest_path, encoding="utf-8") as fp:
+                for line in fp.readlines():
+                    if "=" not in line:
+                        continue
+                    key, value = line.split("=", 1)
+                    manifest[key.strip()] = value.strip()
+
+        if src_manifest:
+            if "name" not in manifest:
+                manifest['name'] = src_manifest['name']
+            if "version" in src_manifest:
+                manifest['version'] = src_manifest['version']
+            manifest['__src_url'] = src_manifest['url']
+
+        if "name" not in manifest:
+            manifest['name'] = basename(pkg_dir)
+        if "version" not in manifest:
+            manifest['version'] = "0.0.0"
+
+        manifest['__pkg_dir'] = pkg_dir
+        self.cache_set(cache_key, manifest)
+        return manifest
+
+    def get_installed(self):
+        items = []
+        for pkg_dir in self.read_dirs(self.package_dir):
+            manifest = self.load_manifest(pkg_dir)
+            if not manifest:
+                continue
+            assert "name" in manifest
+            items.append(manifest)
+        return items
+
+    def get_package(self, name, requirements=None, url=None):
+        pkg_id = int(name[3:]) if name.startswith("id=") else 0
+        best = None
+        for manifest in self.get_installed():
+            if url:
+                if manifest.get("__src_url") != url:
+                    continue
+            elif pkg_id and manifest.get("id") != pkg_id:
+                continue
+            elif not pkg_id and manifest['name'] != name:
+                continue
+
+            # strict version or VCS HASH
+            if requirements and requirements == manifest['version']:
+                return manifest
+
+            try:
+                if requirements and not semantic_version.Spec(
+                        requirements).match(
+                            semantic_version.Version(
+                                manifest['version'], partial=True)):
+                    continue
+                elif not best or (semantic_version.Version(
+                        manifest['version'], partial=True) >
+                                  semantic_version.Version(
+                                      best['version'], partial=True)):
+                    best = manifest
+            except ValueError:
+                pass
+
+        return best
+
+    def get_package_dir(self, name, requirements=None, url=None):
+        manifest = self.get_package(name, requirements, url)
+        return manifest.get("__pkg_dir") if manifest and isdir(
+            manifest.get("__pkg_dir")) else None
+
+    def find_pkg_root(self, src_dir):
+        if self.manifest_exists(src_dir):
+            return src_dir
+        for root, _, _ in os.walk(src_dir):
             if self.manifest_exists(root):
                 return root
-
-        raise exception.MissingPackageManifest(self.manifest_name)
+        raise exception.MissingPackageManifest(", ".join(self.manifest_names))
 
     def _install_from_piorepo(self, name, requirements):
         pkg_dir = None
@@ -179,18 +341,24 @@ class PkgInstallerMixin(object):
                                                     util.get_systype())
         return pkg_dir
 
-    def _install_from_url(self, name, url, requirements=None, sha1=None):
-        pkg_dir = None
-        tmp_dir = mkdtemp("-package", "installing-", self.package_dir)
+    def _install_from_url(self,
+                          name,
+                          url,
+                          requirements=None,
+                          sha1=None,
+                          track=False):
+        tmp_dir = mkdtemp("-package", "_tmp_installing-", self.package_dir)
+        src_manifest_dir = None
+        src_manifest = {"name": name, "url": url, "requirements": requirements}
 
         try:
             if url.startswith("file://"):
-                url = url[7:]
-                if isfile(url):
-                    self.unpack(url, tmp_dir)
+                _url = url[7:]
+                if isfile(_url):
+                    self.unpack(_url, tmp_dir)
                 else:
                     util.rmtree_(tmp_dir)
-                    shutil.copytree(url, tmp_dir)
+                    shutil.copytree(_url, tmp_dir)
             elif url.startswith(("http://", "https://")):
                 dlpath = self.download(url, tmp_dir, sha1)
                 assert isfile(dlpath)
@@ -199,71 +367,113 @@ class PkgInstallerMixin(object):
             else:
                 vcs = VCSClientFactory.newClient(tmp_dir, url)
                 assert vcs.export()
-                with open(join(vcs.storage_dir, self.VCS_MANIFEST_NAME),
-                          "w") as fp:
-                    json.dump({
-                        "name": name,
-                        "version": vcs.get_current_revision(),
-                        "url": url,
-                        "requirements": requirements
-                    }, fp)
+                src_manifest_dir = vcs.storage_dir
+                src_manifest['version'] = vcs.get_current_revision()
 
-            pkg_dir = self.check_pkg_structure(tmp_dir)
-            pkg_dir = self._install_from_tmp_dir(pkg_dir, requirements)
+            _tmp_dir = tmp_dir
+            if not src_manifest_dir:
+                _tmp_dir = self.find_pkg_root(tmp_dir)
+                src_manifest_dir = join(_tmp_dir, ".pio")
+
+            # write source data to a special manifest
+            if track:
+                self._update_src_manifest(src_manifest, src_manifest_dir)
+
+            return self._install_from_tmp_dir(_tmp_dir, requirements)
         finally:
             if isdir(tmp_dir):
                 util.rmtree_(tmp_dir)
-        return pkg_dir
+        return
 
-    def _install_from_tmp_dir(self, tmp_dir, requirements=None):
-        tmpmanifest = self.load_manifest(tmp_dir)
-        assert set(["name", "version"]) <= set(tmpmanifest.keys())
-        name = tmpmanifest['name']
-        pkg_dir = join(self.package_dir, name)
-        if "id" in tmpmanifest:
-            name += "_ID%d" % tmpmanifest['id']
-            pkg_dir = join(self.package_dir, name)
+    def _update_src_manifest(self, data, src_dir):
+        if not isdir(src_dir):
+            os.makedirs(src_dir)
+        src_manifest_path = join(src_dir, self.SRC_MANIFEST_NAME)
+        _data = {}
+        if isfile(src_manifest_path):
+            _data = util.load_json(src_manifest_path)
+        _data.update(data)
+        with open(src_manifest_path, "w") as fp:
+            json.dump(_data, fp)
+
+    def _install_from_tmp_dir(  # pylint: disable=too-many-branches
+            self, tmp_dir, requirements=None):
+        tmp_manifest = self.load_manifest(tmp_dir)
+        assert set(["name", "version"]) <= set(tmp_manifest.keys())
+
+        pkg_dirname = self.get_install_dirname(tmp_manifest)
+        pkg_dir = join(self.package_dir, pkg_dirname)
+        cur_manifest = self.load_manifest(pkg_dir)
+
+        tmp_semver = None
+        cur_semver = None
+        try:
+            tmp_semver = semantic_version.Version(
+                tmp_manifest['version'], partial=True)
+            if cur_manifest:
+                cur_semver = semantic_version.Version(
+                    cur_manifest['version'], partial=True)
+        except ValueError:
+            pass
 
         # package should satisfy requirements
         if requirements:
             mismatch_error = (
-                "Package version %s doesn't satisfy requirements %s" % (
-                    tmpmanifest['version'], requirements))
+                "Package version %s doesn't satisfy requirements %s" %
+                (tmp_manifest['version'], requirements))
             try:
-                reqspec = semantic_version.Spec(requirements)
-                tmpmanver = semantic_version.Version(
-                    tmpmanifest['version'], partial=True)
-                assert tmpmanver in reqspec, mismatch_error
+                assert tmp_semver and tmp_semver in semantic_version.Spec(
+                    requirements), mismatch_error
+            except (AssertionError, ValueError):
+                assert tmp_manifest['version'] == requirements, mismatch_error
 
-                if self.manifest_exists(pkg_dir):
-                    curmanifest = self.load_manifest(pkg_dir)
-                    curmanver = semantic_version.Version(
-                        curmanifest['version'], partial=True)
-                    # if current package version < new package, backup it
-                    if tmpmanver > curmanver:
-                        os.rename(pkg_dir,
-                                  join(self.package_dir, "%s@%s" %
-                                       (name, curmanifest['version'])))
-                    elif tmpmanver < curmanver:
-                        pkg_dir = join(self.package_dir, "%s@%s" %
-                                       (name, tmpmanifest['version']))
-            except ValueError:
-                assert tmpmanifest['version'] == requirements, mismatch_error
+        # check if package already exists
+        if cur_manifest:
+            # 0-overwrite, 1-rename, 2-fix to a version
+            action = 0
+            if "__src_url" in cur_manifest:
+                if cur_manifest['__src_url'] != tmp_manifest.get("__src_url"):
+                    action = 1
+            elif "__src_url" in tmp_manifest:
+                action = 2
+            else:
+                if tmp_semver and (not cur_semver or tmp_semver > cur_semver):
+                    action = 1
+                elif tmp_semver and cur_semver and tmp_semver != cur_semver:
+                    action = 2
+
+            # rename
+            if action == 1:
+                target_dirname = "%s@%s" % (pkg_dirname,
+                                            cur_manifest['version'])
+                if "__src_url" in cur_manifest:
+                    target_dirname = "%s@src-%s" % (
+                        pkg_dirname,
+                        hashlib.md5(cur_manifest['__src_url']).hexdigest())
+                os.rename(pkg_dir, join(self.package_dir, target_dirname))
+            # fix to a version
+            elif action == 2:
+                target_dirname = "%s@%s" % (pkg_dirname,
+                                            tmp_manifest['version'])
+                if "__src_url" in tmp_manifest:
+                    target_dirname = "%s@src-%s" % (
+                        pkg_dirname,
+                        hashlib.md5(tmp_manifest['__src_url']).hexdigest())
+                pkg_dir = join(self.package_dir, target_dirname)
 
         # remove previous/not-satisfied package
         if isdir(pkg_dir):
             util.rmtree_(pkg_dir)
         os.rename(tmp_dir, pkg_dir)
         assert isdir(pkg_dir)
+        self.cache_reset()
         return pkg_dir
 
 
 class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
 
-    _INSTALLED_CACHE = {}
-
-    FILE_CACHE_VALID = "1m"  # 1 month
-    FILE_CACHE_MAX_SIZE = 1024 * 1024
+    # Handle circle dependencies
+    INSTALL_HISTORY = None
 
     def __init__(self, package_dir, repositories=None):
         self.repositories = repositories
@@ -273,57 +483,27 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         assert isdir(self.package_dir)
 
     @property
-    def manifest_name(self):
+    def manifest_names(self):
         raise NotImplementedError()
 
-    def download(self, url, dest_dir, sha1=None):
-        cache_key_fname = app.ContentCache.key_from_args(url, "fname")
-        cache_key_data = app.ContentCache.key_from_args(url, "data")
-        if self.FILE_CACHE_VALID:
-            with app.ContentCache() as cc:
-                fname = cc.get(cache_key_fname)
-                cache_path = cc.get_cache_path(cache_key_data)
-                if fname and isfile(cache_path):
-                    dst_path = join(dest_dir, fname)
-                    shutil.copy(cache_path, dst_path)
-                    return dst_path
-
-        fd = FileDownloader(url, dest_dir)
-        fd.start()
-        if sha1:
-            fd.verify(sha1)
-        dst_path = fd.get_filepath()
-        if not self.FILE_CACHE_VALID or getsize(
-                dst_path) > BasePkgManager.FILE_CACHE_MAX_SIZE:
-            return dst_path
-
-        with app.ContentCache() as cc:
-            cc.set(cache_key_fname, basename(dst_path), self.FILE_CACHE_VALID)
-            cc.set(cache_key_data, "DUMMY", self.FILE_CACHE_VALID)
-            shutil.copy(dst_path, cc.get_cache_path(cache_key_data))
-        return dst_path
-
-    @staticmethod
-    def unpack(source_path, dest_dir):
-        fu = FileUnpacker(source_path, dest_dir)
-        return fu.start()
-
-    def reset_cache(self):
-        if self.package_dir in BasePkgManager._INSTALLED_CACHE:
-            del BasePkgManager._INSTALLED_CACHE[self.package_dir]
-
     def print_message(self, message, nl=True):
         click.echo("%s: %s" % (self.__class__.__name__, message), nl=nl)
 
     @staticmethod
-    def parse_pkg_name(  # pylint: disable=too-many-branches
+    def parse_pkg_input(  # pylint: disable=too-many-branches
             text, requirements=None):
         text = str(text)
-        url_marker = "://"
-        if not any([
-                requirements, "@" not in text, text.startswith("git@"),
-                url_marker in text
-        ]):
+        # git@github.com:user/package.git
+        url_marker = text[:4]
+        if url_marker not in ("git@", "git+") or ":" not in text:
+            url_marker = "://"
+
+        req_conditions = [
+            not requirements,
+            "@" in text,
+            not url_marker.startswith("git")
+        ]  # yapf: disable
+        if all(req_conditions):
             text, requirements = text.rsplit("@", 1)
         if text.isdigit():
             text = "id=" + text
@@ -336,25 +516,21 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             # Handle GitHub URL (https://github.com/user/package)
             url.startswith("https://github.com/") and not url.endswith(
                 (".zip", ".tar.gz")),
-            url.startswith("http") and
-            (url.split("#", 1)[0] if "#" in url else url).endswith(".git")
+            url.startswith("http")
+            and (url.split("#", 1)[0] if "#" in url else url).endswith(".git")
         ]
-
         if any(git_conditions):
             url = "git+" + url
+
         # Handle Developer Mbed URL
         # (https://developer.mbed.org/users/user/code/package/)
-        elif url.startswith("https://developer.mbed.org"):
+        if url.startswith("https://developer.mbed.org"):
             url = "hg+" + url
 
-        # git@github.com:user/package.git
-        if url.startswith("git@"):
-            url_marker = "git@"
-
         if any([s in url for s in ("\\", "/")]) and url_marker not in url:
             if isfile(url) or isdir(url):
                 url = "file://" + url
-            elif url.count("/") == 1 and not url.startswith("git@"):
+            elif url.count("/") == 1 and "git" not in url_marker:
                 url = "git+https://github.com/" + url
 
         # determine name
@@ -364,91 +540,73 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 _url = _url[:-1]
             name = basename(_url)
             if "." in name and not name.startswith("."):
-                name = name.split(".", 1)[0]
+                name = name.rsplit(".", 1)[0]
 
         if url_marker not in url:
             url = None
         return (name or text, requirements, url)
 
-    def get_installed(self):
-        if self.package_dir in BasePkgManager._INSTALLED_CACHE:
-            return BasePkgManager._INSTALLED_CACHE[self.package_dir]
-        items = []
-        for p in sorted(os.listdir(self.package_dir)):
-            pkg_dir = join(self.package_dir, p)
-            if not isdir(pkg_dir):
-                continue
-            manifest = self.load_manifest(pkg_dir)
-            if not manifest:
-                continue
-            assert set(["name", "version"]) <= set(manifest.keys())
-            items.append(manifest)
-        BasePkgManager._INSTALLED_CACHE[self.package_dir] = items
-        return items
+    def outdated(self, pkg_dir, requirements=None):
+        """
+        Has 3 different results:
+        `None` - unknown package, VCS is fixed to commit
+        `False` - package is up-to-date
+        `String` - a found latest version
+        """
+        assert isdir(pkg_dir)
+        latest = None
+        manifest = self.load_manifest(pkg_dir)
+        # skip a fixed package to a specific version
+        if "@" in pkg_dir and "__src_url" not in manifest:
+            return None
 
-    def get_package(self, name, requirements=None, url=None):
-        pkg_id = int(name[3:]) if name.startswith("id=") else 0
-        best = None
-        reqspec = None
-        if requirements:
+        if "__src_url" in manifest:
             try:
-                reqspec = semantic_version.Spec(requirements)
-            except ValueError:
-                pass
-
-        for manifest in self.get_installed():
-            if pkg_id and manifest.get("id") != pkg_id:
-                continue
-            elif not pkg_id and manifest['name'] != name:
-                continue
-            elif not reqspec and requirements:
-                if requirements == manifest['version']:
-                    best = manifest
-                    break
-                continue
-            try:
-                if reqspec and not reqspec.match(
-                        semantic_version.Version(
-                            manifest['version'], partial=True)):
-                    continue
-                elif not best or (semantic_version.Version(
-                        manifest['version'], partial=True) >
-                                  semantic_version.Version(
-                                      best['version'], partial=True)):
-                    best = manifest
-            except ValueError:
-                pass
-        if best:
-            # check that URL is the same in installed package (VCS)
-            if url and best.get("url") != url:
+                vcs = VCSClientFactory.newClient(
+                    pkg_dir, manifest['__src_url'], silent=True)
+            except (AttributeError, exception.PlatformioException):
+                return None
+            if not vcs.can_be_updated:
+                return None
+            latest = vcs.get_latest_revision()
+        else:
+            try:
+                latest = self.get_latest_repo_version(
+                    "id=%d" % manifest['id']
+                    if "id" in manifest else manifest['name'],
+                    requirements,
+                    silent=True)
+            except (exception.PlatformioException, ValueError):
                 return None
-            return best
-        return None
 
-    def get_package_dir(self, name, requirements=None, url=None):
-        package = self.get_package(name, requirements, url)
-        return package.get("__pkg_dir") if package else None
+        if not latest:
+            return None
 
-    def is_outdated(self, name, requirements=None):
-        package_dir = self.get_package_dir(name, requirements)
-        if not package_dir:
-            click.secho(
-                "%s @ %s is not installed" % (name, requirements or "*"),
-                fg="yellow")
-            return
-        if self.get_vcs_manifest_path(package_dir):
-            return False
-        manifest = self.load_manifest(package_dir)
-        return manifest['version'] != self.get_latest_repo_version(
-            name, requirements)
+        up_to_date = False
+        try:
+            assert "__src_url" not in manifest
+            up_to_date = (semantic_version.Version.coerce(manifest['version'])
+                          >= semantic_version.Version.coerce(latest))
+        except (AssertionError, ValueError):
+            up_to_date = latest == manifest['version']
+
+        return False if up_to_date else latest
 
     def install(self,
                 name,
                 requirements=None,
                 silent=False,
-                trigger_event=True,
-                interactive=False):  # pylint: disable=unused-argument
-        name, requirements, url = self.parse_pkg_name(name, requirements)
+                trigger_event=True):
+
+        # avoid circle dependencies
+        if not self.INSTALL_HISTORY:
+            self.INSTALL_HISTORY = []
+        history_key = "%s-%s" % (name, requirements) if requirements else name
+        if history_key in self.INSTALL_HISTORY:
+            return
+        self.INSTALL_HISTORY.append(history_key)
+
+        name, requirements, url = self.parse_pkg_input(name, requirements)
         package_dir = self.get_package_dir(name, requirements, url)
 
         if not package_dir or not silent:
@@ -465,15 +623,16 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             return package_dir
 
         if url:
-            pkg_dir = self._install_from_url(name, url, requirements)
+            pkg_dir = self._install_from_url(
+                name, url, requirements, track=True)
         else:
             pkg_dir = self._install_from_piorepo(name, requirements)
         if not pkg_dir or not self.manifest_exists(pkg_dir):
             raise exception.PackageInstallError(name, requirements or "*",
                                                 util.get_systype())
 
-        self.reset_cache()
         manifest = self.load_manifest(pkg_dir)
+        assert manifest
 
         if trigger_event:
             telemetry.on_event(
@@ -481,37 +640,48 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 action="Install",
                 label=manifest['name'])
 
-        click.secho(
-            "{name} @ {version} has been successfully installed!".format(
-                **manifest),
-            fg="green")
+        if not silent:
+            click.secho(
+                "{name} @ {version} has been successfully installed!".format(
+                    **manifest),
+                fg="green")
 
         return pkg_dir
 
-    def uninstall(self, name, requirements=None, trigger_event=True):
-        name, requirements, url = self.parse_pkg_name(name, requirements)
-        package_dir = self.get_package_dir(name, requirements, url)
-        if not package_dir:
-            click.secho(
-                "%s @ %s is not installed" % (name, requirements or "*"),
-                fg="yellow")
-            return
+    def uninstall(self, package, requirements=None, trigger_event=True):
+        if isdir(package):
+            pkg_dir = package
+        else:
+            name, requirements, url = self.parse_pkg_input(
+                package, requirements)
+            pkg_dir = self.get_package_dir(name, requirements, url)
 
-        manifest = self.load_manifest(package_dir)
+        if not pkg_dir:
+            raise exception.UnknownPackage("%s @ %s" % (package,
+                                                        requirements or "*"))
+
+        manifest = self.load_manifest(pkg_dir)
         click.echo(
-            "Uninstalling %s @ %s: \t" % (click.style(
-                manifest['name'], fg="cyan"), manifest['version']),
+            "Uninstalling %s @ %s: \t" %
+            (click.style(manifest['name'], fg="cyan"), manifest['version']),
             nl=False)
 
-        if isdir(package_dir):
-            if islink(package_dir):
-                os.unlink(package_dir)
-            else:
-                util.rmtree_(package_dir)
+        if islink(pkg_dir):
+            os.unlink(pkg_dir)
+        else:
+            util.rmtree_(pkg_dir)
+        self.cache_reset()
+
+        # unfix package with the same name
+        pkg_dir = self.get_package_dir(manifest['name'])
+        if pkg_dir and "@" in pkg_dir:
+            os.rename(pkg_dir,
+                      join(self.package_dir,
+                           self.get_install_dirname(manifest)))
+            self.cache_reset()
 
         click.echo("[%s]" % click.style("OK", fg="green"))
 
-        self.reset_cache()
         if trigger_event:
             telemetry.on_event(
                 category=self.__class__.__name__,
@@ -521,77 +691,49 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
 
     def update(  # pylint: disable=too-many-return-statements
             self,
-            name,
+            package,
             requirements=None,
             only_check=False):
-        name, requirements, url = self.parse_pkg_name(name, requirements)
-        package_dir = self.get_package_dir(name, None, url)
-        if not package_dir:
-            click.secho(
-                "%s @ %s is not installed" % (name, requirements or "*"),
-                fg="yellow")
+        if isdir(package):
+            pkg_dir = package
+        else:
+            pkg_dir = self.get_package_dir(*self.parse_pkg_input(package))
+
+        if not pkg_dir:
+            raise exception.UnknownPackage("%s @ %s" % (package,
+                                                        requirements or "*"))
+
+        manifest = self.load_manifest(pkg_dir)
+        name = manifest['name']
+
+        click.echo(
+            "{} {:<40} @ {:<15}".format(
+                "Checking" if only_check else "Updating",
+                click.style(manifest['name'], fg="cyan"), manifest['version']),
+            nl=False)
+        if not util.internet_on():
+            click.echo("[%s]" % (click.style("Off-line", fg="yellow")))
             return
 
-        is_vcs_pkg = False
-        if self.get_vcs_manifest_path(package_dir):
-            is_vcs_pkg = True
-            manifest_path = self.get_vcs_manifest_path(package_dir)
+        latest = self.outdated(pkg_dir, requirements)
+        if latest:
+            click.echo("[%s]" % (click.style(latest, fg="red")))
+        elif latest is False:
+            click.echo("[%s]" % (click.style("Up-to-date", fg="green")))
         else:
-            manifest_path = self.get_manifest_path(package_dir)
+            click.echo("[%s]" % (click.style("Skip", fg="yellow")))
 
-        manifest = self.load_manifest(manifest_path)
-        click.echo(
-            "%s %s @ %s: \t" % ("Checking"
-                                if only_check else "Updating", click.style(
-                                    manifest['name'], fg="cyan"),
-                                manifest['version']),
-            nl=False)
-        if is_vcs_pkg:
-            if only_check:
-                click.echo("[%s]" % (click.style("Skip", fg="yellow")))
-                return
-            click.echo("[%s]" % (click.style("VCS", fg="yellow")))
-            vcs = VCSClientFactory.newClient(package_dir, manifest['url'])
-            if not vcs.can_be_updated:
-                click.secho(
-                    "Skip update because repository is fixed "
-                    "to %s revision" % manifest['version'],
-                    fg="yellow")
-                return
+        if only_check or not latest:
+            return
+
+        if "__src_url" in manifest:
+            vcs = VCSClientFactory.newClient(pkg_dir, manifest['__src_url'])
             assert vcs.update()
-            with open(manifest_path, "w") as fp:
-                manifest['version'] = vcs.get_current_revision()
-                json.dump(manifest, fp)
+            self._update_src_manifest(
+                dict(version=vcs.get_current_revision()), vcs.storage_dir)
         else:
-            latest_version = None
-            try:
-                latest_version = self.get_latest_repo_version(name,
-                                                              requirements)
-            except exception.PlatformioException:
-                pass
-            if not latest_version:
-                click.echo("[%s]" % (click.style(
-                    "Off-line" if not util.internet_on() else "Unknown",
-                    fg="yellow")))
-                return
-
-            up_to_date = False
-            try:
-                up_to_date = (
-                    semantic_version.Version.coerce(manifest['version']) >=
-                    semantic_version.Version.coerce(latest_version))
-            except ValueError:
-                up_to_date = latest_version == manifest['version']
-
-            if up_to_date:
-                click.echo("[%s]" % (click.style("Up-to-date", fg="green")))
-                return
-
-            click.echo("[%s]" % (click.style("Out-of-date", fg="red")))
-            if only_check:
-                return
-            self.uninstall(name, manifest['version'], trigger_event=False)
-            self.install(name, latest_version, trigger_event=False)
+            self.uninstall(pkg_dir, trigger_event=False)
+            self.install(name, latest, trigger_event=False)
 
         telemetry.on_event(
             category=self.__class__.__name__,
@@ -605,5 +747,5 @@ class PackageManager(BasePkgManager):
     FILE_CACHE_VALID = None  # disable package caching
 
     @property
-    def manifest_name(self):
-        return "package.json"
+    def manifest_names(self):
+        return ["package.json"]

platformio/managers/platform.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,8 +20,10 @@ from multiprocessing import cpu_count
 from os.path import basename, dirname, isdir, isfile, join
 
 import click
+import semantic_version
 
-from platformio import app, exception, util
+from platformio import __version__, app, exception, util
+from platformio.managers.core import get_core_package_dir
 from platformio.managers.package import BasePkgManager, PackageManager
 
 
@@ -36,13 +38,22 @@ class PlatformManager(BasePkgManager):
                 "{0}://dl.platformio.org/platforms/manifest.json".format(
                     "https" if app.get_setting("enable_ssl") else "http")
             ]
-        BasePkgManager.__init__(self, package_dir or
-                                join(util.get_home_dir(), "platforms"),
+        BasePkgManager.__init__(self, package_dir
+                                or join(util.get_home_dir(), "platforms"),
                                 repositories)
 
     @property
-    def manifest_name(self):
-        return "platform.json"
+    def manifest_names(self):
+        return ["platform.json"]
+
+    def get_manifest_path(self, pkg_dir):
+        if not isdir(pkg_dir):
+            return None
+        for name in self.manifest_names:
+            manifest_path = join(pkg_dir, name)
+            if isfile(manifest_path):
+                return manifest_path
+        return None
 
     def install(self,
                 name,
@@ -50,50 +61,80 @@ class PlatformManager(BasePkgManager):
                 with_packages=None,
                 without_packages=None,
                 skip_default_package=False,
-                **_):  # pylint: disable=too-many-arguments
-        platform_dir = BasePkgManager.install(self, name, requirements)
-        p = PlatformFactory.newPlatform(self.get_manifest_path(platform_dir))
-        p.install_packages(with_packages, without_packages,
-                           skip_default_package)
+                trigger_event=True,
+                silent=False,
+                **_):  # pylint: disable=too-many-arguments, arguments-differ
+        platform_dir = BasePkgManager.install(
+            self, name, requirements, silent=silent)
+        p = PlatformFactory.newPlatform(platform_dir)
+
+        # @Hook: when 'update' operation (trigger_event is False),
+        # don't cleanup packages or install them
+        if not trigger_event:
+            return True
+        p.install_packages(
+            with_packages,
+            without_packages,
+            skip_default_package,
+            silent=silent)
         self.cleanup_packages(p.packages.keys())
         return True
 
-    def uninstall(self, name, requirements=None, trigger_event=True):
-        name, requirements, _ = self.parse_pkg_name(name, requirements)
-        p = PlatformFactory.newPlatform(name, requirements)
-        BasePkgManager.uninstall(self, name, requirements)
-        # trigger event is disabled when upgrading operation
-        # don't cleanup packages, "install" will do that
-        if trigger_event:
-            self.cleanup_packages(p.packages.keys())
+    def uninstall(self, package, requirements=None, trigger_event=True):
+        if isdir(package):
+            pkg_dir = package
+        else:
+            name, requirements, url = self.parse_pkg_input(
+                package, requirements)
+            pkg_dir = self.get_package_dir(name, requirements, url)
+
+        p = PlatformFactory.newPlatform(pkg_dir)
+        BasePkgManager.uninstall(self, pkg_dir, requirements)
+
+        # @Hook: when 'update' operation (trigger_event is False),
+        # don't cleanup packages or install them
+        if not trigger_event:
+            return True
+
+        self.cleanup_packages(p.packages.keys())
         return True
 
     def update(  # pylint: disable=arguments-differ
             self,
-            name,
+            package,
             requirements=None,
-            only_packages=False,
-            only_check=False):
-        name, requirements, _ = self.parse_pkg_name(name, requirements)
+            only_check=False,
+            only_packages=False):
+        if isdir(package):
+            pkg_dir = package
+        else:
+            name, requirements, url = self.parse_pkg_input(
+                package, requirements)
+            pkg_dir = self.get_package_dir(name, requirements, url)
+
+        p = PlatformFactory.newPlatform(pkg_dir)
+        pkgs_before = pkgs_after = p.get_installed_packages().keys()
+
         if not only_packages:
-            BasePkgManager.update(self, name, requirements, only_check)
-        p = PlatformFactory.newPlatform(name, requirements)
+            BasePkgManager.update(self, pkg_dir, requirements, only_check)
+            p = PlatformFactory.newPlatform(pkg_dir)
+            pkgs_after = p.get_installed_packages().keys()
+
         p.update_packages(only_check)
         self.cleanup_packages(p.packages.keys())
+
+        pkgs_missed = set(pkgs_before) - set(pkgs_after)
+        if pkgs_missed:
+            p.install_packages(
+                with_packages=pkgs_missed, skip_default_package=True)
+
         return True
 
-    def is_outdated(self, name, requirements=None):
-        if BasePkgManager.is_outdated(self, name, requirements):
-            return True
-        p = PlatformFactory.newPlatform(name, requirements)
-        return p.are_outdated_packages()
-
     def cleanup_packages(self, names):
-        self.reset_cache()
+        self.cache_reset()
         deppkgs = {}
         for manifest in PlatformManager().get_installed():
-            p = PlatformFactory.newPlatform(manifest['name'],
-                                            manifest['version'])
+            p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
             for pkgname, pkgmanifest in p.get_installed_packages().items():
                 if pkgname not in deppkgs:
                     deppkgs[pkgname] = set()
@@ -103,34 +144,36 @@ class PlatformManager(BasePkgManager):
         for manifest in pm.get_installed():
             if manifest['name'] not in names:
                 continue
-            if (manifest['name'] not in deppkgs or
-                    manifest['version'] not in deppkgs[manifest['name']]):
-                pm.uninstall(
-                    manifest['name'], manifest['version'], trigger_event=False)
+            if (manifest['name'] not in deppkgs
+                    or manifest['version'] not in deppkgs[manifest['name']]):
+                pm.uninstall(manifest['__pkg_dir'], trigger_event=False)
 
-        self.reset_cache()
+        self.cache_reset()
         return True
 
     def get_installed_boards(self):
         boards = []
         for manifest in self.get_installed():
-            p = PlatformFactory.newPlatform(
-                self.get_manifest_path(manifest['__pkg_dir']))
+            p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
             for config in p.get_boards().values():
-                boards.append(config.get_brief_data())
+                board = config.get_brief_data()
+                if board not in boards:
+                    boards.append(board)
         return boards
 
     @staticmethod
     @util.memoized
     def get_registered_boards():
-        return util.get_api_result("/boards", cache_valid="365d")
+        return util.get_api_result("/boards", cache_valid="30d")
 
-    def board_config(self, id_):
+    def board_config(self, id_, platform=None):
         for manifest in self.get_installed_boards():
-            if manifest['id'] == id_:
+            if manifest['id'] == id_ and (not platform
+                                          or manifest['platform'] == platform):
                 return manifest
         for manifest in self.get_registered_boards():
-            if manifest['id'] == id_:
+            if manifest['id'] == id_ and (not platform
+                                          or manifest['platform'] == platform):
                 return manifest
         raise exception.UnknownBoard(id_)
 
@@ -155,14 +198,17 @@ class PlatformFactory(object):
     @classmethod
     def newPlatform(cls, name, requirements=None):
         platform_dir = None
-        if name.endswith("platform.json") and isfile(name):
+        if isdir(name):
+            platform_dir = name
+            name = PlatformManager().load_manifest(platform_dir)['name']
+        elif name.endswith("platform.json") and isfile(name):
             platform_dir = dirname(name)
             name = util.load_json(name)['name']
         else:
             if not requirements and "@" in name:
                 name, requirements = name.rsplit("@", 1)
-            platform_dir = PlatformManager().get_package_dir(name,
-                                                             requirements)
+            platform_dir = PlatformManager().get_package_dir(
+                name, requirements)
 
         if not platform_dir:
             raise exception.UnknownPlatform(name if not requirements else
@@ -189,8 +235,8 @@ class PlatformPackagesMixin(object):
                          without_packages=None,
                          skip_default_package=False,
                          silent=False):
-        with_packages = set(self.pkg_types_to_names(with_packages or []))
-        without_packages = set(self.pkg_types_to_names(without_packages or []))
+        with_packages = set(self.find_pkg_names(with_packages or []))
+        without_packages = set(self.find_pkg_names(without_packages or []))
 
         upkgs = with_packages | without_packages
         ppkgs = set(self.packages.keys())
@@ -198,44 +244,84 @@ class PlatformPackagesMixin(object):
             raise exception.UnknownPackage(", ".join(upkgs - ppkgs))
 
         for name, opts in self.packages.items():
+            version = opts.get("version", "")
             if name in without_packages:
                 continue
             elif (name in with_packages or
                   not (skip_default_package or opts.get("optional", False))):
-                if any([s in opts.get("version", "") for s in ("\\", "/")]):
-                    self.pm.install(
-                        "%s=%s" % (name, opts['version']), silent=silent)
+                if self.is_valid_requirements(version):
+                    self.pm.install(name, version, silent=silent)
                 else:
-                    self.pm.install(name, opts.get("version"), silent=silent)
+                    requirements = None
+                    if "@" in version:
+                        version, requirements = version.rsplit("@", 1)
+                    self.pm.install(
+                        "%s=%s" % (name, version), requirements, silent=silent)
 
         return True
 
-    def get_installed_packages(self):
-        items = {}
-        for name, opts in self.packages.items():
-            package = self.pm.get_package(name, opts['version'])
-            if package:
-                items[name] = package
-        return items
+    def find_pkg_names(self, items):
+        result = []
+        for item in items:
+            candidate = item
+
+            # lookup by package types
+            for _name, _opts in self.packages.items():
+                if _opts.get("type") == item:
+                    candidate = _name
+
+            if (self.frameworks and item.startswith("framework-")
+                    and item[10:] in self.frameworks):
+                candidate = self.frameworks[item[10:]]['package']
+
+            result.append(candidate)
+        return result
 
     def update_packages(self, only_check=False):
-        for name in self.get_installed_packages():
-            self.pm.update(name, self.packages[name]['version'], only_check)
+        for name, manifest in self.get_installed_packages().items():
+            version = self.packages[name].get("version", "")
+            if "@" in version:
+                _, version = version.rsplit("@", 1)
+            self.pm.update(manifest['__pkg_dir'], version, only_check)
+
+    def get_installed_packages(self):
+        items = {}
+        for name in self.packages:
+            pkg_dir = self.get_package_dir(name)
+            if pkg_dir:
+                items[name] = self.pm.load_manifest(pkg_dir)
+        return items
 
     def are_outdated_packages(self):
-        for name, opts in self.get_installed_packages().items():
-            if (opts['version'] != self.pm.get_latest_repo_version(
-                    name, self.packages[name].get("version"))):
+        for name, manifest in self.get_installed_packages().items():
+            version = self.packages[name].get("version", "")
+            if "@" in version:
+                _, version = version.rsplit("@", 1)
+            if self.pm.outdated(manifest['__pkg_dir'], version):
                 return True
         return False
 
     def get_package_dir(self, name):
-        return self.pm.get_package_dir(name,
-                                       self.packages[name].get("version"))
+        version = self.packages[name].get("version", "")
+        if self.is_valid_requirements(version):
+            return self.pm.get_package_dir(name, version)
+        return self.pm.get_package_dir(*self._parse_pkg_input(name, version))
 
     def get_package_version(self, name):
-        package = self.pm.get_package(name, self.packages[name].get("version"))
-        return package['version'] if package else None
+        pkg_dir = self.get_package_dir(name)
+        if not pkg_dir:
+            return None
+        return self.pm.load_manifest(pkg_dir).get("version")
+
+    @staticmethod
+    def is_valid_requirements(requirements):
+        return requirements and "://" not in requirements
+
+    def _parse_pkg_input(self, name, version):
+        requirements = None
+        if "@" in version:
+            version, requirements = version.rsplit("@", 1)
+        return self.pm.parse_pkg_input("%s=%s" % (name, version), requirements)
 
 
 class PlatformRunMixin(object):
@@ -270,9 +356,10 @@ class PlatformRunMixin(object):
     def _run_scons(self, variables, targets):
         cmd = [
             util.get_pythonexe_path(),
-            join(self.get_package_dir("tool-scons"), "script", "scons"), "-Q",
+            join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q",
             "-j %d" % self.get_job_nums(), "--warn=no-no-parallel-support",
-            "-f", join(util.get_source_dir(), "builder", "main.py")
+            "-f",
+            join(util.get_source_dir(), "builder", "main.py")
         ]
         cmd.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
         cmd += targets
@@ -316,8 +403,10 @@ class PlatformRunMixin(object):
             return 1
 
 
-class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
+class PlatformBase(  # pylint: disable=too-many-public-methods
+        PlatformPackagesMixin, PlatformRunMixin):
 
+    PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__))
     _BOARDS_CACHE = {}
 
     def __init__(self, manifest_path):
@@ -332,6 +421,12 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
         self.silent = False
         self.verbose = False
 
+        if self.engines and "platformio" in self.engines:
+            if self.PIO_VERSION not in semantic_version.Spec(
+                    self.engines['platformio']):
+                raise exception.IncompatiblePlatform(self.name,
+                                                     str(self.PIO_VERSION))
+
     @property
     def name(self):
         return self._manifest['name']
@@ -356,6 +451,10 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
     def vendor_url(self):
         return self._manifest.get("url")
 
+    @property
+    def repository_url(self):
+        return self._manifest.get("repository", {}).get("url")
+
     @property
     def license(self):
         return self._manifest.get("license")
@@ -364,6 +463,10 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
     def frameworks(self):
         return self._manifest.get("frameworks")
 
+    @property
+    def engines(self):
+        return self._manifest.get("engines")
+
     @property
     def manifest(self):
         return self._manifest
@@ -395,8 +498,8 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
             config = PlatformBoardConfig(manifest_path)
             if "platform" in config and config.get("platform") != self.name:
                 return
-            elif ("platforms" in config and
-                  self.name not in config.get("platforms")):
+            elif ("platforms" in config
+                  and self.name not in config.get("platforms")):
                 return
             config.manifest['platform'] = self.name
             self._BOARDS_CACHE[board_id] = config
@@ -435,20 +538,6 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
     def get_package_type(self, name):
         return self.packages[name].get("type")
 
-    def pkg_types_to_names(self, types):
-        names = []
-        for type_ in types:
-            name = type_
-            # lookup by package types
-            for _name, _opts in self.packages.items():
-                if _opts.get("type") == type_:
-                    name = None
-                    names.append(_name)
-            # if type is the right name
-            if name:
-                names.append(name)
-        return names
-
     def configure_default_packages(self, variables, targets):
         # enable used frameworks
         frameworks = variables.get("pioframework", [])
@@ -460,8 +549,9 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
             framework = framework.lower().strip()
             if not framework or framework not in self.frameworks:
                 continue
-            _pkg_name = self.frameworks[framework]['package']
-            self.packages[_pkg_name]['optional'] = False
+            _pkg_name = self.frameworks[framework].get("package")
+            if _pkg_name:
+                self.packages[_pkg_name]['optional'] = False
 
         # enable upload tools for upload targets
         if any(["upload" in t for t in targets] + ["program" in targets]):
@@ -472,16 +562,31 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
                     # skip all packages, allow only upload tools
                     self.packages[_name]['optional'] = True
 
-        if "__test" in targets and "tool-unity" not in self.packages:
-            self.packages['tool-unity'] = {
-                "version": "~1.20302.1",
-                "optional": False
-            }
-        if "tool-scons" not in self.packages:
-            self.packages['tool-scons'] = {
-                "version": "~3.20501.2",
-                "optional": False
-            }
+    def get_lib_storages(self):
+        storages = []
+        for opts in (self.frameworks or {}).values():
+            if "package" not in opts:
+                continue
+            pkg_dir = self.get_package_dir(opts['package'])
+            if not pkg_dir or not isdir(join(pkg_dir, "libraries")):
+                continue
+            libs_dir = join(pkg_dir, "libraries")
+            storages.append({"name": opts['package'], "path": libs_dir})
+            libcores_dir = join(libs_dir, "__cores__")
+            if not isdir(libcores_dir):
+                continue
+            for item in os.listdir(libcores_dir):
+                libcore_dir = join(libcores_dir, item)
+                if not isdir(libcore_dir):
+                    continue
+                storages.append({
+                    "name":
+                    "%s-core-%s" % (opts['package'], item),
+                    "path":
+                    libcore_dir
+                })
+
+        return storages
 
 
 class PlatformBoardConfig(object):
@@ -540,7 +645,20 @@ class PlatformBoardConfig(object):
             int(self._manifest.get("build", {}).get("f_cpu", "0L")[:-1]),
             "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0),
             "rom": self._manifest.get("upload", {}).get("maximum_size", 0),
+            "connectivity": self._manifest.get("connectivity"),
             "frameworks": self._manifest.get("frameworks"),
+            "debug": self.get_debug_data(),
             "vendor": self._manifest['vendor'],
             "url": self._manifest['url']
         }
+
+    def get_debug_data(self):
+        if not self._manifest.get("debug", {}).get("tools"):
+            return
+        tools = {}
+        for name, options in self._manifest['debug']['tools'].items():
+            tools[name] = {}
+            for key, value in options.items():
+                if key in ("default", "onboard"):
+                    tools[name][key] = value
+        return {"tools": tools}

platformio/pioplus.py

@@ -1,94 +0,0 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import subprocess
-import sys
-from os.path import join
-
-from platformio import exception, util
-from platformio.managers.package import PackageManager
-
-PACKAGE_DEPS = {
-    "pysite": {
-        "name": "pysite-pioplus",
-        "requirements": ">=0.3.0,<2"
-    },
-    "tool": {
-        "name": "tool-pioplus",
-        "requirements": ">=0.6.6,<2"
-    }
-}
-
-AUTO_UPDATES_MAX = 100
-
-
-class PioPlusPackageManager(PackageManager):
-
-    def __init__(self):
-        PackageManager.__init__(
-            self,
-            join(util.get_home_dir(), "packages"), [
-                "https://dl.bintray.com/platformio/dl-packages/manifest.json",
-                "http%s://dl.platformio.org/packages/manifest.json" %
-                ("" if sys.version_info < (2, 7, 9) else "s")
-            ])
-
-
-def pioplus_install():
-    pm = PioPlusPackageManager()
-    for item in PACKAGE_DEPS.values():
-        pm.install(item['name'], item['requirements'], silent=True)
-
-
-def pioplus_update():
-    pm = PioPlusPackageManager()
-    for item in PACKAGE_DEPS.values():
-        package_dir = pm.get_package_dir(item['name'])
-        if package_dir:
-            pm.update(item['name'], item['requirements'])
-
-
-def pioplus_call(args, **kwargs):
-    pioplus_install()
-    pm = PioPlusPackageManager()
-    pioplus_path = join(
-        pm.get_package_dir(PACKAGE_DEPS['tool']['name'],
-                           PACKAGE_DEPS['tool']['requirements']), "pioplus")
-    os.environ['PYTHONEXEPATH'] = util.get_pythonexe_path()
-    os.environ['PYTHONPYSITEDIR'] = pm.get_package_dir(
-        PACKAGE_DEPS['pysite']['name'], PACKAGE_DEPS['pysite']['requirements'])
-    util.copy_pythonpath_to_osenv()
-    code = subprocess.call([pioplus_path] + args, **kwargs)
-
-    # handle remote update request
-    if code == 13:
-        count_attr = "_update_count"
-        try:
-            count_value = getattr(pioplus_call, count_attr)
-        except AttributeError:
-            count_value = 0
-            setattr(pioplus_call, count_attr, 1)
-        count_value += 1
-        setattr(pioplus_call, count_attr, count_value)
-        if count_value < AUTO_UPDATES_MAX:
-            pioplus_update()
-            return pioplus_call(args, **kwargs)
-
-    # handle reload request
-    elif code == 14:
-        return pioplus_call(args, **kwargs)
-
-    if code != 0:
-        raise exception.ReturnErrorCode(1)

platformio/telemetry.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -92,12 +92,26 @@ class MeasurementProtocol(TelemetryBase):
         self['an'] = " ".join(dpdata)
 
     def _prefill_custom_data(self):
+
+        def _filter_args(items):
+            result = []
+            stop = False
+            for item in items:
+                item = str(item).lower()
+                result.append(item)
+                if stop:
+                    break
+                if item == "account":
+                    stop = True
+            return result
+
         caller_id = str(app.get_session_var("caller_id"))
         self['cd1'] = util.get_systype()
         self['cd2'] = "Python/%s %s" % (platform.python_version(),
                                         platform.platform())
-        self['cd4'] = 1 if (not util.is_ci() and
-                            (caller_id or not util.is_container())) else 0
+        self['cd3'] = " ".join(_filter_args(sys.argv[1:]))
+        self['cd4'] = 1 if (not util.is_ci()
+                            and (caller_id or not util.is_container())) else 0
         if caller_id:
             self['cd5'] = caller_id.lower()
 
@@ -109,7 +123,6 @@ class MeasurementProtocol(TelemetryBase):
                     return _arg
             return None
 
-        self['cd3'] = " ".join([str(s).lower() for s in sys.argv[1:]])
         if not app.get_session_var("command_ctx"):
             return
         ctx_args = app.get_session_var("command_ctx").args
@@ -121,8 +134,8 @@ class MeasurementProtocol(TelemetryBase):
                        "settings", "account"):
             cmd_path = args[:2]
         if args[0] == "lib" and len(args) > 1:
-            lib_subcmds = ("install", "list", "register", "search", "show",
-                           "uninstall", "update")
+            lib_subcmds = ("builtin", "install", "list", "register", "search",
+                           "show", "stats", "uninstall", "update")
             sub_cmd = _first_arg_from_list(args[1:], lib_subcmds)
             if sub_cmd:
                 cmd_path.append(sub_cmd)
@@ -227,8 +240,13 @@ class MPDataPusher(object):
                 timeout=1)
             r.raise_for_status()
             return True
+        except requests.exceptions.HTTPError as e:
+            # skip Bad Request
+            if 400 >= e.response.status_code < 500:
+                return True
         except:  # pylint: disable=W0702
-            self._http_offline = True
+            pass
+        self._http_offline = True
         return False
 
 
@@ -250,8 +268,9 @@ def measure_ci():
             "label": getenv("APPVEYOR_REPO_NAME")
         },
         "CIRCLECI": {
-            "label": "%s/%s" % (getenv("CIRCLE_PROJECT_USERNAME"),
-                                getenv("CIRCLE_PROJECT_REPONAME"))
+            "label":
+            "%s/%s" % (getenv("CIRCLE_PROJECT_USERNAME"),
+                       getenv("CIRCLE_PROJECT_REPONAME"))
         },
         "TRAVIS": {
             "label": getenv("TRAVIS_REPO_SLUG")
@@ -273,7 +292,10 @@ def measure_ci():
 
 
 def on_run_environment(options, targets):
-    opts = ["%s=%s" % (opt, value) for opt, value in sorted(options.items())]
+    opts = [
+        "%s=%s" % (opt, value.replace("\n", ", ") if "\n" in value else value)
+        for opt, value in sorted(options.items())
+    ]
     targets = [t.title() for t in targets or ["run"]]
     on_event("Env", " ".join(targets), "&".join(opts))
 
@@ -292,19 +314,23 @@ def on_event(category, action, label=None, value=None, screen_name=None):
 
 
 def on_exception(e):
-    skip = any([
+    skip_conditions = [
         isinstance(e, cls)
         for cls in (IOError, exception.AbortedByUser,
-                    exception.NotGlobalLibDir, exception.InternetIsOffline)
-    ])
-    if skip:
+                    exception.NotGlobalLibDir, exception.InternetIsOffline,
+                    exception.NotPlatformIOProject,
+                    exception.UserSideException)
+    ]
+    skip_conditions.append("[API] Account: " in str(e))
+    if any(skip_conditions):
         return
     is_crash = any([
         not isinstance(e, exception.PlatformioException),
         "Error" in e.__class__.__name__
     ])
     mp = MeasurementProtocol()
-    mp['exd'] = "%s: %s" % (type(e).__name__, format_exc() if is_crash else e)
+    mp['exd'] = ("%s: %s" % (type(e).__name__, format_exc()
+                             if is_crash else e))[:2048]
     mp['exf'] = 1 if is_crash else 0
     mp.send("exception")
 

platformio/unpacker.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

platformio/util.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ import socket
 import stat
 import subprocess
 import sys
+from contextlib import contextmanager
 from glob import glob
 from os.path import (abspath, basename, dirname, expanduser, isdir, isfile,
                      join, normpath, splitdrive)
@@ -46,7 +47,7 @@ class ProjectConfig(ConfigParser):
 
     VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}")
 
-    def items(self, section, **_):
+    def items(self, section, **_):  # pylint: disable=arguments-differ
         items = []
         for option in ConfigParser.options(self, section):
             items.append((option, self.get(section, option)))
@@ -130,10 +131,9 @@ class memoized(object):
             return self.func(*args)
         if args in self.cache:
             return self.cache[args]
-        else:
-            value = self.func(*args)
-            self.cache[args] = value
-            return value
+        value = self.func(*args)
+        self.cache[args] = value
+        return value
 
     def __repr__(self):
         '''Return the function's docstring.'''
@@ -141,7 +141,12 @@ class memoized(object):
 
     def __get__(self, obj, objtype):
         '''Support instance methods.'''
-        return functools.partial(self.__call__, obj)
+        fn = functools.partial(self.__call__, obj)
+        fn.reset = self._reset
+        return fn
+
+    def _reset(self):
+        self.cache = {}
 
 
 def singleton(cls):
@@ -156,9 +161,21 @@ def singleton(cls):
     return get_instance
 
 
+@contextmanager
+def capture_stdout(output):
+    stdout = sys.stdout
+    sys.stdout = output
+    yield
+    sys.stdout = stdout
+
+
 def load_json(file_path):
-    with open(file_path, "r") as f:
-        return json.load(f)
+    try:
+        with open(file_path, "r") as f:
+            return json.load(f)
+    except ValueError:
+        raise exception.PlatformioException(
+            "Could not load broken JSON: %s" % file_path)
 
 
 def get_systype():
@@ -183,8 +200,8 @@ def get_project_optional_dir(name, default=None):
     else:
         try:
             config = load_project_config()
-            if (config.has_section("platformio") and
-                    config.has_option("platformio", name)):
+            if (config.has_section("platformio")
+                    and config.has_option("platformio", name)):
                 data = config.get("platformio", name)
         except exception.NotPlatformIOProject:
             pass
@@ -203,15 +220,19 @@ def get_project_optional_dir(name, default=None):
 def get_home_dir():
     home_dir = get_project_optional_dir("home_dir",
                                         join(expanduser("~"), ".platformio"))
-
+    win_home_dir = None
     if "windows" in get_systype():
-        try:
-            home_dir.encode("utf8")
-        except UnicodeDecodeError:
-            home_dir = splitdrive(home_dir)[0] + "\\.platformio"
+        win_home_dir = splitdrive(home_dir)[0] + "\\.platformio"
+        if isdir(win_home_dir):
+            home_dir = win_home_dir
 
     if not isdir(home_dir):
-        os.makedirs(home_dir)
+        try:
+            os.makedirs(home_dir)
+        except:  # pylint: disable=bare-except
+            if win_home_dir:
+                os.makedirs(win_home_dir)
+                home_dir = win_home_dir
 
     assert isdir(home_dir)
     return home_dir
@@ -261,8 +282,8 @@ def get_projectsrc_dir():
 
 
 def get_projecttest_dir():
-    return get_project_optional_dir("test_dir",
-                                    join(get_project_dir(), "test"))
+    return get_project_optional_dir("test_dir", join(get_project_dir(),
+                                                     "test"))
 
 
 def get_projectboards_dir():
@@ -290,8 +311,8 @@ URL=http://docs.platformio.org/page/projectconf.html#envs-dir
 
 
 def get_projectdata_dir():
-    return get_project_optional_dir("data_dir",
-                                    join(get_project_dir(), "data"))
+    return get_project_optional_dir("data_dir", join(get_project_dir(),
+                                                     "data"))
 
 
 def load_project_config(path=None):
@@ -306,6 +327,15 @@ def load_project_config(path=None):
     return cp
 
 
+def parse_conf_multi_values(items):
+    if not items:
+        return []
+    return [
+        item.strip() for item in items.split("\n" if "\n" in items else ", ")
+        if item.strip()
+    ]
+
+
 def change_filemtime(path, time):
     os.utime(path, (time, time))
 
@@ -456,13 +486,14 @@ def _get_api_result(
                 data=data,
                 headers=headers,
                 auth=auth,
-                verify=disable_ssl_check)
+                verify=not disable_ssl_check)
         else:
-            r = _api_request_session().get(url,
-                                           params=params,
-                                           headers=headers,
-                                           auth=auth,
-                                           verify=disable_ssl_check)
+            r = _api_request_session().get(
+                url,
+                params=params,
+                headers=headers,
+                auth=auth,
+                verify=not disable_ssl_check)
         result = r.json()
         r.raise_for_status()
     except requests.exceptions.HTTPError as e:
@@ -473,8 +504,8 @@ def _get_api_result(
         else:
             raise exception.APIRequestError(e)
     except ValueError:
-        raise exception.APIRequestError("Invalid response: %s" %
-                                        r.text.encode("utf-8"))
+        raise exception.APIRequestError(
+            "Invalid response: %s" % r.text.encode("utf-8"))
     finally:
         if r:
             r.close()
@@ -518,14 +549,15 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
 
 
 def internet_on(timeout=3):
-    host = "8.8.8.8"
-    port = 53
-    try:
-        socket.setdefaulttimeout(timeout)
-        socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
-        return True
-    except:  # pylint: disable=bare-except
-        return False
+    socket.setdefaulttimeout(timeout)
+    for host in ("dl.bintray.com", "dl.platformio.org"):
+        try:
+            socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host,
+                                                                       80))
+            return True
+        except:  # pylint: disable=bare-except
+            pass
+    return False
 
 
 def get_pythonexe_path():
@@ -558,7 +590,7 @@ def where_is_program(program, envpath=None):
 
 
 def pepver_to_semver(pepver):
-    return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2", pepver, 1)
+    return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1)
 
 
 def rmtree_(path):
@@ -568,3 +600,28 @@ def rmtree_(path):
         os.remove(name)
 
     return rmtree(path, onerror=_onerror)
+
+
+#
+# Glob.Escape from Python 3.4
+# https://github.com/python/cpython/blob/master/Lib/glob.py#L161
+#
+
+try:
+    from glob import escape as glob_escape  # pylint: disable=unused-import
+except ImportError:
+    magic_check = re.compile('([*?[])')
+    magic_check_bytes = re.compile(b'([*?[])')
+
+    def glob_escape(pathname):
+        """Escape all special characters.
+        """
+        # Escaping is done by wrapping any of "*?[" between square brackets.
+        # Metacharacters do not work in the drive part and shouldn't be
+        # escaped.
+        drive, pathname = os.path.splitdrive(pathname)
+        if isinstance(pathname, bytes):
+            pathname = magic_check_bytes.sub(br'[\1]', pathname)
+        else:
+            pathname = magic_check.sub(r'[\1]', pathname)
+        return drive + pathname

platformio/vcsclient.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,27 +19,28 @@ from sys import modules
 from urlparse import urlparse
 
 from platformio import util
-from platformio.exception import PlatformioException
+from platformio.exception import PlatformioException, UserSideException
 
 
 class VCSClientFactory(object):
 
     @staticmethod
-    def newClient(src_dir, remote_url):
+    def newClient(src_dir, remote_url, silent=False):
         result = urlparse(remote_url)
         type_ = result.scheme
+        tag = None
         if not type_ and remote_url.startswith("git@"):
             type_ = "git"
         elif "+" in result.scheme:
             type_, _ = result.scheme.split("+", 1)
             remote_url = remote_url[len(type_) + 1:]
-        if result.fragment:
-            remote_url = remote_url.rsplit("#", 1)[0]
+        if "#" in remote_url:
+            remote_url, tag = remote_url.rsplit("#", 1)
         if not type_:
-            raise PlatformioException("VCS: Unknown repository type %s" %
-                                      remote_url)
+            raise PlatformioException(
+                "VCS: Unknown repository type %s" % remote_url)
         obj = getattr(modules[__name__], "%sClient" % type_.title())(
-            src_dir, remote_url, result.fragment)
+            src_dir, remote_url, tag, silent)
         assert isinstance(obj, VCSClientBase)
         return obj
 
@@ -48,18 +49,22 @@ class VCSClientBase(object):
 
     command = None
 
-    def __init__(self, src_dir, remote_url=None, tag=None):
+    def __init__(self, src_dir, remote_url=None, tag=None, silent=False):
         self.src_dir = src_dir
         self.remote_url = remote_url
         self.tag = tag
+        self.silent = silent
         self.check_client()
 
     def check_client(self):
         try:
             assert self.command
-            assert self.run_cmd(["--version"])
-        except (AssertionError, OSError):
-            raise PlatformioException(
+            if self.silent:
+                self.get_cmd_output(["--version"])
+            else:
+                assert self.run_cmd(["--version"])
+        except (AssertionError, OSError, PlatformioException):
+            raise UserSideException(
                 "VCS: `%s` client is not installed in your system" %
                 self.command)
         return True
@@ -81,6 +86,9 @@ class VCSClientBase(object):
     def get_current_revision(self):
         raise NotImplementedError
 
+    def get_latest_revision(self):
+        return None if self.can_be_updated else self.get_current_revision()
+
     def run_cmd(self, args, **kwargs):
         args = [self.command] + args
         if "cwd" not in kwargs:
@@ -95,8 +103,8 @@ class VCSClientBase(object):
         if result['returncode'] == 0:
             return result['out'].strip()
         raise PlatformioException(
-            "VCS: Could not receive an output from `%s` command (%s)" % (
-                args, result))
+            "VCS: Could not receive an output from `%s` command (%s)" %
+            (args, result))
 
 
 class GitClient(VCSClientBase):
@@ -108,6 +116,16 @@ class GitClient(VCSClientBase):
         output = output.replace("*", "")  # fix active branch
         return [b.strip() for b in output.split("\n")]
 
+    def get_current_branch(self):
+        output = self.get_cmd_output(["branch"])
+        for line in output.split("\n"):
+            line = line.strip()
+            if line.startswith("*"):
+                branch = line[1:].strip()
+                if branch != "(no branch)":
+                    return branch
+        return None
+
     def get_tags(self):
         output = self.get_cmd_output(["tag", "-l"])
         return [t.strip() for t in output.split("\n")]
@@ -134,12 +152,25 @@ class GitClient(VCSClientBase):
         return True
 
     def update(self):
-        args = ["pull"]
+        args = ["pull", "--recurse-submodules"]
         return self.run_cmd(args)
 
     def get_current_revision(self):
         return self.get_cmd_output(["rev-parse", "--short", "HEAD"])
 
+    def get_latest_revision(self):
+        if not self.can_be_updated:
+            return self.get_current_revision()
+        branch = self.get_current_branch()
+        if not branch:
+            return self.get_current_revision()
+        result = self.get_cmd_output(["ls-remote"])
+        for line in result.split("\n"):
+            ref_pos = line.strip().find("refs/heads/" + branch)
+            if ref_pos > 0:
+                return line[:ref_pos].strip()[:7]
+        return None
+
 
 class HgClient(VCSClientBase):
 
@@ -159,6 +190,11 @@ class HgClient(VCSClientBase):
     def get_current_revision(self):
         return self.get_cmd_output(["identify", "--id"])
 
+    def get_latest_revision(self):
+        if not self.can_be_updated:
+            return self.get_latest_revision()
+        return self.get_cmd_output(["identify", "--id", self.remote_url])
+
 
 class SvnClient(VCSClientBase):
 
@@ -177,9 +213,8 @@ class SvnClient(VCSClientBase):
         return self.run_cmd(args)
 
     def get_current_revision(self):
-        output = self.get_cmd_output([
-            "info", "--non-interactive", "--trust-server-cert", "-r", "HEAD"
-        ])
+        output = self.get_cmd_output(
+            ["info", "--non-interactive", "--trust-server-cert", "-r", "HEAD"])
         for line in output.split("\n"):
             line = line.strip()
             if line.startswith("Revision:"):

scripts/98-openocd-udev.rules

@@ -0,0 +1,163 @@
+# UDEV Rules for debug adapters/boards supported by OpenOCD
+
+#
+# INSTALLATION
+#
+
+#
+# The latest version of this file may be found at:
+# 	https://github.com/platformio/platformio-core/blob/develop/scripts/98-openocd-udev.rules
+#
+# This file must be placed at:
+# 	/etc/udev/rules.d/98-openocd-udev.rules    (preferred location)
+#   	or
+# 	/lib/udev/rules.d/98-openocd-udev.rules    (req'd on some broken systems)
+#
+# To install, type this command in a terminal:
+# 	sudo cp 98-openocd-udev.rules /etc/udev/rules.d/98-openocd-udev.rules
+#
+# Restart "udev" management tool:
+#	sudo service udev restart
+#		or
+# 	sudo udevadm control --reload-rules
+#	sudo udevadm trigger
+#
+# Ubuntu/Debian users may need to add own “username” to the “dialout” group if
+# they are not “root”, doing this issuing a
+#   sudo usermod -a -G dialout $USER
+#   sudo usermod -a -G plugdev $USER
+#
+# After this file is installed, physically unplug and reconnect your adapter/board.
+
+ACTION!="add|change", GOTO="openocd_rules_end"
+SUBSYSTEM!="usb|tty|hidraw", GOTO="openocd_rules_end"
+
+# Please keep this list sorted by VID:PID
+
+# opendous and estick
+ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Original FT232/FT245 VID:PID
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Original FT2232 VID:PID
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Original FT4232 VID:PID
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Original FT232H VID:PID
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# DISTORTEC JTAG-lock-pick Tiny 2
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# TUMPA, TUMPA Lite
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# XDS100v2
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE)
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# TI/Luminary Stellaris Evaluation Board FTDI (several)
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# TI/Luminary Stellaris In-Circuit Debug Interface FTDI (ICDI) Board
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# egnite Turtelizer 2
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Section5 ICEbear
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Amontec JTAGkey and JTAGkey-tiny
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# TI ICDI
+ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# STLink v1
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# STLink v2
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# STLink v2-1
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Hilscher NXHX Boards
+ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Hitex STR9-comStick
+ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Hitex STM32-PerformanceStick
+ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Altera USB Blaster
+ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Amontec JTAGkey-HiSpeed
+ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# SEGGER J-Link
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Raisonance RLink
+ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Debug Board for Neo1973
+ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Olimex ARM-USB-OCD
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Olimex ARM-USB-OCD-TINY
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Olimex ARM-JTAG-EW
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Olimex ARM-USB-OCD-TINY-H
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Olimex ARM-USB-OCD-H
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# USBprog with OpenOCD firmware
+ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# TI/Luminary Stellaris In-Circuit Debug Interface (ICDI) Board
+ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Marvell Sheevaplug
+ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Keil Software, Inc. ULink
+ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# CMSIS-DAP compatible adapters
+ATTRS{product}=="*CMSIS-DAP*", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+LABEL="openocd_rules_end"

scripts/99-platformio-udev.rules

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#
+# INSTALLATION
+#
+
 # UDEV Rules for PlatformIO supported boards, http://platformio.org/boards
 #
 # The latest version of this file may be found at:
@@ -31,6 +35,11 @@
 # 	sudo udevadm control --reload-rules
 #	sudo udevadm trigger
 #
+# Ubuntu/Debian users may need to add own “username” to the “dialout” group if
+# they are not “root”, doing this issuing a
+#   sudo usermod -a -G dialout $USER
+#   sudo usermod -a -G plugdev $USER
+#
 # After this file is installed, physically unplug and reconnect your board.
 
 # CP210X USB UART
@@ -65,6 +74,9 @@ SUBSYSTEMS=="usb", ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374?", MODE:="066
 # USBtiny
 SUBSYSTEMS=="usb", ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE="0666"
 
+# USBasp V2.0
+SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="05dc", MODE:="0666"
+
 # Teensy boards
 ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", ENV{ID_MM_DEVICE_IGNORE}="1"
 ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", ENV{MTP_NO_PROBE}="1"

scripts/docspregen.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,7 +36,7 @@ def is_compat_platform_and_framework(platform, framework):
     return False
 
 
-def generate_boards(boards):
+def generate_boards(boards, extend_debug=False):
 
     def _round_memory_size(size):
         if size == 1:
@@ -48,6 +48,8 @@ def generate_boards(boards):
                 return int(ceil(size / b) * b)
         assert NotImplemented()
 
+    platforms = {m['name']: m['title'] for m in PLATFORM_MANIFESTS}
+
     lines = []
 
     lines.append("""
@@ -56,22 +58,44 @@ def generate_boards(boards):
 
     * - ID
       - Name
+      - Platform
+      - Debug
       - Microcontroller
       - Frequency
       - Flash
       - RAM""")
 
     for data in sorted(boards, key=lambda item: item['id']):
+        debug = [":ref:`Yes <piodebug>`" if data['debug'] else ""]
+        if extend_debug and data['debug']:
+            debug = []
+            for name, options in data['debug']['tools'].items():
+                attrs = []
+                if options.get("default"):
+                    attrs.append("default")
+                if options.get("onboard"):
+                    attrs.append("on-board")
+                tool = ":ref:`debugging_tool_%s`" % name
+                if attrs:
+                    debug.append("%s (%s)" % (tool, ", ".join(attrs)))
+                else:
+                    debug.append(tool)
+
         board_ram = float(data['ram']) / 1024
         lines.append("""
     * - ``{id}``
       - `{name} <{url}>`_
+      - :ref:`{platform_title} <platform_{platform}>`
+      - {debug}
       - {mcu}
       - {f_cpu:d} MHz
       - {rom} Kb
       - {ram} Kb""".format(
             id=data['id'],
             name=data['name'],
+            platform=data['platform'],
+            platform_title=platforms[data['platform']],
+            debug=", ".join(debug),
             url=data['url'],
             mcu=data['mcu'].upper(),
             f_cpu=int(data['fcpu']) / 1000000,
@@ -93,45 +117,44 @@ Packages
     :header-rows:  1
 
     * - Name
-      - Contents""")
+      - Description""")
     for name in sorted(packagenames):
         assert name in API_PACKAGES, name
-        contitems = [
-            "`{name} <{url}>`_".format(**item) for item in API_PACKAGES[name]
-        ]
         lines.append("""
-    * - ``{name}``
-      - {contents}""".format(
-            name=name, contents=", ".join(contitems)))
+    * - `{name} <{url}>`__
+      - {description}""".format(
+            name=name,
+            url=API_PACKAGES[name]['url'],
+            description=API_PACKAGES[name]['description']))
 
     if is_embedded:
         lines.append("""
 .. warning::
     **Linux Users**:
 
-    * Ubuntu/Debian users may need to add own "username" to the "dialout"
-      group if they are not "root", doing this issuing a
-      ``sudo usermod -a -G dialout yourusername``.
-    * Install "udev" rules file `99-platformio-udev.rules <https://github.com/platformio/platformio-core/blob/develop/scripts/99-platformio-udev.rules>`_
-      (an instruction is located in the file).
-    * Raspberry Pi users, please read this article
-      `Enable serial port on Raspberry Pi <https://hallard.me/enable-serial-port-on-raspberry-pi/>`__.
+        * Install "udev" rules file `99-platformio-udev.rules <https://github.com/platformio/platformio-core/blob/develop/scripts/99-platformio-udev.rules>`_
+          (an instruction is located inside a file).
+        * Raspberry Pi users, please read this article
+          `Enable serial port on Raspberry Pi <https://hallard.me/enable-serial-port-on-raspberry-pi/>`__.
 """)
 
         if platform == "teensy":
             lines.append("""
-    **Windows Users:** Teensy programming uses only Windows built-in HID
-    drivers. When Teensy is programmed to act as a USB Serial device,
-    Windows XP, Vista, 7 and 8 require `this serial driver
-    <http://www.pjrc.com/teensy/serial_install.exe>`_
-    is needed to access the COM port your program uses. No special driver
-    installation is necessary on Windows 10.
+    **Windows Users:**
+
+        Teensy programming uses only Windows built-in HID
+        drivers. When Teensy is programmed to act as a USB Serial device,
+        Windows XP, Vista, 7 and 8 require `this serial driver
+        <http://www.pjrc.com/teensy/serial_install.exe>`_
+        is needed to access the COM port your program uses. No special driver
+        installation is necessary on Windows 10.
 """)
         else:
             lines.append("""
-    **Windows Users:** Please check that you have correctly installed USB
-    driver from board manufacturer
+    **Windows Users:**
 
+        Please check that you have a correctly installed USB driver from board
+        manufacturer
 """)
 
     return "\n".join(lines)
@@ -142,7 +165,7 @@ def generate_platform(name):
     lines = []
 
     lines.append(
-        """..  Copyright 2014-present PlatformIO <contact@platformio.org>
+        """..  Copyright (c) 2014-present PlatformIO <contact@platformio.org>
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at
@@ -153,27 +176,28 @@ def generate_platform(name):
     See the License for the specific language governing permissions and
     limitations under the License.
 """)
+    p = PlatformFactory.newPlatform(name)
 
-    lines.append(".. _platform_%s:" % name)
+    lines.append(".. _platform_%s:" % p.name)
     lines.append("")
 
-    _title = "Platform ``%s``" % name
-    lines.append(_title)
-    lines.append("=" * len(_title))
-
-    p = PlatformFactory.newPlatform(name)
+    lines.append(p.title)
+    lines.append("=" * len(p.title))
+    lines.append(":ref:`projectconf_env_platform` = ``%s``" % p.name)
+    lines.append("")
     lines.append(p.description)
     lines.append("""
 For more detailed information please visit `vendor site <%s>`_.""" %
                  p.vendor_url)
     lines.append("""
-.. contents::""")
+.. contents:: Contents
+    :local:""")
 
     #
     # Packages
     #
-    _packages_content = generate_packages(name, p.packages.keys(),
-                                          p.is_embedded())
+    _packages_content = generate_packages(name,
+                                          p.packages.keys(), p.is_embedded())
     if _packages_content:
         lines.append(_packages_content)
 
@@ -248,7 +272,7 @@ def generate_framework(type_, data):
     lines = []
 
     lines.append(
-        """..  Copyright 2014-present PlatformIO <contact@platformio.org>
+        """..  Copyright (c) 2014-present PlatformIO <contact@platformio.org>
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at
@@ -263,15 +287,18 @@ def generate_framework(type_, data):
     lines.append(".. _framework_%s:" % type_)
     lines.append("")
 
-    _title = "Framework ``%s``" % type_
-    lines.append(_title)
-    lines.append("=" * len(_title))
+    lines.append(data['title'])
+    lines.append("=" * len(data['title']))
+    lines.append(":ref:`projectconf_env_framework` = ``%s``" % type_)
+    lines.append("")
     lines.append(data['description'])
     lines.append("""
 For more detailed information please visit `vendor site <%s>`_.
 """ % data['url'])
 
-    lines.append(".. contents::")
+    lines.append("""
+.. contents:: Contents
+    :local:""")
 
     lines.append("""
 Platforms
@@ -288,10 +315,11 @@ Platforms
             continue
         _found_platform = True
         p = PlatformFactory.newPlatform(manifest['name'])
-        lines.append("""
+        lines.append(
+            """
     * - :ref:`platform_{type_}`
-      - {description}""".format(
-            type_=manifest['name'], description=p.description))
+      - {description}"""
+            .format(type_=manifest['name'], description=p.description))
     if not _found_platform:
         del lines[-1]
 
@@ -347,19 +375,21 @@ Packages
     :header-rows:  1
 
     * - Name
-      - Contents""")
+      - Description""")
     for name, items in sorted(API_PACKAGES.iteritems()):
-        contitems = ["`{name} <{url}>`_".format(**item) for item in items]
         lines.append("""
-    * - ``{name}``
-      - {contents}""".format(
-            name=name, contents=", ".join(contitems)))
+    * - `{name} <{url}>`__
+      - {description}""".format(
+            name=name,
+            url=API_PACKAGES[name]['url'],
+            description=API_PACKAGES[name]['description']))
 
     with open(
             join(util.get_source_dir(), "..", "docs", "platforms",
                  "creating_platform.rst"), "r+") as fp:
         content = fp.read()
-        fp.seek(0, 0)
+        fp.seek(0)
+        fp.truncate()
         fp.write(content[:content.index(".. _platform_creating_packages:")] +
                  "\n".join(lines) + "\n\n" + content[content.index(
                      ".. _platform_creating_manifest_file:"):])
@@ -369,7 +399,7 @@ def update_embedded_boards():
     lines = []
 
     lines.append(
-        """..  Copyright 2014-present PlatformIO <contact@platformio.org>
+        """..  Copyright (c) 2014-present PlatformIO <contact@platformio.org>
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
     You may obtain a copy of the License at
@@ -392,14 +422,16 @@ Rapid Embedded Development, Continuous and IDE integration in a few
 steps with PlatformIO thanks to built-in project generator for the most
 popular embedded boards and IDE.
 
-* You can list pre-configured boards using :ref:`cmd_boards` command or
-  `PlatformIO Boards Explorer <http://platformio.org/boards>`_
-* For more detailed ``board`` information please scroll tables below by
-  horizontal.
+.. note::
+    * You can list pre-configured boards by :ref:`cmd_boards` command or
+      `PlatformIO Boards Explorer <http://platformio.org/boards>`_
+    * For more detailed ``board`` information please scroll tables below by horizontal.
 """)
 
-    lines.append(".. contents::")
-    lines.append("")
+    lines.append("""
+.. contents:: Vendors
+    :local:
+    """)
 
     vendors = {}
     for data in BOARDS:
@@ -421,11 +453,88 @@ popular embedded boards and IDE.
         f.write("\n".join(lines))
 
 
+def update_debugging():
+    vendors = {}
+    platforms = []
+    frameworks = []
+    for data in BOARDS:
+        if not data['debug']:
+            continue
+        platforms.append(data['platform'])
+        frameworks.extend(data['frameworks'])
+        vendor = data['vendor']
+        if vendor in vendors:
+            vendors[vendor].append(data)
+        else:
+            vendors[vendor] = [data]
+
+    lines = []
+    # Platforms
+    lines.append(""".. _debugging_platforms:
+
+Platforms
+---------
+.. list-table::
+    :header-rows:  1
+
+    * - Name
+      - Description""")
+
+    for manifest in PLATFORM_MANIFESTS:
+        if manifest['name'] not in platforms:
+            continue
+        p = PlatformFactory.newPlatform(manifest['name'])
+        lines.append(
+            """
+    * - :ref:`platform_{type_}`
+      - {description}"""
+            .format(type_=manifest['name'], description=p.description))
+
+    # Frameworks
+    lines.append("""
+Frameworks
+----------
+.. list-table::
+    :header-rows:  1
+
+    * - Name
+      - Description""")
+    for framework in API_FRAMEWORKS:
+        if framework['name'] not in frameworks:
+            continue
+        lines.append("""
+    * - :ref:`framework_{name}`
+      - {description}""".format(**framework))
+
+    # Boards
+    lines.append("""
+Boards
+------
+
+.. note::
+    For more detailed ``board`` information please scroll tables below by horizontal.
+""")
+    for vendor, boards in sorted(vendors.iteritems()):
+        lines.append(str(vendor))
+        lines.append("~" * len(vendor))
+        lines.append(generate_boards(boards, extend_debug=True))
+
+    with open(
+            join(util.get_source_dir(), "..", "docs", "plus",
+                 "debugging.rst"), "r+") as fp:
+        content = fp.read()
+        fp.seek(0)
+        fp.truncate()
+        fp.write(content[:content.index(".. _debugging_platforms:")] +
+                 "\n".join(lines))
+
+
 def main():
     update_create_platform_doc()
     update_platform_docs()
     update_framework_docs()
     update_embedded_boards()
+    update_debugging()
 
 
 if __name__ == "__main__":

scripts/fixsymlink.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

scripts/get-platformio.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -112,27 +112,25 @@ def install_pip():
 
 def install_platformio():
     r = None
-    cmd = ["-m", "pip.__main__" if sys.version_info < (2, 7, 0) else "pip"]
+    cmd = ["pip", "install", "-U", "platformio"]
+    # cmd = [
+    #     "pip", "install", "-U",
+    #     "https://github.com/platformio/platformio-core/archive/develop.zip"
+    # ]
     try:
-        # r = exec_python_cmd(cmd + ["install", "-U", "platformio"])
-        r = exec_python_cmd(cmd + [
-            "install", "-U",
-            "https://github.com/platformio/platformio-core/archive/develop.zip"
-        ])
+        r = exec_python_cmd(cmd)
         assert r['returncode'] == 0
     except AssertionError:
-        r = exec_python_cmd(cmd + ["--no-cache-dir", "install", "-U",
-                                   "platformio"])
+        cmd.insert(1, "--no-cache-dir")
+        r = exec_python_cmd(cmd)
     if r:
         print_exec_result(r)
 
 
 def main():
-    steps = [
-        ("Fixing Windows %PATH% Environment", fix_winpython_pathenv),
-        ("Installing Python Package Manager", install_pip),
-        ("Installing PlatformIO and dependencies", install_platformio)
-    ]
+    steps = [("Fixing Windows %PATH% Environment", fix_winpython_pathenv),
+             ("Installing Python Package Manager", install_pip),
+             ("Installing PlatformIO and dependencies", install_platformio)]
 
     if not IS_WINDOWS:
         del steps[0]
@@ -161,7 +159,7 @@ Permission denied
 You need the `sudo` permission to install Python packages. Try
 
 $ sudo python -c "$(curl -fsSL
-https://raw.githubusercontent.com/platformio/platformio/master/scripts/get-platformio.py)"
+https://raw.githubusercontent.com/platformio/platformio/develop/scripts/get-platformio.py)"
 """)
 
     if is_error:

setup.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,13 +18,14 @@ from platformio import (__author__, __description__, __email__, __license__,
                         __title__, __url__, __version__)
 
 install_requires = [
+    "arrow<1",
     "bottle<0.13",
     "click>=5,<6",
-    "lockfile>=0.9.1,<0.13",
-    "requests>=2.4.0,<3",
-    "semantic_version>=2.5.0",
     "colorama",
-    "pyserial>=3,<4"
+    "lockfile>=0.9.1,<0.13",
+    "pyserial>=3,<4,!=3.3",
+    "requests>=2.4.0,<3",
+    "semantic_version>=2.5.0"
 ]
 
 setup(
@@ -50,6 +51,7 @@ setup(
     entry_points={
         "console_scripts": [
             "pio = platformio.__main__:main",
+            "piodebuggdb = platformio.__main__:debug_gdb_main",
             "platformio = platformio.__main__:main"
         ]
     },
@@ -66,9 +68,8 @@ setup(
         "Topic :: Software Development :: Compilers"
     ],
     keywords=[
-        "iot", "ide", "build", "compile", "library manager",
-        "embedded", "ci", "continuous integration", "arduino", "mbed",
-        "esp8266", "framework", "ide", "ide integration", "library.json",
-        "make", "cmake", "makefile", "mk", "pic32", "fpga"
-    ]
-)
+        "iot", "embedded", "arduino", "mbed", "esp8266", "esp32", "fpga",
+        "firmware", "continuous-integration", "cloud-ide", "avr", "arm",
+        "ide", "unit-testing", "hardware", "verilog", "microcontroller",
+        "debug"
+    ])

tests/commands/test_boards.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

tests/commands/test_ci.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,15 +25,14 @@ def test_ci_empty(clirunner):
 
 def test_ci_boards(clirunner, validate_cliresult):
     result = clirunner.invoke(cmd_ci, [
-        join("examples", "atmelavr-and-arduino", "arduino-internal-libs",
-             "src", "ChatServer.ino"), "-b", "uno", "-b", "leonardo"
+        join("examples", "atmelavr", "arduino-internal-libs", "src",
+             "ChatServer.ino"), "-b", "uno", "-b", "leonardo"
     ])
     validate_cliresult(result)
 
 
 def test_ci_project_conf(clirunner, validate_cliresult):
-    project_dir = join("examples", "atmelavr-and-arduino",
-                       "arduino-internal-libs")
+    project_dir = join("examples", "atmelavr", "arduino-internal-libs")
     result = clirunner.invoke(cmd_ci, [
         join(project_dir, "src", "ChatServer.ino"), "--project-conf",
         join(project_dir, "platformio.ini")
@@ -43,8 +42,7 @@ def test_ci_project_conf(clirunner, validate_cliresult):
 
 
 def test_ci_lib_and_board(clirunner, validate_cliresult):
-    example_dir = join("examples", "atmelavr-and-arduino",
-                       "arduino-external-libs")
+    example_dir = join("examples", "atmelavr", "arduino-external-libs")
     result = clirunner.invoke(cmd_ci, [
         join(example_dir, "lib", "OneWire", "examples", "DS2408_Switch",
              "DS2408_Switch.pde"), "-l", join(example_dir, "lib", "OneWire"),

tests/commands/test_init.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -73,7 +73,7 @@ def test_init_ide_atom(clirunner, validate_cliresult, tmpdir):
 
         # switch to NodeMCU
         result = clirunner.invoke(
-            cmd_init, ["--ide", "atom", "-b", "nodemcuv2", "-b", "uno"])
+            cmd_init, ["--ide", "atom", "-b", "nodemcuv2"])
         validate_cliresult(result)
         validate_pioproject(str(tmpdir))
         assert "arduinoespressif" in tmpdir.join(".clang_complete").read()

tests/commands/test_lib.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ import json
 import re
 from os.path import basename
 
-from platformio import util
+from platformio import exception, util
 from platformio.commands.init import cli as cmd_init
 from platformio.commands.lib import cli as cmd_lib
 
@@ -37,15 +37,35 @@ def test_search(clirunner, validate_cliresult):
 def test_global_install_registry(clirunner, validate_cliresult,
                                  isolated_pio_home):
     result = clirunner.invoke(cmd_lib, [
-        "-g", "install", "58", "OneWire",
+        "-g", "install", "58", "547@2.2.4", "DallasTemperature",
         "http://dl.platformio.org/libraries/archives/3/5174.tar.gz",
-        "ArduinoJson@5.6.7", "ArduinoJson@>5.6"
+        "ArduinoJson@5.6.7", "ArduinoJson@~5.7.0", "1089@fee16e880b"
     ])
     validate_cliresult(result)
+
+    # check lib with duplicate URL
+    result = clirunner.invoke(cmd_lib, [
+        "-g", "install",
+        "http://dl.platformio.org/libraries/archives/3/5174.tar.gz"
+    ])
+    validate_cliresult(result)
+    assert "is already installed" in result.output
+
+    # check lib with duplicate ID
+    result = clirunner.invoke(cmd_lib, ["-g", "install", "305"])
+    validate_cliresult(result)
+    assert "is already installed" in result.output
+
+    # install unknown library
+    result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"])
+    assert result.exit_code != 0
+    assert isinstance(result.exception, exception.LibNotFound)
+
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
     items2 = [
-        "DHT22_ID58", "ArduinoJson_ID64", "ArduinoJson_ID64@5.6.7",
-        "OneWire_ID1", "ESPAsyncTCP_ID305"
+        "ArduinoJson_ID64", "ArduinoJson_ID64@5.6.7", "DallasTemperature_ID54",
+        "DHT22_ID58", "ESPAsyncTCP_ID305", "NeoPixelBus_ID547", "OneWire_ID1",
+        "IRremoteESP8266_ID1089"
     ]
     assert set(items1) == set(items2)
 
@@ -55,11 +75,29 @@ def test_global_install_archive(clirunner, validate_cliresult,
     result = clirunner.invoke(cmd_lib, [
         "-g", "install", "https://github.com/adafruit/Adafruit-ST7735-Library/"
         "archive/master.zip",
+        "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip",
+        "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip",
+        "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2"
+    ])
+    validate_cliresult(result)
+
+    # incorrect requirements
+    result = clirunner.invoke(cmd_lib, [
+        "-g", "install",
+        "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@1.2.3"
+    ])
+    assert result.exit_code != 0
+
+    # check lib with duplicate URL
+    result = clirunner.invoke(cmd_lib, [
+        "-g", "install",
         "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip"
     ])
     validate_cliresult(result)
+    assert "is already installed" in result.output
+
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
-    items2 = ["Adafruit ST7735 Library", "RadioHead"]
+    items2 = ["Adafruit ST7735 Library", "RadioHead-1.62"]
     assert set(items1) >= set(items2)
 
 
@@ -71,14 +109,29 @@ def test_global_install_repository(clirunner, validate_cliresult,
             "-g",
             "install",
             "https://github.com/gioblu/PJON.git#3.0",
+            "https://github.com/gioblu/PJON.git#6.2",
+            "https://github.com/bblanchon/ArduinoJson.git",
             "https://gitlab.com/ivankravets/rs485-nodeproto.git",
+            "https://github.com/platformio/platformio-libmirror.git",
             # "https://developer.mbed.org/users/simon/code/TextLCD/",
             "knolleary/pubsubclient"
         ])
     validate_cliresult(result)
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
-    items2 = ["PJON", "ESPAsyncTCP", "PubSubClient"]
-    assert set(items2) & set(items1)
+    items2 = [
+        "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d",
+        "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "rs485-nodeproto",
+        "PubSubClient"
+    ]
+    assert set(items1) >= set(items2)
+
+    # check lib with duplicate URL
+    result = clirunner.invoke(cmd_lib, [
+        "-g", "install",
+        "https://github.com/platformio/platformio-libmirror.git"
+    ])
+    validate_cliresult(result)
+    assert "is already installed" in result.output
 
 
 def test_global_lib_list(clirunner, validate_cliresult, isolated_pio_home):
@@ -89,81 +142,120 @@ def test_global_lib_list(clirunner, validate_cliresult, isolated_pio_home):
     result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
     assert all([
         n in result.output
-        for n in ("PJON", "git+https://github.com/knolleary/pubsubclient")
+        for n in ("PJON", "git+https://github.com/knolleary/pubsubclient",
+                  "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip"
+                  )
     ])
     items1 = [i['name'] for i in json.loads(result.output)]
     items2 = [
         "OneWire", "DHT22", "PJON", "ESPAsyncTCP", "ArduinoJson",
-        "pubsubclient", "rs485-nodeproto", "Adafruit ST7735 Library",
-        "RadioHead"
+        "PubSubClient", "rs485-nodeproto", "Adafruit ST7735 Library",
+        "RadioHead-1.62", "DallasTemperature", "NeoPixelBus",
+        "IRremoteESP8266", "platformio-libmirror"
     ]
     assert set(items1) == set(items2)
 
 
-def test_global_lib_show(clirunner, validate_cliresult, isolated_pio_home):
-    result = clirunner.invoke(cmd_lib, ["-g", "show", "64@5.6.7"])
+def test_global_lib_update_check(clirunner, validate_cliresult,
+                                 isolated_pio_home):
+    result = clirunner.invoke(
+        cmd_lib, ["-g", "update", "--only-check", "--json-output"])
     validate_cliresult(result)
-    assert all([
-        s in result.output for s in ("Json", "arduino", "atmelavr", "5.6.7")
-    ])
-
-    result = clirunner.invoke(cmd_lib, ["-g", "show", "ArduinoJson@>5.6.7"])
-    validate_cliresult(result)
-    assert all(
-        [s in result.output for s in ("ArduinoJson", "arduino", "atmelavr")])
-    assert "5.6.7" not in result.output
-
-    result = clirunner.invoke(cmd_lib, ["-g", "show", "1"])
-    validate_cliresult(result)
-    assert "OneWire" in result.output
+    output = json.loads(result.output)
+    assert set(["ArduinoJson", "IRremoteESP8266", "NeoPixelBus"]) == set(
+        [l['name'] for l in output])
 
 
 def test_global_lib_update(clirunner, validate_cliresult, isolated_pio_home):
+    # update library using package directory
+    result = clirunner.invoke(
+        cmd_lib,
+        ["-g", "update", "NeoPixelBus", "--only-check", "--json-output"])
+    validate_cliresult(result)
+    oudated = json.loads(result.output)
+    assert len(oudated) == 1
+    assert "__pkg_dir" in oudated[0]
+    result = clirunner.invoke(cmd_lib,
+                              ["-g", "update", oudated[0]['__pkg_dir']])
+    validate_cliresult(result)
+    assert "Uninstalling NeoPixelBus @ 2.2.4" in result.output
+
+    # update rest libraries
     result = clirunner.invoke(cmd_lib, ["-g", "update"])
     validate_cliresult(result)
-    assert all([s in result.output for s in ("[Up-to-date]", "[VCS]")])
+    validate_cliresult(result)
+    assert result.output.count("[Skip]") == 5
+    assert result.output.count("[Up-to-date]") == 10
+    assert "Uninstalling ArduinoJson @ 5.7.3" in result.output
+    assert "Uninstalling IRremoteESP8266 @ fee16e880b" in result.output
+
+    # update unknown library
+    result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"])
+    assert result.exit_code != 0
+    assert isinstance(result.exception, exception.UnknownPackage)
 
 
 def test_global_lib_uninstall(clirunner, validate_cliresult,
                               isolated_pio_home):
+    # uninstall using package directory
+    result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
+    validate_cliresult(result)
+    items = json.loads(result.output)
+    result = clirunner.invoke(cmd_lib,
+                              ["-g", "uninstall", items[0]['__pkg_dir']])
+    validate_cliresult(result)
+    assert "Uninstalling Adafruit ST7735 Library" in result.output
+
+    # uninstall the rest libraries
     result = clirunner.invoke(cmd_lib, [
-        "-g", "uninstall", "1", "ArduinoJson@!=5.6.7", "TextLCD",
-        "Adafruit ST7735 Library"
+        "-g", "uninstall", "1", "ArduinoJson@!=5.6.7",
+        "https://github.com/bblanchon/ArduinoJson.git", "IRremoteESP8266@>=0.2"
     ])
     validate_cliresult(result)
+
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
     items2 = [
-        "DHT22_ID58", "ArduinoJson_ID64@5.6.7", "ESPAsyncTCP_ID305",
-        "pubsubclient", "PJON", "rs485-nodeproto", "RadioHead_ID124"
+        "ArduinoJson", "ArduinoJson_ID64@5.6.7", "DallasTemperature_ID54",
+        "DHT22_ID58", "ESPAsyncTCP_ID305", "NeoPixelBus_ID547", "PJON",
+        "PJON@src-79de467ebe19de18287becff0a1fb42d", "PubSubClient",
+        "RadioHead-1.62", "rs485-nodeproto", "platformio-libmirror"
     ]
     assert set(items1) == set(items2)
 
+    # uninstall unknown library
+    result = clirunner.invoke(cmd_lib, ["-g", "uninstall", "Unknown"])
+    assert result.exit_code != 0
+    assert isinstance(result.exception, exception.UnknownPackage)
 
-def test_project_lib_complex(clirunner, validate_cliresult, tmpdir):
-    with tmpdir.as_cwd():
-        # init
-        result = clirunner.invoke(cmd_init)
-        validate_cliresult(result)
 
-        # isntall
-        result = clirunner.invoke(cmd_lib, ["install", "54", "ArduinoJson"])
-        validate_cliresult(result)
-        items1 = [
-            d.basename
-            for d in tmpdir.join(basename(util.get_projectlibdeps_dir()))
-            .listdir()
-        ]
-        items2 = ["DallasTemperature_ID54", "OneWire_ID1", "ArduinoJson_ID64"]
-        assert set(items1) == set(items2)
+def test_lib_show(clirunner, validate_cliresult, isolated_pio_home):
+    result = clirunner.invoke(cmd_lib, ["show", "64"])
+    validate_cliresult(result)
+    assert all(
+        [s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")])
+    result = clirunner.invoke(cmd_lib, ["show", "OneWire"])
+    validate_cliresult(result)
+    assert "OneWire" in result.output
 
-        # list
-        result = clirunner.invoke(cmd_lib, ["list", "--json-output"])
-        validate_cliresult(result)
-        items1 = [i['name'] for i in json.loads(result.output)]
-        items2 = ["DallasTemperature", "OneWire", "ArduinoJson"]
-        assert set(items1) == set(items2)
 
-        # update
-        result = clirunner.invoke(cmd_lib, ["update"])
-        validate_cliresult(result)
-        assert "[Up-to-date]" in result.output
+def test_lib_builtin(clirunner, validate_cliresult, isolated_pio_home):
+    result = clirunner.invoke(cmd_lib, ["builtin"])
+    validate_cliresult(result)
+    result = clirunner.invoke(cmd_lib, ["builtin", "--json-output"])
+    validate_cliresult(result)
+
+
+def test_lib_stats(clirunner, validate_cliresult, isolated_pio_home):
+    result = clirunner.invoke(cmd_lib, ["stats"])
+    validate_cliresult(result)
+    assert all([
+        s in result.output
+        for s in ("UPDATED", "ago", "http://platformio.org/lib/show")
+    ])
+
+    result = clirunner.invoke(cmd_lib, ["stats", "--json-output"])
+    validate_cliresult(result)
+    assert set([
+        "dlweek", "added", "updated", "topkeywords", "dlmonth", "dlday",
+        "lastkeywords"
+    ]) == set(json.loads(result.output).keys())

tests/commands/test_platform.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,30 +13,12 @@
 # limitations under the License.
 
 import json
-import os
-from os.path import join
 
-from platformio import exception, util
+from platformio import exception
 from platformio.commands import platform as cli_platform
 
 
-def test_list_json_output(clirunner, validate_cliresult):
-    result = clirunner.invoke(cli_platform.platform_list, ["--json-output"])
-    validate_cliresult(result)
-    list_result = json.loads(result.output)
-    assert isinstance(list_result, list)
-    assert len(list_result)
-    platforms = [item['name'] for item in list_result]
-    assert "titiva" in platforms
-
-
-def test_list_raw_output(clirunner, validate_cliresult):
-    result = clirunner.invoke(cli_platform.platform_list)
-    validate_cliresult(result)
-    assert "teensy" in result.output
-
-
-def test_search_json_output(clirunner, validate_cliresult):
+def test_search_json_output(clirunner, validate_cliresult, isolated_pio_home):
     result = clirunner.invoke(cli_platform.platform_search,
                               ["arduino", "--json-output"])
     validate_cliresult(result)
@@ -47,73 +29,86 @@ def test_search_json_output(clirunner, validate_cliresult):
     assert "atmelsam" in platforms
 
 
-def test_search_raw_output(clirunner, validate_cliresult):
+def test_search_raw_output(clirunner, validate_cliresult, isolated_pio_home):
     result = clirunner.invoke(cli_platform.platform_search, ["arduino"])
     validate_cliresult(result)
     assert "teensy" in result.output
 
 
-def test_install_uknown_from_registry(clirunner, validate_cliresult):
-    result = clirunner.invoke(cli_platform.platform_install,
-                              ["uknown-platform"])
-    assert result.exit_code == -1
-    assert isinstance(result.exception, exception.UnknownPackage)
-
-
-def test_install_uknown_version(clirunner, validate_cliresult):
+def test_install_unknown_version(clirunner, validate_cliresult,
+                                 isolated_pio_home):
     result = clirunner.invoke(cli_platform.platform_install,
                               ["atmelavr@99.99.99"])
     assert result.exit_code == -1
     assert isinstance(result.exception, exception.UndefinedPackageVersion)
 
 
-def test_complex(clirunner, validate_cliresult):
-    with clirunner.isolated_filesystem():
-        os.environ["PLATFORMIO_HOME_DIR"] = os.getcwd()
-        try:
-            result = clirunner.invoke(
-                cli_platform.platform_install,
-                ["teensy", "--with-package", "framework-arduinoteensy"])
-            validate_cliresult(result)
-            assert all([
-                s in result.output
-                for s in ("teensy", "Downloading", "Unpacking")
-            ])
+def test_install_unknown_from_registry(clirunner, validate_cliresult,
+                                       isolated_pio_home):
+    result = clirunner.invoke(cli_platform.platform_install,
+                              ["unknown-platform"])
+    assert result.exit_code == -1
+    assert isinstance(result.exception, exception.UnknownPackage)
 
-            # show platform information
-            result = clirunner.invoke(cli_platform.platform_show, ["teensy"])
-            validate_cliresult(result)
-            assert "teensy" in result.output
 
-            # list platforms
-            result = clirunner.invoke(cli_platform.platform_list,
-                                      ["--json-output"])
-            validate_cliresult(result)
-            list_result = json.loads(result.output)
-            assert isinstance(list_result, list)
-            assert len(list_result) == 1
-            assert list_result[0]["name"] == "teensy"
-            assert list_result[0]["packages"] == ["framework-arduinoteensy"]
+def test_install_known_version(clirunner, validate_cliresult,
+                               isolated_pio_home):
+    result = clirunner.invoke(cli_platform.platform_install, [
+        "atmelavr@1.1.0", "--skip-default-package", "--with-package",
+        "tool-avrdude"
+    ])
+    validate_cliresult(result)
+    assert "atmelavr @ 1.1.0" in result.output
+    assert "Installing tool-avrdude @" in result.output
+    assert len(isolated_pio_home.join("packages").listdir()) == 1
 
-            # try to install again
-            result = clirunner.invoke(cli_platform.platform_install,
-                                      ["teensy"])
-            validate_cliresult(result)
-            assert "is already installed" in result.output
 
-            # try to update
-            for _ in range(2):
-                result = clirunner.invoke(cli_platform.platform_update)
-                validate_cliresult(result)
-                assert "teensy" in result.output
-                assert "Up-to-date" in result.output
-                assert "Out-of-date" not in result.output
+def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home):
+    result = clirunner.invoke(cli_platform.platform_install, [
+        "https://github.com/platformio/"
+        "platform-espressif8266.git#feature/stage", "--skip-default-package"
+    ])
+    validate_cliresult(result)
+    assert "espressif8266_stage" in result.output
 
-            # try to uninstall
-            result = clirunner.invoke(cli_platform.platform_uninstall,
-                                      ["teensy"])
-            validate_cliresult(result)
-            for folder in ("platforms", "packages"):
-                assert len(os.listdir(join(util.get_home_dir(), folder))) == 0
-        finally:
-            del os.environ["PLATFORMIO_HOME_DIR"]
+
+def test_list_json_output(clirunner, validate_cliresult, isolated_pio_home):
+    result = clirunner.invoke(cli_platform.platform_list, ["--json-output"])
+    validate_cliresult(result)
+    list_result = json.loads(result.output)
+    assert isinstance(list_result, list)
+    assert len(list_result)
+    platforms = [item['name'] for item in list_result]
+    assert set(["atmelavr", "espressif8266_stage"]) == set(platforms)
+
+
+def test_list_raw_output(clirunner, validate_cliresult, isolated_pio_home):
+    result = clirunner.invoke(cli_platform.platform_list)
+    validate_cliresult(result)
+    assert all(
+        [s in result.output for s in ("atmelavr", "espressif8266_stage")])
+
+
+def test_update_check(clirunner, validate_cliresult, isolated_pio_home):
+    result = clirunner.invoke(cli_platform.platform_update,
+                              ["--only-check", "--json-output"])
+    validate_cliresult(result)
+    output = json.loads(result.output)
+    assert len(output) == 1
+    assert output[0]['name'] == "atmelavr"
+    assert len(isolated_pio_home.join("packages").listdir()) == 1
+
+
+def test_update_raw(clirunner, validate_cliresult, isolated_pio_home):
+    result = clirunner.invoke(cli_platform.platform_update)
+    validate_cliresult(result)
+    assert "Uninstalling atmelavr @ 1.1.0:" in result.output
+    assert "PlatformManager: Installing atmelavr @" in result.output
+    assert len(isolated_pio_home.join("packages").listdir()) == 1
+
+
+def test_uninstall(clirunner, validate_cliresult, isolated_pio_home):
+    result = clirunner.invoke(cli_platform.platform_uninstall,
+                              ["atmelavr", "espressif8266_stage"])
+    validate_cliresult(result)
+    assert len(isolated_pio_home.join("platforms").listdir()) == 0

tests/commands/test_settings.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

tests/commands/test_test.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,10 +20,11 @@ from platformio import util
 
 
 def test_local_env():
-    result = util.exec_command(["platformio", "test", "-d",
-                                join("examples", "unit-testing", "calculator"),
-                                "-e", "native"])
+    result = util.exec_command([
+        "platformio", "test", "-d",
+        join("examples", "unit-testing", "calculator"), "-e", "native"
+    ])
     if result['returncode'] != 1:
         pytest.fail(result)
-    assert all(
-        [s in result['out'] for s in ("PASSED", "IGNORED", "FAILED")])
+    assert all([s in result['out']
+                for s in ("PASSED", "IGNORED", "FAILED")]), result['out']

tests/commands/test_update.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

tests/conftest.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,11 +15,8 @@
 import os
 
 import pytest
-import requests
 from click.testing import CliRunner
 
-requests.packages.urllib3.disable_warnings()
-
 
 @pytest.fixture(scope="module")
 def clirunner():

tests/ino2cpp/basic/basic.ino

@@ -26,12 +26,15 @@ Foo foo(&fooCallback);
 
 //
 
+template<class T> T Add(T n1, T n2) {
+    return n1 + n2;
+}
+
 void setup() {
     struct Item item1;
     myFunction(&item1);
 }
 
-
 void loop() {
 
 }
@@ -40,7 +43,7 @@ void myFunction(struct Item *item) {
 
 }
 
-#warning "Line number is 43"
+#warning "Line number is 46"
 
 void fooCallback(){
 

tests/test_examples.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

tests/test_ino2cpp.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,7 +42,7 @@ def test_warning_line(clirunner, validate_cliresult):
     validate_cliresult(result)
     assert ('basic.ino:16:14: warning: #warning "Line number is 16"' in
             result.output)
-    assert ('basic.ino:43:2: warning: #warning "Line number is 43"' in
+    assert ('basic.ino:46:2: warning: #warning "Line number is 46"' in
             result.output)
     result = clirunner.invoke(
         cmd_ci, [join(INOTEST_DIR, "strmultilines"), "-b", "uno"])

tests/test_maintenance.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import json
+import re
 from time import time
 
 from platformio import app, maintenance
@@ -135,7 +136,8 @@ def test_check_and_update_libraries(clirunner, validate_cliresult,
     assert ("There are the new updates for libraries (ArduinoJson)" in
             result.output)
     assert "Please wait while updating libraries" in result.output
-    assert "[Out-of-date]" in result.output
+    assert re.search(r"Updating ArduinoJson\s+@ 5.6.7\s+\[[\d\.]+\]",
+                     result.output)
 
     # check updated version
     result = clirunner.invoke(cli_pio, ["lib", "-g", "list", "--json-output"])
@@ -154,7 +156,7 @@ def test_check_platform_updates(clirunner, validate_cliresult,
     manifest['version'] = "0.0.0"
     manifest_path.write(json.dumps(manifest))
     # reset cached manifests
-    PlatformManager().reset_cache()
+    PlatformManager().cache_reset()
 
     # reset check time
     interval = int(app.get_setting("check_platforms_interval")) * 3600 * 24
@@ -188,7 +190,8 @@ def test_check_and_update_platforms(clirunner, validate_cliresult,
     validate_cliresult(result)
     assert "There are the new updates for platforms (native)" in result.output
     assert "Please wait while updating platforms" in result.output
-    assert "[Out-of-date]" in result.output
+    assert re.search(r"Updating native\s+@ 0.0.0\s+\[[\d\.]+\]",
+                     result.output)
 
     # check updated version
     result = clirunner.invoke(cli_pio, ["platform", "list", "--json-output"])

tests/test_managers.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,77 +12,196 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+from os.path import join
+
 from platformio import util
-from platformio.managers.package import BasePkgManager
+from platformio.managers.package import PackageManager
 
 
-def test_pkg_name_parser():
+def test_pkg_input_parser():
     items = [
         ["PkgName", ("PkgName", None, None)],
         [("PkgName", "!=1.2.3,<2.0"), ("PkgName", "!=1.2.3,<2.0", None)],
         ["PkgName@1.2.3", ("PkgName", "1.2.3", None)],
         [("PkgName@1.2.3", "1.2.5"), ("PkgName@1.2.3", "1.2.5", None)],
         ["id:13", ("id:13", None, None)],
-        ["id:13@~1.2.3", ("id:13", "~1.2.3", None)], [
+        ["id:13@~1.2.3", ("id:13", "~1.2.3", None)],
+        [
             util.get_home_dir(),
             (".platformio", None, "file://" + util.get_home_dir())
-        ], [
+        ],
+        [
             "LocalName=" + util.get_home_dir(),
             ("LocalName", None, "file://" + util.get_home_dir())
-        ], [
+        ],
+        [
+            "LocalName=%s@>2.3.0" % util.get_home_dir(),
+            ("LocalName", ">2.3.0", "file://" + util.get_home_dir())
+        ],
+        [
             "https://github.com/user/package.git",
             ("package", None, "git+https://github.com/user/package.git")
-        ], [
-            "https://gitlab.com/user/package.git",
-            ("package", None, "git+https://gitlab.com/user/package.git")
-        ], [
+        ],
+        [
+            "MyPackage=https://gitlab.com/user/package.git",
+            ("MyPackage", None, "git+https://gitlab.com/user/package.git")
+        ],
+        [
+            "MyPackage=https://gitlab.com/user/package.git@3.2.1,!=2",
+            ("MyPackage", "3.2.1,!=2",
+             "git+https://gitlab.com/user/package.git")
+        ],
+        [
+            "https://somedomain.com/path/LibraryName-1.2.3.zip",
+            ("LibraryName-1.2.3", None,
+             "https://somedomain.com/path/LibraryName-1.2.3.zip")
+        ],
+        [
             "https://github.com/user/package/archive/branch.zip",
             ("branch", None,
              "https://github.com/user/package/archive/branch.zip")
-        ], [
+        ],
+        [
+            "https://github.com/user/package/archive/branch.zip@~1.2.3",
+            ("branch", "~1.2.3",
+             "https://github.com/user/package/archive/branch.zip")
+        ],
+        [
             "https://github.com/user/package/archive/branch.tar.gz",
-            ("branch", None,
+            ("branch.tar", None,
              "https://github.com/user/package/archive/branch.tar.gz")
-        ], [
+        ],
+        [
+            "https://github.com/user/package/archive/branch.tar.gz@!=5",
+            ("branch.tar", "!=5",
+             "https://github.com/user/package/archive/branch.tar.gz")
+        ],
+        [
             "https://developer.mbed.org/users/user/code/package/",
             ("package", None,
              "hg+https://developer.mbed.org/users/user/code/package/")
-        ], [
+        ],
+        [
             "https://github.com/user/package#v1.2.3",
             ("package", None, "git+https://github.com/user/package#v1.2.3")
-        ], [
+        ],
+        [
             "https://github.com/user/package.git#branch",
             ("package", None, "git+https://github.com/user/package.git#branch")
-        ], [
+        ],
+        [
             "PkgName=https://github.com/user/package.git#a13d344fg56",
             ("PkgName", None,
              "git+https://github.com/user/package.git#a13d344fg56")
-        ], [
+        ],
+        [
+            "user/package",
+            ("package", None, "git+https://github.com/user/package")
+        ],
+        [
             "PkgName=user/package",
             ("PkgName", None, "git+https://github.com/user/package")
-        ], [
+        ],
+        [
             "PkgName=user/package#master",
             ("PkgName", None, "git+https://github.com/user/package#master")
-        ], [
+        ],
+        [
             "git+https://github.com/user/package",
             ("package", None, "git+https://github.com/user/package")
-        ], [
+        ],
+        [
             "hg+https://example.com/user/package",
             ("package", None, "hg+https://example.com/user/package")
-        ], [
+        ],
+        [
             "git@github.com:user/package.git",
             ("package", None, "git@github.com:user/package.git")
-        ], [
+        ],
+        [
             "git@github.com:user/package.git#v1.2.0",
             ("package", None, "git@github.com:user/package.git#v1.2.0")
-        ], [
+        ],
+        [
             "git+ssh://git@gitlab.private-server.com/user/package#1.2.0",
             ("package", None,
              "git+ssh://git@gitlab.private-server.com/user/package#1.2.0")
+        ],
+        [
+            "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0",
+            ("package", None,
+             "git+ssh://user@gitlab.private-server.com:1234/package#1.2.0")
         ]
     ]
     for params, result in items:
         if isinstance(params, tuple):
-            assert BasePkgManager.parse_pkg_name(*params) == result
+            assert PackageManager.parse_pkg_input(*params) == result
         else:
-            assert BasePkgManager.parse_pkg_name(params) == result
+            assert PackageManager.parse_pkg_input(params) == result
+
+
+def test_install_packages(isolated_pio_home, tmpdir):
+    packages = [
+        dict(id=1, name="name_1", version="shasum"),
+        dict(id=1, name="name_1", version="2.0.0"),
+        dict(id=1, name="name_1", version="2.1.0"),
+        dict(id=1, name="name_1", version="1.2.0"),
+        dict(id=1, name="name_1", version="1.0.0"),
+        dict(name="name_2", version="1.0.0"),
+        dict(name="name_2", version="2.0.0",
+             __src_url="git+https://github.com"),
+        dict(name="name_2", version="3.0.0",
+             __src_url="git+https://github2.com"),
+        dict(name="name_2", version="4.0.0",
+             __src_url="git+https://github2.com")
+    ]
+
+    pm = PackageManager(join(util.get_home_dir(), "packages"))
+    for package in packages:
+        tmp_dir = tmpdir.mkdir("tmp-package")
+        tmp_dir.join("package.json").write(json.dumps(package))
+        pm._install_from_url(package['name'], "file://%s" % str(tmp_dir))
+        tmp_dir.remove(rec=1)
+
+    assert len(pm.get_installed()) == len(packages) - 1
+
+    pkg_dirnames = [
+        'name_1_ID1', 'name_1_ID1@1.0.0', 'name_1_ID1@1.2.0',
+        'name_1_ID1@2.0.0', 'name_1_ID1@shasum', 'name_2',
+        'name_2@src-177cbce1f0705580d17790fda1cc2ef5',
+        'name_2@src-f863b537ab00f4c7b5011fc44b120e1f'
+    ]
+    assert set([p.basename for p in isolated_pio_home.join(
+        "packages").listdir()]) == set(pkg_dirnames)
+
+
+def test_get_package(isolated_pio_home):
+    tests = [
+        [("unknown", ), None],
+        [("1", ), None],
+        [("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")],
+        [("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")],
+        [("id=1", "^1"), dict(id=1, name="name_1", version="1.2.0")],
+        [("id=1", "^1"), dict(id=1, name="name_1", version="1.2.0")],
+        [("name_1", "<2"), dict(id=1, name="name_1", version="1.2.0")],
+        [("name_1", ">2"), None],
+        [("name_1", "2-0-0"), dict(id=1, name="name_1", version="2.1.0")],
+        [("name_1", "2-0-0"), dict(id=1, name="name_1", version="2.1.0")],
+        [("name_2", ), dict(name="name_2", version="4.0.0")],
+        [("url_has_higher_priority", None, "git+https://github.com"),
+         dict(name="name_2", version="2.0.0",
+              __src_url="git+https://github.com")],
+        [("name_2", None, "git+https://github.com"),
+         dict(name="name_2", version="2.0.0",
+              __src_url="git+https://github.com")],
+    ]
+
+    pm = PackageManager(join(util.get_home_dir(), "packages"))
+    for test in tests:
+        manifest = pm.get_package(*test[0])
+        if test[1] is None:
+            assert manifest is None, test
+            continue
+        for key, value in test[1].items():
+            assert manifest[key] == value, test

tests/test_pkgmanifest.py

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,19 +16,6 @@ import pytest
 import requests
 
 
-def pytest_generate_tests(metafunc):
-    if "package_data" not in metafunc.fixturenames:
-        return
-    pkgs_manifest = requests.get(
-        "https://dl.bintray.com/platformio/dl-packages/manifest.json").json()
-    assert isinstance(pkgs_manifest, dict)
-    packages = []
-    for _, variants in pkgs_manifest.iteritems():
-        for item in variants:
-            packages.append(item)
-    metafunc.parametrize("package_data", packages)
-
-
 def validate_response(req):
     assert req.status_code == 200
     assert int(req.headers['Content-Length']) > 0
@@ -36,13 +23,22 @@ def validate_response(req):
                                            "application/octet-stream")
 
 
-def test_package(package_data):
-    assert package_data['url'].endswith(".tar.gz")
+def test_packages():
+    pkgs_manifest = requests.get(
+        "https://dl.bintray.com/platformio/dl-packages/manifest.json").json()
+    assert isinstance(pkgs_manifest, dict)
+    items = []
+    for _, variants in pkgs_manifest.iteritems():
+        for item in variants:
+            items.append(item)
 
-    r = requests.head(package_data['url'], allow_redirects=True)
-    validate_response(r)
+    for item in items:
+        assert item['url'].endswith(".tar.gz"), item
 
-    if "X-Checksum-Sha1" not in r.headers:
-        return pytest.skip("X-Checksum-Sha1 is not provided")
+        r = requests.head(item['url'], allow_redirects=True)
+        validate_response(r)
 
-    assert package_data['sha1'] == r.headers.get("X-Checksum-Sha1")
+        if "X-Checksum-Sha1" not in r.headers:
+            return pytest.skip("X-Checksum-Sha1 is not provided")
+
+        assert item['sha1'] == r.headers.get("X-Checksum-Sha1"), item

tox.ini

@@ -1,4 +1,4 @@
-# Copyright 2014-present PlatformIO <contact@platformio.org>
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ usedevelop = True
 deps =
     isort
     flake8
-    yapf
+    yapf<0.17
     pylint
     pytest
 commands = python --version
@@ -31,6 +31,7 @@ basepython = python2.7
 deps =
     sphinx
     sphinx_rtd_theme
+    restructuredtext-lint
 commands =
     sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/_build/html
     sphinx-build -W -b latex -d {envtmpdir}/doctrees docs docs/_build/latex
@@ -61,6 +62,13 @@ commands =
     {envpython} --version
     py.test -v --basetemp="{envtmpdir}" tests
 
+[testenv:skipexamples]
+basepython = python2.7
+deps =
+    pytest
+commands =
+    py.test -v --basetemp="{envtmpdir}" tests --ignore tests/test_examples.py
+
 [testenv:coverage]
 basepython = python2.7
 passenv = *