Merge branch 'release/v3.5.3'

* Additional fix to 7354515

* Update CMakeListsPrivate.txt.tpl

Also perform `replace('"', '\\"')`

.gitignore

@@ -9,3 +9,4 @@ build
 coverage.xml
 .coverage
 htmlcov
+.pytest_cache

.isort.cfg

@@ -1,3 +1,3 @@
 [settings]
 line_length=79
-known_third_party=arrow,bottle,click,lockfile,pytest,requests,SCons,semantic_version,serial
+known_third_party=bottle,click,lockfile,python-dateutil,pytest,requests,SCons,semantic_version,serial

HISTORY.rst

@@ -4,6 +4,90 @@ Release Notes
 PlatformIO 3.0
 --------------
 
+3.5.3 (2018-06-01)
+~~~~~~~~~~~~~~~~~~
+
+* `PlatformIO Home <http://docs.platformio.org/page/home/index.html>`__ -
+  interact with PlatformIO ecosystem using modern and cross-platform GUI:
+
+  - "Recent News" block on "Welcome" page
+  - Direct import of development platform's example
+
+* Simplify configuration for `PIO Unit Testing <http://docs.platformio.org/page/plus/unit-testing.html>`__: separate main program from a test build process, drop
+  requirement for ``#ifdef UNIT_TEST`` guard
+* Override any option from board manifest in `Project Configuration File "platformio.ini" <http://docs.platformio.org/page/projectconf/section_env_board.html#more-options>`__
+  (`issue #1612 <https://github.com/platformio/platformio-core/issues/1612>`_)
+* Configure a custom path to SVD file using `debug_svd_path <http://docs.platformio.org/page/projectconf/section_env_debug.html#debug-svd-path>`__
+  option
+* Custom project `description <http://docs.platformio.org/en/latest/projectconf/section_platformio.html#description>`_
+  which will be used by `PlatformIO Home <http://docs.platformio.org/page/home/index.html>`_
+* Updated Unity tool to 2.4.3
+* Improved support for Black Magic Probe in "uploader" mode
+* Renamed "monitor_baud" option to "monitor_speed"
+* Fixed issue when a custom `lib_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#lib-dir>`__
+  was not handled correctly
+  (`issue #1473 <https://github.com/platformio/platformio-core/issues/1473>`_)
+* Fixed issue with useless project rebuilding for case insensitive file
+  systems (Windows)
+* Fixed issue with ``build_unflags`` option when a macro contains value
+  (e.g., ``-DNAME=VALUE``)
+* Fixed issue which did not allow to override runtime build environment using
+  extra POST script
+* Fixed "RuntimeError: maximum recursion depth exceeded" for library manager
+  (`issue #1528 <https://github.com/platformio/platformio-core/issues/1528>`_)
+
+3.5.2 (2018-03-13)
+~~~~~~~~~~~~~~~~~~
+
+* `PlatformIO Home <http://docs.platformio.org/page/home/index.html>`__ -
+  interact with PlatformIO ecosystem using modern and cross-platform GUI:
+
+  - Multiple themes (Dark & Light)
+  - Ability to specify a name for new project
+
+* Control `PIO Unified Debugger <http://docs.platformio.org/en/latest/plus/debugging.html>`__
+  and its firmware loading mode using
+  `debug_load_mode <http://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-load-mode>`__ option
+* Added aliases (off, light, strict) for
+  `LDF Compatibility Mode <http://docs.platformio.org/page/librarymanager/ldf.html>`__
+* Search for a library using PIO Library Registry ID ``id:X`` (e.g. ``pio lib search id:13``)
+* Show device system information (MCU, Frequency, RAM, Flash, Debugging tools)
+  in a build log
+* Show all available upload protocols before firmware uploading in a build log
+* Handle "os.mbed.com" URL as a Mercurial (hg) repository
+* Improved support for old mbed libraries without manifest
+* Fixed project generator for Qt Creator IDE
+  (`issue #1303 <https://github.com/platformio/platformio-core/issues/1303>`_,
+  `issue #1323 <https://github.com/platformio/platformio-core/issues/1323>`_)
+* Mark project source and library directories for CLion IDE
+  (`issue #1359 <https://github.com/platformio/platformio-core/issues/1359>`_,
+  `issue #1345 <https://github.com/platformio/platformio-core/issues/1345>`_,
+  `issue #897 <https://github.com/platformio/platformio-core/issues/897>`_)
+* Fixed issue with duplicated "include" records when generating data for IDE
+  (`issue #1301 <https://github.com/platformio/platformio-core/issues/1301>`_)
+
+3.5.1 (2018-01-18)
+~~~~~~~~~~~~~~~~~~
+
+* New ``test_speed`` option to control a communication baudrate/speed between
+  `PIO Unit Testing <http://docs.platformio.org/page/plus/unit-testing.html>`__
+  engine and a target device
+  (`issue #1273 <https://github.com/platformio/platformio-core/issues/1273>`_)
+* Show full library version in "Library Dependency Graph" including VCS
+  information
+  (`issue #1274 <https://github.com/platformio/platformio-core/issues/1274>`_)
+* Configure a custom firmware/program name in build directory (`example <http://docs.platformio.org/page/projectconf/advanced_scripting.html#custom-firmware-program-name>`__)
+* Renamed ``envs_dir`` option to ``build_dir``
+  in `Project Configuration File "platformio.ini" <http://docs.platformio.org/page/projectconf/section_platformio.html#build-dir>`__
+* Refactored code without "arrow" dependency (resolve issue with "ImportError:
+  No module named backports.functools_lru_cache")
+* Improved support of PIO Unified Debugger for Eclipse Oxygen
+* Improved a work in off-line mode
+* Fixed project generator for CLion and Qt Creator IDE
+  (`issue #1299 <https://github.com/platformio/platformio-core/issues/1299>`_)
+* Fixed PIO Unified Debugger for mbed framework
+* Fixed library updates when a version is declared in VCS format (not SemVer)
+
 3.5.0 (2017-12-28)
 ~~~~~~~~~~~~~~~~~~
 
@@ -28,7 +112,7 @@ PlatformIO 3.0
 * New `include <http://docs.platformio.org/page/projectconf/section_platformio.html#include-dir>`__
   folder for project's header files
   (`issue #1107 <https://github.com/platformio/platformio-core/issues/1107>`_)
-* Depend on development platform using VSC URL (Git, Mercurial and Subversion)
+* Depend on development platform using VCS URL (Git, Mercurial and Subversion)
   instead of a name in `Project Configuration File "platformio.ini" <http://docs.platformio.org/page/projectconf/section_env_general.html#platform>`__.
   Drop support for ``*_stage`` dev/platform names (use VCS URL instead).
 * Reinstall/redownload package with a new ``-f, --force`` option for
@@ -470,7 +554,7 @@ PlatformIO 2.0
 * Added support for `emonPi <https://github.com/openenergymonitor/emonpi>`__,
   the OpenEnergyMonitor system
   (`issue #687 <https://github.com/platformio/platformio-core/issues/687>`_)
-* Added support for `SPL <http://platformio.org/frameworks/spl>`__
+* Added support for `SPL <https://platformio.org/frameworks/spl>`__
   framework for STM32F0 boards
   (`issue #683 <https://github.com/platformio/platformio-core/issues/683>`_)
 * Added support for `Arduboy DevKit <https://www.arduboy.com>`__, the game system
@@ -1161,7 +1245,7 @@ PlatformIO 1.0
 1.5.0 (2015-05-15)
 ~~~~~~~~~~~~~~~~~~
 
-* Added support of `Framework mbed <http://platformio.org/frameworks/mbed>`_
+* Added support of `Framework mbed <https://platformio.org/frameworks/mbed>`_
   for Teensy 3.1
   (`issue #183 <https://github.com/platformio/platformio-core/issues/183>`_)
 * Added GDB as alternative uploader to `ststm32 <http://docs.platformio.org/page/platforms/ststm32.html>`__ platform
@@ -1246,9 +1330,9 @@ PlatformIO 1.0
   `ststm32 <http://docs.platformio.org/page/platforms/ststm32.html>`__
   development platform
 * Created new `Frameworks <http://docs.platformio.org/page/frameworks/index.html>`__
-  page in documentation and added to `PlatformIO Web Site <http://platformio.org>`_
+  page in documentation and added to `PlatformIO Web Site <https://platformio.org>`_
   (`issue #115 <https://github.com/platformio/platformio-core/issues/115>`_)
-* Introduced online `Embedded Boards Explorer <http://platformio.org/boards>`_
+* Introduced online `Embedded Boards Explorer <https://platformio.org/boards>`_
 * Automatically append define ``-DPLATFORMIO=%version%`` to
   builder (`issue #105 <https://github.com/platformio/platformio-core/issues/105>`_)
 * Renamed ``stm32`` development platform to

Makefile

@@ -10,7 +10,7 @@ yapf:
 	yapf --recursive --in-place platformio/
 
 test:
-	py.test -v -s tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py
+	py.test -v -s -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py
 
 before-commit: isort yapf lint test
 

README.rst

@@ -18,101 +18,104 @@ PlatformIO
     :alt:  License
 .. image:: https://img.shields.io/PlatformIO/Community.png
    :alt: Community Forums
-   :target: https://community.platformio.org
+   :target: https://community.platformio.org?utm_source=github&utm_medium=core
 .. image:: https://img.shields.io/PlatformIO/Plus.png?color=orange
    :alt: PlatformIO Plus: Professional solutions for an awesome open source PlatformIO ecosystem
-   :target: https://pioplus.com
+   :target: https://platformio.org/pricing?utm_source=github&utm_medium=core
 
-**Quick Links:** `Home Page <http://platformio.org>`_ |
-`PlatformIO Plus <https://pioplus.com>`_ |
-`PlatformIO IDE <http://platformio.org/platformio-ide>`_ |
+**Quick Links:** `Home Page <https://platformio.org?utm_source=github&utm_medium=core>`_ |
+`PlatformIO Plus <https://platformio.org/pricing?utm_source=github&utm_medium=core>`_ |
+`PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_ |
 `Project Examples <https://github.com/platformio/platformio-examples/>`_ |
-`Docs <http://docs.platformio.org>`_ |
-`Donate <http://platformio.org/donate>`_ |
-`Contact Us <https://pioplus.com/contact.html>`_
+`Docs <http://docs.platformio.org?utm_source=github&utm_medium=core>`_ |
+`Donate <https://platformio.org/donate?utm_source=github&utm_medium=core>`_ |
+`Contact Us <https://platformio.org/contact?utm_source=github&utm_medium=core>`_
 
 **Social:** `Twitter <https://twitter.com/PlatformIO_Org>`_ |
 `Facebook <https://www.facebook.com/platformio>`_ |
 `Hackaday <https://hackaday.io/project/7980-platformio>`_ |
 `Bintray <https://bintray.com/platformio>`_ |
-`Community <https://community.platformio.org>`_
+`Community <https://community.platformio.org?utm_source=github&utm_medium=core>`_
 
 .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png
-    :target: http://platformio.org
+    :target: https://platformio.org?utm_source=github&utm_medium=core
 
-`PlatformIO <http://platformio.org>`_ is an open source ecosystem for IoT
+`PlatformIO <https://platformio.org?utm_source=github&utm_medium=core>`_ is an open source ecosystem for IoT
 development. Cross-platform IDE and unified debugger. Remote unit testing and
 firmware updates.
 
 Get Started
 -----------
 
-* `What is PlatformIO? <http://docs.platformio.org/page/what-is-platformio.html>`_
+* `What is PlatformIO? <http://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
 
 Products
 --------
 
-* `PlatformIO IDE <http://platformio.org/platformio-ide>`_
-* `PlatformIO Core <http://docs.platformio.org/page/core.html>`_
-* `PIO Remote™ <http://docs.platformio.org/page/plus/pio-remote.html>`_
-* `PIO Unified Debugger <http://docs.platformio.org/page/plus/debugging.html>`_
-* `PIO Unit Testing <http://docs.platformio.org/page/plus/unit-testing.html>`_
-* `PIO Delivery™ <http://platformio.org/pricing#solution-pio-delivery>`_
-* `Cloud Builder <http://platformio.org/pricing#solution-cloud-builder>`_
+* `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_
+* `PlatformIO Core (CLI) <http://docs.platformio.org/en/latest/core.html?utm_source=github&utm_medium=core>`_
+* `PIO Remote™ <http://docs.platformio.org/en/latest/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
+* `PIO Unified Debugger <http://docs.platformio.org/en/latest/plus/debugging.html?utm_source=github&utm_medium=core>`_
+* `PIO Unit Testing <http://docs.platformio.org/en/latest/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
+* `PIO Delivery™ <https://platformio.org/pricing?utm_source=github&utm_medium=core#solution-pio-delivery>`_
+* `Cloud Builder <https://platformio.org/pricing?utm_source=github&utm_medium=core#solution-cloud-builder>`_
 
 Registry
 --------
 
-* `Libraries <http://platformio.org/lib>`_
-* `Development Platforms <http://platformio.org/platforms>`_
-* `Frameworks <http://platformio.org/frameworks>`_
-* `Embedded Boards <http://platformio.org/boards>`_
+* `Libraries <https://platformio.org/lib?utm_source=github&utm_medium=core>`_
+* `Development Platforms <https://platformio.org/platforms?utm_source=github&utm_medium=core>`_
+* `Frameworks <https://platformio.org/frameworks?utm_source=github&utm_medium=core>`_
+* `Embedded Boards <https://platformio.org/boards?utm_source=github&utm_medium=core>`_
 
 Solutions
 ---------
 
-* `Library Manager <http://docs.platformio.org/page/librarymanager/index.html>`_
-* `Cloud IDEs Integration <http://platformio.org/pricing#solution-cloud-ide>`_
-* `Standalone IDEs Integration <http://docs.platformio.org/page/ide.html#other-ide>`_
-* `Continuous Integration <http://docs.platformio.org/page/ci/index.html>`_
+* `Library Manager <http://docs.platformio.org/en/latest/librarymanager/index.html?utm_source=github&utm_medium=core>`_
+* `Cloud IDEs Integration <https://platformio.org/pricing?utm_source=github&utm_medium=core#solution-cloud-ide>`_
+* `Standalone IDEs Integration <http://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core#other-ide>`_
+* `Continuous Integration <http://docs.platformio.org/en/latest/ci/index.html?utm_source=github&utm_medium=core>`_
 
 Development Platforms
 ---------------------
 
-* `Atmel AVR <http://platformio.org/platforms/atmelavr>`_
-* `Atmel SAM <http://platformio.org/platforms/atmelsam>`_
-* `Espressif 32 <http://platformio.org/platforms/espressif32>`_
-* `Espressif 8266 <http://platformio.org/platforms/espressif8266>`_
-* `Freescale Kinetis <http://platformio.org/platforms/freescalekinetis>`_
-* `Intel ARC32 <http://platformio.org/platforms/intel_arc32>`_
-* `Lattice iCE40 <http://platformio.org/platforms/lattice_ice40>`_
-* `Maxim 32 <http://platformio.org/platforms/maxim32>`_
-* `Microchip PIC32 <http://platformio.org/platforms/microchippic32>`_
-* `Nordic nRF51 <http://platformio.org/platforms/nordicnrf51>`_
-* `Nordic nRF52 <http://platformio.org/platforms/nordicnrf52>`_
-* `NXP LPC <http://platformio.org/platforms/nxplpc>`_
-* `Silicon Labs EFM32 <http://platformio.org/platforms/siliconlabsefm32>`_
-* `ST STM32 <http://platformio.org/platforms/ststm32>`_
-* `Teensy <http://platformio.org/platforms/teensy>`_
-* `TI MSP430 <http://platformio.org/platforms/timsp430>`_
-* `TI Tiva <http://platformio.org/platforms/titiva>`_
-* `WIZNet W7500 <http://platformio.org/platforms/wiznet7500>`_
+* `Atmel AVR <https://platformio.org/platforms/atmelavr?utm_source=github&utm_medium=core>`_
+* `Atmel SAM <https://platformio.org/platforms/atmelsam?utm_source=github&utm_medium=core>`_
+* `Espressif 32 <https://platformio.org/platforms/espressif32?utm_source=github&utm_medium=core>`_
+* `Espressif 8266 <https://platformio.org/platforms/espressif8266?utm_source=github&utm_medium=core>`_
+* `Freescale Kinetis <https://platformio.org/platforms/freescalekinetis?utm_source=github&utm_medium=core>`_
+* `Intel ARC32 <https://platformio.org/platforms/intel_arc32?utm_source=github&utm_medium=core>`_
+* `Lattice iCE40 <https://platformio.org/platforms/lattice_ice40?utm_source=github&utm_medium=core>`_
+* `Maxim 32 <https://platformio.org/platforms/maxim32?utm_source=github&utm_medium=core>`_
+* `Microchip PIC32 <https://platformio.org/platforms/microchippic32?utm_source=github&utm_medium=core>`_
+* `Nordic nRF51 <https://platformio.org/platforms/nordicnrf51?utm_source=github&utm_medium=core>`_
+* `Nordic nRF52 <https://platformio.org/platforms/nordicnrf52?utm_source=github&utm_medium=core>`_
+* `NXP LPC <https://platformio.org/platforms/nxplpc?utm_source=github&utm_medium=core>`_
+* `Silicon Labs EFM32 <https://platformio.org/platforms/siliconlabsefm32?utm_source=github&utm_medium=core>`_
+* `ST STM32 <https://platformio.org/platforms/ststm32?utm_source=github&utm_medium=core>`_
+* `Teensy <https://platformio.org/platforms/teensy?utm_source=github&utm_medium=core>`_
+* `TI MSP430 <https://platformio.org/platforms/timsp430?utm_source=github&utm_medium=core>`_
+* `TI Tiva <https://platformio.org/platforms/titiva?utm_source=github&utm_medium=core>`_
+* `WIZNet W7500 <https://platformio.org/platforms/wiznet7500?utm_source=github&utm_medium=core>`_
 
 Frameworks
 ----------
 
-* `Arduino <http://platformio.org/frameworks/arduino>`_
-* `ARTIK SDK <http://platformio.org/frameworks/artik-sdk>`_
-* `CMSIS <http://platformio.org/frameworks/cmsis>`_
-* `Energia <http://platformio.org/frameworks/energia>`_
-* `ESP-IDF <http://platformio.org/frameworks/espidf>`_
-* `libOpenCM3 <http://platformio.org/frameworks/libopencm3>`_
-* `mbed <http://platformio.org/frameworks/mbed>`_
-* `Pumbaa <http://platformio.org/frameworks/pumbaa>`_
-* `Simba <http://platformio.org/frameworks/simba>`_
-* `SPL <http://platformio.org/frameworks/spl>`_
-* `STM32Cube <http://platformio.org/frameworks/stm32cube>`_
-* `WiringPi <http://platformio.org/frameworks/wiringpi>`_
+* `Arduino <https://platformio.org/frameworks/arduino?utm_source=github&utm_medium=core>`_
+* `ARTIK SDK <https://platformio.org/frameworks/artik-sdk?utm_source=github&utm_medium=core>`_
+* `CMSIS <https://platformio.org/frameworks/cmsis?utm_source=github&utm_medium=core>`_
+* `Energia <https://platformio.org/frameworks/energia?utm_source=github&utm_medium=core>`_
+* `ESP-IDF <https://platformio.org/frameworks/espidf?utm_source=github&utm_medium=core>`_
+* `ESP8266 Non-OS SDK <https://platformio.org/frameworks/esp8266-nonos-sdk?utm_source=github&utm_medium=core>`_
+* `ESP8266 RTOS SDK <https://platformio.org/frameworks/esp8266-rtos-sdk?utm_source=github&utm_medium=core>`_
+* `libOpenCM3 <https://platformio.org/frameworks/libopencm3?utm_source=github&utm_medium=core>`_
+* `mbed <https://platformio.org/frameworks/mbed?utm_source=github&utm_medium=core>`_
+* `Pumbaa <https://platformio.org/frameworks/pumbaa?utm_source=github&utm_medium=core>`_
+* `Simba <https://platformio.org/frameworks/simba?utm_source=github&utm_medium=core>`_
+* `SPL <https://platformio.org/frameworks/spl?utm_source=github&utm_medium=core>`_
+* `STM32Cube <https://platformio.org/frameworks/stm32cube?utm_source=github&utm_medium=core>`_
+* `Tizen RT <https://platformio.org/frameworks/tizenrt?utm_source=github&utm_medium=core>`_
+* `WiringPi <https://platformio.org/frameworks/wiringpi?utm_source=github&utm_medium=core>`_
 
 Contributing
 ------------

platformio/__init__.py

@@ -14,15 +14,17 @@
 
 import sys
 
-VERSION = (3, 5, 0)
+VERSION = (3, 5, 3)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
-__description__ = ("An open source ecosystem for IoT development. "
-                   "Cross-platform build system and library manager. "
-                   "Continuous and IDE integration. "
-                   "Arduino, ESP8266 and ARM mbed compatible")
-__url__ = "http://platformio.org"
+__description__ = (
+    "An open source ecosystem for IoT development. "
+    "Cross-platform IDE and unified debugger. "
+    "Remote unit testing and firmware updates. "
+    "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, "
+    "FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3")
+__url__ = "https://platformio.org"
 
 __author__ = "Ivan Kravets"
 __email__ = "me@ikravets.com"
@@ -36,5 +38,5 @@ if sys.version_info < (2, 7, 0) or sys.version_info >= (3, 0, 0):
     msg = ("PlatformIO Core v%s does not run under Python version %s.\n"
            "Minimum supported version is 2.7, please upgrade Python.\n"
            "Python 3 is not yet supported.\n")
-    sys.stderr.write(msg % (__version__, sys.version.split()[0]))
+    sys.stderr.write(msg % (__version__, sys.version))
     sys.exit(1)

platformio/app.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import codecs
 import hashlib
 import json
 import os
@@ -106,7 +107,7 @@ class State(object):
     def __exit__(self, type_, value, traceback):
         if self._prev_state != self._state:
             try:
-                with open(self.path, "w") as fp:
+                with codecs.open(self.path, "w", encoding="utf8") as fp:
                     if "dev" in __version__:
                         json.dump(self._state, fp, indent=4)
                     else:
@@ -141,7 +142,7 @@ class ContentCache(object):
         self._db_path = None
         self._lockfile = None
 
-        self.cache_dir = cache_dir or join(util.get_home_dir(), ".cache")
+        self.cache_dir = cache_dir or util.get_cache_dir()
         self._db_path = join(self.cache_dir, "db.data")
 
     def __enter__(self):
@@ -187,11 +188,8 @@ class ContentCache(object):
         cache_path = self.get_cache_path(key)
         if not isfile(cache_path):
             return None
-        with open(cache_path, "rb") as fp:
-            data = fp.read()
-            if data and data[0] in ("{", "["):
-                return json.loads(data)
-            return data
+        with codecs.open(cache_path, "rb", encoding="utf8") as fp:
+            return fp.read()
 
     def set(self, key, data, valid):
         if not get_setting("enable_cache"):
@@ -212,13 +210,17 @@ class ContentCache(object):
 
         if not isdir(dirname(cache_path)):
             os.makedirs(dirname(cache_path))
-        with open(cache_path, "wb") as fp:
-            if isinstance(data, (dict, list)):
-                json.dump(data, fp)
-            else:
-                fp.write(str(data))
-        with open(self._db_path, "a") as fp:
-            fp.write("%s=%s\n" % (str(expire_time), cache_path))
+        try:
+            with codecs.open(cache_path, "wb", encoding="utf8") as fp:
+                fp.write(data)
+            with open(self._db_path, "a") as fp:
+                fp.write("%s=%s\n" % (str(expire_time), cache_path))
+        except UnicodeError:
+            if isfile(cache_path):
+                try:
+                    remove(cache_path)
+                except OSError:
+                    pass
 
         return self._unlock_dbindex()
 

platformio/builder/main.py

@@ -20,7 +20,7 @@ from os.path import expanduser, join
 from time import time
 
 from SCons.Script import (ARGUMENTS, COMMAND_LINE_TARGETS, DEFAULT_TARGETS,
-                          AllowSubstExceptions, AlwaysBuild,
+                          Action, AllowSubstExceptions, AlwaysBuild,
                           DefaultEnvironment, Variables)
 
 from platformio import util
@@ -54,17 +54,23 @@ commonvars.AddVariables(
 
     # board options
     ("BOARD",),
+    # deprecated options, use board_{object.path} instead
     ("BOARD_MCU",),
     ("BOARD_F_CPU",),
     ("BOARD_F_FLASH",),
     ("BOARD_FLASH_MODE",),
+    # end of deprecated options
 
     # upload options
     ("UPLOAD_PORT",),
     ("UPLOAD_PROTOCOL",),
     ("UPLOAD_SPEED",),
     ("UPLOAD_FLAGS",),
-    ("UPLOAD_RESETMETHOD",)
+    ("UPLOAD_RESETMETHOD",),
+
+    # debug options
+    ("DEBUG_TOOL",),
+    ("DEBUG_SVD_PATH",),
 
 )  # yapf: disable
 
@@ -75,7 +81,7 @@ MULTILINE_VARS = [
 
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
-        "ar", "as", "gcc", "g++", "gnulink", "platformio", "pioplatform",
+        "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform",
         "piowinhooks", "piolib", "pioupload", "piomisc", "pioide"
     ],  # yapf: disable
     toolpath=[join(util.get_source_dir(), "builder", "tools")],
@@ -91,10 +97,11 @@ DEFAULT_ENV_OPTIONS = dict(
     PROJECTSRC_DIR=util.get_projectsrc_dir(),
     PROJECTTEST_DIR=util.get_projecttest_dir(),
     PROJECTDATA_DIR=util.get_projectdata_dir(),
-    PROJECTPIOENVS_DIR=util.get_projectpioenvs_dir(),
-    BUILD_DIR=join("$PROJECTPIOENVS_DIR", "$PIOENV"),
+    PROJECTBUILD_DIR=util.get_projectbuild_dir(),
+    BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"),
     BUILDSRC_DIR=join("$BUILD_DIR", "src"),
     BUILDTEST_DIR=join("$BUILD_DIR", "test"),
+    LIBPATH=["$BUILD_DIR"],
     LIBSOURCE_DIRS=[
         util.get_projectlib_dir(),
         util.get_projectlibdeps_dir(),
@@ -150,9 +157,9 @@ env['LIBSOURCE_DIRS'] = [
 env.LoadPioPlatform(commonvars)
 
 env.SConscriptChdir(0)
-env.SConsignFile(join("$PROJECTPIOENVS_DIR", ".sconsign.dblite"))
+env.SConsignFile(join("$PROJECTBUILD_DIR", ".sconsign.dblite"))
 
-for item in env.GetPreExtraScripts():
+for item in env.GetExtraScripts("pre"):
     env.SConscript(item, exports="env")
 
 env.SConscript("$BUILD_SCRIPT")
@@ -161,9 +168,9 @@ AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS + ["size"]))
 AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS + ["size"]))
 
 if "UPLOAD_FLAGS" in env:
-    env.Append(UPLOADERFLAGS=["$UPLOAD_FLAGS"])
+    env.Prepend(UPLOADERFLAGS=["$UPLOAD_FLAGS"])
 
-for item in env.GetPostExtraScripts():
+for item in env.GetExtraScripts("post"):
     env.SConscript(item, exports="env")
 
 if "envdump" in COMMAND_LINE_TARGETS:
@@ -182,3 +189,7 @@ if "idedata" in COMMAND_LINE_TARGETS:
             "See explanation in FAQ > Troubleshooting > Building\n"
             "http://docs.platformio.org/page/faq.html\n\n")
         env.Exit(1)
+
+env.AddPreAction(["upload", "program"],
+                 Action(lambda source, target, env: env.PrintUploadInfo(),
+                        "Configuring upload protocol..."))

platformio/builder/tools/pioide.py

@@ -16,7 +16,7 @@ from __future__ import absolute_import
 
 from glob import glob
 from os import environ
-from os.path import join
+from os.path import abspath, isfile, join
 
 from SCons.Defaults import processDefines
 
@@ -53,7 +53,13 @@ def _dump_includes(env):
     if unity_dir:
         includes.append(unity_dir)
 
-    return includes
+    # remove duplicates
+    result = []
+    for item in includes:
+        if item not in result:
+            result.append(abspath(item))
+
+    return result
 
 
 def _get_gcc_defines(env):
@@ -95,12 +101,34 @@ def _dump_defines(env):
                     .replace("ATMEGA", "ATmega").replace("ATTINY", "ATtiny")))
 
     # built-in GCC marcos
-    if env.GetCompilerType() == "gcc":
-        defines.extend(_get_gcc_defines(env))
+    # if env.GetCompilerType() == "gcc":
+    #     defines.extend(_get_gcc_defines(env))
 
     return defines
 
 
+def _get_svd_path(env):
+    svd_path = env.subst("$DEBUG_SVD_PATH")
+    if svd_path:
+        return abspath(svd_path)
+
+    if "BOARD" not in env:
+        return None
+    try:
+        svd_path = env.BoardConfig().get("debug.svd_path")
+        assert svd_path
+    except (AssertionError, KeyError):
+        return None
+    # custom path to SVD file
+    if isfile(svd_path):
+        return svd_path
+    # default file from ./platform/misc/svd folder
+    p = env.PioPlatform()
+    if isfile(join(p.get_dir(), "misc", "svd", svd_path)):
+        return abspath(join(p.get_dir(), "misc", "svd", svd_path))
+    return None
+
+
 def DumpIDEData(env):
     LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS"
     LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS"
@@ -124,6 +152,8 @@ def DumpIDEData(env):
         util.where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
         "prog_path":
         env.subst("$PROG_PATH"),
+        "svd_path":
+        _get_svd_path(env),
         "compiler_type":
         env.GetCompilerType()
     }

platformio/builder/tools/piolib.py

@@ -20,16 +20,17 @@ from __future__ import absolute_import
 import hashlib
 import os
 import sys
+from glob import glob
 from os.path import (basename, commonprefix, dirname, isdir, isfile, join,
                      realpath, sep)
-from platform import system
 
 import SCons.Scanner
 from SCons.Script import ARGUMENTS, COMMAND_LINE_TARGETS, DefaultEnvironment
 
-from platformio import util
+from platformio import exception, util
 from platformio.builder.tools import platformio as piotool
 from platformio.managers.lib import LibraryManager
+from platformio.managers.package import PackageManager
 
 
 class LibBuilderFactory(object):
@@ -55,10 +56,9 @@ class LibBuilderFactory(object):
 
     @staticmethod
     def get_used_frameworks(env, path):
-        if any([
+        if any(
                 isfile(join(path, fname))
-                for fname in ("library.properties", "keywords.txt")
-        ]):
+                for fname in ("library.properties", "keywords.txt")):
             return ["arduino"]
 
         if isfile(join(path, "module.json")):
@@ -81,14 +81,21 @@ class LibBuilderFactory(object):
 
 class LibBuilderBase(object):
 
+    IS_WINDOWS = "windows" in util.get_systype()
+
     LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"]
     LDF_MODE_DEFAULT = "chain"
 
-    COMPAT_MODES = [0, 1, 2]
-    COMPAT_MODE_DEFAULT = 1
+    COMPAT_MODES = ["off", "soft", "strict"]
+    COMPAT_MODE_DEFAULT = "soft"
 
     CLASSIC_SCANNER = SCons.Scanner.C.CScanner()
-    ADVANCED_SCANNER = SCons.Scanner.C.CScanner(advanced=True)
+    CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner()
+    # Max depth of nested includes:
+    # -1 = unlimited
+    # 0 - disabled nesting
+    # >0 - number of allowed nested includes
+    CCONDITIONAL_SCANNER_DEPTH = 99
     PARSE_SRC_BY_H_NAME = True
 
     _INCLUDE_DIRS_CACHE = None
@@ -118,7 +125,7 @@ class LibBuilderBase(object):
     def __contains__(self, path):
         p1 = self.path
         p2 = path
-        if system() == "Windows":
+        if self.IS_WINDOWS:
             p1 = p1.lower()
             p2 = p2.lower()
         return commonprefix((p1 + sep, p2)) == p1 + sep
@@ -131,6 +138,13 @@ class LibBuilderBase(object):
     def version(self):
         return self._manifest.get("version")
 
+    @property
+    def vcs_info(self):
+        items = glob(join(self.path, ".*", PackageManager.SRC_MANIFEST_NAME))
+        if not items:
+            return None
+        return util.load_json(items[0])
+
     @property
     def dependencies(self):
         return LibraryManager.normalize_dependencies(
@@ -147,7 +161,7 @@ class LibBuilderBase(object):
 
     @property
     def include_dir(self):
-        if not all([isdir(join(self.path, d)) for d in ("include", "src")]):
+        if not all(isdir(join(self.path, d)) for d in ("include", "src")):
             return None
         return join(self.path, "include")
 
@@ -221,25 +235,15 @@ class LibBuilderBase(object):
 
     @staticmethod
     def validate_compat_mode(mode):
-        try:
-            mode = int(mode)
-            assert mode in LibBuilderBase.COMPAT_MODES
+        if isinstance(mode, basestring):
+            mode = mode.strip().lower()
+        if mode in LibBuilderBase.COMPAT_MODES:
             return mode
-        except (AssertionError, ValueError):
-            return LibBuilderBase.COMPAT_MODE_DEFAULT
-
-    @staticmethod
-    def items_to_list(items):
-        if not isinstance(items, list):
-            items = [i.strip() for i in items.split(",")]
-        return [i.lower() for i in items if i]
-
-    def items_in_list(self, items, ilist):
-        items = self.items_to_list(items)
-        ilist = self.items_to_list(ilist)
-        if "*" in items or "*" in ilist:
-            return True
-        return set(items) & set(ilist)
+        try:
+            return LibBuilderBase.COMPAT_MODES[int(mode)]
+        except (IndexError, ValueError):
+            pass
+        return LibBuilderBase.COMPAT_MODE_DEFAULT
 
     def is_platforms_compatible(self, platforms):
         return True
@@ -273,7 +277,7 @@ class LibBuilderBase(object):
                 if env_key not in self.env:
                     continue
                 if (key in item and
-                        not self.items_in_list(self.env[env_key], item[key])):
+                        not util.items_in_list(self.env[env_key], item[key])):
                     if self.verbose:
                         sys.stderr.write(
                             "Skip %s incompatible dependency %s\n" % (key[:-1],
@@ -304,9 +308,8 @@ class LibBuilderBase(object):
 
     def get_search_files(self):
         items = [
-            join(self.src_dir, item)
-            for item in self.env.MatchSourceFiles(self.src_dir,
-                                                  self.src_filter)
+            join(self.src_dir, item) for item in self.env.MatchSourceFiles(
+                self.src_dir, self.src_filter)
         ]
         include_dir = self.include_dir
         if include_dir:
@@ -345,17 +348,18 @@ class LibBuilderBase(object):
         for path in self._validate_search_files(search_files):
             try:
                 assert "+" in self.lib_ldf_mode
-                incs = self.env.File(path).get_found_includes(
-                    self.env, LibBuilderBase.ADVANCED_SCANNER,
-                    tuple(include_dirs))
+                incs = LibBuilderBase.CCONDITIONAL_SCANNER(
+                    self.env.File(path),
+                    self.env,
+                    tuple(include_dirs),
+                    depth=self.CCONDITIONAL_SCANNER_DEPTH)
             except Exception as e:  # pylint: disable=broad-except
                 if self.verbose and "+" in self.lib_ldf_mode:
                     sys.stderr.write(
                         "Warning! Classic Pre Processor is used for `%s`, "
                         "advanced has failed with `%s`\n" % (path, e))
-                _incs = self.env.File(path).get_found_includes(
-                    self.env, LibBuilderBase.CLASSIC_SCANNER,
-                    tuple(include_dirs))
+                _incs = LibBuilderBase.CLASSIC_SCANNER(
+                    self.env.File(path), self.env, tuple(include_dirs))
                 incs = []
                 for inc in _incs:
                     incs.append(inc)
@@ -496,7 +500,7 @@ class ArduinoLibBuilder(LibBuilderBase):
         return src_filter
 
     def is_frameworks_compatible(self, frameworks):
-        return self.items_in_list(frameworks, ["arduino", "energia"])
+        return util.items_in_list(frameworks, ["arduino", "energia"])
 
 
 class MbedLibBuilder(LibBuilderBase):
@@ -522,12 +526,24 @@ class MbedLibBuilder(LibBuilderBase):
         include_dirs = LibBuilderBase.get_include_dirs(self)
         if self.path not in include_dirs:
             include_dirs.append(self.path)
+
+        # library with module.json
         for p in self._manifest.get("extraIncludes", []):
             include_dirs.append(join(self.path, p))
+
+        # old mbed library without manifest, add to CPPPATH all folders
+        if not self._manifest:
+            for root, _, __ in os.walk(self.path):
+                part = root.replace(self.path, "").lower()
+                if any(s in part for s in ("%s." % sep, "test", "example")):
+                    continue
+                if root not in include_dirs:
+                    include_dirs.append(root)
+
         return include_dirs
 
     def is_frameworks_compatible(self, frameworks):
-        return self.items_in_list(frameworks, ["mbed"])
+        return util.items_in_list(frameworks, ["mbed"])
 
 
 class PlatformIOLibBuilder(LibBuilderBase):
@@ -541,7 +557,7 @@ class PlatformIOLibBuilder(LibBuilderBase):
         if "platforms" in manifest:
             manifest['platforms'] = [
                 "espressif8266" if p == "espressif" else p
-                for p in self.items_to_list(manifest['platforms'])
+                for p in util.items_to_list(manifest['platforms'])
             ]
 
         return manifest
@@ -610,13 +626,13 @@ class PlatformIOLibBuilder(LibBuilderBase):
         items = self._manifest.get("platforms")
         if not items:
             return LibBuilderBase.is_platforms_compatible(self, platforms)
-        return self.items_in_list(platforms, items)
+        return util.items_in_list(platforms, items)
 
     def is_frameworks_compatible(self, frameworks):
         items = self._manifest.get("frameworks")
         if not items:
             return LibBuilderBase.is_frameworks_compatible(self, frameworks)
-        return self.items_in_list(frameworks, items)
+        return util.items_in_list(frameworks, items)
 
     def get_include_dirs(self):
         include_dirs = LibBuilderBase.get_include_dirs(self)
@@ -646,7 +662,9 @@ class ProjectAsLibBuilder(LibBuilderBase):
 
     def get_include_dirs(self):
         include_dirs = LibBuilderBase.get_include_dirs(self)
-        include_dirs.append(self.env.subst("$PROJECTINCLUDE_DIR"))
+        project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR")
+        if isdir(project_include_dir):
+            include_dirs.append(project_include_dir)
         return include_dirs
 
     def get_search_files(self):
@@ -655,9 +673,9 @@ class ProjectAsLibBuilder(LibBuilderBase):
         # test files
         if "__test" in COMMAND_LINE_TARGETS:
             items.extend([
-                join("$PROJECTTEST_DIR", item)
-                for item in self.env.MatchSourceFiles("$PROJECTTEST_DIR",
-                                                      "$PIOTEST_SRC_FILTER")
+                join("$PROJECTTEST_DIR",
+                     item) for item in self.env.MatchSourceFiles(
+                         "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER")
             ])
         return items
 
@@ -734,13 +752,13 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             if verbose:
                 sys.stderr.write("Ignored library %s\n" % lb.path)
             return None
-        if compat_mode > 1 and not lb.is_platforms_compatible(
+        if compat_mode == "strict" and not lb.is_platforms_compatible(
                 env['PIOPLATFORM']):
             if verbose:
                 sys.stderr.write(
                     "Platform incompatible library %s\n" % lb.path)
             return False
-        if compat_mode > 0 and "PIOFRAMEWORK" in env and \
+        if compat_mode == "soft" and "PIOFRAMEWORK" in env and \
            not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])):
             if verbose:
                 sys.stderr.write(
@@ -759,7 +777,7 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             try:
                 lb = LibBuilderFactory.new(
                     env, join(libs_dir, item), verbose=verbose)
-            except ValueError:
+            except exception.InvalidJSONFile:
                 if verbose:
                     sys.stderr.write("Skip library with broken manifest: %s\n"
                                      % join(libs_dir, item))
@@ -786,9 +804,8 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
 
 
 def BuildProjectLibraries(env):
-    lib_builders = env.GetLibBuilders()
 
-    def correct_found_libs():
+    def correct_found_libs(lib_builders):
         # build full dependency graph
         found_lbs = [lb for lb in lib_builders if lb.dependent]
         for lb in lib_builders:
@@ -803,10 +820,15 @@ def BuildProjectLibraries(env):
         margin = "|   " * (level)
         for lb in root.depbuilders:
             title = "<%s>" % lb.name
+            vcs_info = lb.vcs_info
             if lb.version:
                 title += " v%s" % lb.version
+            if vcs_info and vcs_info.get("version"):
+                title += " #%s" % vcs_info.get("version")
             sys.stdout.write("%s|-- %s" % (margin, title))
             if int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                if vcs_info:
+                    sys.stdout.write(" [%s]" % vcs_info.get("url"))
                 sys.stdout.write(" (")
                 sys.stdout.write(lb.path)
                 sys.stdout.write(")")
@@ -814,20 +836,25 @@ def BuildProjectLibraries(env):
             if lb.depbuilders:
                 print_deps_tree(lb, level + 1)
 
-    print "Collected %d compatible libraries" % len(lib_builders)
-    print "Looking for dependencies..."
-
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
     project.env = env
+    ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project)
+
+    print "Library Dependency Finder -> http://bit.ly/configure-pio-ldf"
+    print "LDF MODES: FINDER(%s) COMPATIBILITY(%s)" % (ldf_mode,
+                                                       project.lib_compat_mode)
+
+    lib_builders = env.GetLibBuilders()
+    print "Collected %d compatible libraries" % len(lib_builders)
+
+    print "Scanning dependencies..."
     project.search_deps_recursive()
 
-    if (LibBuilderBase.validate_ldf_mode(
-            env.get("LIB_LDF_MODE", LibBuilderBase.LDF_MODE_DEFAULT))
-            .startswith("chain") and project.depbuilders):
-        correct_found_libs()
+    if ldf_mode.startswith("chain") and project.depbuilders:
+        correct_found_libs(lib_builders)
 
     if project.depbuilders:
-        print "Library Dependency Graph ( http://bit.ly/configure-pio-ldf )"
+        print "Dependency Graph"
         print_deps_tree(project)
     else:
         print "No dependencies"

platformio/builder/tools/piomisc.py

@@ -18,7 +18,7 @@ import atexit
 import re
 import sys
 from os import environ, remove, walk
-from os.path import basename, isdir, isfile, join, relpath, sep
+from os.path import basename, isdir, isfile, join, realpath, relpath, sep
 from tempfile import mkstemp
 
 from SCons.Action import Action
@@ -199,7 +199,7 @@ def _delete_file(path):
         pass
 
 
-@util.memoized
+@util.memoized()
 def _get_compiler_type(env):
     try:
         sysenv = environ.copy()
@@ -295,25 +295,21 @@ def ProcessTest(env):
         src_filter.append("+<%s%s>" % (env['PIOTEST'], sep))
     env.Replace(PIOTEST_SRC_FILTER=src_filter)
 
-    return env.CollectBuildFiles(
-        "$BUILDTEST_DIR",
-        "$PROJECTTEST_DIR",
-        "$PIOTEST_SRC_FILTER",
-        duplicate=False)
+    return env.CollectBuildFiles("$BUILDTEST_DIR", "$PROJECTTEST_DIR",
+                                 "$PIOTEST_SRC_FILTER")
 
 
-def GetPreExtraScripts(env):
-    return [
-        item[4:] for item in env.get("EXTRA_SCRIPTS", [])
-        if item.startswith("pre:")
-    ]
-
-
-def GetPostExtraScripts(env):
-    return [
-        item[5:] if item.startswith("post:") else item
-        for item in env.get("EXTRA_SCRIPTS", []) if not item.startswith("pre:")
-    ]
+def GetExtraScripts(env, scope):
+    items = []
+    for item in env.get("EXTRA_SCRIPTS", []):
+        if scope == "post" and ":" not in item:
+            items.append(item)
+        elif item.startswith("%s:" % scope):
+            items.append(item[len(scope) + 1:])
+    if not items:
+        return items
+    with util.cd(env.subst("$PROJECT_DIR")):
+        return [realpath(item) for item in items]
 
 
 def exists(_):
@@ -328,6 +324,5 @@ def generate(env):
     env.AddMethod(PioClean)
     env.AddMethod(ProcessDebug)
     env.AddMethod(ProcessTest)
-    env.AddMethod(GetPreExtraScripts)
-    env.AddMethod(GetPostExtraScripts)
+    env.AddMethod(GetExtraScripts)
     return env

platformio/builder/tools/pioplatform.py

@@ -14,6 +14,7 @@
 
 from __future__ import absolute_import
 
+import base64
 import sys
 from os.path import isdir, isfile, join
 
@@ -22,8 +23,10 @@ from SCons.Script import COMMAND_LINE_TARGETS
 from platformio import exception, util
 from platformio.managers.platform import PlatformFactory
 
+# pylint: disable=too-many-branches
 
-@util.memoized
+
+@util.memoized()
 def initPioPlatform(name):
     return PlatformFactory.newPlatform(name)
 
@@ -41,8 +44,9 @@ def PioPlatform(env):
 def BoardConfig(env, board=None):
     p = initPioPlatform(env['PLATFORM_MANIFEST'])
     try:
-        assert env.get("BOARD", board), "BoardConfig: Board is not defined"
-        config = p.board_config(board if board else env.get("BOARD"))
+        board = board or env.get("BOARD")
+        assert board, "BoardConfig: Board is not defined"
+        config = p.board_config(board)
     except (AssertionError, exception.UnknownBoard) as e:
         sys.stderr.write("Error: %s\n" % str(e))
         env.Exit(1)
@@ -68,7 +72,7 @@ def LoadPioPlatform(env, variables):
     # Add toolchains and uploaders to $PATH
     for name in installed_packages:
         type_ = p.get_package_type(name)
-        if type_ not in ("toolchain", "uploader"):
+        if type_ not in ("toolchain", "uploader", "debugger"):
             continue
         path = p.get_package_dir(name)
         if isdir(join(path, "bin")):
@@ -80,23 +84,86 @@ def LoadPioPlatform(env, variables):
         env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")])
 
     if "BOARD" not in env:
+        # handle _MCU and _F_CPU variables for AVR native
+        for key, value in variables.UnknownVariables().items():
+            if not key.startswith("BOARD_"):
+                continue
+            env.Replace(
+                **{key.upper().replace("BUILD.", ""): base64.b64decode(value)})
         return
 
+    # update board manifest with a custom data
     board_config = env.BoardConfig()
-    for k in variables.keys():
-        if k in env or \
-                not any([k.startswith("BOARD_"), k.startswith("UPLOAD_")]):
+    for key, value in variables.UnknownVariables().items():
+        if not key.startswith("BOARD_"):
             continue
-        _opt, _val = k.lower().split("_", 1)
+        board_config.update(key.lower()[6:], base64.b64decode(value))
+
+    # update default environment variables
+    for key in variables.keys():
+        if key in env or \
+                not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]):
+            continue
+        _opt, _val = key.lower().split("_", 1)
         if _opt == "board":
             _opt = "build"
         if _val in board_config.get(_opt):
-            env.Replace(**{k: board_config.get("%s.%s" % (_opt, _val))})
+            env.Replace(**{key: board_config.get("%s.%s" % (_opt, _val))})
 
     if "build.ldscript" in board_config:
         env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript"))
 
 
+def PrintConfiguration(env):
+    platform_data = ["PLATFORM: %s >" % env.PioPlatform().title]
+    system_data = ["SYSTEM:"]
+    mcu = env.subst("$BOARD_MCU")
+    f_cpu = env.subst("$BOARD_F_CPU")
+    if mcu:
+        system_data.append(mcu.upper())
+    if f_cpu:
+        f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()]))
+        system_data.append("%dMHz" % (f_cpu / 1000000))
+
+    debug_tools = None
+    if "BOARD" in env:
+        board_config = env.BoardConfig()
+        platform_data.append(board_config.get("name"))
+
+        debug_tools = board_config.get("debug", {}).get("tools")
+        ram = board_config.get("upload", {}).get("maximum_ram_size")
+        flash = board_config.get("upload", {}).get("maximum_size")
+        system_data.append("%s RAM (%s Flash)" % (util.format_filesize(ram),
+                                                  util.format_filesize(flash)))
+
+    if platform_data:
+        print " ".join(platform_data)
+    if system_data:
+        print " ".join(system_data)
+
+    # Debugging
+    if not debug_tools:
+        return
+
+    data = [
+        "CURRENT(%s)" % board_config.get_debug_tool_name(
+            env.subst("$DEBUG_TOOL"))
+    ]
+    onboard = []
+    external = []
+    for key, value in debug_tools.items():
+        if value.get("onboard"):
+            onboard.append(key)
+        else:
+            external.append(key)
+    if onboard:
+        data.append("ON-BOARD(%s)" % ", ".join(sorted(onboard)))
+    if external:
+        data.append("EXTERNAL(%s)" % ", ".join(sorted(external)))
+
+    print "DEBUG: %s" % " ".join(data)
+
+
 def exists(_):
     return True
 
@@ -106,4 +173,5 @@ def generate(env):
     env.AddMethod(BoardConfig)
     env.AddMethod(GetFrameworkScript)
     env.AddMethod(LoadPioPlatform)
+    env.AddMethod(PrintConfiguration)
     return env

platformio/builder/tools/pioupload.py

@@ -18,7 +18,6 @@ import sys
 from fnmatch import fnmatch
 from os import environ
 from os.path import isfile, join
-from platform import system
 from shutil import copyfile
 from time import sleep
 
@@ -114,10 +113,10 @@ def AutodetectUploadPort(*args, **kwargs):  # pylint: disable=unused-argument
             mbed_pages = [
                 join(item['path'], n) for n in ("mbed.htm", "mbed.html")
             ]
-            if any([isfile(p) for p in mbed_pages]):
+            if any(isfile(p) for p in mbed_pages):
                 return item['path']
             if item['name'] \
-                    and any([l in item['name'].lower() for l in msdlabels]):
+                    and any(l in item['name'].lower() for l in msdlabels):
                 return item['path']
         return None
 
@@ -131,9 +130,12 @@ def AutodetectUploadPort(*args, **kwargs):  # pylint: disable=unused-argument
             if not _is_match_pattern(item['port']):
                 continue
             port = item['port']
-            if upload_protocol.startswith("blackmagic") \
-                    and "GDB" in item['description']:
-                return port
+            if upload_protocol.startswith("blackmagic"):
+                if "windows" in util.get_systype() and \
+                        port.startswith("COM") and len(port) > 4:
+                    port = "\\\\.\\%s" % port
+                if "GDB" in item['description']:
+                    return port
             for hwid in board_hwids:
                 hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
                 if hwid_str in item['hwid']:
@@ -144,19 +146,20 @@ def AutodetectUploadPort(*args, **kwargs):  # pylint: disable=unused-argument
         print env.subst("Use manually specified: $UPLOAD_PORT")
         return
 
-    if "mbed" in env.subst("$PIOFRAMEWORK") \
-            and not env.subst("$UPLOAD_PROTOCOL"):
+    if (env.subst("$UPLOAD_PROTOCOL") == "mbed"
+            or ("mbed" in env.subst("$PIOFRAMEWORK")
+                and not env.subst("$UPLOAD_PROTOCOL"))):
         env.Replace(UPLOAD_PORT=_look_for_mbed_disk())
     else:
-        if (system() == "Linux" and not any([
+        if ("linux" in util.get_systype() and not any([
                 isfile("/etc/udev/rules.d/99-platformio-udev.rules"),
                 isfile("/lib/udev/rules.d/99-platformio-udev.rules")
         ])):
             sys.stderr.write(
                 "\nWarning! Please install `99-platformio-udev.rules` and "
                 "check that your board's PID and VID are listed in the rules."
-                "\n https://raw.githubusercontent.com/platformio/platformio"
-                "/develop/scripts/99-platformio-udev.rules\n")
+                "\n http://docs.platformio.org/en/latest/faq.html"
+                "#platformio-udev-rules\n")
         env.Replace(UPLOAD_PORT=_look_for_serial_port())
 
     if env.subst("$UPLOAD_PORT"):
@@ -212,6 +215,18 @@ def CheckUploadSize(_, target, source, env):  # pylint: disable=W0613,W0621
         env.Exit(1)
 
 
+def PrintUploadInfo(env):
+    configured = env.subst("$UPLOAD_PROTOCOL")
+    available = [configured] if configured else []
+    if "BOARD" in env:
+        available.extend(env.BoardConfig().get("upload", {}).get(
+            "protocols", []))
+    if available:
+        print "AVAILABLE: %s" % ", ".join(sorted(set(available)))
+    if configured:
+        print "CURRENT: upload_protocol = %s" % configured
+
+
 def exists(_):
     return True
 
@@ -223,4 +238,5 @@ def generate(env):
     env.AddMethod(AutodetectUploadPort)
     env.AddMethod(UploadToDisk)
     env.AddMethod(CheckUploadSize)
+    env.AddMethod(PrintUploadInfo)
     return env

platformio/builder/tools/platformio.py

@@ -20,10 +20,9 @@ from glob import glob
 from os import sep, walk
 from os.path import basename, dirname, isdir, join, realpath
 
-from SCons.Action import Action
+from SCons import Builder, Util
 from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild,
                           DefaultEnvironment, SConscript)
-from SCons.Util import case_sensitive_suffixes, is_Sequence
 
 from platformio.util import glob_escape, pioversion_to_intstr
 
@@ -31,6 +30,15 @@ SRC_HEADER_EXT = ["h", "hpp"]
 SRC_C_EXT = ["c", "cc", "cpp"]
 SRC_BUILD_EXT = SRC_C_EXT + ["S", "spp", "SPP", "sx", "s", "asm", "ASM"]
 SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep]
+SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>")
+
+
+def scons_patched_match_splitext(path, suffixes=None):
+    """Patch SCons Builder, append $OBJSUFFIX to the end of each target"""
+    tokens = Util.splitext(path)
+    if suffixes and tokens[1] and tokens[1] in suffixes:
+        return (path, tokens[1])
+    return tokens
 
 
 def BuildProgram(env):
@@ -42,15 +50,18 @@ def BuildProgram(env):
 
     _append_pio_macros()
 
-    # fix ASM handling under non-casitive OS
-    if not case_sensitive_suffixes(".s", ".S"):
+    env.PrintConfiguration()
+
+    # fix ASM handling under non case-sensitive OS
+    if not Util.case_sensitive_suffixes(".s", ".S"):
         env.Replace(AS="$CC", ASCOM="$ASPPCOM")
 
+    if "__debug" in COMMAND_LINE_TARGETS:
+        env.ProcessDebug()
+
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
         env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
-    # remove base flags
-    env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
     # apply user flags
     env.ProcessFlags(env.get("BUILD_FLAGS"))
 
@@ -60,24 +71,15 @@ def BuildProgram(env):
     # restore PIO macros if it was deleted by framework
     _append_pio_macros()
 
-    # Search for project source files
-    env.Append(
-        LIBPATH=["$BUILD_DIR"],
-        PIOBUILDFILES=env.CollectBuildFiles(
-            "$BUILDSRC_DIR", "$PROJECTSRC_DIR", "$SRC_FILTER",
-            duplicate=False))
+    # remove specified flags
+    env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
-    if "__debug" in COMMAND_LINE_TARGETS:
-        env.ProcessDebug()
-    if "__test" in COMMAND_LINE_TARGETS:
-        env.Append(PIOBUILDFILES=env.ProcessTest())
-
-    # build dependent libs
-    env.Append(LIBS=env.BuildProjectLibraries())
+    # build dependent libs; place them before built-in libs
+    env.Prepend(LIBS=env.BuildProjectLibraries())
 
     # append specified LD_SCRIPT
     if ("LDSCRIPT_PATH" in env
-            and not any(["-Wl,-T" in f for f in env['LINKFLAGS']])):
+            and not any("-Wl,-T" in f for f in env['LINKFLAGS'])):
         env.Append(LINKFLAGS=['-Wl,-T"$LDSCRIPT_PATH"'])
 
     # enable "cyclic reference" for linker
@@ -88,7 +90,16 @@ def BuildProgram(env):
     # Handle SRC_BUILD_FLAGS
     env.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
 
-    if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
+    if "__test" in COMMAND_LINE_TARGETS:
+        env.Append(PIOBUILDFILES=env.ProcessTest())
+    else:
+        env.Append(
+            PIOBUILDFILES=env.CollectBuildFiles(
+                "$BUILDSRC_DIR",
+                "$PROJECTSRC_DIR",
+                src_filter=env.get("SRC_FILTER")))
+
+    if not env['PIOBUILDFILES'] and not COMMAND_LINE_TARGETS:
         sys.stderr.write(
             "Error: Nothing to build. Please put your source code files "
             "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR"))
@@ -97,7 +108,8 @@ def BuildProgram(env):
     program = env.Program(
         join("$BUILD_DIR", env.subst("$PROGNAME")), env['PIOBUILDFILES'])
 
-    checksize_action = Action(env.CheckUploadSize, "Checking program size")
+    checksize_action = env.VerboseAction(env.CheckUploadSize,
+                                         "Checking program size")
     AlwaysBuild(env.Alias("checkprogsize", program, checksize_action))
     if set(["upload", "program"]) & set(COMMAND_LINE_TARGETS):
         env.AddPostAction(program, checksize_action)
@@ -105,38 +117,47 @@ def BuildProgram(env):
     return program
 
 
-def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
-    if not flags:
-        return
+def ParseFlagsExtended(env, flags):
     if isinstance(flags, list):
         flags = " ".join(flags)
-    parsed_flags = env.ParseFlags(str(flags))
-    for flag in parsed_flags.pop("CPPDEFINES"):
-        if not is_Sequence(flag):
-            env.Append(CPPDEFINES=flag)
+    result = env.ParseFlags(str(flags))
+
+    cppdefines = []
+    for item in result['CPPDEFINES']:
+        if not Util.is_Sequence(item):
+            cppdefines.append(item)
             continue
-        _key, _value = flag[:2]
-        if '\"' in _value:
-            _value = _value.replace('\"', '\\\"')
-        elif _value.isdigit():
-            _value = int(_value)
-        elif _value.replace(".", "", 1).isdigit():
-            _value = float(_value)
-        env.Append(CPPDEFINES=(_key, _value))
-    env.Append(**parsed_flags)
+        name, value = item[:2]
+        if '\"' in value:
+            value = value.replace('\"', '\\\"')
+        elif value.isdigit():
+            value = int(value)
+        elif value.replace(".", "", 1).isdigit():
+            value = float(value)
+        cppdefines.append((name, value))
+    result['CPPDEFINES'] = cppdefines
 
     # fix relative CPPPATH & LIBPATH
     for k in ("CPPPATH", "LIBPATH"):
-        for i, p in enumerate(env.get(k, [])):
+        for i, p in enumerate(result.get(k, [])):
             if isdir(p):
-                env[k][i] = realpath(p)
+                result[k][i] = realpath(p)
+
     # fix relative path for "-include"
-    for i, f in enumerate(env.get("CCFLAGS", [])):
+    for i, f in enumerate(result.get("CCFLAGS", [])):
         if isinstance(f, tuple) and f[0] == "-include":
-            env['CCFLAGS'][i] = (f[0], env.File(realpath(f[1].get_path())))
+            result['CCFLAGS'][i] = (f[0], env.File(realpath(f[1].get_path())))
+
+    return result
+
+
+def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
+    if not flags:
+        return
+    env.Append(**env.ParseFlagsExtended(flags))
 
     # Cancel any previous definition of name, either built in or
-    # provided with a -D option // Issue #191
+    # provided with a -U option // Issue #191
     undefines = [
         u for u in env.get("CCFLAGS", [])
         if isinstance(u, basestring) and u.startswith("-U")
@@ -150,19 +171,16 @@ def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
 def ProcessUnFlags(env, flags):
     if not flags:
         return
-    if isinstance(flags, list):
-        flags = " ".join(flags)
-    parsed_flags = env.ParseFlags(str(flags))
-    all_flags = []
-    for items in parsed_flags.values():
-        all_flags.extend(items)
-    all_flags = set(all_flags)
-
-    for key in parsed_flags:
-        cur_flags = set(env.Flatten(env.get(key, [])))
-        for item in cur_flags & all_flags:
-            while item in env[key]:
-                env[key].remove(item)
+    for key, unflags in env.ParseFlagsExtended(flags).items():
+        for unflag in unflags:
+            for current in env.get(key, []):
+                conditions = [
+                    unflag == current,
+                    isinstance(current, (tuple, list))
+                    and unflag[0] == current[0]
+                ]
+                if any(conditions):
+                    env[key].remove(current)
 
 
 def IsFileWithExt(env, file_, ext):  # pylint: disable=W0613
@@ -176,8 +194,6 @@ def IsFileWithExt(env, file_, ext):  # pylint: disable=W0613
 
 def MatchSourceFiles(env, src_dir, src_filter=None):
 
-    SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>")
-
     def _append_build_item(items, item, src_dir):
         if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT):
             items.add(item.replace(src_dir + sep, ""))
@@ -254,6 +270,8 @@ def BuildFrameworks(env, frameworks):
 
     for f in frameworks:
         if f in ("arduino", "energia"):
+            # Arduino IDE appends .o the end of filename
+            Builder.match_splitext = scons_patched_match_splitext
             env.ConvertInoToCpp()
 
         if f in board_frameworks:
@@ -265,15 +283,14 @@ def BuildFrameworks(env, frameworks):
 
 
 def BuildLibrary(env, variant_dir, src_dir, src_filter=None):
-    lib = env.Clone()
-    return lib.StaticLibrary(
-        lib.subst(variant_dir),
-        lib.CollectBuildFiles(variant_dir, src_dir, src_filter))
+    return env.StaticLibrary(
+        env.subst(variant_dir),
+        env.CollectBuildFiles(variant_dir, src_dir, src_filter))
 
 
 def BuildSources(env, variant_dir, src_dir, src_filter=None):
-    DefaultEnvironment().Append(PIOBUILDFILES=env.Clone().CollectBuildFiles(
-        variant_dir, src_dir, src_filter))
+    DefaultEnvironment().Append(
+        PIOBUILDFILES=env.CollectBuildFiles(variant_dir, src_dir, src_filter))
 
 
 def exists(_):
@@ -282,6 +299,7 @@ def exists(_):
 
 def generate(env):
     env.AddMethod(BuildProgram)
+    env.AddMethod(ParseFlagsExtended)
     env.AddMethod(ProcessFlags)
     env.AddMethod(ProcessUnFlags)
     env.AddMethod(IsFileWithExt)

platformio/commands/boards.py

@@ -16,6 +16,7 @@ import json
 
 import click
 
+from platformio import util
 from platformio.managers.platform import PlatformManager
 
 
@@ -60,22 +61,13 @@ def print_boards(boards):
     click.echo("-" * terminal_width)
 
     for board in boards:
-        ram_size = board['ram']
-        if ram_size >= 1024:
-            if ram_size % 1024:
-                ram_size = "%.1fkB" % (ram_size / 1024.0)
-            else:
-                ram_size = "%dkB" % (ram_size / 1024)
-        else:
-            ram_size = "%dB" % ram_size
-
         click.echo(
             BOARDLIST_TPL.format(
                 type=click.style(board['id'], fg="cyan"),
                 mcu=board['mcu'],
-                frequency="%dMhz" % (board['fcpu'] / 1000000),
-                flash="%dkB" % (board['rom'] / 1024),
-                ram=ram_size,
+                frequency="%dMHz" % (board['fcpu'] / 1000000),
+                flash=util.format_filesize(board['rom']),
+                ram=util.format_filesize(board['ram']),
                 name=board['name']))
 
 

platformio/commands/device.py

@@ -165,8 +165,10 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
                                               kwargs['environment'])
         monitor_options = {k: v for k, v in project_options or []}
         if monitor_options:
-            for k in ("port", "baud", "rts", "dtr"):
+            for k in ("port", "baud", "speed", "rts", "dtr"):
                 k2 = "monitor_%s" % k
+                if k == "speed":
+                    k = "baud"
                 if kwargs[k] is None and k2 in monitor_options:
                     kwargs[k] = monitor_options[k2]
                     if k != "port":

platformio/commands/init.py

@@ -139,15 +139,12 @@ def init_base_project(project_dir):
             join(util.get_source_dir(), "projectconftpl.ini"),
             join(project_dir, "platformio.ini"))
 
-    lib_dir = join(project_dir, "lib")
-    src_dir = join(project_dir, "src")
-    config = util.load_project_config(project_dir)
-    if config.has_option("platformio", "src_dir"):
-        src_dir = join(project_dir, config.get("platformio", "src_dir"))
-
-    for d in (src_dir, lib_dir):
-        if not isdir(d):
-            makedirs(d)
+    with util.cd(project_dir):
+        lib_dir = util.get_projectlib_dir()
+        src_dir = util.get_projectsrc_dir()
+        for d in (src_dir, lib_dir):
+            if not isdir(d):
+                makedirs(d)
 
     init_lib_readme(lib_dir)
     init_ci_conf(project_dir)
@@ -168,16 +165,21 @@ The source code of each library should be placed in separate directory, like
 For example, see how can be organized `Foo` and `Bar` libraries:
 
 |--lib
+|  |
 |  |--Bar
 |  |  |--docs
 |  |  |--examples
 |  |  |--src
 |  |     |- Bar.c
 |  |     |- Bar.h
+|  |  |- library.json (optional, custom build options, etc) http://docs.platformio.org/page/librarymanager/config.html
+|  |
 |  |--Foo
 |  |  |- Foo.c
 |  |  |- Foo.h
+|  |
 |  |- readme.txt --> THIS FILE
+|
 |- platformio.ini
 |--src
    |- main.c
@@ -231,8 +233,14 @@ def init_ci_conf(project_dir):
 # python:
 #     - "2.7"
 #
+# sudo: false
+# cache:
+#     directories:
+#         - "~/.platformio"
+#
 # install:
 #     - pip install -U platformio
+#     - platformio update
 #
 # script:
 #     - platformio run
@@ -246,6 +254,11 @@ def init_ci_conf(project_dir):
 # python:
 #     - "2.7"
 #
+# sudo: false
+# cache:
+#     directories:
+#         - "~/.platformio"
+#
 # env:
 #     - PLATFORMIO_CI_SRC=path/to/test/file.c
 #     - PLATFORMIO_CI_SRC=examples/file.ino
@@ -253,6 +266,7 @@ def init_ci_conf(project_dir):
 #
 # install:
 #     - pip install -U platformio
+#     - platformio update
 #
 # script:
 #     - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N

platformio/commands/lib.py

@@ -15,16 +15,14 @@
 # pylint: disable=too-many-branches, too-many-locals
 
 import json
+import time
 from os.path import isdir, join
-from time import sleep
 from urllib import quote
 
-import arrow
 import click
 
 from platformio import exception, util
-from platformio.managers.lib import LibraryManager
-from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.managers.lib import LibraryManager, get_builtin_libs
 from platformio.util import get_api_result
 
 
@@ -99,7 +97,7 @@ def cli(ctx, **options):
     help="Reinstall/redownload library if exists")
 @click.pass_obj
 def lib_install(lm, libraries, silent, interactive, force):
-    # @TODO "save" option
+    # @TODO: "save" option
     for library in libraries:
         lm.install(
             library, silent=silent, interactive=interactive, force=force)
@@ -188,6 +186,7 @@ def print_lib_item(item):
 @click.argument("query", required=False, nargs=-1)
 @click.option("--json-output", is_flag=True)
 @click.option("--page", type=click.INT, default=1)
+@click.option("--id", multiple=True)
 @click.option("-n", "--name", multiple=True)
 @click.option("-a", "--author", multiple=True)
 @click.option("-k", "--keyword", multiple=True)
@@ -252,13 +251,14 @@ def lib_search(query, json_output, page, noninteractive, **filters):
                 result['perpage'],
                 fg="yellow")
             click.echo()
-            sleep(5)
+            time.sleep(5)
         elif not click.confirm("Show next libraries?"):
             break
         result = get_api_result(
-            "/v2/lib/search",
-            {"query": " ".join(query),
-             "page": int(result['page']) + 1},
+            "/v2/lib/search", {
+                "query": " ".join(query),
+                "page": int(result['page']) + 1
+            },
             cache_valid="1d")
 
 
@@ -280,25 +280,6 @@ def lib_list(lm, json_output):
     return True
 
 
-@util.memoized
-def get_builtin_libs(storage_names=None):
-    items = []
-    storage_names = storage_names or []
-    pm = PlatformManager()
-    for manifest in pm.get_installed():
-        p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
-        for storage in p.get_lib_storages():
-            if storage_names and storage['name'] not in storage_names:
-                continue
-            lm = LibraryManager(storage['path'])
-            items.append({
-                "name": storage['name'],
-                "path": storage['path'],
-                "items": lm.get_installed()
-            })
-    return items
-
-
 @cli.command("builtin", short_help="List built-in libraries")
 @click.option("--storage", multiple=True)
 @click.option("--json-output", is_flag=True)
@@ -326,8 +307,13 @@ def lib_builtin(storage, json_output):
 def lib_show(library, json_output):
     lm = LibraryManager()
     name, requirements, _ = lm.parse_pkg_uri(library)
-    lib_id = lm.get_pkg_id_by_name(
-        name, requirements, silent=json_output, interactive=not json_output)
+    lib_id = lm.search_lib_id(
+        {
+            "name": name,
+            "requirements": requirements
+        },
+        silent=json_output,
+        interactive=not json_output)
     lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
     if json_output:
         return click.echo(json.dumps(lib))
@@ -338,9 +324,10 @@ def lib_show(library, json_output):
     click.echo(lib['description'])
     click.echo()
 
-    click.echo("Version: %s, released %s" %
-               (lib['version']['name'],
-                arrow.get(lib['version']['released']).humanize()))
+    click.echo(
+        "Version: %s, released %s" %
+        (lib['version']['name'],
+         time.strftime("%c", util.parse_date(lib['version']['released']))))
     click.echo("Manifest: %s" % lib['confurl'])
     for key in ("homepage", "repository", "license"):
         if key not in lib or not lib[key]:
@@ -376,7 +363,8 @@ def lib_show(library, json_output):
     blocks.append(("Headers", lib['headers']))
     blocks.append(("Examples", lib['examples']))
     blocks.append(("Versions", [
-        "%s, released %s" % (v['name'], arrow.get(v['released']).humanize())
+        "%s, released %s" %
+        (v['name'], time.strftime("%c", util.parse_date(v['released'])))
         for v in lib['versions']
     ]))
     blocks.append(("Unique Downloads", [
@@ -439,10 +427,10 @@ def lib_stats(json_output):
                     if "date" in item else printitem_tpl).format(
                         name=click.style(item['name'], fg="cyan"),
                         date=str(
-                            arrow.get(item['date']).humanize()
+                            time.strftime("%c", util.parse_date(item['date']))
                             if "date" in item else ""),
                         url=click.style(
-                            "http://platformio.org/lib/show/%s/%s" %
+                            "https://platformio.org/lib/show/%s/%s" %
                             (item['id'], quote(item['name'])),
                             fg="blue")))
 
@@ -451,7 +439,7 @@ def lib_stats(json_output):
             printitem_tpl.format(
                 name=click.style(name, fg="cyan"),
                 url=click.style(
-                    "http://platformio.org/lib/search?query=" +
+                    "https://platformio.org/lib/search?query=" +
                     quote("keyword:%s" % name),
                     fg="blue")))
 

platformio/commands/platform.py

@@ -85,6 +85,7 @@ def _get_installed_platform_data(platform,
         homepage=p.homepage,
         repository=p.repository_url,
         url=p.vendor_url,
+        docs=p.docs_url,
         license=p.license,
         forDesktop=not p.is_embedded(),
         frameworks=sorted(p.frameworks.keys() if p.frameworks else []),
@@ -195,7 +196,7 @@ def platform_frameworks(query, json_output):
         if query and query.lower() not in search_data.lower():
             continue
         framework['homepage'] = (
-            "http://platformio.org/frameworks/" + framework['name'])
+            "https://platformio.org/frameworks/" + framework['name'])
         framework['platforms'] = [
             platform['name'] for platform in _get_registry_platforms()
             if framework['name'] in platform['frameworks']

platformio/commands/run.py

@@ -23,10 +23,9 @@ import click
 from platformio import __version__, exception, telemetry, util
 from platformio.commands.device import device_monitor as cmd_device_monitor
 from platformio.commands.lib import lib_install as cmd_lib_install
-from platformio.commands.lib import get_builtin_libs
 from platformio.commands.platform import \
     platform_install as cmd_platform_install
-from platformio.managers.lib import LibraryManager
+from platformio.managers.lib import LibraryManager, is_builtin_lib
 from platformio.managers.platform import PlatformFactory
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
@@ -60,26 +59,26 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
         raise exception.NotPlatformIOProject(project_dir)
 
     with util.cd(project_dir):
-        # clean obsolete .pioenvs dir
+        # clean obsolete build dir
         if not disable_auto_clean:
             try:
-                _clean_pioenvs_dir(util.get_projectpioenvs_dir())
+                _clean_build_dir(util.get_projectbuild_dir())
             except:  # pylint: disable=bare-except
                 click.secho(
                     "Can not remove temporary directory `%s`. Please remove "
-                    "`.pioenvs` directory from the project manually to avoid "
-                    "build issues" % util.get_projectpioenvs_dir(force=True),
+                    "it manually to avoid build issues" %
+                    util.get_projectbuild_dir(force=True),
                     fg="yellow")
 
         config = util.load_project_config()
-        check_project_defopts(config)
-        assert check_project_envs(config, environment)
-
         env_default = None
         if config.has_option("platformio", "env_default"):
             env_default = util.parse_conf_multi_values(
                 config.get("platformio", "env_default"))
 
+        check_project_defopts(config)
+        check_project_envs(config, environment or env_default)
+
         results = []
         start_time = time()
         for section in config.sections():
@@ -112,7 +111,7 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
                     "nobuild" not in ep.get_build_targets():
                 ctx.invoke(cmd_device_monitor)
 
-        found_error = any([status is False for (_, status) in results])
+        found_error = any(status is False for (_, status) in results)
 
         if (found_error or not silent) and len(results) > 1:
             click.echo()
@@ -125,32 +124,46 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
 
 class EnvironmentProcessor(object):
 
-    KNOWN_OPTIONS = ("platform", "framework", "board", "board_mcu",
-                     "board_f_cpu", "board_f_flash", "board_flash_mode",
-                     "build_flags", "src_build_flags", "build_unflags",
-                     "src_filter", "extra_scripts", "targets", "upload_port",
-                     "upload_protocol", "upload_speed", "upload_flags",
-                     "upload_resetmethod", "lib_deps", "lib_ignore",
-                     "lib_extra_dirs", "lib_ldf_mode", "lib_compat_mode",
-                     "lib_archive", "piotest", "test_transport", "test_filter",
-                     "test_ignore", "test_port", "debug_tool", "debug_port",
-                     "debug_init_cmds", "debug_extra_cmds", "debug_server",
-                     "debug_init_break", "debug_load_cmd", "monitor_port",
-                     "monitor_baud", "monitor_rts", "monitor_dtr")
+    DEFAULT_DUMP_OPTIONS = ("platform", "framework", "board")
+
+    KNOWN_PLATFORMIO_OPTIONS = ("description", "env_default", "home_dir",
+                                "lib_dir", "libdeps_dir", "include_dir",
+                                "src_dir", "build_dir", "data_dir", "test_dir",
+                                "boards_dir", "lib_extra_dirs")
+
+    KNOWN_ENV_OPTIONS = ("platform", "framework", "board", "build_flags",
+                         "src_build_flags", "build_unflags", "src_filter",
+                         "extra_scripts", "targets", "upload_port",
+                         "upload_protocol", "upload_speed", "upload_flags",
+                         "upload_resetmethod", "lib_deps", "lib_ignore",
+                         "lib_extra_dirs", "lib_ldf_mode", "lib_compat_mode",
+                         "lib_archive", "piotest", "test_transport",
+                         "test_filter", "test_ignore", "test_port",
+                         "test_speed", "debug_tool", "debug_port",
+                         "debug_init_cmds", "debug_extra_cmds", "debug_server",
+                         "debug_init_break", "debug_load_cmd",
+                         "debug_load_mode", "debug_svd_path", "monitor_port",
+                         "monitor_speed", "monitor_rts", "monitor_dtr")
 
     IGNORE_BUILD_OPTIONS = ("test_transport", "test_filter", "test_ignore",
-                            "test_port", "debug_tool", "debug_port",
+                            "test_port", "test_speed", "debug_port",
                             "debug_init_cmds", "debug_extra_cmds",
                             "debug_server", "debug_init_break",
-                            "debug_load_cmd", "monitor_port", "monitor_baud",
-                            "monitor_rts", "monitor_dtr")
+                            "debug_load_cmd", "debug_load_mode",
+                            "monitor_port", "monitor_speed", "monitor_rts",
+                            "monitor_dtr")
 
     REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"}
 
     RENAMED_OPTIONS = {
         "lib_use": "lib_deps",
         "lib_force": "lib_deps",
-        "extra_script": "extra_scripts"
+        "extra_script": "extra_scripts",
+        "monitor_baud": "monitor_speed",
+        "board_mcu": "board_build.mcu",
+        "board_f_cpu": "board_build.f_cpu",
+        "board_f_flash": "board_build.f_flash",
+        "board_flash_mode": "board_build.flash_mode"
     }
 
     RENAMED_PLATFORMS = {"espressif": "espressif8266"}
@@ -175,19 +188,19 @@ class EnvironmentProcessor(object):
     def process(self):
         terminal_width, _ = click.get_terminal_size()
         start_time = time()
+        env_dump = []
 
         for k, v in self.options.items():
             self.options[k] = self.options[k].strip()
+            if self.verbose or k in self.DEFAULT_DUMP_OPTIONS:
+                env_dump.append(
+                    "%s: %s" % (k, ", ".join(util.parse_conf_multi_values(v))))
 
         if not self.silent:
             click.echo("[%s] Processing %s (%s)" %
                        (datetime.now().strftime("%c"),
                         click.style(self.name, fg="cyan", bold=True),
-                        "; ".join([
-                            "%s: %s" %
-                            (k, ", ".join(util.parse_conf_multi_values(v)))
-                            for k, v in self.options.items()
-                        ])))
+                        "; ".join(env_dump)))
             click.secho("-" * terminal_width, bold=True)
 
         self.options = self._validate_options(self.options)
@@ -228,7 +241,11 @@ class EnvironmentProcessor(object):
                 v = self.RENAMED_PLATFORMS[v]
 
             # warn about unknown options
-            if k not in self.KNOWN_OPTIONS and not k.startswith("custom_"):
+            unknown_conditions = [
+                k not in self.KNOWN_ENV_OPTIONS, not k.startswith("custom_"),
+                not k.startswith("board_")
+            ]
+            if all(unknown_conditions):
                 click.secho(
                     "Detected non-PlatformIO `%s` option in `[env:%s]` section"
                     % (k, self.name),
@@ -307,36 +324,31 @@ def _autoinstall_libdeps(ctx, libraries, verbose=False):
         try:
             ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose)
         except exception.LibNotFound as e:
-            if not _is_builtin_lib(lib):
+            if verbose or not is_builtin_lib(lib):
                 click.secho("Warning! %s" % e, fg="yellow")
+        except exception.InternetIsOffline as e:
+            click.secho(str(e), fg="yellow")
 
 
-def _is_builtin_lib(lib_name):
-    for storage in get_builtin_libs():
-        if any([l.get("name") == lib_name for l in storage['items']]):
-            return True
-    return False
-
-
-def _clean_pioenvs_dir(pioenvs_dir):
-    structhash_file = join(pioenvs_dir, "structure.hash")
+def _clean_build_dir(build_dir):
+    structhash_file = join(build_dir, "structure.hash")
     proj_hash = calculate_project_hash()
 
     # if project's config is modified
-    if (isdir(pioenvs_dir)
+    if (isdir(build_dir)
             and getmtime(join(util.get_project_dir(),
-                              "platformio.ini")) > getmtime(pioenvs_dir)):
-        util.rmtree_(pioenvs_dir)
+                              "platformio.ini")) > getmtime(build_dir)):
+        util.rmtree_(build_dir)
 
     # check project structure
-    if isdir(pioenvs_dir) and isfile(structhash_file):
+    if isdir(build_dir) and isfile(structhash_file):
         with open(structhash_file) as f:
             if f.read() == proj_hash:
                 return
-        util.rmtree_(pioenvs_dir)
+        util.rmtree_(build_dir)
 
-    if not isdir(pioenvs_dir):
-        makedirs(pioenvs_dir)
+    if not isdir(build_dir):
+        makedirs(build_dir)
 
     with open(structhash_file, "w") as f:
         f.write(proj_hash)
@@ -383,38 +395,42 @@ def print_summary(results, start_time):
 def check_project_defopts(config):
     if not config.has_section("platformio"):
         return True
-    known = ("env_default", "home_dir", "lib_dir", "libdeps_dir", "src_dir",
-             "envs_dir", "data_dir", "test_dir", "boards_dir",
-             "lib_extra_dirs")
-    unknown = set([k for k, _ in config.items("platformio")]) - set(known)
+    unknown = set([k for k, _ in config.items("platformio")]) - set(
+        EnvironmentProcessor.KNOWN_PLATFORMIO_OPTIONS)
     if not unknown:
         return True
     click.secho(
-        "Warning! Ignore unknown `%s` option from `[platformio]` section" %
+        "Warning! Ignore unknown `%s` option in `[platformio]` section" %
         ", ".join(unknown),
         fg="yellow")
     return False
 
 
-def check_project_envs(config, environments):
+def check_project_envs(config, environments=None):
     if not config.sections():
         raise exception.ProjectEnvsNotAvailable()
 
     known = set([s[4:] for s in config.sections() if s.startswith("env:")])
-    unknown = set(environments) - known
+    unknown = set(environments or []) - known
     if unknown:
         raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known))
     return True
 
 
 def calculate_project_hash():
-    structure = [__version__]
+    check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
+    chunks = [__version__]
     for d in (util.get_projectsrc_dir(), util.get_projectlib_dir()):
         if not isdir(d):
             continue
         for root, _, files in walk(d):
             for f in files:
                 path = join(root, f)
-                if not any([s in path for s in (".git", ".svn", ".pioenvs")]):
-                    structure.append(path)
-    return sha1(",".join(sorted(structure))).hexdigest() if structure else ""
+                if path.endswith(check_suffixes):
+                    chunks.append(path)
+    chunks_to_str = ",".join(sorted(chunks))
+    if "windows" in util.get_systype():
+        # Fix issue with useless project rebuilding for case insensitive FS.
+        # A case of disk drive can differ...
+        chunks_to_str = chunks_to_str.lower()
+    return sha1(chunks_to_str).hexdigest()

platformio/commands/upgrade.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import re
+from zipfile import ZipFile
 
 import click
 import requests
@@ -36,12 +38,9 @@ def cli(dev):
     # kill all PIO Home servers, they block `pioplus` binary
     shutdown_servers()
 
-    to_develop = dev or not all([c.isdigit() for c in __version__ if c != "."])
-    cmds = ([
-        "pip", "install", "--upgrade",
-        "https://github.com/platformio/platformio-core/archive/develop.zip"
-        if to_develop else "platformio"
-    ], ["platformio", "--version"])
+    to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".")
+    cmds = (["pip", "install", "--upgrade",
+             get_pip_package(to_develop)], ["platformio", "--version"])
 
     cmd = None
     r = None
@@ -69,7 +68,7 @@ def cli(dev):
         if not r:
             raise exception.UpgradeError("\n".join([str(cmd), str(e)]))
         permission_errors = ("permission denied", "not permitted")
-        if (any([m in r['err'].lower() for m in permission_errors])
+        if (any(m in r['err'].lower() for m in permission_errors)
                 and "windows" not in util.get_systype()):
             click.secho(
                 """
@@ -92,6 +91,29 @@ WARNING! Don't use `sudo` for the rest PlatformIO commands.
     return True
 
 
+def get_pip_package(to_develop):
+    if not to_develop:
+        return "platformio"
+    dl_url = ("https://github.com/platformio/"
+              "platformio-core/archive/develop.zip")
+    cache_dir = util.get_cache_dir()
+    if not os.path.isdir(cache_dir):
+        os.makedirs(cache_dir)
+    pkg_name = os.path.join(cache_dir, "piocoredevelop.zip")
+    try:
+        with open(pkg_name, "w") as fp:
+            r = util.exec_command(
+                ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True)
+            assert r['returncode'] == 0
+        # check ZIP structure
+        with ZipFile(pkg_name) as zp:
+            assert zp.testzip() is None
+        return pkg_name
+    except:  # pylint: disable=bare-except
+        pass
+    return dl_url
+
+
 def get_latest_version():
     try:
         if not str(VERSION[2]).isdigit():

platformio/downloader.py

@@ -21,7 +21,7 @@ from time import mktime
 import click
 import requests
 
-from platformio import app, util
+from platformio import util
 from platformio.exception import (FDSHASumMismatch, FDSizeMismatch,
                                   FDUnrecognizedStatusCode)
 
@@ -48,9 +48,8 @@ class FileDownloader(object):
                     "'", "")
             self._fname = self._fname.encode("utf8")
         else:
-            self._fname = url.split("/")[-1]
+            self._fname = [p for p in url.split("/") if p][-1]
 
-        self._progressbar = None
         self._destination = self._fname
         if dest_dir:
             self.set_destination(
@@ -70,12 +69,12 @@ class FileDownloader(object):
             return -1
         return int(self._request.headers['content-length'])
 
-    def start(self):
+    def start(self, with_progress=True):
         label = "Downloading"
         itercontent = self._request.iter_content(chunk_size=self.CHUNK_SIZE)
         f = open(self._destination, "wb")
         try:
-            if app.is_disabled_progressbar() or self.get_size() == -1:
+            if not with_progress or self.get_size() == -1:
                 click.echo("%s..." % label)
                 for chunk in itercontent:
                     if chunk:
@@ -85,12 +84,6 @@ class FileDownloader(object):
                 with click.progressbar(length=chunks, label=label) as pb:
                     for _ in pb:
                         f.write(next(itercontent))
-        except IOError as e:
-            click.secho(
-                "Error: Please read http://bit.ly/package-manager-ioerror",
-                fg="red",
-                err=True)
-            raise e
         finally:
             f.close()
             self._request.close()
@@ -98,6 +91,8 @@ class FileDownloader(object):
         if self.get_lmtime():
             self._preserve_filemtime(self.get_lmtime())
 
+        return True
+
     def verify(self, sha1=None):
         _dlsize = getsize(self._destination)
         if self.get_size() != -1 and _dlsize != self.get_size():

platformio/exception.py

@@ -97,10 +97,9 @@ class UndefinedPackageVersion(PlatformioException):
 
 class PackageInstallError(PlatformioException):
 
-    MESSAGE = (
-        "Could not install '{0}' with version requirements '{1}' for your "
-        "system '{2}'.\n If you use Antivirus, it can block PlatformIO "
-        "Package Manager. Try to disable it for a while.")
+    MESSAGE = ("Could not install '{0}' with version requirements '{1}' "
+               "for your system '{2}'.\n\n"
+               "Please try this solution -> http://bit.ly/faq-package-manager")
 
 
 class FDUnrecognizedStatusCode(PlatformioException):
@@ -208,6 +207,11 @@ class InvalidSettingValue(PlatformioException):
     MESSAGE = "Invalid value '{0}' for the setting '{1}'"
 
 
+class InvalidJSONFile(PlatformioException):
+
+    MESSAGE = "Could not load broken JSON: {0}"
+
+
 class CIBuildEnvsEmpty(PlatformioException):
 
     MESSAGE = ("Can't find PlatformIO build environments.\n"
@@ -239,3 +243,15 @@ class CygwinEnvDetected(PlatformioException):
 
     MESSAGE = ("PlatformIO does not work within Cygwin environment. "
                "Use native Terminal instead.")
+
+
+class DebugSupportError(PlatformioException):
+
+    MESSAGE = ("Currently, PlatformIO does not support debugging for `{0}`.\n"
+               "Please mail contact@pioplus.com or visit "
+               "< http://docs.platformio.org/page/plus/debugging.html >")
+
+
+class DebugInvalidOptions(PlatformioException):
+
+    pass

platformio/ide/projectgenerator.py

@@ -30,11 +30,8 @@ class ProjectGenerator(object):
         self.project_dir = project_dir
         self.ide = ide
         self.env_name = env_name
+
         self._tplvars = {}
-
-        with util.cd(self.project_dir):
-            self.project_src_dir = util.get_projectsrc_dir()
-
         self._gather_tplvars()
 
     @staticmethod
@@ -43,7 +40,7 @@ class ProjectGenerator(object):
         return sorted(
             [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))])
 
-    @util.memoized
+    @util.memoized()
     def get_project_env(self):
         data = {}
         config = util.load_project_config(self.project_dir)
@@ -57,7 +54,6 @@ class ProjectGenerator(object):
                 data[k] = v
         return data
 
-    @util.memoized
     def get_project_build_data(self):
         data = {
             "defines": [],
@@ -92,7 +88,7 @@ class ProjectGenerator(object):
     def get_src_files(self):
         result = []
         with util.cd(self.project_dir):
-            for root, _, files in os.walk(self.project_src_dir):
+            for root, _, files in os.walk(util.get_projectsrc_dir()):
                 for f in files:
                     result.append(relpath(join(root, f)))
         return result
@@ -153,26 +149,21 @@ class ProjectGenerator(object):
     def _gather_tplvars(self):
         self._tplvars.update(self.get_project_env())
         self._tplvars.update(self.get_project_build_data())
-        self._tplvars.update({
-            "project_name":
-            self.get_project_name(),
-            "src_files":
-            self.get_src_files(),
-            "user_home_dir":
-            abspath(expanduser("~")),
-            "project_dir":
-            self.project_dir,
-            "project_src_dir":
-            self.project_src_dir,
-            "systype":
-            util.get_systype(),
-            "platformio_path":
-            self._fix_os_path(util.where_is_program("platformio")),
-            "env_pathsep":
-            os.pathsep,
-            "env_path":
-            self._fix_os_path(os.getenv("PATH"))
-        })
+        with util.cd(self.project_dir):
+            self._tplvars.update({
+                "project_name": self.get_project_name(),
+                "src_files": self.get_src_files(),
+                "user_home_dir": abspath(expanduser("~")),
+                "project_dir": self.project_dir,
+                "project_src_dir": util.get_projectsrc_dir(),
+                "project_lib_dir": util.get_projectlib_dir(),
+                "project_libdeps_dir": util.get_projectlibdeps_dir(),
+                "systype": util.get_systype(),
+                "platformio_path": self._fix_os_path(
+                    util.where_is_program("platformio")),
+                "env_pathsep": os.pathsep,
+                "env_path": self._fix_os_path(os.getenv("PATH"))
+            })  # yapf: disable
 
     @staticmethod
     def _fix_os_path(path):

platformio/ide/tpls/clion/.idea/misc.xml.tpl

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CMakeWorkspace" PROJECT_DIR="$PROJECT_DIR$" />
+  <component name="CidrRootsConfiguration">
+    <sourceRoots>
+      <file path="$PROJECT_DIR$/src" />
+    </sourceRoots>
+    <libraryRoots>
+      <file path="$PROJECT_DIR$/lib" />
+      <file path="$PROJECT_DIR$/.piolibdeps" />
+    </libraryRoots>
+    <excludeRoots>
+      <file path="$PROJECT_DIR$/.pioenvs" />
+    </excludeRoots>
+  </component>
+</project>

platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -7,8 +7,9 @@ SET(CMAKE_CXX_FLAGS_DISTRIBUTION "{{cxx_flags}}")
 SET(CMAKE_C_FLAGS_DISTRIBUTION "{{cc_flags}}")
 set(CMAKE_CXX_STANDARD 11)
 
+% import re
 % for define in defines:
-add_definitions(-D{{!define}})
+add_definitions(-D'{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}')
 % end
 
 % for include in includes:
@@ -23,4 +24,4 @@ include_directories("{{include.replace("\\", "/")}}")
 % end
 % end
 
-FILE(GLOB_RECURSE SRC_LIST "{{project_src_dir.replace("\\", "/")}}/*.*")
+FILE(GLOB_RECURSE SRC_LIST "{{project_src_dir.replace("\\", "/")}}/*.*" "{{project_lib_dir.replace("\\", "/")}}/*.*" "{{project_libdeps_dir.replace("\\", "/")}}/*.*")

platformio/ide/tpls/eclipse/.cproject.tpl

@@ -5,13 +5,13 @@
 			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="0.910961921" moduleId="org.eclipse.cdt.core.settings" name="Default">
 				<externalSettings/>
 				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
 					<extension id="org.eclipse.cdt.core.VCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
-					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
 				</extensions>
 			</storageModule>
 			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
@@ -99,6 +99,104 @@
 			</storageModule>
 			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
 		</cconfiguration>
+		<cconfiguration id="0.910961921.1363900502">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="0.910961921.1363900502" moduleId="org.eclipse.cdt.core.settings" name="Debug">
+				<externalSettings/>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.VCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration artifactName="mbed" buildProperties="" description="" id="0.910961921.1363900502" name="Debug" parent="org.eclipse.cdt.build.core.prefbase.cfg">
+					<folderInfo id="0.910961921.1363900502." name="/" resourcePath="">
+						<toolChain id="org.eclipse.cdt.build.core.prefbase.toolchain.2116690625" name="No ToolChain" resourceTypeBasedDiscovery="false" superClass="org.eclipse.cdt.build.core.prefbase.toolchain">
+							<targetPlatform binaryParser="org.eclipse.cdt.core.ELF" id="org.eclipse.cdt.build.core.prefbase.toolchain.2116690625.848954921" name=""/>
+							<builder arguments="-f -c eclipse debug" cleanBuildTarget="run --target clean" command="platformio" enableCleanBuild="false" id="org.eclipse.cdt.build.core.settings.default.builder.985867833" incrementalBuildTarget="" keepEnvironmentInBuildfile="false" managedBuildOn="false" name="Gnu Make Builder" superClass="org.eclipse.cdt.build.core.settings.default.builder"/>
+							<tool id="org.eclipse.cdt.build.core.settings.holder.libs.1855678035" name="holder for library settings" superClass="org.eclipse.cdt.build.core.settings.holder.libs"/>
+							<tool id="org.eclipse.cdt.build.core.settings.holder.30528994" name="Assembly" superClass="org.eclipse.cdt.build.core.settings.holder">
+								<option id="org.eclipse.cdt.build.core.settings.holder.incpaths.794801023" name="Include Paths" superClass="org.eclipse.cdt.build.core.settings.holder.incpaths" valueType="includePath">
+									% for include in includes:
+									% if "toolchain" in include:
+									% continue
+									% end
+                                    % if include.startswith(user_home_dir):
+                                    % if "windows" in systype:
+									<listOptionValue builtIn="false" value="${USERPROFILE}{{include.replace(user_home_dir, '')}}"/>
+									% else:
+									<listOptionValue builtIn="false" value="${HOME}{{include.replace(user_home_dir, '')}}"/>
+									% end
+                                    % else:
+									<listOptionValue builtIn="false" value="{{include}}"/>
+									% end
+									% end
+								</option>
+								<option id="org.eclipse.cdt.build.core.settings.holder.symbols.1743427839" name="Symbols" superClass="org.eclipse.cdt.build.core.settings.holder.symbols" valueType="definedSymbols">
+									% for define in defines:
+									<listOptionValue builtIn="false" value="{{define}}"/>
+									% end
+								</option>
+								<inputType id="org.eclipse.cdt.build.core.settings.holder.inType.919136836" languageId="org.eclipse.cdt.core.assembly" languageName="Assembly" sourceContentType="org.eclipse.cdt.core.asmSource" superClass="org.eclipse.cdt.build.core.settings.holder.inType"/>
+							</tool>
+							<tool id="org.eclipse.cdt.build.core.settings.holder.1146422798" name="GNU C++" superClass="org.eclipse.cdt.build.core.settings.holder">
+								<option id="org.eclipse.cdt.build.core.settings.holder.incpaths.650084869" name="Include Paths" superClass="org.eclipse.cdt.build.core.settings.holder.incpaths" useByScannerDiscovery="false" valueType="includePath">
+									% for include in includes:
+									% if "toolchain" in include:
+									% continue
+									% end
+                                    % if include.startswith(user_home_dir):
+                                    % if "windows" in systype:
+									<listOptionValue builtIn="false" value="${USERPROFILE}{{include.replace(user_home_dir, '')}}"/>
+									% else:
+									<listOptionValue builtIn="false" value="${HOME}{{include.replace(user_home_dir, '')}}"/>
+									% end
+                                    % else:
+									<listOptionValue builtIn="false" value="{{include}}"/>
+									% end
+									% end
+								</option>
+								<option id="org.eclipse.cdt.build.core.settings.holder.symbols.2055633423" name="Symbols" superClass="org.eclipse.cdt.build.core.settings.holder.symbols" useByScannerDiscovery="false" valueType="definedSymbols">
+									% for define in defines:
+									<listOptionValue builtIn="false" value="{{define}}"/>
+									% end
+								</option>
+								<inputType id="org.eclipse.cdt.build.core.settings.holder.inType.445650141" languageId="org.eclipse.cdt.core.g++" languageName="GNU C++" sourceContentType="org.eclipse.cdt.core.cxxSource,org.eclipse.cdt.core.cxxHeader" superClass="org.eclipse.cdt.build.core.settings.holder.inType"/>
+							</tool>
+							<tool id="org.eclipse.cdt.build.core.settings.holder.1637357529" name="GNU C" superClass="org.eclipse.cdt.build.core.settings.holder">
+								<option id="org.eclipse.cdt.build.core.settings.holder.incpaths.1246337321" name="Include Paths" superClass="org.eclipse.cdt.build.core.settings.holder.incpaths" useByScannerDiscovery="false" valueType="includePath">
+									% for include in includes:
+									% if "toolchain" in include:
+									% continue
+									% end
+                                    % if include.startswith(user_home_dir):
+                                    % if "windows" in systype:
+									<listOptionValue builtIn="false" value="${USERPROFILE}{{include.replace(user_home_dir, '')}}"/>
+									% else:
+									<listOptionValue builtIn="false" value="${HOME}{{include.replace(user_home_dir, '')}}"/>
+									% end
+                                    % else:
+									<listOptionValue builtIn="false" value="{{include}}"/>
+									% end
+									% end
+								</option>
+								<option id="org.eclipse.cdt.build.core.settings.holder.symbols.2122043341" name="Symbols" superClass="org.eclipse.cdt.build.core.settings.holder.symbols" useByScannerDiscovery="false" valueType="definedSymbols">
+									% for define in defines:
+									<listOptionValue builtIn="false" value="{{define}}"/>
+									% end
+								</option>
+								<inputType id="org.eclipse.cdt.build.core.settings.holder.inType.207004812" languageId="org.eclipse.cdt.core.gcc" languageName="GNU C" sourceContentType="org.eclipse.cdt.core.cSource,org.eclipse.cdt.core.cHeader" superClass="org.eclipse.cdt.build.core.settings.holder.inType"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+		</cconfiguration>
 	</storageModule>
 	<storageModule moduleId="cdtBuildSystem" version="4.0.0">
 		<project id="{{project_name}}.null.189551033" name="{{project_name}}"/>

platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl

@@ -2,7 +2,7 @@
 <launchConfiguration type="org.eclipse.cdt.launch.applicationLaunchType">
 <booleanAttribute key="org.eclipse.cdt.dsf.gdb.AUTO_SOLIB" value="true"/>
 <listAttribute key="org.eclipse.cdt.dsf.gdb.AUTO_SOLIB_LIST"/>
-<stringAttribute key="org.eclipse.cdt.dsf.gdb.DEBUG_NAME" value="platformio -c eclipse debug -d ${project_loc} --interface=gdb"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.DEBUG_NAME" value="piodebuggdb"/>
 <booleanAttribute key="org.eclipse.cdt.dsf.gdb.DEBUG_ON_FORK" value="false"/>
 <stringAttribute key="org.eclipse.cdt.dsf.gdb.GDB_INIT" value=".pioinit"/>
 <booleanAttribute key="org.eclipse.cdt.dsf.gdb.NON_STOP" value="false"/>
@@ -12,17 +12,17 @@
 <stringAttribute key="org.eclipse.cdt.dsf.gdb.TRACEPOINT_MODE" value="TP_NORMAL_ONLY"/>
 <booleanAttribute key="org.eclipse.cdt.dsf.gdb.UPDATE_THREADLIST_ON_SUSPEND" value="false"/>
 <booleanAttribute key="org.eclipse.cdt.dsf.gdb.internal.ui.launching.LocalApplicationCDebuggerTab.DEFAULTS_SET" value="true"/>
-<intAttribute key="org.eclipse.cdt.launch.ATTR_BUILD_BEFORE_LAUNCH_ATTR" value="0"/>
+<intAttribute key="org.eclipse.cdt.launch.ATTR_BUILD_BEFORE_LAUNCH_ATTR" value="1"/>
 <stringAttribute key="org.eclipse.cdt.launch.COREFILE_PATH" value=""/>
 <stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_ID" value="gdb"/>
 <stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_REGISTER_GROUPS" value=""/>
 <stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_START_MODE" value="run"/>
-<booleanAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN" value="true"/>
-<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN_SYMBOL" value="main"/>
-<stringAttribute key="org.eclipse.cdt.launch.PROGRAM_NAME" value=".pioenvs/{{env_name}}/firmware.elf"/>
+<booleanAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN" value="false"/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN_SYMBOL" value=""/>
+<stringAttribute key="org.eclipse.cdt.launch.PROGRAM_NAME" value="{{prog_path}}"/>
 <stringAttribute key="org.eclipse.cdt.launch.PROJECT_ATTR" value="{{project_name}}"/>
 <booleanAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_AUTO_ATTR" value="false"/>
-<stringAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_ID_ATTR" value=""/>
+<stringAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_ID_ATTR" value="0.910961921.1363900502"/>
 <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
 <listEntry value="/{{project_name}}"/>
 </listAttribute>

platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl

@@ -15,4 +15,19 @@
 			</provider>
 		</extension>
 	</configuration>
+	<configuration id="0.910961921.1363900502" name="Debug">
+		<extension point="org.eclipse.cdt.core.LanguageSettingsProvider">
+			<provider copy-of="extension" id="org.eclipse.cdt.ui.UserLanguageSettingsProvider"/>
+			<provider-reference id="org.eclipse.cdt.core.ReferencedProjectsLanguageSettingsProvider" ref="shared-provider"/>
+			<provider-reference id="org.eclipse.cdt.managedbuilder.core.MBSLanguageSettingsProvider" ref="shared-provider"/>
+			% if "windows" in systype:
+			<provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="1291887707783033084" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="${USERPROFILE}{{cxx_path.replace(user_home_dir, '')}} ${FLAGS} -E -P -v -dD &quot;${INPUTS}&quot;" prefer-non-shared="true">
+			% else:
+			<provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="-869785120007741010" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="${HOME}{{cxx_path.replace(user_home_dir, '')}} ${FLAGS} -E -P -v -dD &quot;${INPUTS}&quot;" prefer-non-shared="true">
+			% end
+				<language-scope id="org.eclipse.cdt.core.gcc"/>
+				<language-scope id="org.eclipse.cdt.core.g++"/>
+			</provider>
+		</extension>
+	</configuration>
 </project>

platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl

@@ -3,4 +3,9 @@ environment/project/0.910961921/PATH/delimiter={{env_pathsep.replace(":", "\\:")
 environment/project/0.910961921/PATH/operation=replace
 environment/project/0.910961921/PATH/value={{env_path.replace(":", "\\:")}}
 environment/project/0.910961921/append=true
-environment/project/0.910961921/appendContributed=true
+environment/project/0.910961921/appendContributed=true
+environment/project/0.910961921.1363900502/PATH/delimiter={{env_pathsep.replace(":", "\\:")}}
+environment/project/0.910961921.1363900502/PATH/operation=replace
+environment/project/0.910961921.1363900502/PATH/value={{env_path.replace(":", "\\:")}}
+environment/project/0.910961921.1363900502/append=true
+environment/project/0.910961921.1363900502/appendContributed=true

platformio/ide/tpls/qtcreator/platformio.pro.tpl

@@ -14,7 +14,8 @@ INCLUDEPATH += "{{include}}"
 % end
 
 % for define in defines:
-DEFINES += "{{define}}"
+% tokens = define.split("##", 1)
+DEFINES += "{{tokens[0].strip()}}"
 % end
 
 OTHER_FILES += platformio.ini

platformio/ide/tpls/visualstudio/platformio.vcxproj.filters.tpl

@@ -15,7 +15,7 @@
   </ItemGroup>
   % for file in src_files:
   <ItemGroup>
-    % if any([file.endswith(".%s" % e) for e in ("h", "hh", "hpp", "inc")]):
+    % if any(file.endswith(".%s" % e) for e in ("h", "hh", "hpp", "inc")):
     <ClInclude Include="{{file}}">
       <Filter>Header Files</Filter>
     </ClInclude>

platformio/ide/tpls/visualstudio/platformio.vcxproj.tpl

@@ -60,7 +60,7 @@
   </ItemGroup>
   % for file in src_files:
   <ItemGroup>
-    % if any([file.endswith(".%s" % e) for e in ("h", "hh", "hpp", "inc")]):
+    % if any(file.endswith(".%s" % e) for e in ("h", "hh", "hpp", "inc")):
     <ClInclude Include="{{file}}">
       <Filter>Header Files</Filter>
     </ClInclude>

platformio/ide/tpls/vscode/.gitignore.tpl

@@ -1,3 +1,5 @@
 .pioenvs
 .piolibdeps
+.vscode/.browse.c_cpp.db*
 .vscode/c_cpp_properties.json
+.vscode/launch.json

platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl

@@ -1,8 +1,19 @@
 {
+    "!!! WARNING !!!": "PLEASE DO NOT MODIFY THIS FILE! USE http://docs.platformio.org/page/projectconf/section_env_build.html#build-flags",
     "configurations": [
         {
 % import platform
+% from os.path import commonprefix, dirname
+%
 % systype = platform.system().lower()
+%
+% cleaned_includes = []
+% for include in includes:
+%   if "toolchain-" not in dirname(commonprefix([include, cc_path])):
+%       cleaned_includes.append(include)
+%   end
+% end
+%
 % if systype == "windows":
             "name": "Win32",
 % elif systype == "darwin":
@@ -11,16 +22,16 @@
             "name": "Linux",
 % end
             "includePath": [
-% for include in includes:
+% for include in cleaned_includes:
                 "{{include.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
 % end
                 ""
             ],
             "browse": {
                 "limitSymbolsToIncludedHeaders": true,
-                "databaseFilename": "",
+                "databaseFilename": "${workspaceRoot}/.vscode/.browse.c_cpp.db",
                 "path": [
-% for include in includes:
+% for include in cleaned_includes:
                     "{{include.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
 % end
                     ""
@@ -32,7 +43,19 @@
 % end
                 ""
             ],
-            "intelliSenseMode": "clang-x64"
+            "intelliSenseMode": "clang-x64",
+% import re
+% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
+% cc_stds = STD_RE.findall(cc_flags)
+% cxx_stds = STD_RE.findall(cxx_flags)
+%
+% if cc_stds:
+            "cStandard": "c{{ cc_stds[-1] }}",
+% end
+% if cxx_stds:
+            "cppStandard": "c++{{ cxx_stds[-1] }}",
+% end
+            "compilerPath": "{{ cc_path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"') }}"
         }
     ]
 }

platformio/ide/tpls/vscode/.vscode/extensions.json.tpl

@@ -0,0 +1,7 @@
+{
+	// See http://go.microsoft.com/fwlink/?LinkId=827846
+	// for the documentation about the extensions.json format
+	"recommendations": [
+		"platformio.platformio-ide"
+	]
+}

platformio/ide/tpls/vscode/.vscode/launch.json.tpl

@@ -1,15 +1,41 @@
+// AUTOMATICALLY GENERATED FILE. PLEASE DO NOT MODIFY IT MANUALLY
+
+// PIO Unified Debugger
+//
+// Documentation: http://docs.platformio.org/page/plus/debugging.html
+// Configuration: http://docs.platformio.org/page/projectconf/section_env_debug.html
+
 % from os.path import dirname, join
+%
+% def _escape_path(path):
+%   return path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')
+% end
+%
 {
     "version": "0.2.0",
     "configurations": [
         {
-            "type": "gdb",
+            "type": "platformio-debug",
             "request": "launch",
-            "cwd": "${workspaceRoot}",
             "name": "PlatformIO Debugger",
-            "target": "{{prog_path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
-            "gdbpath": "{{join(dirname(platformio_path), "piodebuggdb").replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}",
-            "autorun": [ "source .pioinit" ]
+            "executable": "{{ _escape_path(prog_path) }}",
+            "toolchainBinDir": "{{ _escape_path(dirname(gdb_path)) }}",
+% if svd_path:
+            "svdPath": "{{ _escape_path(svd_path) }}",
+% end
+            "preLaunchTask": "PlatformIO: Pre-Debug",
+            "internalConsoleOptions": "openOnSessionStart"
+        },
+        {
+            "type": "platformio-debug",
+            "request": "launch",
+            "name": "PlatformIO Debugger (Skip Pre-Debug)",
+            "executable": "{{ _escape_path(prog_path) }}",
+            "toolchainBinDir": "{{ _escape_path(dirname(gdb_path)) }}",
+% if svd_path:
+            "svdPath": "{{ _escape_path(svd_path) }}",
+% end
+            "internalConsoleOptions": "openOnSessionStart"
         }
     ]
 }

platformio/maintenance.py

@@ -222,7 +222,7 @@ def after_upgrade(ctx):
         click.echo(
             "- %s PlatformIO IDE for IoT development > %s" %
             (click.style("try", fg="cyan"),
-             click.style("http://platformio.org/platformio-ide", fg="cyan")))
+             click.style("https://platformio.org/platformio-ide", fg="cyan")))
     if not util.is_ci():
         click.echo("- %s us with PlatformIO Plus > %s" %
                    (click.style("support", fg="cyan"),

platformio/managers/core.py

@@ -21,11 +21,11 @@ from platformio import __version__, exception, util
 from platformio.managers.package import PackageManager
 
 CORE_PACKAGES = {
-    "contrib-piohome": ">=0.6.0,<2",
-    "contrib-pysite": ">=0.1.2,<2",
-    "tool-pioplus": ">=0.12.1,<2",
-    "tool-unity": "~1.20302.1",
-    "tool-scons": "~3.20501.2"
+    "contrib-piohome": ">=0.9.5,<2",
+    "contrib-pysite": ">=0.2.0,<2",
+    "tool-pioplus": ">=1.3.1,<2",
+    "tool-unity": "~1.20403.0",
+    "tool-scons": "~2.20501.4"
 }
 
 PIOPLUS_AUTO_UPDATES_MAX = 100
@@ -69,7 +69,7 @@ class CorePackageManager(PackageManager):
             if manifest['name'] not in best_pkg_versions:
                 continue
             if manifest['version'] != best_pkg_versions[manifest['name']]:
-                self.uninstall(manifest['__pkg_dir'], trigger_event=False)
+                self.uninstall(manifest['__pkg_dir'], after_update=True)
         self.cache_reset()
         return True
 
@@ -101,8 +101,7 @@ def pioplus_call(args, **kwargs):
         raise exception.PlatformioException(
             "PlatformIO Core Plus v%s does not run under Python version %s.\n"
             "Minimum supported version is 2.7.6, please upgrade Python.\n"
-            "Python 3 is not yet supported.\n" % (__version__,
-                                                  sys.version.split()[0]))
+            "Python 3 is not yet supported.\n" % (__version__, sys.version))
 
     pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus")
     pythonexe_path = util.get_pythonexe_path()

platformio/managers/lib.py

@@ -13,17 +13,18 @@
 # limitations under the License.
 
 # pylint: disable=too-many-arguments, too-many-locals, too-many-branches
+# pylint: disable=too-many-return-statements
 
 import json
 import re
 from glob import glob
 from os.path import isdir, join
 
-import arrow
 import click
 
 from platformio import app, commands, exception, util
 from platformio.managers.package import BasePkgManager
+from platformio.managers.platform import PlatformFactory, PlatformManager
 
 
 class LibraryManager(BasePkgManager):
@@ -155,8 +156,8 @@ class LibraryManager(BasePkgManager):
     def max_satisfying_repo_version(self, versions, requirements=None):
 
         def _cmp_dates(datestr1, datestr2):
-            date1 = arrow.get(datestr1)
-            date2 = arrow.get(datestr2)
+            date1 = util.parse_date(datestr1)
+            date2 = util.parse_date(datestr2)
             if date1 == date2:
                 return 0
             return -1 if date1 < date2 else 1
@@ -186,29 +187,15 @@ class LibraryManager(BasePkgManager):
     def get_latest_repo_version(self, name, requirements, silent=False):
         item = self.max_satisfying_repo_version(
             util.get_api_result(
-                "/lib/info/%d" % self.get_pkg_id_by_name(
-                    name, requirements, silent=silent),
+                "/lib/info/%d" % self.search_lib_id(
+                    {
+                        "name": name,
+                        "requirements": requirements
+                    },
+                    silent=silent),
                 cache_valid="1h")['versions'], requirements)
         return item['name'] if item else None
 
-    def get_pkg_id_by_name(self,
-                           name,
-                           requirements,
-                           silent=False,
-                           interactive=False):
-        if name.startswith("id="):
-            return int(name[3:])
-        # try to find ID from installed packages
-        package_dir = self.get_package_dir(name, requirements)
-        if package_dir:
-            manifest = self.load_manifest(package_dir)
-            if "id" in manifest:
-                return int(manifest['id'])
-        return int(
-            self.search_for_library({
-                "name": name
-            }, silent, interactive)['id'])
-
     def _install_from_piorepo(self, name, requirements):
         assert name.startswith("id="), name
         version = self.get_latest_repo_version(name, requirements)
@@ -225,88 +212,20 @@ class LibraryManager(BasePkgManager):
             "http://", "https://") if app.get_setting("enable_ssl") else
                                       dl_data['url'], requirements)
 
-    def install(  # pylint: disable=arguments-differ
+    def search_lib_id(  # pylint: disable=too-many-branches
             self,
-            name,
-            requirements=None,
-            silent=False,
-            trigger_event=True,
-            interactive=False,
-            force=False):
-        pkg_dir = None
-        try:
-            _name, _requirements, _url = self.parse_pkg_uri(name, requirements)
-            if not _url:
-                name = "id=%d" % self.get_pkg_id_by_name(
-                    _name,
-                    _requirements,
-                    silent=silent,
-                    interactive=interactive)
-                requirements = _requirements
-            pkg_dir = BasePkgManager.install(
-                self,
-                name,
-                requirements,
-                silent=silent,
-                trigger_event=trigger_event,
-                force=force)
-        except exception.InternetIsOffline as e:
-            if not silent:
-                click.secho(str(e), fg="yellow")
-            return None
-
-        if not pkg_dir:
-            return None
-
-        manifest = self.load_manifest(pkg_dir)
-        if "dependencies" not in manifest:
-            return pkg_dir
-
-        if not silent:
-            click.secho("Installing dependencies", fg="yellow")
-
-        for filters in self.normalize_dependencies(manifest['dependencies']):
-            assert "name" in filters
-            if any([s in filters.get("version", "") for s in ("\\", "/")]):
-                self.install(
-                    "{name}={version}".format(**filters),
-                    silent=silent,
-                    trigger_event=trigger_event,
-                    interactive=interactive,
-                    force=force)
-            else:
-                try:
-                    lib_info = self.search_for_library(filters, silent,
-                                                       interactive)
-                except exception.LibNotFound as e:
-                    if not silent:
-                        click.secho("Warning! %s" % e, fg="yellow")
-                    continue
-
-                if filters.get("version"):
-                    self.install(
-                        lib_info['id'],
-                        filters.get("version"),
-                        silent=silent,
-                        trigger_event=trigger_event,
-                        interactive=interactive,
-                        force=force)
-                else:
-                    self.install(
-                        lib_info['id'],
-                        silent=silent,
-                        trigger_event=trigger_event,
-                        interactive=interactive,
-                        force=force)
-        return pkg_dir
-
-    @staticmethod
-    def search_for_library(  # pylint: disable=too-many-branches
             filters,
             silent=False,
             interactive=False):
         assert isinstance(filters, dict)
         assert "name" in filters
+
+        # try to find ID within installed packages
+        lib_id = self._get_lib_id_from_installed(filters)
+        if lib_id:
+            return lib_id
+
+        # looking in PIO Library Registry
         if not silent:
             click.echo("Looking for %s library in registry" % click.style(
                 filters['name'], fg="cyan"))
@@ -363,7 +282,153 @@ class LibraryManager(BasePkgManager):
                 raise exception.LibNotFound(str(filters))
         if not silent:
             click.echo("Found: %s" % click.style(
-                "http://platformio.org/lib/show/{id}/{name}".format(
+                "https://platformio.org/lib/show/{id}/{name}".format(
                     **lib_info),
                 fg="blue"))
-        return lib_info
+        return int(lib_info['id'])
+
+    def _get_lib_id_from_installed(self, filters):
+        if filters['name'].startswith("id="):
+            return int(filters['name'][3:])
+        package_dir = self.get_package_dir(filters['name'],
+                                           filters.get("requirements",
+                                                       filters.get("version")))
+        if not package_dir:
+            return None
+        manifest = self.load_manifest(package_dir)
+        if "id" not in manifest:
+            return None
+
+        for key in ("frameworks", "platforms"):
+            if key not in filters:
+                continue
+            if key not in manifest:
+                return None
+            if not util.items_in_list(
+                    util.items_to_list(filters[key]),
+                    util.items_to_list(manifest[key])):
+                return None
+
+        if "authors" in filters:
+            if "authors" not in manifest:
+                return None
+            manifest_authors = manifest['authors']
+            if not isinstance(manifest_authors, list):
+                manifest_authors = [manifest_authors]
+            manifest_authors = [
+                a['name'] for a in manifest_authors
+                if isinstance(a, dict) and "name" in a
+            ]
+            filter_authors = filters['authors']
+            if not isinstance(filter_authors, list):
+                filter_authors = [filter_authors]
+            if not set(filter_authors) <= set(manifest_authors):
+                return None
+
+        return int(manifest['id'])
+
+    def install(  # pylint: disable=arguments-differ
+            self,
+            name,
+            requirements=None,
+            silent=False,
+            after_update=False,
+            interactive=False,
+            force=False):
+        _name, _requirements, _url = self.parse_pkg_uri(name, requirements)
+        if not _url:
+            name = "id=%d" % self.search_lib_id(
+                {
+                    "name": _name,
+                    "requirements": _requirements
+                },
+                silent=silent,
+                interactive=interactive)
+            requirements = _requirements
+        pkg_dir = BasePkgManager.install(
+            self,
+            name,
+            requirements,
+            silent=silent,
+            after_update=after_update,
+            force=force)
+
+        if not pkg_dir:
+            return None
+
+        manifest = self.load_manifest(pkg_dir)
+        if "dependencies" not in manifest:
+            return pkg_dir
+
+        if not silent:
+            click.secho("Installing dependencies", fg="yellow")
+
+        for filters in self.normalize_dependencies(manifest['dependencies']):
+            assert "name" in filters
+
+            # avoid circle dependencies
+            if not self.INSTALL_HISTORY:
+                self.INSTALL_HISTORY = []
+            history_key = str(filters)
+            if history_key in self.INSTALL_HISTORY:
+                continue
+            self.INSTALL_HISTORY.append(history_key)
+
+            if any(s in filters.get("version", "") for s in ("\\", "/")):
+                self.install(
+                    "{name}={version}".format(**filters),
+                    silent=silent,
+                    after_update=after_update,
+                    interactive=interactive,
+                    force=force)
+            else:
+                try:
+                    lib_id = self.search_lib_id(filters, silent, interactive)
+                except exception.LibNotFound as e:
+                    if not silent or is_builtin_lib(filters['name']):
+                        click.secho("Warning! %s" % e, fg="yellow")
+                    continue
+
+                if filters.get("version"):
+                    self.install(
+                        lib_id,
+                        filters.get("version"),
+                        silent=silent,
+                        after_update=after_update,
+                        interactive=interactive,
+                        force=force)
+                else:
+                    self.install(
+                        lib_id,
+                        silent=silent,
+                        after_update=after_update,
+                        interactive=interactive,
+                        force=force)
+        return pkg_dir
+
+
+@util.memoized()
+def get_builtin_libs(storage_names=None):
+    items = []
+    storage_names = storage_names or []
+    pm = PlatformManager()
+    for manifest in pm.get_installed():
+        p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
+        for storage in p.get_lib_storages():
+            if storage_names and storage['name'] not in storage_names:
+                continue
+            lm = LibraryManager(storage['path'])
+            items.append({
+                "name": storage['name'],
+                "path": storage['path'],
+                "items": lm.get_installed()
+            })
+    return items
+
+
+@util.memoized()
+def is_builtin_lib(name):
+    for storage in get_builtin_libs():
+        if any(l.get("name") == name for l in storage['items']):
+            return True
+    return False

platformio/managers/package.py

@@ -134,6 +134,7 @@ class PkgRepoMixin(object):
 class PkgInstallerMixin(object):
 
     SRC_MANIFEST_NAME = ".piopkgmanager.json"
+    TMP_FOLDER_PREFIX = "_tmp_installing-"
 
     FILE_CACHE_VALID = "1m"  # 1 month
     FILE_CACHE_MAX_SIZE = 1024 * 1024
@@ -176,8 +177,25 @@ class PkgInstallerMixin(object):
                     shutil.copy(cache_path, dst_path)
                     return dst_path
 
-        fd = FileDownloader(url, dest_dir)
-        fd.start()
+        with_progress = not app.is_disabled_progressbar()
+        try:
+            fd = FileDownloader(url, dest_dir)
+            fd.start(with_progress=with_progress)
+        except IOError as e:
+            raise_error = not with_progress
+            if with_progress:
+                try:
+                    fd = FileDownloader(url, dest_dir)
+                    fd.start(with_progress=False)
+                except IOError:
+                    raise_error = True
+            if raise_error:
+                click.secho(
+                    "Error: Please read http://bit.ly/package-manager-ioerror",
+                    fg="red",
+                    err=True)
+                raise e
+
         if sha1:
             fd.verify(sha1)
         dst_path = fd.get_filepath()
@@ -193,8 +211,15 @@ class PkgInstallerMixin(object):
 
     @staticmethod
     def unpack(source_path, dest_dir):
-        with FileUnpacker(source_path) as fu:
-            return fu.unpack(dest_dir)
+        with_progress = not app.is_disabled_progressbar()
+        try:
+            with FileUnpacker(source_path) as fu:
+                return fu.unpack(dest_dir, with_progress=with_progress)
+        except IOError as e:
+            if not with_progress:
+                raise e
+            with FileUnpacker(source_path) as fu:
+                return fu.unpack(dest_dir, with_progress=False)
 
     @staticmethod
     def parse_semver_spec(value, raise_exception=False):
@@ -211,6 +236,8 @@ class PkgInstallerMixin(object):
             try:
                 return semantic_version.Version(value)
             except ValueError:
+                if "." not in str(value) and not str(value).isdigit():
+                    raise ValueError("Invalid SemVer version %s" % value)
                 return semantic_version.Version.coerce(value)
         except ValueError as e:
             if raise_exception:
@@ -292,6 +319,8 @@ class PkgInstallerMixin(object):
     def get_installed(self):
         items = []
         for pkg_dir in self.read_dirs(self.package_dir):
+            if self.TMP_FOLDER_PREFIX in pkg_dir:
+                continue
             manifest = self.load_manifest(pkg_dir)
             if not manifest:
                 continue
@@ -361,7 +390,7 @@ class PkgInstallerMixin(object):
                 break
             except Exception as e:  # pylint: disable=broad-except
                 click.secho("Warning! Package Mirror: %s" % e, fg="yellow")
-                click.secho("Looking for other mirror...", fg="yellow")
+                click.secho("Looking for another mirror...", fg="yellow")
 
         if versions is None:
             raise exception.UnknownPackage(name)
@@ -376,7 +405,7 @@ class PkgInstallerMixin(object):
                           requirements=None,
                           sha1=None,
                           track=False):
-        tmp_dir = mkdtemp("-package", "_tmp_installing-", self.package_dir)
+        tmp_dir = mkdtemp("-package", self.TMP_FOLDER_PREFIX, self.package_dir)
         src_manifest_dir = None
         src_manifest = {"name": name, "url": url, "requirements": requirements}
 
@@ -473,7 +502,7 @@ class PkgInstallerMixin(object):
                     target_dirname = "%s@src-%s" % (
                         pkg_dirname,
                         hashlib.md5(cur_manifest['__src_url']).hexdigest())
-                os.rename(pkg_dir, join(self.package_dir, target_dirname))
+                shutil.move(pkg_dir, join(self.package_dir, target_dirname))
             # fix to a version
             elif action == 2:
                 target_dirname = "%s@%s" % (pkg_dirname,
@@ -487,7 +516,7 @@ class PkgInstallerMixin(object):
         # remove previous/not-satisfied package
         if isdir(pkg_dir):
             util.rmtree_(pkg_dir)
-        os.rename(tmp_dir, pkg_dir)
+        shutil.move(tmp_dir, pkg_dir)
         assert isdir(pkg_dir)
         self.cache_reset()
         return pkg_dir
@@ -548,7 +577,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             hg_conditions = [
                 # Handle Developer Mbed URL
                 # (https://developer.mbed.org/users/user/code/package/)
-                text.startswith("https://developer.mbed.org")
+                # (https://os.mbed.com/users/user/code/package/)
+                text.startswith("https://developer.mbed.org"),
+                text.startswith("https://os.mbed.com")
             ]
             if any(git_conditions):
                 url = "git+" + text
@@ -626,7 +657,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 name,
                 requirements=None,
                 silent=False,
-                trigger_event=True,
+                after_update=False,
                 force=False):
         name, requirements, url = self.parse_pkg_uri(name, requirements)
         package_dir = self.get_package_dir(name, requirements, url)
@@ -661,6 +692,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
                 name, url, requirements, track=True)
         else:
             pkg_dir = self._install_from_piorepo(name, requirements)
+
         if not pkg_dir or not self.manifest_exists(pkg_dir):
             raise exception.PackageInstallError(name, requirements or "*",
                                                 util.get_systype())
@@ -668,7 +700,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         manifest = self.load_manifest(pkg_dir)
         assert manifest
 
-        if trigger_event:
+        if not after_update:
             telemetry.on_event(
                 category=self.__class__.__name__,
                 action="Install",
@@ -682,7 +714,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
 
         return pkg_dir
 
-    def uninstall(self, package, requirements=None, trigger_event=True):
+    def uninstall(self, package, requirements=None, after_update=False):
         if isdir(package):
             pkg_dir = package
         else:
@@ -708,14 +740,14 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
         # unfix package with the same name
         pkg_dir = self.get_package_dir(manifest['name'])
         if pkg_dir and "@" in pkg_dir:
-            os.rename(pkg_dir,
-                      join(self.package_dir,
-                           self.get_install_dirname(manifest)))
+            shutil.move(pkg_dir,
+                        join(self.package_dir,
+                             self.get_install_dirname(manifest)))
             self.cache_reset()
 
         click.echo("[%s]" % click.style("OK", fg="green"))
 
-        if trigger_event:
+        if not after_update:
             telemetry.on_event(
                 category=self.__class__.__name__,
                 action="Uninstall",
@@ -761,8 +793,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             self._update_src_manifest(
                 dict(version=vcs.get_current_revision()), vcs.storage_dir)
         else:
-            self.uninstall(pkg_dir, trigger_event=False)
-            self.install(name, latest, trigger_event=False)
+            self.uninstall(pkg_dir, after_update=True)
+            self.install(name, latest, after_update=True)
 
         telemetry.on_event(
             category=self.__class__.__name__,

platformio/managers/platform.py

@@ -30,7 +30,7 @@ from platformio.managers.package import BasePkgManager, PackageManager
 
 class PlatformManager(BasePkgManager):
 
-    FILE_CACHE_VALID = None  # disable platform caching
+    FILE_CACHE_VALID = None  # disable platform download caching
 
     def __init__(self, package_dir=None, repositories=None):
         if not repositories:
@@ -62,7 +62,7 @@ class PlatformManager(BasePkgManager):
                 with_packages=None,
                 without_packages=None,
                 skip_default_package=False,
-                trigger_event=True,
+                after_update=False,
                 silent=False,
                 force=False,
                 **_):  # pylint: disable=too-many-arguments, arguments-differ
@@ -70,20 +70,20 @@ class PlatformManager(BasePkgManager):
             self, name, requirements, silent=silent, force=force)
         p = PlatformFactory.newPlatform(platform_dir)
 
-        # @Hook: when 'update' operation (trigger_event is False),
-        # don't cleanup packages or install them
-        if not trigger_event:
+        # don't cleanup packages or install them after update
+        # we check packages for updates in def update()
+        if after_update:
             return True
+
         p.install_packages(
             with_packages,
             without_packages,
             skip_default_package,
             silent=silent,
             force=force)
-        self.cleanup_packages(p.packages.keys())
-        return True
+        return self.cleanup_packages(p.packages.keys())
 
-    def uninstall(self, package, requirements=None, trigger_event=True):
+    def uninstall(self, package, requirements=None, after_update=False):
         if isdir(package):
             pkg_dir = package
         else:
@@ -96,13 +96,12 @@ class PlatformManager(BasePkgManager):
         p = PlatformFactory.newPlatform(pkg_dir)
         BasePkgManager.uninstall(self, pkg_dir, requirements)
 
-        # @Hook: when 'update' operation (trigger_event is False),
-        # don't cleanup packages or install them
-        if not trigger_event:
+        # don't cleanup packages or install them after update
+        # we check packages for updates in def update()
+        if after_update:
             return True
 
-        self.cleanup_packages(p.packages.keys())
-        return True
+        return self.cleanup_packages(p.packages.keys())
 
     def update(  # pylint: disable=arguments-differ
             self,
@@ -154,11 +153,15 @@ class PlatformManager(BasePkgManager):
                 continue
             if (manifest['name'] not in deppkgs
                     or manifest['version'] not in deppkgs[manifest['name']]):
-                pm.uninstall(manifest['__pkg_dir'], trigger_event=False)
+                try:
+                    pm.uninstall(manifest['__pkg_dir'], after_update=True)
+                except exception.UnknownPackage:
+                    pass
 
         self.cache_reset()
         return True
 
+    @util.memoized(expire=5000)
     def get_installed_boards(self):
         boards = []
         for manifest in self.get_installed():
@@ -170,7 +173,7 @@ class PlatformManager(BasePkgManager):
         return boards
 
     @staticmethod
-    @util.memoized
+    @util.memoized()
     def get_registered_boards():
         return util.get_api_result("/boards", cache_valid="7d")
 
@@ -280,21 +283,25 @@ class PlatformPackagesMixin(object):
 
         return True
 
-    def find_pkg_names(self, items):
+    def find_pkg_names(self, candidates):
         result = []
-        for item in items:
-            candidate = item
+        for candidate in candidates:
+            found = False
 
             # lookup by package types
             for _name, _opts in self.packages.items():
-                if _opts.get("type") == item:
-                    candidate = _name
+                if _opts.get("type") == candidate:
+                    result.append(_name)
+                    found = True
 
-            if (self.frameworks and item.startswith("framework-")
-                    and item[10:] in self.frameworks):
-                candidate = self.frameworks[item[10:]]['package']
+            if (self.frameworks and candidate.startswith("framework-")
+                    and candidate[10:] in self.frameworks):
+                result.append(self.frameworks[candidate[10:]]['package'])
+                found = True
+
+            if not found:
+                result.append(candidate)
 
-            result.append(candidate)
         return result
 
     def update_packages(self, only_check=False):
@@ -427,10 +434,10 @@ class PlatformRunMixin(object):
 """.format(filename=filename,
            filename_styled=click.style(filename, fg="cyan"),
            link=click.style(
-               "http://platformio.org/lib/search?query=header:%s" % quote(
+               "https://platformio.org/lib/search?query=header:%s" % quote(
                    filename, safe=""),
                fg="blue"),
-           dots="*" * (55 + len(filename)))
+           dots="*" * (56 + len(filename)))
         click.echo(banner, err=True)
 
     @staticmethod
@@ -489,6 +496,10 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
     def vendor_url(self):
         return self._manifest.get("url")
 
+    @property
+    def docs_url(self):
+        return self._manifest.get("docs")
+
     @property
     def repository_url(self):
         return self._manifest.get("repository", {}).get("url")
@@ -563,9 +574,9 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
                     if not isdir(boards_dir):
                         continue
                     manifest_path = join(boards_dir, "%s.json" % id_)
-                    if not isfile(manifest_path):
-                        continue
-                    _append_board(id_, manifest_path)
+                    if isfile(manifest_path):
+                        _append_board(id_, manifest_path)
+                        break
             if id_ not in self._BOARDS_CACHE:
                 raise exception.UnknownBoard(id_)
         return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE
@@ -654,6 +665,15 @@ class PlatformBoardConfig(object):
             else:
                 raise KeyError("Invalid board option '%s'" % path)
 
+    def update(self, path, value):
+        newdict = None
+        for key in path.split(".")[::-1]:
+            if newdict is None:
+                newdict = {key: value}
+            else:
+                newdict = {key: newdict}
+        util.merge_dicts(self._manifest, newdict)
+
     def __contains__(self, key):
         try:
             self.get(key)
@@ -684,9 +704,11 @@ class PlatformBoardConfig(object):
             "mcu":
             self._manifest.get("build", {}).get("mcu", "").upper(),
             "fcpu":
-            int(
-                re.sub(r"[^\d]+", "",
-                       self._manifest.get("build", {}).get("f_cpu", "0L"))),
+            int("".join([
+                c for c in str(
+                    self._manifest.get("build", {}).get("f_cpu", "0L"))
+                if c.isdigit()
+            ])),
             "ram":
             self._manifest.get("upload", {}).get("maximum_ram_size", 0),
             "rom":
@@ -713,3 +735,33 @@ class PlatformBoardConfig(object):
                 if key in ("default", "onboard"):
                     tools[name][key] = value
         return {"tools": tools}
+
+    def get_debug_tool_name(self, custom=None):
+        debug_tools = self._manifest.get("debug", {}).get("tools")
+        tool_name = custom
+        if tool_name == "custom":
+            return tool_name
+        if not debug_tools:
+            raise exception.DebugSupportError(self._manifest['name'])
+        if tool_name:
+            if tool_name in debug_tools:
+                return tool_name
+            raise exception.DebugInvalidOptions(
+                "Unknown debug tool `%s`. Please use one of `%s` or `custom`" %
+                (tool_name, ", ".join(sorted(debug_tools.keys()))))
+
+        # automatically select best tool
+        data = {"default": [], "onboard": [], "external": []}
+        for key, value in debug_tools.items():
+            if value.get("default"):
+                data['default'].append(key)
+            elif value.get("onboard"):
+                data['onboard'].append(key)
+            data['external'].append(key)
+
+        for key, value in data.items():
+            if not value:
+                continue
+            return sorted(value)[0]
+
+        assert any(item for item in data)

platformio/telemetry.py

@@ -16,6 +16,7 @@ import atexit
 import platform
 import Queue
 import re
+import sys
 import threading
 from collections import deque
 from os import getenv, sep
@@ -152,16 +153,22 @@ class MeasurementProtocol(TelemetryBase):
                         cmd_path.append(sub_cmd)
         self['screen_name'] = " ".join([p.title() for p in cmd_path])
 
-    def send(self, hittype):
+    @staticmethod
+    def _ignore_hit():
         if not app.get_setting("enable_telemetry"):
+            return True
+        if app.get_session_var("caller_id") and \
+                all(c in sys.argv for c in ("run", "idedata")):
+            return True
+        return False
+
+    def send(self, hittype):
+        if self._ignore_hit():
             return
-
         self['t'] = hittype
-
         # correct queue time
         if "qt" in self._params and isinstance(self['qt'], float):
             self['qt'] = int((time() - self['qt']) * 1000)
-
         MPDataPusher().push(self._params)
 
 

platformio/unpacker.py

@@ -20,7 +20,7 @@ from zipfile import ZipFile
 
 import click
 
-from platformio import app, util
+from platformio import util
 from platformio.exception import UnsupportedArchiveType
 
 
@@ -96,9 +96,9 @@ class FileUnpacker(object):
         if self._unpacker:
             self._unpacker.close()
 
-    def unpack(self, dest_dir="."):
+    def unpack(self, dest_dir=".", with_progress=True):
         assert self._unpacker
-        if app.is_disabled_progressbar():
+        if not with_progress:
             click.echo("Unpacking...")
             for item in self._unpacker.get_items():
                 self._unpacker.extract_item(item, dest_dir)

platformio/util.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import collections
-import functools
 import json
 import os
 import platform
@@ -22,13 +20,13 @@ import socket
 import stat
 import subprocess
 import sys
+import time
 from functools import wraps
 from glob import glob
 from os.path import (abspath, basename, dirname, expanduser, isdir, isfile,
                      join, normpath, splitdrive)
 from shutil import rmtree
 from threading import Thread
-from time import sleep, time
 
 import click
 import requests
@@ -113,40 +111,23 @@ class cd(object):
 
 
 class memoized(object):
-    '''
-    Decorator. Caches a function's return value each time it is called.
-    If called later with the same arguments, the cached value is returned
-    (not reevaluated).
-    https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize
-    '''
 
-    def __init__(self, func):
-        self.func = func
+    def __init__(self, expire=0):
+        self.expire = expire / 1000  # milliseconds
         self.cache = {}
 
-    def __call__(self, *args):
-        if not isinstance(args, collections.Hashable):
-            # uncacheable. a list, for instance.
-            # better to not cache than blow up.
-            return self.func(*args)
-        if args in self.cache:
-            return self.cache[args]
-        value = self.func(*args)
-        self.cache[args] = value
-        return value
+    def __call__(self, func):
 
-    def __repr__(self):
-        '''Return the function's docstring.'''
-        return self.func.__doc__
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            key = str(args) + str(kwargs)
+            if (key not in self.cache
+                    or (self.expire > 0
+                        and self.cache[key][0] < time.time() - self.expire)):
+                self.cache[key] = (time.time(), func(*args, **kwargs))
+            return self.cache[key][1]
 
-    def __get__(self, obj, objtype):
-        '''Support instance methods.'''
-        fn = functools.partial(self.__call__, obj)
-        fn.reset = self._reset
-        return fn
-
-    def _reset(self):
-        self.cache = {}
+        return wrapper
 
 
 class throttle(object):
@@ -155,15 +136,15 @@ class throttle(object):
         self.threshhold = threshhold  # milliseconds
         self.last = 0
 
-    def __call__(self, fn):
+    def __call__(self, func):
 
-        @wraps(fn)
+        @wraps(func)
         def wrapper(*args, **kwargs):
-            diff = int(round((time() - self.last) * 1000))
+            diff = int(round((time.time() - self.last) * 1000))
             if diff < self.threshhold:
-                sleep((self.threshhold - diff) * 0.001)
-            self.last = time()
-            return fn(*args, **kwargs)
+                time.sleep((self.threshhold - diff) * 0.001)
+            self.last = time.time()
+            return func(*args, **kwargs)
 
         return wrapper
 
@@ -189,8 +170,7 @@ def load_json(file_path):
         with open(file_path, "r") as f:
             return json.load(f)
     except ValueError:
-        raise exception.PlatformioException(
-            "Could not load broken JSON: %s" % file_path)
+        raise exception.InvalidJSONFile(file_path)
 
 
 def get_systype():
@@ -253,6 +233,11 @@ def get_home_dir():
     return home_dir
 
 
+def get_cache_dir():
+    return get_project_optional_dir("cache_dir", join(get_home_dir(),
+                                                      ".cache"))
+
+
 def get_source_dir():
     curpath = abspath(__file__)
     if not isfile(curpath):
@@ -311,8 +296,8 @@ def get_projectboards_dir():
                                     join(get_project_dir(), "boards"))
 
 
-def get_projectpioenvs_dir(force=False):
-    path = get_project_optional_dir("envs_dir",
+def get_projectbuild_dir(force=False):
+    path = get_project_optional_dir("build_dir",
                                     join(get_project_dir(), ".pioenvs"))
     try:
         if not isdir(path):
@@ -322,7 +307,7 @@ def get_projectpioenvs_dir(force=False):
             with open(dontmod_path, "w") as fp:
                 fp.write("""
 [InternetShortcut]
-URL=http://docs.platformio.org/page/projectconf.html#envs-dir
+URL=http://docs.platformio.org/page/projectconf/section_platformio.html#build-dir
 """)
     except Exception as e:  # pylint: disable=broad-except
         if not force:
@@ -330,6 +315,10 @@ URL=http://docs.platformio.org/page/projectconf.html#envs-dir
     return path
 
 
+# compatibility with PIO Core+
+get_projectpioenvs_dir = get_projectbuild_dir
+
+
 def get_projectdata_dir():
     return get_project_optional_dir("data_dir", join(get_project_dir(),
                                                      "data"))
@@ -427,7 +416,7 @@ def get_serial_ports(filter_hwid=False):
     for p, d, h in comports():
         if not p:
             continue
-        if platform.system() == "Windows":
+        if "windows" in get_systype():
             try:
                 d = unicode(d, errors="ignore")
             except TypeError:
@@ -439,15 +428,19 @@ def get_serial_ports(filter_hwid=False):
         return result
 
     # fix for PySerial
-    if not result and platform.system() == "Darwin":
+    if not result and "darwin" in get_systype():
         for p in glob("/dev/tty.*"):
             result.append({"port": p, "description": "n/a", "hwid": "n/a"})
     return result
 
 
+# Backward compatibility for PIO Core <3.5
+get_serialports = get_serial_ports
+
+
 def get_logical_devices():
     items = []
-    if platform.system() == "Windows":
+    if "windows" in get_systype():
         try:
             result = exec_command(
                 ["wmic", "logicaldisk", "get", "name,VolumeName"]).get(
@@ -483,14 +476,6 @@ def get_logical_devices():
     return items
 
 
-### Backward compatibility for PIO Core <3.5
-get_serialports = get_serial_ports
-get_logicaldisks = lambda: [{
-    "disk": d['path'],
-    "name": d['name']
-} for d in get_logical_devices()]
-
-
 def get_mdns_services():
     try:
         import zeroconf
@@ -541,8 +526,16 @@ def get_mdns_services():
 
     items = []
     with mDNSListener() as mdns:
-        sleep(3)
+        time.sleep(3)
         for service in mdns.get_services():
+            properties = None
+            try:
+                if service.properties:
+                    json.dumps(service.properties)
+                properties = service.properties
+            except UnicodeDecodeError:
+                pass
+
             items.append({
                 "type":
                 service.type,
@@ -553,7 +546,7 @@ def get_mdns_services():
                 "port":
                 service.port,
                 "properties":
-                service.properties
+                properties
             })
     return items
 
@@ -563,7 +556,7 @@ def get_request_defheaders():
     return {"User-Agent": "PlatformIO/%s CI/%d %s" % data}
 
 
-@memoized
+@memoized(expire=10000)
 def _api_request_session():
     return requests.Session()
 
@@ -604,6 +597,7 @@ def _get_api_result(
                 verify=verify_ssl)
         result = r.json()
         r.raise_for_status()
+        return r.text
     except requests.exceptions.HTTPError as e:
         if result and "message" in result:
             raise exception.APIRequestError(result['message'])
@@ -617,11 +611,10 @@ def _get_api_result(
     finally:
         if r:
             r.close()
-    return result
+    return None
 
 
 def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
-    internet_on(raise_exception=True)
     from platformio.app import ContentCache
     total = 0
     max_retries = 5
@@ -633,12 +626,16 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
                 if cache_key:
                     result = cc.get(cache_key)
                     if result is not None:
-                        return result
+                        return json.loads(result)
+
+            # check internet before and resolve issue with 60 seconds timeout
+            internet_on(raise_exception=True)
+
             result = _get_api_result(url, params, data)
             if cache_valid:
                 with ContentCache() as cc:
                     cc.set(cache_key, result, cache_valid)
-            return result
+            return json.loads(result)
         except (requests.exceptions.ConnectionError,
                 requests.exceptions.Timeout) as e:
             from platformio.maintenance import in_silence
@@ -648,7 +645,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
                     "[API] ConnectionError: {0} (incremented retry: max={1}, "
                     "total={2})".format(e, max_retries, total),
                     fg="yellow")
-            sleep(2 * total)
+            time.sleep(2 * total)
 
     raise exception.APIRequestError(
         "Could not connect to PlatformIO API Service. "
@@ -662,7 +659,7 @@ PING_INTERNET_IPS = [
 ]
 
 
-@memoized
+@memoized(expire=5000)
 def _internet_on():
     timeout = 2
     socket.setdefaulttimeout(timeout)
@@ -720,6 +717,55 @@ def pepver_to_semver(pepver):
     return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1)
 
 
+def items_to_list(items):
+    if not isinstance(items, list):
+        items = [i.strip() for i in items.split(",")]
+    return [i.lower() for i in items if i]
+
+
+def items_in_list(needle, haystack):
+    needle = items_to_list(needle)
+    haystack = items_to_list(haystack)
+    if "*" in needle or "*" in haystack:
+        return True
+    return set(needle) & set(haystack)
+
+
+def parse_date(datestr):
+    if "T" in datestr and "Z" in datestr:
+        return time.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ")
+    return time.strptime(datestr)
+
+
+def format_filesize(filesize):
+    base = 1024
+    unit = 0
+    suffix = "B"
+    filesize = float(filesize)
+    if filesize < base:
+        return "%d%s" % (filesize, suffix)
+    for i, suffix in enumerate("KMGTPEZY"):
+        unit = base**(i + 2)
+        if filesize >= unit:
+            continue
+        if filesize % (base**(i + 1)):
+            return "%.2f%sB" % ((base * filesize / unit), suffix)
+        break
+    return "%d%sB" % ((base * filesize / unit), suffix)
+
+
+def merge_dicts(d1, d2, path=None):
+    if path is None:
+        path = []
+    for key in d2:
+        if (key in d1 and isinstance(d1[key], dict)
+                and isinstance(d2[key], dict)):
+            merge_dicts(d1[key], d2[key], path + [str(key)])
+        else:
+            d1[key] = d2[key]
+    return d1
+
+
 def rmtree_(path):
 
     def _onerror(_, name, __):

scripts/98-openocd-udev.rules

@@ -1,163 +0,0 @@
-# UDEV Rules for debug adapters/boards supported by OpenOCD
-
-#
-# INSTALLATION
-#
-
-#
-# The latest version of this file may be found at:
-# 	https://github.com/platformio/platformio-core/blob/develop/scripts/98-openocd-udev.rules
-#
-# This file must be placed at:
-# 	/etc/udev/rules.d/98-openocd-udev.rules    (preferred location)
-#   	or
-# 	/lib/udev/rules.d/98-openocd-udev.rules    (req'd on some broken systems)
-#
-# To install, type this command in a terminal:
-# 	sudo cp 98-openocd-udev.rules /etc/udev/rules.d/98-openocd-udev.rules
-#
-# Restart "udev" management tool:
-#	sudo service udev restart
-#		or
-# 	sudo udevadm control --reload-rules
-#	sudo udevadm trigger
-#
-# Ubuntu/Debian users may need to add own “username” to the “dialout” group if
-# they are not “root”, doing this issuing a
-#   sudo usermod -a -G dialout $USER
-#   sudo usermod -a -G plugdev $USER
-#
-# After this file is installed, physically unplug and reconnect your adapter/board.
-
-ACTION!="add|change", GOTO="openocd_rules_end"
-SUBSYSTEM!="usb|tty|hidraw", GOTO="openocd_rules_end"
-
-# Please keep this list sorted by VID:PID
-
-# opendous and estick
-ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Original FT232/FT245 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Original FT2232 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Original FT4232 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Original FT232H VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# DISTORTEC JTAG-lock-pick Tiny 2
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# TUMPA, TUMPA Lite
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# XDS100v2
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE)
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# TI/Luminary Stellaris Evaluation Board FTDI (several)
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# TI/Luminary Stellaris In-Circuit Debug Interface FTDI (ICDI) Board
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# egnite Turtelizer 2
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Section5 ICEbear
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Amontec JTAGkey and JTAGkey-tiny
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# TI ICDI
-ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# STLink v1
-ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# STLink v2
-ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# STLink v2-1
-ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Hilscher NXHX Boards
-ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Hitex STR9-comStick
-ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Hitex STM32-PerformanceStick
-ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Altera USB Blaster
-ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Amontec JTAGkey-HiSpeed
-ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# SEGGER J-Link
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="660", GROUP="plugdev", TAG+="uaccess"
-ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Raisonance RLink
-ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Debug Board for Neo1973
-ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Olimex ARM-USB-OCD
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Olimex ARM-USB-OCD-TINY
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Olimex ARM-JTAG-EW
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Olimex ARM-USB-OCD-TINY-H
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Olimex ARM-USB-OCD-H
-ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# USBprog with OpenOCD firmware
-ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# TI/Luminary Stellaris In-Circuit Debug Interface (ICDI) Board
-ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Marvell Sheevaplug
-ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# Keil Software, Inc. ULink
-ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-# CMSIS-DAP compatible adapters
-ATTRS{product}=="*CMSIS-DAP*", MODE="660", GROUP="plugdev", TAG+="uaccess"
-
-LABEL="openocd_rules_end"

scripts/99-platformio-udev.rules

@@ -12,35 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#####################################################################################
 #
 # INSTALLATION
 #
+# Please visit > http://docs.platformio.org/en/latest/faq.html#platformio-udev-rules
+#
+#####################################################################################
 
-# UDEV Rules for PlatformIO supported boards, http://platformio.org/boards
 #
-# The latest version of this file may be found at:
-# 	https://github.com/platformio/platformio-core/blob/develop/scripts/99-platformio-udev.rules
+# Boards
 #
-# This file must be placed at:
-# 	/etc/udev/rules.d/99-platformio-udev.rules    (preferred location)
-#   	or
-# 	/lib/udev/rules.d/99-platformio-udev.rules    (req'd on some broken systems)
-#
-# To install, type this command in a terminal:
-# 	sudo cp 99-platformio-udev.rules /etc/udev/rules.d/99-platformio-udev.rules
-#
-# Restart "udev" management tool:
-#	sudo service udev restart
-#		or
-# 	sudo udevadm control --reload-rules
-#	sudo udevadm trigger
-#
-# Ubuntu/Debian users may need to add own “username” to the “dialout” group if
-# they are not “root”, doing this issuing a
-#   sudo usermod -a -G dialout $USER
-#   sudo usermod -a -G plugdev $USER
-#
-# After this file is installed, physically unplug and reconnect your board.
 
 # CP210X USB UART
 SUBSYSTEMS=="usb", ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea60", MODE:="0666"
@@ -89,9 +71,137 @@ SUBSYSTEMS=="usb", ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666
 #TI MSP430 Launchpad
 SUBSYSTEMS=="usb", ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666"
 
-# CMSIS-DAP compatible adapters
-ATTRS{product}=="*CMSIS-DAP*", MODE="664", GROUP="plugdev"
+
+#
+# Debuggers
+#
 
 # Black Magic Probe
 SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic GDB Server"
 SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port"
+
+# opendous and estick
+ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Original FT232/FT245 VID:PID
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Original FT2232 VID:PID
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Original FT4232 VID:PID
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Original FT232H VID:PID
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# DISTORTEC JTAG-lock-pick Tiny 2
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# TUMPA, TUMPA Lite
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# XDS100v2
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE)
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# TI/Luminary Stellaris Evaluation Board FTDI (several)
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# TI/Luminary Stellaris In-Circuit Debug Interface FTDI (ICDI) Board
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# egnite Turtelizer 2
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Section5 ICEbear
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Amontec JTAGkey and JTAGkey-tiny
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# TI ICDI
+ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# STLink v1
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# STLink v2
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# STLink v2-1
+ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Hilscher NXHX Boards
+ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Hitex STR9-comStick
+ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Hitex STM32-PerformanceStick
+ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Altera USB Blaster
+ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Amontec JTAGkey-HiSpeed
+ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# SEGGER J-Link
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="660", GROUP="plugdev", TAG+="uaccess"
+ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Raisonance RLink
+ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Debug Board for Neo1973
+ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Olimex ARM-USB-OCD
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Olimex ARM-USB-OCD-TINY
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Olimex ARM-JTAG-EW
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Olimex ARM-USB-OCD-TINY-H
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Olimex ARM-USB-OCD-H
+ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# USBprog with OpenOCD firmware
+ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# TI/Luminary Stellaris In-Circuit Debug Interface (ICDI) Board
+ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Marvell Sheevaplug
+ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# Keil Software, Inc. ULink
+ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="660", GROUP="plugdev", TAG+="uaccess"
+
+# CMSIS-DAP compatible adapters
+ATTRS{product}=="*CMSIS-DAP*", MODE="660", GROUP="plugdev", TAG+="uaccess"

scripts/docspregen.py

@@ -12,8 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from math import ceil
-from os.path import dirname, isfile, join, realpath
+import os
+import urlparse
+from os.path import dirname, isdir, isfile, join, realpath
 from sys import exit as sys_exit
 from sys import path
 
@@ -36,39 +37,48 @@ def is_compat_platform_and_framework(platform, framework):
     return False
 
 
-def generate_boards(boards, extend_debug=False):
+def campaign_url(url, source="platformio", medium="docs"):
+    data = urlparse.urlparse(url)
+    query = data.query
+    if query:
+        query += "&"
+    query += "utm_source=%s&utm_medium=%s" % (source, medium)
+    return urlparse.urlunparse(
+        urlparse.ParseResult(data.scheme, data.netloc, data.path, data.params,
+                             query, data.fragment))
 
-    def _round_memory_size(size):
-        if size == 1:
-            return 1
-
-        size = ceil(size)
-        for b in (64, 32, 16, 8, 4, 2, 1):
-            if b < size:
-                return int(ceil(size / b) * b)
-        assert NotImplemented()
 
+def generate_boards(boards, extend_debug=False, skip_columns=None):
+    columns = [
+        ("ID", "``{id}``"),
+        ("Name", "`{name} <{url}>`_"),
+        ("Platform", ":ref:`{platform_title} <platform_{platform}>`"),
+        ("Debug", "{debug}"),
+        ("MCU", "{mcu}"),
+        ("Frequency", "{f_cpu:d}MHz"),
+        ("Flash", "{rom}"),
+        ("RAM", "{ram}"),
+    ]
     platforms = {m['name']: m['title'] for m in PLATFORM_MANIFESTS}
-
     lines = []
 
     lines.append("""
 .. list-table::
     :header-rows:  1
+""")
 
-    * - ID
-      - Name
-      - Platform
-      - Debug
-      - Microcontroller
-      - Frequency
-      - Flash
-      - RAM""")
+    # add header
+    for (name, template) in columns:
+        if skip_columns and name in skip_columns:
+            continue
+        prefix = "    * - " if name == "ID" else "      - "
+        lines.append(prefix + name)
 
     for data in sorted(boards, key=lambda item: item['id']):
-        debug = [":ref:`Yes <piodebug>`" if data['debug'] else ""]
+        debug = [":ref:`Yes <piodebug>`" if data['debug'] else "No"]
         if extend_debug and data['debug']:
-            debug = []
+            debug_onboard = []
+            debug_external = []
             for name, options in data['debug']['tools'].items():
                 attrs = []
                 if options.get("default"):
@@ -77,32 +87,81 @@ def generate_boards(boards, extend_debug=False):
                     attrs.append("on-board")
                 tool = ":ref:`debugging_tool_%s`" % name
                 if attrs:
-                    debug.append("%s (%s)" % (tool, ", ".join(attrs)))
+                    tool = "%s (%s)" % (tool, ", ".join(attrs))
+                if options.get("onboard"):
+                    debug_onboard.append(tool)
                 else:
-                    debug.append(tool)
+                    debug_external.append(tool)
+            debug = sorted(debug_onboard) + sorted(debug_external)
 
-        board_ram = float(data['ram']) / 1024
-        lines.append("""
-    * - ``{id}``
-      - `{name} <{url}>`_
-      - :ref:`{platform_title} <platform_{platform}>`
-      - {debug}
-      - {mcu}
-      - {f_cpu:d} MHz
-      - {rom} Kb
-      - {ram} Kb""".format(
+        variables = dict(
             id=data['id'],
             name=data['name'],
             platform=data['platform'],
             platform_title=platforms[data['platform']],
             debug=", ".join(debug),
-            url=data['url'],
+            url=campaign_url(data['url']),
             mcu=data['mcu'].upper(),
             f_cpu=int(data['fcpu']) / 1000000,
-            ram=int(board_ram) if board_ram % 1 == 0 else board_ram,
-            rom=_round_memory_size(data['rom'] / 1024)))
+            ram=util.format_filesize(data['ram']),
+            rom=util.format_filesize(data['rom']))
 
-    return "\n".join(lines + [""])
+        for (name, template) in columns:
+            if skip_columns and name in skip_columns:
+                continue
+            prefix = "    * - " if name == "ID" else "      - "
+            lines.append(prefix + template.format(**variables))
+
+    if lines:
+        lines.append("")
+
+    return lines
+
+
+def generate_debug_boards(boards, skip_columns=None):
+    lines = []
+    onboard_debug = [
+        b for b in boards if b['debug'] and any(
+            t.get("onboard") for (_, t) in b['debug']['tools'].items())
+    ]
+    external_debug = [
+        b for b in boards if b['debug'] and b not in onboard_debug
+    ]
+    if onboard_debug or external_debug:
+        lines.append("""
+Debugging
+---------
+
+:ref:`piodebug` - "1-click" solution for debugging with a zero configuration.
+
+Supported debugging tools are listed in "Debug" column. For more detailed
+information, please scroll table by horizontal.
+You can switch between debugging :ref:`debugging_tools` using
+:ref:`projectconf_debug_tool` options.
+""")
+    if onboard_debug:
+        lines.append("""
+On-Board tools
+~~~~~~~~~~~~~~
+
+Boards listed below have on-board debugging tools and **ARE READY** for debugging!
+You do not need to use/buy external debugger.
+""")
+        lines.extend(
+            generate_boards(
+                onboard_debug, extend_debug=True, skip_columns=skip_columns))
+    if external_debug:
+        lines.append("""
+External tools
+~~~~~~~~~~~~~~
+
+Boards listed below are compatible with :ref:`piodebug` but depend on external
+debugging tools. See "Debug" column for compatible debugging tools.
+""")
+        lines.extend(
+            generate_boards(
+                external_debug, extend_debug=True, skip_columns=skip_columns))
+    return lines
 
 
 def generate_packages(platform, packagenames, is_embedded):
@@ -124,7 +183,7 @@ Packages
     * - `{name} <{url}>`__
       - {description}""".format(
             name=name,
-            url=API_PACKAGES[name]['url'],
+            url=campaign_url(API_PACKAGES[name]['url']),
             description=API_PACKAGES[name]['description']))
 
     if is_embedded:
@@ -132,8 +191,7 @@ Packages
 .. warning::
     **Linux Users**:
 
-        * Install "udev" rules file `99-platformio-udev.rules <https://github.com/platformio/platformio-core/blob/develop/scripts/99-platformio-udev.rules>`_
-          (an instruction is located inside a file).
+        * Install "udev" rules :ref:`faq_udev_rules`
         * Raspberry Pi users, please read this article
           `Enable serial port on Raspberry Pi <https://hallard.me/enable-serial-port-on-raspberry-pi/>`__.
 """)
@@ -160,8 +218,13 @@ Packages
     return "\n".join(lines)
 
 
-def generate_platform(name):
+def generate_platform(name, has_extra=False):
     print "Processing platform: %s" % name
+
+    compatible_boards = [
+        board for board in BOARDS if name in board['platform']
+    ]
+
     lines = []
 
     lines.append(
@@ -188,16 +251,89 @@ def generate_platform(name):
     lines.append(p.description)
     lines.append("""
 For more detailed information please visit `vendor site <%s>`_.""" %
-                 p.vendor_url)
+                 campaign_url(p.vendor_url))
     lines.append("""
 .. contents:: Contents
-    :local:""")
+    :local:
+    :depth: 1
+""")
+
+    #
+    # Extra
+    #
+    if has_extra:
+        lines.append(".. include:: %s_extra.rst" % p.name)
+
+    #
+    # Examples
+    #
+    lines.append("""
+Examples
+--------
+
+Examples are listed from `%s development platform repository <%s>`_:
+""" % (p.title,
+       campaign_url(
+           "https://github.com/platformio/platform-%s/tree/develop/examples" %
+           p.name)))
+    examples_dir = join(p.get_dir(), "examples")
+    if isdir(examples_dir):
+        for eitem in os.listdir(examples_dir):
+            if not isdir(join(examples_dir, eitem)):
+                continue
+            url = ("https://github.com/platformio/platform-%s"
+                   "/tree/develop/examples/%s" % (p.name, eitem))
+            lines.append("* `%s <%s>`_" % (eitem, campaign_url(url)))
+
+    #
+    # Debugging
+    #
+    if compatible_boards:
+        lines.extend(
+            generate_debug_boards(
+                compatible_boards, skip_columns=["Platform"]))
+
+    #
+    # Development version of dev/platform
+    #
+    lines.append("""
+Stable and upstream versions
+----------------------------
+
+You can switch between `stable releases <https://github.com/platformio/platform-{name}/releases>`__
+of {title} development platform and the latest upstream version using
+:ref:`projectconf_env_platform` option in :ref:`projectconf` as described below.
+
+Stable
+~~~~~~
+
+.. code-block:: ini
+
+    ; Latest stable version
+    [env:latest_stable]
+    platform = {name}
+    board = ...
+
+    ; Custom stable version
+    [env:custom_stable]
+    platform = {name}@x.y.z
+    board = ...
+
+Upstream
+~~~~~~~~
+
+.. code-block:: ini
+
+    [env:upstream_develop]
+    platform = https://github.com/platformio/platform-{name}.git
+    board = ...
+""".format(name=p.name, title=p.title))
 
     #
     # Packages
     #
-    _packages_content = generate_packages(name,
-                                          p.packages.keys(), p.is_embedded())
+    _packages_content = generate_packages(name, p.packages.keys(),
+                                          p.is_embedded())
     if _packages_content:
         lines.append(_packages_content)
 
@@ -226,31 +362,28 @@ Frameworks
     #
     # Boards
     #
-    vendors = {}
-    for board in BOARDS:
-        vendor = board['vendor']
-        if name in board['platform']:
-            if vendor in vendors:
-                vendors[vendor].append(board)
-            else:
-                vendors[vendor] = [board]
+    if compatible_boards:
+        vendors = {}
+        for board in compatible_boards:
+            if board['vendor'] not in vendors:
+                vendors[board['vendor']] = []
+            vendors[board['vendor']].append(board)
 
-    if vendors:
         lines.append("""
 Boards
 ------
 
 .. note::
     * You can list pre-configured boards by :ref:`cmd_boards` command or
-      `PlatformIO Boards Explorer <http://platformio.org/boards>`_
+      `PlatformIO Boards Explorer <https://platformio.org/boards>`_
     * For more detailed ``board`` information please scroll tables below by
       horizontal.
 """)
 
-    for vendor, boards in sorted(vendors.iteritems()):
-        lines.append(str(vendor))
-        lines.append("~" * len(vendor))
-        lines.append(generate_boards(boards))
+        for vendor, boards in sorted(vendors.items()):
+            lines.append(str(vendor))
+            lines.append("~" * len(vendor))
+            lines.extend(generate_boards(boards, skip_columns=["Platform"]))
 
     return "\n".join(lines)
 
@@ -262,13 +395,24 @@ def update_platform_docs():
             dirname(realpath(__file__)), "..", "docs", "platforms")
         rst_path = join(platforms_dir, "%s.rst" % name)
         with open(rst_path, "w") as f:
-            f.write(generate_platform(name))
-            if isfile(join(platforms_dir, "%s_extra.rst" % name)):
-                f.write("\n.. include:: %s_extra.rst\n" % name)
+            f.write(
+                generate_platform(name,
+                                  isfile(
+                                      join(platforms_dir,
+                                           "%s_extra.rst" % name))))
 
 
-def generate_framework(type_, data):
+def generate_framework(type_, data, has_extra=False):
     print "Processing framework: %s" % type_
+
+    compatible_platforms = [
+        m for m in PLATFORM_MANIFESTS
+        if is_compat_platform_and_framework(m['name'], type_)
+    ]
+    compatible_boards = [
+        board for board in BOARDS if type_ in board['frameworks']
+    ]
+
     lines = []
 
     lines.append(
@@ -294,13 +438,38 @@ def generate_framework(type_, data):
     lines.append(data['description'])
     lines.append("""
 For more detailed information please visit `vendor site <%s>`_.
-""" % data['url'])
+""" % campaign_url(data['url']))
 
     lines.append("""
 .. contents:: Contents
-    :local:""")
+    :local:
+    :depth: 1""")
 
-    lines.append("""
+    # Extra
+    if has_extra:
+        lines.append(".. include:: %s_extra.rst" % type_)
+
+    #
+    # Debugging
+    #
+    if compatible_boards:
+        lines.extend(generate_debug_boards(compatible_boards))
+
+    if compatible_platforms:
+        # examples
+        lines.append("""
+Examples
+--------
+""")
+        for manifest in compatible_platforms:
+            lines.append("* `%s for %s <%s>`_" % (
+                data['title'], manifest['title'],
+                campaign_url(
+                    "https://github.com/platformio/platform-%s/tree/develop/examples"
+                    % manifest['name'])))
+
+        # Platforms
+        lines.append("""
 Platforms
 ---------
 .. list-table::
@@ -309,43 +478,35 @@ Platforms
     * - Name
       - Description""")
 
-    _found_platform = False
-    for manifest in PLATFORM_MANIFESTS:
-        if not is_compat_platform_and_framework(manifest['name'], type_):
-            continue
-        _found_platform = True
-        p = PlatformFactory.newPlatform(manifest['name'])
-        lines.append(
-            """
+        for manifest in compatible_platforms:
+            p = PlatformFactory.newPlatform(manifest['name'])
+            lines.append("""
     * - :ref:`platform_{type_}`
-      - {description}"""
-            .format(type_=manifest['name'], description=p.description))
-    if not _found_platform:
-        del lines[-1]
+      - {description}""".format(
+                type_=manifest['name'], description=p.description))
 
-    lines.append("""
+    #
+    # Boards
+    #
+    if compatible_boards:
+        vendors = {}
+        for board in compatible_boards:
+            if board['vendor'] not in vendors:
+                vendors[board['vendor']] = []
+            vendors[board['vendor']].append(board)
+        lines.append("""
 Boards
 ------
 
 .. note::
     * You can list pre-configured boards by :ref:`cmd_boards` command or
-      `PlatformIO Boards Explorer <http://platformio.org/boards>`_
+      `PlatformIO Boards Explorer <https://platformio.org/boards>`_
     * For more detailed ``board`` information please scroll tables below by horizontal.
 """)
-
-    vendors = {}
-    for data in BOARDS:
-        frameworks = data['frameworks'] or []
-        vendor = data['vendor']
-        if type_ in frameworks:
-            if vendor in vendors:
-                vendors[vendor].append(data)
-            else:
-                vendors[vendor] = [data]
-    for vendor, boards in sorted(vendors.iteritems()):
-        lines.append(str(vendor))
-        lines.append("~" * len(vendor))
-        lines.append(generate_boards(boards))
+        for vendor, boards in sorted(vendors.items()):
+            lines.append(str(vendor))
+            lines.append("~" * len(vendor))
+            lines.extend(generate_boards(boards))
     return "\n".join(lines)
 
 
@@ -356,9 +517,11 @@ def update_framework_docs():
             dirname(realpath(__file__)), "..", "docs", "frameworks")
         rst_path = join(frameworks_dir, "%s.rst" % name)
         with open(rst_path, "w") as f:
-            f.write(generate_framework(name, framework))
-            if isfile(join(frameworks_dir, "%s_extra.rst" % name)):
-                f.write("\n.. include:: %s_extra.rst\n" % name)
+            f.write(
+                generate_framework(name, framework,
+                                   isfile(
+                                       join(frameworks_dir,
+                                            "%s_extra.rst" % name))))
 
 
 def update_create_platform_doc():
@@ -424,7 +587,7 @@ popular embedded boards and IDE.
 
 .. note::
     * You can list pre-configured boards by :ref:`cmd_boards` command or
-      `PlatformIO Boards Explorer <http://platformio.org/boards>`_
+      `PlatformIO Boards Explorer <https://platformio.org/boards>`_
     * For more detailed ``board`` information please scroll tables below by horizontal.
 """)
 
@@ -444,7 +607,7 @@ popular embedded boards and IDE.
     for vendor, boards in sorted(vendors.iteritems()):
         lines.append(str(vendor))
         lines.append("~" * len(vendor))
-        lines.append(generate_boards(boards))
+        lines.extend(generate_boards(boards))
 
     emboards_rst = join(
         dirname(realpath(__file__)), "..", "docs", "platforms",
@@ -484,11 +647,10 @@ Platforms
         if manifest['name'] not in platforms:
             continue
         p = PlatformFactory.newPlatform(manifest['name'])
-        lines.append(
-            """
+        lines.append("""
     * - :ref:`platform_{type_}`
-      - {description}"""
-            .format(type_=manifest['name'], description=p.description))
+      - {description}""".format(
+            type_=manifest['name'], description=p.description))
 
     # Frameworks
     lines.append("""
@@ -517,11 +679,11 @@ Boards
     for vendor, boards in sorted(vendors.iteritems()):
         lines.append(str(vendor))
         lines.append("~" * len(vendor))
-        lines.append(generate_boards(boards, extend_debug=True))
+        lines.extend(generate_boards(boards, extend_debug=True))
 
     with open(
-            join(util.get_source_dir(), "..", "docs", "plus",
-                 "debugging.rst"), "r+") as fp:
+            join(util.get_source_dir(), "..", "docs", "plus", "debugging.rst"),
+            "r+") as fp:
         content = fp.read()
         fp.seek(0)
         fp.truncate()
@@ -529,12 +691,66 @@ Boards
                  "\n".join(lines))
 
 
+def update_examples_readme():
+    examples_dir = join(util.get_source_dir(), "..", "examples")
+
+    # Platforms
+    embedded = []
+    desktop = []
+    for manifest in PLATFORM_MANIFESTS:
+        p = PlatformFactory.newPlatform(manifest['name'])
+        url = campaign_url(
+            "http://docs.platformio.org/en/latest/platforms/%s.html#examples" %
+            p.name,
+            source="github",
+            medium="examples")
+        line = "* [%s](%s)" % (p.title, url)
+        if p.is_embedded():
+            embedded.append(line)
+        else:
+            desktop.append(line)
+
+    # Frameworks
+    frameworks = []
+    for framework in API_FRAMEWORKS:
+        url = campaign_url(
+            "http://docs.platformio.org/en/latest/frameworks/%s.html#examples"
+            % framework['name'],
+            source="github",
+            medium="examples")
+        frameworks.append("* [%s](%s)" % (framework['title'], url))
+
+    with open(join(examples_dir, "README.md"), "w") as fp:
+        fp.write("""# PlatformIO Project Examples
+
+- [Development platforms](#development-platforms):
+  - [Embedded](#embedded)
+  - [Desktop](#desktop)
+- [Frameworks](#frameworks)
+
+## Development platforms
+
+### Embedded
+
+%s
+
+### Desktop
+
+%s
+
+## Frameworks
+
+%s
+""" % ("\n".join(embedded), "\n".join(desktop), "\n".join(frameworks)))
+
+
 def main():
     update_create_platform_doc()
     update_platform_docs()
     update_framework_docs()
     update_embedded_boards()
     update_debugging()
+    update_examples_readme()
 
 
 if __name__ == "__main__":

scripts/install_devplatforms.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import subprocess
+import sys
+
+
+def main():
+    platforms = json.loads(
+        subprocess.check_output(
+            ["platformio", "platform", "search", "--json-output"]))
+    for platform in platforms:
+        if platform['forDesktop']:
+            continue
+        subprocess.check_call(
+            ["platformio", "platform", "install", platform['repository']])
+
+
+if __name__ == "__main__":
+    sys.exit(main())

setup.py

@@ -18,7 +18,6 @@ from platformio import (__author__, __description__, __email__, __license__,
                         __title__, __url__, __version__)
 
 install_requires = [
-    "arrow>=0.10.0,!=0.11.0",
     "bottle<0.13",
     "click>=5,<6",
     "colorama",
@@ -37,6 +36,7 @@ setup(
     author_email=__email__,
     url=__url__,
     license=__license__,
+    python_requires='>=2.7, <3',
     install_requires=install_requires,
     packages=find_packages(),
     package_data={

tests/commands/test_ci.py

@@ -15,6 +15,7 @@
 from os.path import join
 
 from platformio.commands.ci import cli as cmd_ci
+from platformio.commands.lib import cli as cmd_lib
 
 
 def test_ci_empty(clirunner):
@@ -25,27 +26,31 @@ def test_ci_empty(clirunner):
 
 def test_ci_boards(clirunner, validate_cliresult):
     result = clirunner.invoke(cmd_ci, [
-        join("examples", "atmelavr", "arduino-internal-libs", "src",
-             "ChatServer.ino"), "-b", "uno", "-b", "leonardo"
+        join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", "-b",
+        "leonardo"
     ])
     validate_cliresult(result)
 
 
 def test_ci_project_conf(clirunner, validate_cliresult):
-    project_dir = join("examples", "atmelavr", "arduino-internal-libs")
+    project_dir = join("examples", "wiring-blink")
     result = clirunner.invoke(cmd_ci, [
-        join(project_dir, "src", "ChatServer.ino"), "--project-conf",
+        join(project_dir, "src", "main.cpp"), "--project-conf",
         join(project_dir, "platformio.ini")
     ])
     validate_cliresult(result)
-    assert all([s in result.output for s in ("ethernet", "leonardo", "yun")])
+    assert "uno" in result.output
 
 
-def test_ci_lib_and_board(clirunner, validate_cliresult):
-    example_dir = join("examples", "atmelavr", "arduino-external-libs")
+def test_ci_lib_and_board(clirunner, tmpdir_factory, validate_cliresult):
+    storage_dir = str(tmpdir_factory.mktemp("lib"))
+    result = clirunner.invoke(
+        cmd_lib, ["--storage-dir", storage_dir, "install", "1@2.3.2"])
+    validate_cliresult(result)
+
     result = clirunner.invoke(cmd_ci, [
-        join(example_dir, "lib", "OneWire", "examples", "DS2408_Switch",
-             "DS2408_Switch.pde"), "-l", join(example_dir, "lib", "OneWire"),
-        "-b", "uno"
+        join(storage_dir, "OneWire_ID1", "examples", "DS2408_Switch",
+             "DS2408_Switch.pde"), "-l",
+        join(storage_dir, "OneWire_ID1"), "-b", "uno"
     ])
     validate_cliresult(result)

tests/commands/test_init.py

@@ -54,7 +54,7 @@ def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir):
         assert set(config.sections()) == set(["env:uno"])
 
 
-def test_init_ide_without_board(clirunner, validate_cliresult, tmpdir):
+def test_init_ide_without_board(clirunner, tmpdir):
     with tmpdir.as_cwd():
         result = clirunner.invoke(cmd_init, ["--ide", "atom"])
         assert result.exit_code == -1
@@ -67,13 +67,15 @@ def test_init_ide_atom(clirunner, validate_cliresult, tmpdir):
             cmd_init, ["--ide", "atom", "-b", "uno", "-b", "teensy31"])
         validate_cliresult(result)
         validate_pioproject(str(tmpdir))
-        assert all([tmpdir.join(f).check()
-                    for f in (".clang_complete", ".gcc-flags.json")])
+        assert all([
+            tmpdir.join(f).check()
+            for f in (".clang_complete", ".gcc-flags.json")
+        ])
         assert "arduinoavr" in tmpdir.join(".clang_complete").read()
 
         # switch to NodeMCU
-        result = clirunner.invoke(
-            cmd_init, ["--ide", "atom", "-b", "nodemcuv2"])
+        result = clirunner.invoke(cmd_init,
+                                  ["--ide", "atom", "-b", "nodemcuv2"])
         validate_cliresult(result)
         validate_pioproject(str(tmpdir))
         assert "arduinoespressif" in tmpdir.join(".clang_complete").read()
@@ -104,15 +106,13 @@ def test_init_special_board(clirunner, validate_cliresult):
         boards = json.loads(result.output)
 
         config = util.load_project_config()
-        expected_result = [
-            ("platform", str(boards[0]['platform'])),
-            ("framework", str(boards[0]['frameworks'][0])), ("board", "uno")
-        ]
+        expected_result = [("platform", str(boards[0]['platform'])),
+                           ("framework",
+                            str(boards[0]['frameworks'][0])), ("board", "uno")]
 
         assert config.has_section("env:uno")
-        assert len(
-            set(expected_result).symmetric_difference(
-                set(config.items("env:uno")))) == 0
+        assert not set(expected_result).symmetric_difference(
+            set(config.items("env:uno")))
 
 
 def test_init_enable_auto_uploading(clirunner, validate_cliresult):
@@ -122,14 +122,11 @@ def test_init_enable_auto_uploading(clirunner, validate_cliresult):
         validate_cliresult(result)
         validate_pioproject(getcwd())
         config = util.load_project_config()
-        expected_result = [
-            ("platform", "atmelavr"), ("framework", "arduino"),
-            ("board", "uno"), ("targets", "upload")
-        ]
+        expected_result = [("platform", "atmelavr"), ("framework", "arduino"),
+                           ("board", "uno"), ("targets", "upload")]
         assert config.has_section("env:uno")
-        assert len(
-            set(expected_result).symmetric_difference(
-                set(config.items("env:uno")))) == 0
+        assert not set(expected_result).symmetric_difference(
+            set(config.items("env:uno")))
 
 
 def test_init_custom_framework(clirunner, validate_cliresult):
@@ -139,14 +136,11 @@ def test_init_custom_framework(clirunner, validate_cliresult):
         validate_cliresult(result)
         validate_pioproject(getcwd())
         config = util.load_project_config()
-        expected_result = [
-            ("platform", "teensy"), ("framework", "mbed"),
-            ("board", "teensy31")
-        ]
+        expected_result = [("platform", "teensy"), ("framework", "mbed"),
+                           ("board", "teensy31")]
         assert config.has_section("env:teensy31")
-        assert len(
-            set(expected_result).symmetric_difference(
-                set(config.items("env:teensy31")))) == 0
+        assert not set(expected_result).symmetric_difference(
+            set(config.items("env:teensy31")))
 
 
 def test_init_incorrect_board(clirunner):

tests/commands/test_lib.py

@@ -15,8 +15,7 @@
 import json
 import re
 
-from platformio import exception, util
-from platformio.commands.init import cli as cmd_init
+from platformio import exception
 from platformio.commands.lib import cli as cmd_lib
 
 
@@ -36,25 +35,11 @@ def test_search(clirunner, validate_cliresult):
 def test_global_install_registry(clirunner, validate_cliresult,
                                  isolated_pio_home):
     result = clirunner.invoke(cmd_lib, [
-        "-g", "install", "58", "547@2.2.4", "DallasTemperature",
-        "http://dl.platformio.org/libraries/archives/3/5174.tar.gz",
-        "ArduinoJson@5.6.7", "ArduinoJson@~5.7.0", "168@00589a3250"
+        "-g", "install", "64", "ArduinoJson@~5.10.0", "547@2.2.4",
+        "AsyncMqttClient@<=0.8.2", "999@77d4eb3f8a"
     ])
     validate_cliresult(result)
 
-    # check lib with duplicate URL
-    result = clirunner.invoke(cmd_lib, [
-        "-g", "install",
-        "http://dl.platformio.org/libraries/archives/3/5174.tar.gz"
-    ])
-    validate_cliresult(result)
-    assert "is already installed" in result.output
-
-    # check lib with duplicate ID
-    result = clirunner.invoke(cmd_lib, ["-g", "install", "305"])
-    validate_cliresult(result)
-    assert "is already installed" in result.output
-
     # install unknown library
     result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"])
     assert result.exit_code != 0
@@ -62,9 +47,9 @@ def test_global_install_registry(clirunner, validate_cliresult,
 
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
     items2 = [
-        "ArduinoJson_ID64", "ArduinoJson_ID64@5.6.7", "DallasTemperature_ID54",
-        "DHT22_ID58", "ESPAsyncTCP_ID305", "NeoPixelBus_ID547", "OneWire_ID1",
-        "EspSoftwareSerial_ID168"
+        "ArduinoJson_ID64", "ArduinoJson_ID64@5.10.1", "NeoPixelBus_ID547",
+        "AsyncMqttClient_ID346", "ESPAsyncTCP_ID305", "AsyncTCP_ID1826",
+        "RFcontrol_ID999"
     ]
     assert set(items1) == set(items2)
 
@@ -72,11 +57,12 @@ def test_global_install_registry(clirunner, validate_cliresult,
 def test_global_install_archive(clirunner, validate_cliresult,
                                 isolated_pio_home):
     result = clirunner.invoke(cmd_lib, [
-        "-g", "install", "https://github.com/adafruit/Adafruit-ST7735-Library/"
-        "archive/master.zip",
+        "-g", "install",
         "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip",
         "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip",
-        "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2"
+        "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2",
+        "http://dl.platformio.org/libraries/archives/0/9540.tar.gz",
+        "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip"
     ])
     validate_cliresult(result)
 
@@ -87,16 +73,11 @@ def test_global_install_archive(clirunner, validate_cliresult,
     ])
     assert result.exit_code != 0
 
-    # check lib with duplicate URL
-    result = clirunner.invoke(cmd_lib, [
-        "-g", "install",
-        "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip"
-    ])
-    validate_cliresult(result)
-    assert "is already installed" in result.output
-
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
-    items2 = ["Adafruit ST7735 Library", "RadioHead-1.62"]
+    items2 = [
+        "RadioHead-1.62", "ArduinoJson", "DallasTemperature_ID54",
+        "OneWire_ID1", "ESP32WebServer"
+    ]
     assert set(items1) >= set(items2)
 
 
@@ -113,18 +94,41 @@ def test_global_install_repository(clirunner, validate_cliresult,
             "https://gitlab.com/ivankravets/rs485-nodeproto.git",
             "https://github.com/platformio/platformio-libmirror.git",
             # "https://developer.mbed.org/users/simon/code/TextLCD/",
-            "knolleary/pubsubclient"
+            "knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163"
         ])
     validate_cliresult(result)
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
     items2 = [
         "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d",
         "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "rs485-nodeproto",
-        "PubSubClient"
+        "platformio-libmirror", "PubSubClient"
     ]
     assert set(items1) >= set(items2)
 
-    # check lib with duplicate URL
+
+def test_install_duplicates(clirunner, validate_cliresult, without_internet):
+    # registry
+    result = clirunner.invoke(cmd_lib, [
+        "-g", "install",
+        "http://dl.platformio.org/libraries/archives/0/9540.tar.gz"
+    ])
+    validate_cliresult(result)
+    assert "is already installed" in result.output
+
+    # by ID
+    result = clirunner.invoke(cmd_lib, ["-g", "install", "999"])
+    validate_cliresult(result)
+    assert "is already installed" in result.output
+
+    # archive
+    result = clirunner.invoke(cmd_lib, [
+        "-g", "install",
+        "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip"
+    ])
+    validate_cliresult(result)
+    assert "is already installed" in result.output
+
+    # repository
     result = clirunner.invoke(cmd_lib, [
         "-g", "install",
         "https://github.com/platformio/platformio-libmirror.git"
@@ -136,23 +140,42 @@ def test_global_install_repository(clirunner, validate_cliresult,
 def test_global_lib_list(clirunner, validate_cliresult):
     result = clirunner.invoke(cmd_lib, ["-g", "list"])
     validate_cliresult(result)
-    assert all([n in result.output for n in ("OneWire", "DHT22", "64")])
+    assert all([
+        n in result.output for n in
+        ("Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip",
+         "Version: 5.10.1",
+         "Source: git+https://github.com/gioblu/PJON.git#3.0",
+         "Version: 1fb26fd", "RadioHead-1.62")
+    ])
 
     result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"])
     assert all([
-        n in result.output
-        for n in (
-            "PJON", "git+https://github.com/knolleary/pubsubclient",
-            "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip")
+        n in result.output for n in
+        ("__pkg_dir",
+         '"__src_url": "git+https://gitlab.com/ivankravets/rs485-nodeproto.git"',
+         '"version": "5.10.1"')
     ])
     items1 = [i['name'] for i in json.loads(result.output)]
     items2 = [
-        "OneWire", "DHT22", "PJON", "ESPAsyncTCP", "ArduinoJson",
-        "PubSubClient", "rs485-nodeproto", "Adafruit ST7735 Library",
-        "RadioHead-1.62", "DallasTemperature", "NeoPixelBus",
-        "EspSoftwareSerial", "platformio-libmirror"
+        "ESP32WebServer", "ArduinoJson", "ArduinoJson", "ArduinoJson",
+        "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "DallasTemperature",
+        "ESPAsyncTCP", "NeoPixelBus", "OneWire", "PJON", "PJON",
+        "PubSubClient", "RFcontrol", "RadioHead-1.62", "platformio-libmirror",
+        "rs485-nodeproto"
     ]
-    assert set(items1) == set(items2)
+    assert sorted(items1) == sorted(items2)
+
+    versions1 = [
+        "{name}@{version}".format(**item)
+        for item in json.loads(result.output)
+    ]
+    versions2 = [
+        'ArduinoJson@5.8.2', 'ArduinoJson@5.10.1', 'AsyncMqttClient@0.8.2',
+        'AsyncTCP@1.0.1', 'ESPAsyncTCP@1.1.3', 'NeoPixelBus@2.2.4',
+        'PJON@07fe9aa', 'PJON@1fb26fd', 'PubSubClient@bef5814',
+        'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0'
+    ]
+    assert set(versions1) >= set(versions2)
 
 
 def test_global_lib_update_check(clirunner, validate_cliresult):
@@ -160,7 +183,7 @@ def test_global_lib_update_check(clirunner, validate_cliresult):
         cmd_lib, ["-g", "update", "--only-check", "--json-output"])
     validate_cliresult(result)
     output = json.loads(result.output)
-    assert set(["ArduinoJson", "EspSoftwareSerial",
+    assert set(["RFcontrol",
                 "NeoPixelBus"]) == set([l['name'] for l in output])
 
 
@@ -181,11 +204,9 @@ def test_global_lib_update(clirunner, validate_cliresult):
     # update rest libraries
     result = clirunner.invoke(cmd_lib, ["-g", "update"])
     validate_cliresult(result)
-    validate_cliresult(result)
-    assert result.output.count("[Fixed]") == 5
-    assert result.output.count("[Up-to-date]") == 10
-    assert "Uninstalling ArduinoJson @ 5.7.3" in result.output
-    assert "Uninstalling EspSoftwareSerial @ 00589a3250" in result.output
+    assert result.output.count("[Fixed]") == 6
+    assert result.output.count("[Up-to-date]") == 11
+    assert "Uninstalling RFcontrol @ 77d4eb3f8a" in result.output
 
     # update unknown library
     result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"])
@@ -200,23 +221,24 @@ def test_global_lib_uninstall(clirunner, validate_cliresult,
     validate_cliresult(result)
     items = json.loads(result.output)
     result = clirunner.invoke(cmd_lib,
-                              ["-g", "uninstall", items[0]['__pkg_dir']])
+                              ["-g", "uninstall", items[5]['__pkg_dir']])
     validate_cliresult(result)
-    assert "Uninstalling Adafruit ST7735 Library" in result.output
+    assert "Uninstalling AsyncTCP" in result.output
 
     # uninstall the rest libraries
     result = clirunner.invoke(cmd_lib, [
         "-g", "uninstall", "1", "https://github.com/bblanchon/ArduinoJson.git",
-        "ArduinoJson@!=5.6.7", "EspSoftwareSerial@>=3.3.1"
+        "ArduinoJson@!=5.6.7", "RFcontrol"
     ])
     validate_cliresult(result)
 
     items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()]
     items2 = [
-        "ArduinoJson_ID64", "ArduinoJson_ID64@5.6.7", "DallasTemperature_ID54",
-        "DHT22_ID58", "ESPAsyncTCP_ID305", "NeoPixelBus_ID547", "PJON",
-        "PJON@src-79de467ebe19de18287becff0a1fb42d", "PubSubClient",
-        "RadioHead-1.62", "rs485-nodeproto", "platformio-libmirror"
+        "RadioHead-1.62", "rs485-nodeproto", "platformio-libmirror",
+        "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81",
+        "ESPAsyncTCP_ID305", "DallasTemperature_ID54", "NeoPixelBus_ID547",
+        "PJON", "AsyncMqttClient_ID346", "ArduinoJson_ID64",
+        "PJON@src-79de467ebe19de18287becff0a1fb42d", "ESP32WebServer"
     ]
     assert set(items1) == set(items2)
 
@@ -231,7 +253,7 @@ def test_lib_show(clirunner, validate_cliresult):
     validate_cliresult(result)
     assert all(
         [s in result.output for s in ("ArduinoJson", "Arduino", "Atmel AVR")])
-    result = clirunner.invoke(cmd_lib, ["show", "OneWire"])
+    result = clirunner.invoke(cmd_lib, ["show", "OneWire", "--json-output"])
     validate_cliresult(result)
     assert "OneWire" in result.output
 
@@ -248,7 +270,7 @@ def test_lib_stats(clirunner, validate_cliresult):
     validate_cliresult(result)
     assert all([
         s in result.output
-        for s in ("UPDATED", "ago", "http://platformio.org/lib/show")
+        for s in ("UPDATED", "POPULAR", "https://platformio.org/lib/show")
     ])
 
     result = clirunner.invoke(cmd_lib, ["stats", "--json-output"])

tests/commands/test_platform.py

@@ -24,27 +24,25 @@ def test_search_json_output(clirunner, validate_cliresult, isolated_pio_home):
     validate_cliresult(result)
     search_result = json.loads(result.output)
     assert isinstance(search_result, list)
-    assert len(search_result)
+    assert search_result
     platforms = [item['name'] for item in search_result]
     assert "atmelsam" in platforms
 
 
-def test_search_raw_output(clirunner, validate_cliresult, isolated_pio_home):
+def test_search_raw_output(clirunner, validate_cliresult):
     result = clirunner.invoke(cli_platform.platform_search, ["arduino"])
     validate_cliresult(result)
     assert "teensy" in result.output
 
 
-def test_install_unknown_version(clirunner, validate_cliresult,
-                                 isolated_pio_home):
+def test_install_unknown_version(clirunner):
     result = clirunner.invoke(cli_platform.platform_install,
                               ["atmelavr@99.99.99"])
     assert result.exit_code == -1
     assert isinstance(result.exception, exception.UndefinedPackageVersion)
 
 
-def test_install_unknown_from_registry(clirunner, validate_cliresult,
-                                       isolated_pio_home):
+def test_install_unknown_from_registry(clirunner):
     result = clirunner.invoke(cli_platform.platform_install,
                               ["unknown-platform"])
     assert result.exit_code == -1
@@ -70,19 +68,20 @@ def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home):
     ])
     validate_cliresult(result)
     assert "espressif8266" in result.output
+    assert len(isolated_pio_home.join("packages").listdir()) == 1
 
 
-def test_list_json_output(clirunner, validate_cliresult, isolated_pio_home):
+def test_list_json_output(clirunner, validate_cliresult):
     result = clirunner.invoke(cli_platform.platform_list, ["--json-output"])
     validate_cliresult(result)
     list_result = json.loads(result.output)
     assert isinstance(list_result, list)
-    assert len(list_result)
+    assert list_result
     platforms = [item['name'] for item in list_result]
     assert set(["atmelavr", "espressif8266"]) == set(platforms)
 
 
-def test_list_raw_output(clirunner, validate_cliresult, isolated_pio_home):
+def test_list_raw_output(clirunner, validate_cliresult):
     result = clirunner.invoke(cli_platform.platform_list)
     validate_cliresult(result)
     assert all(
@@ -111,4 +110,4 @@ def test_uninstall(clirunner, validate_cliresult, isolated_pio_home):
     result = clirunner.invoke(cli_platform.platform_uninstall,
                               ["atmelavr", "espressif8266"])
     validate_cliresult(result)
-    assert len(isolated_pio_home.join("platforms").listdir()) == 0
+    assert not isolated_pio_home.join("platforms").listdir()

tests/commands/test_settings.py

@@ -19,6 +19,6 @@ from platformio.commands.settings import cli
 def test_settings_check(clirunner, validate_cliresult):
     result = clirunner.invoke(cli, ["get"])
     validate_cliresult(result)
-    assert len(result.output)
+    assert result.output
     for item in app.DEFAULT_SETTINGS.items():
         assert item[0] in result.output

tests/commands/test_update.py

@@ -16,11 +16,7 @@ from platformio.commands.update import cli as cmd_update
 
 
 def test_update(clirunner, validate_cliresult):
-    matches = (
-        "Platform Manager",
-        "Up-to-date",
-        "Library Manager"
-    )
+    matches = ("Platform Manager", "Up-to-date", "Library Manager")
     result = clirunner.invoke(cmd_update, ["--only-check"])
     validate_cliresult(result)
     assert all([m in result.output for m in matches])

tests/conftest.py

@@ -17,22 +17,24 @@ import os
 import pytest
 from click.testing import CliRunner
 
-
-@pytest.fixture(scope="module")
-def clirunner():
-    return CliRunner()
+from platformio import util
 
 
 @pytest.fixture(scope="session")
 def validate_cliresult():
 
     def decorator(result):
-        assert result.exit_code == 0
-        assert not result.exception
+        assert result.exit_code == 0, result.output
+        assert not result.exception, result.output
 
     return decorator
 
 
+@pytest.fixture(scope="module")
+def clirunner():
+    return CliRunner()
+
+
 @pytest.fixture(scope="module")
 def isolated_pio_home(request, tmpdir_factory):
     home_dir = tmpdir_factory.mktemp(".platformio")
@@ -43,3 +45,8 @@ def isolated_pio_home(request, tmpdir_factory):
 
     request.addfinalizer(fin)
     return home_dir
+
+
+@pytest.fixture(scope="function")
+def without_internet(monkeypatch):
+    monkeypatch.setattr(util, "_internet_on", lambda: False)

tests/ino2cpp/basic/basic.ino

@@ -48,3 +48,5 @@ void myFunction(struct Item *item) {
 void fooCallback(){
 
 }
+
+// юнікод

tests/test_builder.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.commands.run import cli as cmd_run
+
+
+def test_build_flags(clirunner, validate_cliresult, tmpdir):
+    build_flags = [("-D TEST_INT=13", "-DTEST_INT=13"),
+                   ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"),
+                   ('-DTEST_STR_SPACE="Andrew Smith"',
+                    '"-DTEST_STR_SPACE=Andrew Smith"')]
+
+    tmpdir.join("platformio.ini").write("""
+[env:native]
+platform = native
+extra_scripts = extra.py
+build_flags = %s
+    """ % " ".join([f[0] for f in build_flags]))
+
+    tmpdir.join("extra.py").write("""
+Import("env")
+env.Append(CPPDEFINES="POST_SCRIPT_MACRO")
+    """)
+
+    tmpdir.mkdir("src").join("main.cpp").write("""
+#if !defined(TEST_INT) || TEST_INT != 13
+#error "TEST_INT"
+#endif
+
+#ifndef TEST_STR_SPACE
+#error "TEST_STR_SPACE"
+#endif
+
+#ifndef POST_SCRIPT_MACRO
+#error "POST_SCRIPT_MACRO"
+#endif
+
+int main() {
+}
+""")
+
+    result = clirunner.invoke(
+        cmd_run, ["--project-dir", str(tmpdir), "--verbose"])
+    validate_cliresult(result)
+    build_output = result.output[result.output.find(
+        "Scanning dependencies..."):]
+    for flag in build_flags:
+        assert flag[1] in build_output, flag
+
+
+def test_build_unflags(clirunner, validate_cliresult, tmpdir):
+    tmpdir.join("platformio.ini").write("""
+[env:native]
+platform = native
+build_unflags = -DTMP_MACRO1=45 -I. -DNON_EXISTING_MACRO -lunknownLib -Os
+extra_scripts = pre:extra.py
+""")
+
+    tmpdir.join("extra.py").write("""
+Import("env")
+env.Append(CPPPATH="%s")
+env.Append(CPPDEFINES="TMP_MACRO1")
+env.Append(CPPDEFINES=["TMP_MACRO2"])
+env.Append(CPPDEFINES=("TMP_MACRO3", 13))
+env.Append(CCFLAGS=["-Os"])
+env.Append(LIBS=["unknownLib"])
+    """ % str(tmpdir))
+
+    tmpdir.mkdir("src").join("main.c").write("""
+#ifdef TMP_MACRO1
+#error "TMP_MACRO1 should be removed"
+#endif
+
+int main() {
+}
+""")
+
+    result = clirunner.invoke(
+        cmd_run, ["--project-dir", str(tmpdir), "--verbose"])
+    validate_cliresult(result)
+    build_output = result.output[result.output.find(
+        "Scanning dependencies..."):]
+    assert "-DTMP_MACRO1" not in build_output
+    assert "-Os" not in build_output
+    assert str(tmpdir) not in build_output

tests/test_examples.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import random
 from glob import glob
 from os import listdir, walk
 from os.path import dirname, getsize, isdir, isfile, join, normpath
@@ -19,43 +20,66 @@ from os.path import dirname, getsize, isdir, isfile, join, normpath
 import pytest
 
 from platformio import util
+from platformio.managers.platform import PlatformFactory, PlatformManager
 
 
 def pytest_generate_tests(metafunc):
     if "pioproject_dir" not in metafunc.fixturenames:
         return
-    example_dirs = normpath(join(dirname(__file__), "..", "examples"))
-    project_dirs = []
-    for root, _, files in walk(example_dirs):
-        if "platformio.ini" not in files or ".skiptest" in files:
+    examples_dirs = []
+
+    # repo examples
+    examples_dirs.append(normpath(join(dirname(__file__), "..", "examples")))
+
+    # dev/platforms
+    for manifest in PlatformManager().get_installed():
+        p = PlatformFactory.newPlatform(manifest['__pkg_dir'])
+        if not p.is_embedded():
             continue
-        project_dirs.append(root)
+        examples_dir = join(p.get_dir(), "examples")
+        assert isdir(examples_dir)
+        examples_dirs.append(examples_dir)
+
+    project_dirs = []
+    for examples_dir in examples_dirs:
+        for root, _, files in walk(examples_dir):
+            if "platformio.ini" not in files or ".skiptest" in files:
+                continue
+            project_dirs.append(root)
     project_dirs.sort()
     metafunc.parametrize("pioproject_dir", project_dirs)
 
 
 @pytest.mark.examples
 def test_run(pioproject_dir):
-    if isdir(join(pioproject_dir, ".pioenvs")):
-        util.rmtree_(join(pioproject_dir, ".pioenvs"))
+    with util.cd(pioproject_dir):
+        build_dir = util.get_projectbuild_dir()
+        if isdir(build_dir):
+            util.rmtree_(build_dir)
 
-    result = util.exec_command(
-        ["platformio", "--force", "run", "--project-dir", pioproject_dir]
-    )
-    if result['returncode'] != 0:
-        pytest.fail(result)
+        env_names = []
+        for section in util.load_project_config().sections():
+            if section.startswith("env:"):
+                env_names.append(section[4:])
 
-    # check .elf file
-    pioenvs_dir = join(pioproject_dir, ".pioenvs")
-    for item in listdir(pioenvs_dir):
-        if not isdir(item):
-            continue
-        assert isfile(join(pioenvs_dir, item, "firmware.elf"))
-        # check .hex or .bin files
-        firmwares = []
-        for ext in ("bin", "hex"):
-            firmwares += glob(join(pioenvs_dir, item, "firmware*.%s" % ext))
-        if not firmwares:
-            pytest.fail("Missed firmware file")
-        for firmware in firmwares:
-            assert getsize(firmware) > 0
+        result = util.exec_command(
+            ["platformio", "run", "-e",
+             random.choice(env_names)])
+        if result['returncode'] != 0:
+            pytest.fail(result)
+
+        assert isdir(build_dir)
+
+        # check .elf file
+        for item in listdir(build_dir):
+            if not isdir(item):
+                continue
+            assert isfile(join(build_dir, item, "firmware.elf"))
+            # check .hex or .bin files
+            firmwares = []
+            for ext in ("bin", "hex"):
+                firmwares += glob(join(build_dir, item, "firmware*.%s" % ext))
+            if not firmwares:
+                pytest.fail("Missed firmware file")
+            for firmware in firmwares:
+                assert getsize(firmware) > 0

tests/test_maintenance.py

@@ -44,9 +44,7 @@ def test_after_upgrade_2_to_3(clirunner, validate_cliresult,
 
     result = clirunner.invoke(cli_pio, ["settings", "get"])
     validate_cliresult(result)
-    assert "upgraded to 3"
-    assert isolated_pio_home.join("platforms", "native",
-                                  "platform.json").check()
+    assert "upgraded to 3" in result.output
 
     # check PlatformIO 3.0 boards
     assert board_ids == set([p.basename[:-5] for p in boards.listdir()])
@@ -57,8 +55,7 @@ def test_after_upgrade_2_to_3(clirunner, validate_cliresult,
     assert board_ids == set([b['id'] for b in json.loads(result.output)])
 
 
-def test_after_upgrade_silence(clirunner, validate_cliresult,
-                               isolated_pio_home):
+def test_after_upgrade_silence(clirunner, validate_cliresult):
     app.set_state_item("last_version", "2.11.2")
     result = clirunner.invoke(cli_pio, ["boards", "--json-output"])
     validate_cliresult(result)
@@ -66,7 +63,7 @@ def test_after_upgrade_silence(clirunner, validate_cliresult,
     assert any([b['id'] == "uno" for b in boards])
 
 
-def test_check_pio_upgrade(clirunner, validate_cliresult, isolated_pio_home):
+def test_check_pio_upgrade(clirunner, validate_cliresult):
 
     def _patch_pio_version(version):
         maintenance.__version__ = version
@@ -96,7 +93,7 @@ def test_check_pio_upgrade(clirunner, validate_cliresult, isolated_pio_home):
     _patch_pio_version(origin_version)
 
 
-def test_check_lib_updates(clirunner, validate_cliresult, isolated_pio_home):
+def test_check_lib_updates(clirunner, validate_cliresult):
     # install obsolete library
     result = clirunner.invoke(cli_pio,
                               ["lib", "-g", "install", "ArduinoJson@<5.7"])
@@ -113,8 +110,7 @@ def test_check_lib_updates(clirunner, validate_cliresult, isolated_pio_home):
             result.output)
 
 
-def test_check_and_update_libraries(clirunner, validate_cliresult,
-                                    isolated_pio_home):
+def test_check_and_update_libraries(clirunner, validate_cliresult):
     # enable library auto-updates
     result = clirunner.invoke(
         cli_pio, ["settings", "set", "auto_update_libraries", "Yes"])
@@ -168,8 +164,7 @@ def test_check_platform_updates(clirunner, validate_cliresult,
     assert "There are the new updates for platforms (native)" in result.output
 
 
-def test_check_and_update_platforms(clirunner, validate_cliresult,
-                                    isolated_pio_home):
+def test_check_and_update_platforms(clirunner, validate_cliresult):
     # enable library auto-updates
     result = clirunner.invoke(
         cli_pio, ["settings", "set", "auto_update_platforms", "Yes"])
@@ -190,8 +185,7 @@ def test_check_and_update_platforms(clirunner, validate_cliresult,
     validate_cliresult(result)
     assert "There are the new updates for platforms (native)" in result.output
     assert "Please wait while updating platforms" in result.output
-    assert re.search(r"Updating native\s+@ 0.0.0\s+\[[\d\.]+\]",
-                     result.output)
+    assert re.search(r"Updating native\s+@ 0.0.0\s+\[[\d\.]+\]", result.output)
 
     # check updated version
     result = clirunner.invoke(cli_pio, ["platform", "list", "--json-output"])

tests/test_managers.py

@@ -82,6 +82,11 @@ def test_pkg_input_parser():
             ("package", None,
              "hg+https://developer.mbed.org/users/user/code/package/")
         ],
+        [
+            "https://os.mbed.com/users/user/code/package/",
+            ("package", None,
+             "hg+https://os.mbed.com/users/user/code/package/")
+        ],
         [
             "https://github.com/user/package#v1.2.3",
             ("package", None, "git+https://github.com/user/package#v1.2.3")
@@ -186,7 +191,7 @@ def test_install_packages(isolated_pio_home, tmpdir):
         "packages").listdir()]) == set(pkg_dirnames)
 
 
-def test_get_package(isolated_pio_home):
+def test_get_package():
     tests = [
         [("unknown", ), None],
         [("1", ), None],

tests/test_misc.py

@@ -12,11 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pytest
 import requests
 
-from platformio import util
+from platformio import exception, util
 
 
 def test_ping_internet_ips():
     for ip in util.PING_INTERNET_IPS:
         requests.get("http://%s" % ip, allow_redirects=False, timeout=2)
+
+
+def test_api_internet_offline(without_internet, isolated_pio_home):
+    with pytest.raises(exception.InternetIsOffline):
+        util.get_api_result("/stats")
+
+
+def test_api_cache(monkeypatch, isolated_pio_home):
+    api_kwargs = {"url": "/stats", "cache_valid": "10s"}
+    result = util.get_api_result(**api_kwargs)
+    assert result and "boards" in result
+    monkeypatch.setattr(util, '_internet_on', lambda: False)
+    assert util.get_api_result(**api_kwargs) == result

tests/test_pkgmanifest.py

@@ -16,11 +16,11 @@ import pytest
 import requests
 
 
-def validate_response(req):
-    assert req.status_code == 200
-    assert int(req.headers['Content-Length']) > 0
-    assert req.headers['Content-Type'] in ("application/gzip",
-                                           "application/octet-stream")
+def validate_response(r):
+    assert r.status_code == 200, r.url
+    assert int(r.headers['Content-Length']) > 0, r.url
+    assert r.headers['Content-Type'] in ("application/gzip",
+                                         "application/octet-stream")
 
 
 def test_packages():

tox.ini

@@ -23,6 +23,7 @@ deps =
     yapf
     pylint
     pytest
+    pytest-xdist
 commands = python --version
 
 [testenv:docs]
@@ -57,6 +58,9 @@ deps =
     pytest
 commands =
     {envpython} --version
+    {envpython} -c "print 'travis_fold:start:install_devplatforms'"
+    {envpython} scripts/install_devplatforms.py
+    {envpython} -c "print 'travis_fold:end:install_devplatforms'"
     py.test -v --basetemp="{envtmpdir}" tests
 
 [testenv:skipexamples]