Merge branch 'release/v6.0.0'

.gitignore

@@ -1,6 +1,6 @@
 *.egg-info
 *.pyc
-.pioenvs
+__pycache__
 .tox
 docs/_build
 dist

HISTORY.rst

@@ -1,301 +1,114 @@
 Release Notes
 =============
 
+.. |PIOCONF| replace:: `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ configuration file
+.. |LDF| replace:: `LDF <https://docs.platformio.org/en/latest/librarymanager/ldf.html>`__
+
+.. _release_notes_6:
+
+PlatformIO Core 6
+-----------------
+
+**A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.**
+
+6.0.0 (2022-05-16)
+~~~~~~~~~~~~~~~~~~
+
+Please check the `Migration guide from 5.x to 6.0 <https://docs.platformio.org/en/latest/core/migration.html>`__.
+
+* **Package Management**
+
+  - New unified Package Management CLI (``pio pkg``):
+
+    * `pio pkg exec <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_exec.html>`_ - run command from package tool (`issue #4163 <https://github.com/platformio/platformio-core/issues/4163>`_)
+    * `pio pkg install <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html>`_ - install the project dependencies or custom packages
+    * `pio pkg list <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_list.html>`__ - list installed packages
+    * `pio pkg outdated <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_outdated.html>`__ - check for project outdated packages
+    * `pio pkg search <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_search.html>`__ - search for packages
+    * `pio pkg show <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_show.html>`__ - show package information
+    * `pio pkg uninstall <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_uninstall.html>`_ - uninstall the project dependencies or custom packages
+    * `pio pkg update <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_update.html>`__ - update the project dependencies or custom packages
+
+  - Package Manifest
+
+    * Added support for `"scripts" <https://docs.platformio.org/en/latest/librarymanager/config.html#scripts>`__ (`issue #485 <https://github.com/platformio/platformio-core/issues/485>`_)
+    * Added support for `multi-licensed <https://docs.platformio.org/en/latest/librarymanager/config.html#license>`__ packages using SPDX Expressions (`issue #4037 <https://github.com/platformio/platformio-core/issues/4037>`_)
+    * Added support for `"dependencies" <https://docs.platformio.org/en/latest/librarymanager/config.html#dependencies>`__ declared in a "tool" package manifest
+
+  - Added support for `symbolic links <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html#local-folder>`__ allowing pointing the local source folder to the Package Manager (`issue #3348 <https://github.com/platformio/platformio-core/issues/3348>`_)
+  - Automatically install dependencies of the local (private) project libraries (`issue #2910 <https://github.com/platformio/platformio-core/issues/2910>`_)
+  - Improved detection of a package type from the tarball archive (`issue #3828 <https://github.com/platformio/platformio-core/issues/3828>`_)
+  - Ignore files according to the patterns declared in ".gitignore" when using the `pio package pack <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_pack.html>`__ command (`issue #4188 <https://github.com/platformio/platformio-core/issues/4188>`_)
+  - Dropped automatic updates of global libraries and development platforms (`issue #4179 <https://github.com/platformio/platformio-core/issues/4179>`_)
+  - Dropped support for the "pythonPackages" field in "platform.json" manifest in favor of `Extra Python Dependencies <https://docs.platformio.org/en/latest/scripting/examples/extra_python_packages.html>`__
+  - Fixed an issue when manually removed dependencies from the |PIOCONF| were not uninstalled from the storage (`issue #3076 <https://github.com/platformio/platformio-core/issues/3076>`_)
+
+* **Unit Testing**
+
+  - Refactored from scratch `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`_ solution and its documentation
+  - New: `Test Hierarchy <https://docs.platformio.org/en/latest/advanced/unit-testing/structure.html>`_ (`issue #4135 <https://github.com/platformio/platformio-core/issues/4135>`_)
+  - New: `Doctest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/doctest.html>`__ testing framework (`issue #4240 <https://github.com/platformio/platformio-core/issues/4240>`_)
+  - New: `GoogleTest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/googletest.html>`__ testing and mocking framework (`issue #3572 <https://github.com/platformio/platformio-core/issues/3572>`_)
+  - New: `Semihosting <https://docs.platformio.org/en/latest/advanced/unit-testing/semihosting.html>`__ (`issue #3516 <https://github.com/platformio/platformio-core/issues/3516>`_)
+  - New: Hardware `Simulators <https://docs.platformio.org/en/latest/advanced/unit-testing/simulators/index.html>`__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions)
+  - New: ``test`` `build configuration <https://docs.platformio.org/en/latest/projectconf/build_configurations.html>`__
+  - Added support for a `custom testing framework <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/index.html>`_
+  - Added support for a custom `testing command <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-testing-command>`__
+  - Added support for a `custom Unity library <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/examples/custom_unity_library.html>`__ (`issue #3980 <https://github.com/platformio/platformio-core/issues/3980>`_)
+  - Added support for the ``socket://`` and ``rfc2217://`` protocols using `test_port <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-port>`__ option (`issue #4229 <https://github.com/platformio/platformio-core/issues/4229>`_)
+  - List available project tests with a new `pio test --list-tests <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-list-tests>`__ option
+  - Pass extra arguments to the testing program with a new `pio test --program-arg <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-a>`__ option (`issue #3132 <https://github.com/platformio/platformio-core/issues/3132>`_)
+  - Generate reports in JUnit and JSON formats using the `pio test <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html>`__ command (`issue #2891 <https://github.com/platformio/platformio-core/issues/2891>`_)
+  - Provide more information when the native program crashed on a host (errored with a non-zero return code) (`issue #3429 <https://github.com/platformio/platformio-core/issues/3429>`_)
+  - Improved automatic detection of a testing serial port (`issue #4076 <https://github.com/platformio/platformio-core/issues/4076>`_)
+  - Fixed an issue when command line parameters (``--ignore``, ``--filter``) do not override values defined in the |PIOCONF| (`issue #3845 <https://github.com/platformio/platformio-core/issues/3845>`_)
+  - Renamed the "test_build_project_src" project configuration option to the `test_build_src <https://docs.platformio.org/en/latest//projectconf/section_env_test.html#test-build-src>`__
+  - Removed the "test_transport" option in favor of the `Custom "unity_config.h" <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/unity.html>`_
+
+* **Static Code Analysis**
+
+  - Updated analysis tools:
+
+    * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.7 with various checker improvements and fixed false positives
+    * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.18 with improved and updated semantic analysis system
+
+  - Added support for the custom `Clang-Tidy <https://docs.platformio.org/en/latest/plus/check-tools/clang-tidy.html>`__ configuration file (`issue #4186 <https://github.com/platformio/platformio-core/issues/4186>`_)
+  - Added ability to override a tool version using the `platform_packages <https://docs.platformio.org/en/latest/projectconf/section_env_platform.html#platform-packages>`__ option (`issue #3798 <https://github.com/platformio/platformio-core/issues/3798>`_)
+  - Fixed an issue with improper handling of defects that don't specify a source file (`issue #4237 <https://github.com/platformio/platformio-core/issues/4237>`_)
+
+* **Build System**
+
+  - Show project dependency licenses when building in the verbose mode
+  - Fixed an issue when |LDF| ignores the project `lib_deps <https://docs.platformio.org/en/latest/projectconf/section_env_library.html#lib-deps>`__ while resolving library dependencies (`issue #3598 <https://github.com/platformio/platformio-core/issues/3598>`_)
+  - Fixed an issue with calling an extra script located outside a project (`issue #4220 <https://github.com/platformio/platformio-core/issues/4220>`_)
+  - Fixed an issue when GCC preprocessor was applied to the ".s" assembly files on case-sensitive OS such as Window OS (`issue #3917 <https://github.com/platformio/platformio-core/issues/3917>`_)
+  - Fixed an issue when |LDF| ignores `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__ in the "deep+" mode (`issue #4253 <https://github.com/platformio/platformio-core/issues/4253>`_)
+
+* **Integration**
+
+  - Added a new build variable (``COMPILATIONDB_INCLUDE_TOOLCHAIN``) to include toolchain paths in the compilation database (`issue #3735 <https://github.com/platformio/platformio-core/issues/3735>`_)
+  - Changed a default path for compilation database `compile_commands.json <https://docs.platformio.org/en/latest/integration/compile_commands.html>`__ to the project root
+  - Enhanced integration for Qt Creator (`issue #3046 <https://github.com/platformio/platformio-core/issues/3046>`_)
+
+* **Project Configuration**
+
+  - Extended `Interpolation of Values <https://docs.platformio.org/en/latest/projectconf/interpolation.html>`__  with ``${this}`` pattern (`issue #3953 <https://github.com/platformio/platformio-core/issues/3953>`_)
+  - Embed environment name of the current section in the |PIOCONF| using ``${this.__env__}`` pattern
+  - Renamed the "src_build_flags" project configuration option to the `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__
+  - Renamed the "src_filter" project configuration option to the `build_src_filter <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-filter>`__
+
+* **Miscellaneous**
+
+  - Pass extra arguments to the `native <https://docs.platformio.org/en/latest/platforms/native.html>`__ program with a new `pio run --program-arg <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html#cmdoption-pio-run-a>`__ option (`issue #4246 <https://github.com/platformio/platformio-core/issues/4246>`_)
+  - Improved PIO Remote setup on credit-card sized computers (Raspberry Pi, BeagleBon, etc) (`issue #3865 <https://github.com/platformio/platformio-core/issues/3865>`_)
+  - Finally removed all tracks to the Python 2.7, the Python 3.6 is the minimum supported version.
+
 .. _release_notes_5:
 
 PlatformIO Core 5
 -----------------
 
-**A professional collaborative platform for embedded development**
+See `PlatformIO Core 5.0 history <https://github.com/platformio/platformio-core/blob/v5.2.5/HISTORY.rst>`__.
-
-5.2.5 (2022-02-10)
-~~~~~~~~~~~~~~~~~~
-
-- Improved support for private packages in `PlatformIO Registry <https://registry.platformio.org/>`__
-- Improved checking of available Internet connection for IPv6-only workstations (`pull #4151 <https://github.com/platformio/platformio-core/pull/4151>`_)
-- Better detecting of default PlatformIO project directory on Linux OS (`pull #4158 <https://github.com/platformio/platformio-core/pull/4158>`_)
-- Respect disabling debugging server from "platformio.ini" passing an empty value to the `debug_server <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-server>`__ option
-- Fixed a "module 'asyncio' has no attribute 'run'" error when launching PIO Home using Python 3.6 (`issue #4169 <https://github.com/platformio/platformio-core/issues/4169>`_)
-
-5.2.4 (2021-12-15)
-~~~~~~~~~~~~~~~~~~
-
-- Added support for a new ``headers`` field in `library.json <https://docs.platformio.org/en/latest/librarymanager/config.html>`__ (declare a list of header files that can be included in a project source files using ``#include <...>`` directive)
-- Improved tab completion support for Bash, ZSH, and Fish shells (`issue #4114 <https://github.com/platformio/platformio-core/issues/4114>`_)
-- Improved support for projects located on a network share (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_, `issue #3926 <https://github.com/platformio/platformio-core/issues/3926>`_, `issue #4099 <https://github.com/platformio/platformio-core/issues/4099>`_)
-- Improved PIO Remote setup on credit-card sized computers (Raspberry Pi, BeagleBon, etc) (`issue #3865 <https://github.com/platformio/platformio-core/issues/3865>`_)
-- Upgraded build engine to the SCons 4.3 (`release notes <https://github.com/SCons/scons/blob/rel_4.3.0/CHANGES.txt>`__)
-- Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 <https://github.com/platformio/platformio-core/issues/4102>`_)
-- Fixed an issue with the NetBeans project generator when the path to PlatformIO contains a space (`issue #4096 <https://github.com/platformio/platformio-core/issues/4096>`_)
-- Fixed an issue when the system environment variable does not override a project configuration option (`issue #4125 <https://github.com/platformio/platformio-core/issues/4125>`_)
-- Fixed an issue when referencing ``*_dir`` option from a custom project configuration environment (`issue #4110 <https://github.com/platformio/platformio-core/issues/4110>`_)
-- Fixed an issue with the CLion template that generated a broken CMake file if user's home directory contained an unescaped backslash (`issue #4071 <https://github.com/platformio/platformio-core/issues/4071>`_)
-- Fixed an issue with wrong detecting Windows architecture when Python 32bit is used (`issue #4134 <https://github.com/platformio/platformio-core/issues/4134>`_)
-
-5.2.3 (2021-11-05)
-~~~~~~~~~~~~~~~~~~
-
-- Automatically synchronize active projects between IDE and `PlatformIO Home <https://docs.platformio.org/en/latest/home/index.html>`__
-- Added support for custom `device monitor filters <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#filters>`__ (`issue #3924 <https://github.com/platformio/platformio-core/issues/3924>`_)
-- Show human-readable message when infinite recursion is detected while processing `Interpolation of Values <https://docs.platformio.org/en/latest/projectconf/interpolation.html>`__ (`issue #3883 <https://github.com/platformio/platformio-core/issues/3883>`_)
-- Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ configuration file (`issue #3934 <https://github.com/platformio/platformio-core/issues/3934>`_)
-- Ignore resolving of SCons variables (e.g., ``${(SOURCE.get_abspath())}``) when preprocessing interpolations (`issue #3933 <https://github.com/platformio/platformio-core/issues/3933>`_)
-- Added "inc" as a sign that it's the root of the library (`issue #4093 <https://github.com/platformio/platformio-core/issues/4093>`_)
-- Fixed an issue when the ``$PROJECT_DIR`` variable was not properly replaced in the `debug_server <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-server>`__ option (`issue #4086 <https://github.com/platformio/platformio-core/issues/4086>`_)
-- Fixed an issue when `PIO Remote <https://docs.platformio.org/en/latest/plus/pio-remote.html>`__ device monitor crashes on the first keypress (`issue #3832 <https://github.com/platformio/platformio-core/issues/3832>`_)
-- Fixed "Do not know how to make File target 'debug'" issue when debugging project using `CLion IDE <https://docs.platformio.org/en/latest/integration/ide/clion.html>`__ (`pull #4089 <https://github.com/platformio/platformio-core/issues/4089>`_)
-- Fixed "UnicodeEncodeError" when a build output contains non-ASCII characters (`issue #3971 <https://github.com/platformio/platformio-core/issues/3971>`_)
-- Fixed an issue when VSCode's debugger does not the honor default environment (`issue #4098 <https://github.com/platformio/platformio-core/issues/4098>`_)
-
-5.2.2 (2021-10-20)
-~~~~~~~~~~~~~~~~~~
-
-- Override debugging firmware loading mode using ``--load-mode`` option for `pio debug <https://docs.platformio.org/en/latest/core/userguide/cmd_debug.html>`__ command
-- Added support for CLion IDE 2021.3 (`pull #4085 <https://github.com/platformio/platformio-core/issues/4085>`_)
-- Removed debugging "legacy Click" message from CLI (`issue #4083 <https://github.com/platformio/platformio-core/issues/4083>`_)
-- Fixed a "TypeError: sequence item 1: expected str instance, list found" issue when extending configuration option in `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ with the multi-line default value (`issue #4082 <https://github.com/platformio/platformio-core/issues/4082>`_)
-
-5.2.1 (2021-10-11)
-~~~~~~~~~~~~~~~~~~
-
-- Clean a build environment and installed library dependencies using a new ``cleanall`` target (`issue #4062 <https://github.com/platformio/platformio-core/issues/4062>`_)
-- Override a default library builder via a new ``builder`` field in a ``build`` group of `library.json <https://docs.platformio.org/en/latest/librarymanager/config.html#build>`__ manifest (`issue #3957 <https://github.com/platformio/platformio-core/issues/3957>`_)
-- Updated `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.6 with new checks, increased reliability of advanced addons (MISRA/CERT) and various improvements
-- Handle the "test" folder as a part of CLion project (`issue #4005 <https://github.com/platformio/platformio-core/issues/4005>`_)
-- Improved handling of a library root based on "Conan" or "CMake" build systems (`issue #3887 <https://github.com/platformio/platformio-core/issues/3887>`_)
-- Fixed a "KeyError: Invalid board option 'build.cpu'" when using a precompiled library with a board that does not have a CPU field in the manifest (`issue #4056 <https://github.com/platformio/platformio-core/issues/4056>`_)
-- Fixed a "FileExist" error when the `platformio ci <https://docs.platformio.org/en/latest/userguide/cmd_ci.html>`__ command is used in pair with the ``--keep-build-dir`` option (`issue #4011 <https://github.com/platformio/platformio-core/issues/4011>`_)
-- Fixed an issue with draft values of C++ language standards that broke static analysis via Cppcheck (`issue #3944 <https://github.com/platformio/platformio-core/issues/3944>`_)
-
-5.2.0 (2021-09-13)
-~~~~~~~~~~~~~~~~~~
-
-* **PlatformIO Debugging**
-
-  - Boosted `PlatformIO Debugging <https://docs.platformio.org/en/latest/plus/debugging.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
-  - `Debug unit tests <https://docs.platformio.org/en/latest/plus/debugging.html#debug-unit-tests>`__ created with `PlatformIO Unit Testing <https://docs.platformio.org/en/latest/plus/unit-testing.html>`__ solution  (`issue #948 <https://github.com/platformio/platformio-core/issues/948>`_)
-  - Debug native (desktop) applications on a host machine (`issue #980 <https://github.com/platformio/platformio-core/issues/980>`_)
-  - Support debugging on Windows using Windows CMD/CLI (`pio debug <https://docs.platformio.org/en/latest/core/userguide/cmd_debug.html>`__) (`issue #3793 <https://github.com/platformio/platformio-core/issues/3793>`_)
-  - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-server-ready-pattern>`__ option
-  - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 <https://github.com/platformio/platformio-core/issues/3756>`_)
-
-* **Package Management**
-
-  - Improved a package publishing process:
-
-    * Show package details
-    * Check for conflicting names in the PlatformIO Trusted Registry
-    * Check for duplicates and used version
-    * Validate package manifest
-
-  - Added a new option ``--non-interactive`` to `pio package publish <https://docs.platformio.org/en/latest/core/userguide/package/cmd_publish.html>`__ command
-
-* **Build System**
-
-  - Process "precompiled" and "ldflags" properties of the "library.properties" manifest (`issue #3994 <https://github.com/platformio/platformio-core/issues/3994>`_)
-  - Upgraded build engine to the SCons 4.2 (`release notes <https://github.com/SCons/scons/blob/rel_4.2.0/CHANGES.txt>`__)
-  - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 <https://github.com/platformio/platformio-core/issues/3906>`_)
-  - Fixed an issue when PlatformIO archives a library that does not contain C/C++ source files (`issue #4019 <https://github.com/platformio/platformio-core/issues/4019>`_)
-
-* **Static Code Analysis**
-
-  - Updated analysis tools:
-
-    * `Clang-Tidy <https://docs.platformio.org/en/latest/plus/check-tools/clang-tidy.html>`__ v12.0.1 with new modules and extended checks list
-    * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.5.0 with improved code analysis and MISRA improvements
-    * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.14 with support for intermodular analysis, improved MISRA support and new diagnostics
-
-* **Miscellaneous**
-
-  - Ensure that a serial port is ready before running unit tests on a remote target (`issue #3742 <https://github.com/platformio/platformio-core/issues/3742>`_)
-  - Fixed an error "Unknown development platform" when running unit tests on a clean machine (`issue #3901 <https://github.com/platformio/platformio-core/issues/3901>`_)
-  - Fixed an issue when "main.cpp" was generated for a new project for 8-bit development platforms (`issue #3872 <https://github.com/platformio/platformio-core/issues/3872>`_)
-
-5.1.1 (2021-03-17)
-~~~~~~~~~~~~~~~~~~
-
-* Fixed a "The command line is too long" issue with a linking process on Windows (`issue #3827 <https://github.com/platformio/platformio-core/issues/3827>`_)
-* Fixed an issue with `device monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html>`__ when the "send_on_enter" filter didn't send EOL chars (`issue #3787 <https://github.com/platformio/platformio-core/issues/3787>`_)
-* Fixed an issue with silent mode when unwanted data is printed to stdout (`issue #3837 <https://github.com/platformio/platformio-core/issues/3837>`_)
-* Fixed an issue when code inspection fails with "Bad JSON" (`issue #3790 <https://github.com/platformio/platformio-core/issues/3790>`_)
-* Fixed an issue with overriding user-specified debugging configuration information in VSCode (`issue #3824 <https://github.com/platformio/platformio-core/issues/3824>`_)
-
-5.1.0 (2021-01-28)
-~~~~~~~~~~~~~~~~~~
-
-* **PlatformIO Home**
-
-  - Boosted `PlatformIO Home <https://docs.platformio.org/en/latest/home/index.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
-  - Added a new ``--session-id`` option to `pio home <https://docs.platformio.org/en/latest/core/userguide/cmd_home.html>`__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 <https://github.com/platformio/platformio-core/issues/3397>`_)
-
-* **Build System**
-
-  - Upgraded build engine to the SCons 4.1 (`release notes <https://scons.org/scons-410-is-available.html>`_)
-  - Refactored a workaround for a maximum command line character limitation (`issue #3792 <https://github.com/platformio/platformio-core/issues/3792>`_)
-  - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
-
-* **Package Management**
-
-  - New options for `pio system prune <https://docs.platformio.org/en/latest/core/userguide/system/cmd_prune.html>`__ command:
-
-    + ``--dry-run`` option to show data that will be removed
-    + ``--core-packages`` option to remove unnecessary core packages
-    + ``--platform-packages`` option to remove unnecessary development platform packages (`issue #923 <https://github.com/platformio/platformio-core/issues/923>`_)
-
-  - Added new `check_prune_system_threshold <https://docs.platformio.org/en/latest/core/userguide/cmd_settings.html#check-prune-system-threshold>`__ setting
-  - Disabled automatic removal of unnecessary development platform packages (`issue #3708 <https://github.com/platformio/platformio-core/issues/3708>`_, `issue #3770 <https://github.com/platformio/platformio-core/issues/3770>`_)
-  - Fixed an issue when unnecessary packages were removed in  ``update --dry-run`` mode (`issue #3809 <https://github.com/platformio/platformio-core/issues/3809>`_)
-  - Fixed a "ValueError: Invalid simple block" when uninstalling a package with a custom name and external source (`issue #3816 <https://github.com/platformio/platformio-core/issues/3816>`_)
-
-* **Debugging**
-
-  - Configure a custom debug adapter speed using a new `debug_speed <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-speed>`__ option (`issue #3799 <https://github.com/platformio/platformio-core/issues/3799>`_)
-  - Handle debugging server's "ready_pattern" in "stderr" output
-
-* **Miscellaneous**
-
-  - Improved listing of `multicast DNS services <https://docs.platformio.org/en/latest/core/userguide/device/cmd_list.html>`_
-  - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 <https://github.com/platformio/platformio-core/issues/3804>`_)
-  - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 <https://github.com/platformio/platformio-core/issues/3808>`_)
-  - Fixed an issue when `pio device monitor --eol <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#cmdoption-pio-device-monitor-eol>`__ and "send_on_enter" filter do not work properly (`issue #3787 <https://github.com/platformio/platformio-core/issues/3787>`_)
-
-5.0.4 (2020-12-30)
-~~~~~~~~~~~~~~~~~~
-
-- Added "Core" suffix when showing PlatformIO Core version using ``pio --version`` command
-- Improved ".ccls" configuration file for Emacs, Vim, and Sublime Text integrations
-- Updated analysis tools:
-
-  * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.3 with improved C++ parser and several new MISRA rules
-  * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.11 with new diagnostics and updated mass suppression mechanism
-
-- Show a warning message about deprecated support for Python 2 and Python 3.5
-- Do not provide "intelliSenseMode" option when generating configuration for VSCode C/C++ extension
-- Fixed a "git-sh-setup: file not found" error when installing project dependencies from Git VCS (`issue #3740 <https://github.com/platformio/platformio-core/issues/3740>`_)
-- Fixed an issue with package publishing on Windows when Unix permissions are not preserved (`issue #3776 <https://github.com/platformio/platformio-core/issues/3776>`_)
-
-5.0.3 (2020-11-12)
-~~~~~~~~~~~~~~~~~~
-
-- Added an error selector for `Sublime Text <https://docs.platformio.org/en/latest/integration/ide/sublimetext.html>`__ build runner (`issue #3733 <https://github.com/platformio/platformio-core/issues/3733>`_)
-- Generate a working "projectEnvName" for PlatformIO IDE's debugger for VSCode
-- Force VSCode's intelliSenseMode to "gcc-x64" when GCC toolchain is used
-- Print ignored test suites and environments in the test summary report only in verbose mode (`issue #3726 <https://github.com/platformio/platformio-core/issues/3726>`_)
-- Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 <https://github.com/platformio/platformio-core/issues/3662>`_)
-- Fixed an issue when `pio package pack <https://docs.platformio.org/en/latest/core/userguide/package/cmd_pack.html>`__ ignores some folders (`issue #3730 <https://github.com/platformio/platformio-core/issues/3730>`_)
-
-5.0.2 (2020-10-30)
-~~~~~~~~~~~~~~~~~~
-
-- Initialize a new project or update the existing passing working environment name and its options (`issue #3686 <https://github.com/platformio/platformio-core/issues/3686>`_)
-- Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry (`issue #3700 <https://github.com/platformio/platformio-core/issues/3700>`_)
-- Improved "core.call" RPC for PlatformIO Home (`issue #3671 <https://github.com/platformio/platformio-core/issues/3671>`_)
-- Fixed a "PermissionError: [WinError 5]" on Windows when an external repository is used with `lib_deps <https://docs.platformio.org/en/latest/projectconf/section_env_library.html#lib-deps>`__ option (`issue #3664 <https://github.com/platformio/platformio-core/issues/3664>`_)
-- Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 <https://github.com/platformio/platformio-core/issues/3666>`_)
-- Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 <https://github.com/platformio/platformio-core/issues/3669>`_)
-- Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses (`issue #3677 <https://github.com/platformio/platformio-core/issues/3677>`_)
-- Fixed an issue when `pio package publish <https://docs.platformio.org/en/latest/core/userguide/package/cmd_publish.html>`__ command removes original archive after submitting to the registry (`issue #3716 <https://github.com/platformio/platformio-core/issues/3716>`_)
-- Fixed an issue when multiple `pio lib install <https://docs.platformio.org/en/latest/core/userguide/lib/cmd_install.html>`__ command with the same local library results in duplicates in ``lib_deps`` (`issue #3715 <https://github.com/platformio/platformio-core/issues/3715>`_)
-- Fixed an issue with a "wrong" timestamp in device monitor output using `"time" filter <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#filters>`__ (`issue #3712 <https://github.com/platformio/platformio-core/issues/3712>`_)
-
-5.0.1 (2020-09-10)
-~~~~~~~~~~~~~~~~~~
-
-- Added support for "owner" requirement when declaring ``dependencies`` using `library.json <https://docs.platformio.org/en/latest/librarymanager/config.html#dependencies>`__
-- Fixed an issue when using a custom git/ssh package with `platform_packages <https://docs.platformio.org/en/latest/projectconf/section_env_platform.html#platform-packages>`__ option (`issue #3624 <https://github.com/platformio/platformio-core/issues/3624>`_)
-- Fixed an issue with "ImportError: cannot import name '_get_backend' from 'cryptography.hazmat.backends'" when using `Remote Development <https://docs.platformio.org/en/latest/plus/pio-remote.html>`__ on RaspberryPi device (`issue #3652 <https://github.com/platformio/platformio-core/issues/3652>`_)
-- Fixed an issue when `pio package unpublish <https://docs.platformio.org/en/latest/core/userguide/package/cmd_unpublish.html>`__ command crashes (`issue #3660 <https://github.com/platformio/platformio-core/issues/3660>`_)
-- Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 <https://github.com/platformio/platformio-core/issues/3662>`_)
-- Fixed an issue with incorrect value for C++ language standard in IDE projects when an in-progress language standard is used (`issue #3653 <https://github.com/platformio/platformio-core/issues/3653>`_)
-- Fixed an issue with "Invalid simple block (semantic_version)" from library dependency that refs to an external source (repository, ZIP/Tar archives) (`issue #3658 <https://github.com/platformio/platformio-core/issues/3658>`_)
-- Fixed an issue when can not remove update or remove external dev-platform using PlatformIO Home (`issue #3663 <https://github.com/platformio/platformio-core/issues/3663>`_)
-
-5.0.0 (2020-09-03)
-~~~~~~~~~~~~~~~~~~
-
-Please check `Migration guide from 4.x to 5.0 <https://docs.platformio.org/en/latest/core/migration.html>`__.
-
-* Integration with the new **PlatformIO Trusted Registry**
-
-  - Enterprise-grade package storage with high availability (multi replicas)
-  - Secure, fast, and reliable global content delivery network (CDN)
-  - Universal support for all packages:
-
-    * Libraries
-    * Development platforms
-    * Toolchains
-
-  - Built-in fine-grained access control (role-based, teams, organizations)
-  - New CLI commands:
-
-    * `pio package <https://docs.platformio.org/en/latest/core/userguide/package/index.html>`__ – manage packages in the registry
-    * `pio access <https://docs.platformio.org/en/latest/core/userguide/access/index.html>`__ – manage package access for users, teams, and maintainers
-
-* Integration with the new **Account Management System**
-
-  - `Manage organizations <https://docs.platformio.org/en/latest/core/userguide/org/index.html>`__
-  - `Manage teams and team memberships <https://docs.platformio.org/en/latest/core/userguide/team/index.html>`__
-
-* New **Package Management System**
-
-  - Integrated PlatformIO Core with the new PlatformIO Registry
-  - Support for owner-based dependency declaration (resolves name conflicts) (`issue #1824 <https://github.com/platformio/platformio-core/issues/1824>`_)
-  - Automatically save dependencies to `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ when installing using PlatformIO CLI (`issue #2964 <https://github.com/platformio/platformio-core/issues/2964>`_)
-  - Follow SemVer complaint version constraints when checking library updates `issue #1281 <https://github.com/platformio/platformio-core/issues/1281>`_)
-  - Dropped support for "packageRepositories" section in "platform.json" manifest (please publish packages directly to the registry)
-
-* **Build System**
-
-  - Upgraded build engine to the `SCons 4.0 - a next-generation software construction tool <https://scons.org/>`__
-
-    * `Configuration files are Python scripts <https://docs.platformio.org/en/latest/projectconf/advanced_scripting.html>`__ – use the power of a real programming language to solve build problems
-    * Built-in reliable and automatic dependency analysis
-    * Improved support for parallel builds
-    * Ability to `share built files in a cache <https://docs.platformio.org/en/latest/projectconf/section_platformio.html#projectconf-pio-build-cache-dir>`__ to speed up multiple builds
-
-  - New `Custom Targets <https://docs.platformio.org/en/latest/projectconf/advanced_scripting.html#custom-targets>`__
-
-    * Pre/Post processing based on dependent sources (another target, source file, etc.)
-    * Command launcher with own arguments
-    * Launch command with custom options declared in `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__
-    * Python callback as a target (use the power of Python interpreter and PlatformIO Build API)
-    * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html#cmdoption-platformio-run-list-targets>`__ command (`issue #3544 <https://github.com/platformio/platformio-core/issues/3544>`_)
-
-  - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 <https://github.com/platformio/platformio-core/issues/3570>`_)
-  - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) <https://docs.platformio.org/en/latest/librarymanager/ldf.html#dependency-finder-mode>`__ for Arduino library when "library.property" has "depends" field (`issue #3607 <https://github.com/platformio/platformio-core/issues/3607>`_)
-  - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 <https://github.com/platformio/platformio-core/issues/3531>`_)
-  - Fixed an issue with the ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 <https://github.com/platformio/platformio-core/issues/3542>`_)
-
-* **Project Management**
-
-  - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci <https://docs.platformio.org/en/latest/core/userguide/cmd_ci.html>`__, `src_filter <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#src-filter>`__, `check_patterns <https://docs.platformio.org/en/latest/projectconf/section_env_check.html#check-patterns>`__, `library.json > srcFilter <https://docs.platformio.org/en/latest/librarymanager/config.html#srcfilter>`__). Python 3.5+ is required
-  - Added a new ``-e, --environment`` option to `pio project init <https://docs.platformio.org/en/latest/core/userguide/project/cmd_init.html#cmdoption-platformio-project-init-e>`__ command that helps to update a PlatformIO project using the existing environment
-  - Dump build system data intended for IDE extensions/plugins using a new `pio project data <https://docs.platformio.org/en/latest/core/userguide/project/cmd_data.html>`__ command
-  - Do not generate ".travis.yml" for a new project, let the user have a choice
-
-* **Unit Testing**
-
-  - Updated PIO Unit Testing support for Mbed framework and added compatibility with Mbed OS 6
-  - Fixed an issue when running multiple test environments (`issue #3523 <https://github.com/platformio/platformio-core/issues/3523>`_)
-  - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 <https://github.com/platformio/platformio-core/issues/3583>`_)
-
-* **Static Code Analysis**
-
-  - Updated analysis tools:
-
-    * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.1 with a new "soundy" analysis option and improved code parser
-    * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.09 with a new file list analysis mode and an extended list of analysis diagnostics
-
-  - Added Cppcheck package for ARM-based single-board computers (`issue #3559 <https://github.com/platformio/platformio-core/issues/3559>`_)
-  - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 <https://github.com/platformio/platformio-core/issues/3631>`_)
-
-* **Miscellaneous**
-
-  - Display system-wide information using a new `pio system info <https://docs.platformio.org/en/latest/core/userguide/system/cmd_info.html>`__ command (`issue #3521 <https://github.com/platformio/platformio-core/issues/3521>`_)
-  - Remove unused data using a new `pio system prune <https://docs.platformio.org/en/latest/core/userguide/system/cmd_prune.html>`__ command (`issue #3522 <https://github.com/platformio/platformio-core/issues/3522>`_)
-  - Show ignored project environments only in the verbose mode (`issue #3641 <https://github.com/platformio/platformio-core/issues/3641>`_)
-  - Do not escape compiler arguments in VSCode template on Windows
-  - Drop support for Python 2 and 3.5
 
 .. _release_notes_4:
 

Makefile

@@ -11,7 +11,7 @@ format:
 	black ./tests
 
 test:
-	py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
+	py.test --verbose --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
 
 before-commit: isort format lint
 

README.rst

@@ -1,8 +1,12 @@
+.. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg
+    :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md
+    :alt:  SWUbanner
+
 PlatformIO Core
 ===============
 
 .. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg
-    :target: https://docs.platformio.org/page/core/index.html
+    :target: https://docs.platformio.org/en/latest/core/index.html
     :alt:  CI Build for PlatformIO Core
 .. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
     :target: https://github.com/platformio/platformio-examples
@@ -36,7 +40,7 @@ PlatformIO Core
 .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png
     :target: https://platformio.org?utm_source=github&utm_medium=core
 
-`PlatformIO <https://platformio.org?utm_source=github&utm_medium=core>`_ is a professional collaborative platform for embedded development
+`PlatformIO <https://platformio.org>`_ is a professional collaborative platform for embedded development.
 
 **A place where Developers and Teams have true Freedom! No more vendor lock-in!**
 
@@ -49,24 +53,24 @@ PlatformIO Core
 Get Started
 -----------
 
-* `What is PlatformIO? <https://docs.platformio.org/page/what-is-platformio.html?utm_source=github&utm_medium=core>`_
+* `What is PlatformIO? <https://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
 * `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_
-* `PlatformIO Core (CLI) <https://docs.platformio.org/page/core.html?utm_source=github&utm_medium=core>`_
+* `PlatformIO Core (CLI) <https://docs.platformio.org/en/latest/core.html?utm_source=github&utm_medium=core>`_
 * `Project Examples <https://github.com/platformio/platformio-examples?utm_source=github&utm_medium=core>`__
 
 Solutions
 ---------
 
-* `Library Management <https://docs.platformio.org/page/librarymanager/index.html?utm_source=github&utm_medium=core>`_
+* `Library Management <https://docs.platformio.org/en/latest/librarymanager/index.html?utm_source=github&utm_medium=core>`_
-* `Desktop IDEs Integration <https://docs.platformio.org/page/ide.html?utm_source=github&utm_medium=core>`_
+* `Desktop IDEs Integration <https://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core>`_
-* `Continuous Integration <https://docs.platformio.org/page/ci/index.html?utm_source=github&utm_medium=core>`_
+* `Continuous Integration <https://docs.platformio.org/en/latest/ci/index.html?utm_source=github&utm_medium=core>`_
 
 **Advanced**
 
-* `Debugging <https://docs.platformio.org/page/plus/debugging.html?utm_source=github&utm_medium=core>`_
+* `Debugging <https://docs.platformio.org/en/latest/plus/debugging.html?utm_source=github&utm_medium=core>`_
-* `Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
+* `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html?utm_source=github&utm_medium=core>`_
-* `Static Code Analysis <https://docs.platformio.org/page/plus/pio-check.html?utm_source=github&utm_medium=core>`_
+* `Static Code Analysis <https://docs.platformio.org/en/latest/plus/pio-check.html?utm_source=github&utm_medium=core>`_
-* `Remote Development <https://docs.platformio.org/page/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
+* `Remote Development <https://docs.platformio.org/en/latest/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
 
 Registry
 --------
@@ -86,7 +90,7 @@ Telemetry / Privacy Policy
 Share minimal diagnostics and usage information to help us make PlatformIO better.
 It is enabled by default. For more information see:
 
-* `Telemetry Setting <https://docs.platformio.org/page/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
+* `Telemetry Setting <https://docs.platformio.org/en/latest/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
 
 License
 -------

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (5, 2, 5)
+VERSION = (6, 0, 0)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -38,9 +38,9 @@ __license__ = "Apache Software License"
 __copyright__ = "Copyright 2014-present PlatformIO Labs"
 
 __accounts_api__ = "https://api.accounts.platformio.org"
-__registry_api__ = [
+__registry_mirror_hosts__ = [
-    "https://api.registry.platformio.org",
+    "registry.platformio.org",
-    "https://api.registry.ns1.platformio.org",
+    "registry.nm1.platformio.org",
 ]
 __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
 
@@ -49,16 +49,14 @@ __default_requests_timeout__ = (10, None)  # (connect, read)
 __core_packages__ = {
     "contrib-piohome": "~3.4.1",
     "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
-    "tool-unity": "~1.20500.0",
     "tool-scons": "~4.40300.0",
-    "tool-cppcheck": "~1.260.0",
+    "tool-cppcheck": "~1.270.0",
     "tool-clangtidy": "~1.120001.0",
-    "tool-pvs-studio": "~7.14.0",
+    "tool-pvs-studio": "~7.18.0",
 }
 
 __check_internet_hosts__ = [
     "185.199.110.153",  # Github.com
     "88.198.170.159",  # platformio.org
     "github.com",
-    "platformio.org",
+] + __registry_mirror_hosts__
-]

platformio/__main__.py

@@ -12,15 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=import-outside-toplevel
-
 import os
 import sys
 from traceback import format_exc
 
 import click
 
-from platformio import __version__, exception
+from platformio import __version__, exception, maintenance
 from platformio.commands import PlatformioCLI
 from platformio.compat import IS_CYGWIN, ensure_python3
 
@@ -29,7 +27,7 @@ from platformio.compat import IS_CYGWIN, ensure_python3
     cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])
 )
 @click.version_option(__version__, prog_name="PlatformIO Core")
-@click.option("--force", "-f", is_flag=True, help="DEPRECATE")
+@click.option("--force", "-f", is_flag=True, help="DEPRECATED")
 @click.option("--caller", "-c", help="Caller ID (service)")
 @click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters")
 @click.pass_context
@@ -55,29 +53,12 @@ def cli(ctx, force, caller, no_ansi):
     except:  # pylint: disable=bare-except
         pass
 
-    from platformio import maintenance
-
     maintenance.on_platformio_start(ctx, force, caller)
 
 
-try:
+@cli.result_callback()
-
+@click.pass_context
-    @cli.result_callback()
+def process_result(ctx, result, *_, **__):
-    @click.pass_context
-    def process_result(ctx, result, *_, **__):
-        _process_result(ctx, result)
-
-except (AttributeError, TypeError):  # legacy support for CLick > 8.0.1
-
-    @cli.resultcallback()
-    @click.pass_context
-    def process_result(ctx, result, *_, **__):
-        _process_result(ctx, result)
-
-
-def _process_result(ctx, result):
-    from platformio import maintenance
-
     maintenance.on_platformio_end(ctx, result)
 
 
@@ -124,10 +105,7 @@ def main(argv=None):
             exit_code = int(e.code)
     except Exception as e:  # pylint: disable=broad-except
         if not isinstance(e, exception.ReturnErrorCode):
-            if sys.version_info.major != 2:
+            maintenance.on_platformio_exception(e)
-                from platformio import maintenance
-
-                maintenance.on_platformio_exception(e)
             error_str = "Error: "
             if isinstance(e, exception.PlatformioException):
                 error_str += str(e)

platformio/app.py

@@ -35,24 +35,8 @@ def projects_dir_validate(projects_dir):
 
 
 DEFAULT_SETTINGS = {
-    "auto_update_libraries": {
-        "description": "Automatically update libraries (Yes/No)",
-        "value": False,
-    },
-    "auto_update_platforms": {
-        "description": "Automatically update platforms (Yes/No)",
-        "value": False,
-    },
-    "check_libraries_interval": {
-        "description": "Check for the library updates interval (days)",
-        "value": 7,
-    },
     "check_platformio_interval": {
-        "description": "Check for the new PlatformIO interval (days)",
+        "description": "Check for the new PlatformIO Core interval (days)",
-        "value": 3,
-    },
-    "check_platforms_interval": {
-        "description": "Check for the platform updates interval (days)",
         "value": 7,
     },
     "check_prune_system_threshold": {

platformio/builder/main.py

@@ -44,25 +44,28 @@ clivars.AddVariables(
     ("PIOENV",),
     ("PIOTEST_RUNNING_NAME",),
     ("UPLOAD_PORT",),
+    ("PROGRAM_ARGS",),
 )
 
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
         "ar",
-        "as",
         "cc",
         "c++",
         "link",
+        "pioasm",
         "platformio",
-        "piotarget",
-        "pioplatform",
         "pioproject",
+        "pioplatform",
+        "piotest",
+        "piotarget",
         "piomaxlen",
         "piolib",
         "pioupload",
-        "piomisc",
-        "pioide",
         "piosize",
+        "pioino",
+        "piomisc",
+        "piointegration",
     ],
     toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")],
     variables=clivars,
@@ -72,7 +75,7 @@ DEFAULT_ENV_OPTIONS = dict(
     BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
     BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
     BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
-    COMPILATIONDB_PATH=os.path.join("$BUILD_DIR", "compile_commands.json"),
+    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
     LIBPATH=["$BUILD_DIR"],
     PROGNAME="program",
     PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
@@ -223,12 +226,13 @@ if "envdump" in COMMAND_LINE_TARGETS:
     env.Exit(0)
 
 if set(["_idedata", "idedata"]) & set(COMMAND_LINE_TARGETS):
+    projenv = None
     try:
         Import("projenv")
     except:  # pylint: disable=bare-except
         projenv = env
-    data = projenv.DumpIDEData(env)
+    data = projenv.DumpIntegrationData(env)
-    # dump to file for the further reading by project.helpers.load_project_ide_data
+    # dump to file for the further reading by project.helpers.load_build_metadata
     with open(
         projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")),
         mode="w",

platformio/builder/tools/pioasm.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import SCons.Tool.asm  # pylint: disable=import-error
+
+#
+# Resolve https://github.com/platformio/platformio-core/issues/3917
+# Avoid forcing .S to bare assembly on Windows OS
+#
+
+if ".S" in SCons.Tool.asm.ASSuffixes:
+    SCons.Tool.asm.ASSuffixes.remove(".S")
+if ".S" not in SCons.Tool.asm.ASPPSuffixes:
+    SCons.Tool.asm.ASPPSuffixes.append(".S")
+
+
+generate = SCons.Tool.asm.generate
+exists = SCons.Tool.asm.exists

platformio/builder/tools/pioino.py

@@ -0,0 +1,254 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import atexit
+import glob
+import io
+import os
+import re
+import tempfile
+
+import click
+
+from platformio.compat import get_filesystem_encoding, get_locale_encoding
+
+
+class InoToCPPConverter(object):
+
+    PROTOTYPE_RE = re.compile(
+        r"""^(
+        (?:template\<.*\>\s*)?      # template
+        ([a-z_\d\&]+\*?\s+){1,2}    # return type
+        ([a-z_\d]+\s*)              # name of prototype
+        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
+        )\s*(\{|;)                  # must end with `{` or `;`
+        """,
+        re.X | re.M | re.I,
+    )
+    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
+    PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
+
+    def __init__(self, env):
+        self.env = env
+        self._main_ino = None
+        self._safe_encoding = None
+
+    def read_safe_contents(self, path):
+        error_reported = False
+        for encoding in (
+            "utf-8",
+            None,
+            get_filesystem_encoding(),
+            get_locale_encoding(),
+            "latin-1",
+        ):
+            try:
+                with io.open(path, encoding=encoding) as fp:
+                    contents = fp.read()
+                    self._safe_encoding = encoding
+                    return contents
+            except UnicodeDecodeError:
+                if not error_reported:
+                    error_reported = True
+                    click.secho(
+                        "Unicode decode error has occurred, please remove invalid "
+                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
+                        % path,
+                        fg="yellow",
+                        err=True,
+                    )
+        return ""
+
+    def write_safe_contents(self, path, contents):
+        with io.open(
+            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
+        ) as fp:
+            return fp.write(contents)
+
+    def is_main_node(self, contents):
+        return self.DETECTMAIN_RE.search(contents)
+
+    def convert(self, nodes):
+        contents = self.merge(nodes)
+        if not contents:
+            return None
+        return self.process(contents)
+
+    def merge(self, nodes):
+        assert nodes
+        lines = []
+        for node in nodes:
+            contents = self.read_safe_contents(node.get_path())
+            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
+            if self.is_main_node(contents):
+                lines = _lines + lines
+                self._main_ino = node.get_path()
+            else:
+                lines.extend(_lines)
+
+        if not self._main_ino:
+            self._main_ino = nodes[0].get_path()
+
+        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
+
+    def process(self, contents):
+        out_file = self._main_ino + ".cpp"
+        assert self._gcc_preprocess(contents, out_file)
+        contents = self.read_safe_contents(out_file)
+        contents = self._join_multiline_strings(contents)
+        self.write_safe_contents(out_file, self.append_prototypes(contents))
+        return out_file
+
+    def _gcc_preprocess(self, contents, out_file):
+        tmp_path = tempfile.mkstemp()[1]
+        self.write_safe_contents(tmp_path, contents)
+        self.env.Execute(
+            self.env.VerboseAction(
+                '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
+                    out_file, tmp_path
+                ),
+                "Converting " + os.path.basename(out_file[:-4]),
+            )
+        )
+        atexit.register(_delete_file, tmp_path)
+        return os.path.isfile(out_file)
+
+    def _join_multiline_strings(self, contents):
+        if "\\\n" not in contents:
+            return contents
+        newlines = []
+        linenum = 0
+        stropen = False
+        for line in contents.split("\n"):
+            _linenum = self._parse_preproc_line_num(line)
+            if _linenum is not None:
+                linenum = _linenum
+            else:
+                linenum += 1
+
+            if line.endswith("\\"):
+                if line.startswith('"'):
+                    stropen = True
+                    newlines.append(line[:-1])
+                    continue
+                if stropen:
+                    newlines[len(newlines) - 1] += line[:-1]
+                    continue
+            elif stropen and line.endswith(('",', '";')):
+                newlines[len(newlines) - 1] += line
+                stropen = False
+                newlines.append(
+                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
+                )
+                continue
+
+            newlines.append(line)
+
+        return "\n".join(newlines)
+
+    @staticmethod
+    def _parse_preproc_line_num(line):
+        if not line.startswith("#"):
+            return None
+        tokens = line.split(" ", 3)
+        if len(tokens) > 2 and tokens[1].isdigit():
+            return int(tokens[1])
+        return None
+
+    def _parse_prototypes(self, contents):
+        prototypes = []
+        reserved_keywords = set(["if", "else", "while"])
+        for match in self.PROTOTYPE_RE.finditer(contents):
+            if (
+                set([match.group(2).strip(), match.group(3).strip()])
+                & reserved_keywords
+            ):
+                continue
+            prototypes.append(match)
+        return prototypes
+
+    def _get_total_lines(self, contents):
+        total = 0
+        if contents.endswith("\n"):
+            contents = contents[:-1]
+        for line in contents.split("\n")[::-1]:
+            linenum = self._parse_preproc_line_num(line)
+            if linenum is not None:
+                return total + linenum
+            total += 1
+        return total
+
+    def append_prototypes(self, contents):
+        prototypes = self._parse_prototypes(contents) or []
+
+        # skip already declared prototypes
+        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
+        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
+
+        if not prototypes:
+            return contents
+
+        prototype_names = set(m.group(3).strip() for m in prototypes)
+        split_pos = prototypes[0].start()
+        match_ptrs = re.search(
+            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
+            contents[:split_pos],
+            re.M,
+        )
+        if match_ptrs:
+            split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
+
+        result = []
+        result.append(contents[:split_pos].strip())
+        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
+        result.append(
+            '#line %d "%s"'
+            % (
+                self._get_total_lines(contents[:split_pos]),
+                self._main_ino.replace("\\", "/"),
+            )
+        )
+        result.append(contents[split_pos:].strip())
+        return "\n".join(result)
+
+
+def ConvertInoToCpp(env):
+    src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
+    ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
+        os.path.join(src_dir, "*.pde")
+    )
+    if not ino_nodes:
+        return
+    c = InoToCPPConverter(env)
+    out_file = c.convert(ino_nodes)
+
+    atexit.register(_delete_file, out_file)
+
+
+def _delete_file(path):
+    try:
+        if os.path.isfile(path):
+            os.remove(path)
+    except:  # pylint: disable=bare-except
+        pass
+
+
+def generate(env):
+    env.AddMethod(ConvertInoToCpp)
+
+
+def exists(_):
+    return True

platformio/builder/tools/piointegration.py

@@ -20,32 +20,31 @@ import os
 import SCons.Defaults  # pylint: disable=import-error
 import SCons.Subst  # pylint: disable=import-error
 
-from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import exec_command, where_is_program
 
 
-def _dump_includes(env):
+def DumpIntegrationIncludes(env):
-    includes = {}
+    result = dict(build=[], compatlib=[], toolchain=[])
 
-    includes["build"] = [
+    result["build"].extend(
-        env.subst("$PROJECT_INCLUDE_DIR"),
+        [
-        env.subst("$PROJECT_SRC_DIR"),
+            env.subst("$PROJECT_INCLUDE_DIR"),
-    ]
+            env.subst("$PROJECT_SRC_DIR"),
-    includes["build"].extend(
+        ]
+    )
+    result["build"].extend(
         [os.path.abspath(env.subst(item)) for item in env.get("CPPPATH", [])]
     )
 
     # installed libs
-    includes["compatlib"] = []
     for lb in env.GetLibBuilders():
-        includes["compatlib"].extend(
+        result["compatlib"].extend(
             [os.path.abspath(inc) for inc in lb.get_include_dirs()]
         )
 
     # includes from toolchains
     p = env.PioPlatform()
-    includes["toolchain"] = []
+    for pkg in p.get_installed_packages(with_optional=False):
-    for pkg in p.get_installed_packages():
         if p.get_package_type(pkg.metadata.name) != "toolchain":
             continue
         toolchain_dir = glob.escape(pkg.path)
@@ -56,22 +55,9 @@ def _dump_includes(env):
             os.path.join(toolchain_dir, "*", "include*"),
         ]
         for g in toolchain_incglobs:
-            includes["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)])
+            result["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)])
 
-    # include Unity framework if there are tests in project
+    return result
-    includes["unity"] = []
-    auto_install_unity = False
-    test_dir = env.GetProjectConfig().get("platformio", "test_dir")
-    if os.path.isdir(test_dir) and os.listdir(test_dir) != ["README"]:
-        auto_install_unity = True
-    unity_dir = get_core_package_dir(
-        "tool-unity",
-        auto_install=auto_install_unity,
-    )
-    if unity_dir:
-        includes["unity"].append(unity_dir)
-
-    return includes
 
 
 def _get_gcc_defines(env):
@@ -154,14 +140,14 @@ def _subst_cmd(env, cmd):
     return " ".join([SCons.Subst.quote_spaces(arg) for arg in args])
 
 
-def DumpIDEData(env, globalenv):
+def DumpIntegrationData(env, globalenv):
     """env here is `projenv`"""
 
     data = {
         "env_name": env["PIOENV"],
         "libsource_dirs": [env.subst(item) for item in env.GetLibSourceDirs()],
         "defines": _dump_defines(env),
-        "includes": _dump_includes(env),
+        "includes": env.DumpIntegrationIncludes(),
         "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
         "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
         "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
@@ -205,5 +191,6 @@ def exists(_):
 
 
 def generate(env):
-    env.AddMethod(DumpIDEData)
+    env.AddMethod(DumpIntegrationIncludes)
+    env.AddMethod(DumpIntegrationData)
     return env

platformio/builder/tools/piolib.py

@@ -27,14 +27,16 @@ import sys
 import click
 import SCons.Scanner  # pylint: disable=import-error
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
-from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import exception, fs, util
 from platformio.builder.tools import platformio as piotool
 from platformio.clients.http import HTTPClientError, InternetIsOffline
 from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
-from platformio.package.exception import UnknownPackageError
+from platformio.package.exception import (
+    MissingPackageManifestError,
+    UnknownPackageError,
+)
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manifest.parser import (
     ManifestParserError,
@@ -54,9 +56,9 @@ class LibBuilderFactory(object):
             used_frameworks = LibBuilderFactory.get_used_frameworks(env, path)
             common_frameworks = set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks)
             if common_frameworks:
-                clsname = "%sLibBuilder" % list(common_frameworks)[0].title()
+                clsname = "%sLibBuilder" % list(common_frameworks)[0].capitalize()
             elif used_frameworks:
-                clsname = "%sLibBuilder" % used_frameworks[0].title()
+                clsname = "%sLibBuilder" % used_frameworks[0].capitalize()
 
         obj = getattr(sys.modules[__name__], clsname)(env, path, verbose=verbose)
 
@@ -136,9 +138,11 @@ class LibBuilderBase(object):
             )
             self._manifest = {}
 
-        self._is_dependent = False
+        self.is_dependent = False
-        self._is_built = False
+        self.is_built = False
-        self._depbuilders = []
+        self.depbuilders = []
+
+        self._deps_are_processed = False
         self._circular_deps = []
         self._processed_files = []
 
@@ -159,7 +163,12 @@ class LibBuilderBase(object):
             p2 = p2.lower()
         if p1 == p2:
             return True
-        return os.path.commonprefix((p1 + os.path.sep, p2)) == p1 + os.path.sep
+        if os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep:
+            return True
+        # try to resolve paths
+        p1 = os.path.os.path.realpath(p1)
+        p2 = os.path.os.path.realpath(p2)
+        return os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep
 
     @property
     def name(self):
@@ -169,6 +178,11 @@ class LibBuilderBase(object):
     def version(self):
         return self._manifest.get("version")
 
+    @property
+    def dependent(self):
+        """Backward compatibility with ESP-IDF"""
+        return self.is_dependent
+
     @property
     def dependencies(self):
         return self._manifest.get("dependencies")
@@ -225,18 +239,6 @@ class LibBuilderBase(object):
     def extra_script(self):
         return None
 
-    @property
-    def depbuilders(self):
-        return self._depbuilders
-
-    @property
-    def dependent(self):
-        return self._is_dependent
-
-    @property
-    def is_built(self):
-        return self._is_built
-
     @property
     def lib_archive(self):
         return self.env.GetProjectOption("lib_archive")
@@ -296,8 +298,9 @@ class LibBuilderBase(object):
             self.env.ProcessUnFlags(self.build_unflags)
 
     def process_dependencies(self):
-        if not self.dependencies:
+        if not self.dependencies or self._deps_are_processed:
             return
+        self._deps_are_processed = True
         for item in self.dependencies:
             found = False
             for lb in self.env.GetLibBuilders():
@@ -305,7 +308,7 @@ class LibBuilderBase(object):
                     continue
                 found = True
                 if lb not in self.depbuilders:
-                    self.depend_recursive(lb)
+                    self.depend_on(lb)
                 break
 
             if not found and self.verbose:
@@ -400,7 +403,29 @@ class LibBuilderBase(object):
 
         return result
 
-    def depend_recursive(self, lb, search_files=None):
+    def search_deps_recursive(self, search_files=None):
+        self.process_dependencies()
+
+        # when LDF is disabled
+        if self.lib_ldf_mode == "off":
+            return
+
+        if self.lib_ldf_mode.startswith("deep"):
+            search_files = self.get_search_files()
+
+        lib_inc_map = {}
+        for inc in self._get_found_includes(search_files):
+            for lb in self.env.GetLibBuilders():
+                if inc.get_abspath() in lb:
+                    if lb not in lib_inc_map:
+                        lib_inc_map[lb] = []
+                    lib_inc_map[lb].append(inc.get_abspath())
+                    break
+
+        for lb, lb_search_files in lib_inc_map.items():
+            self.depend_on(lb, search_files=lb_search_files)
+
+    def depend_on(self, lb, search_files=None, recursive=True):
         def _already_depends(_lb):
             if self in _lb.depbuilders:
                 return True
@@ -418,38 +443,17 @@ class LibBuilderBase(object):
                         "between `%s` and `%s`\n" % (self.path, lb.path)
                     )
                 self._circular_deps.append(lb)
-            elif lb not in self._depbuilders:
+            elif lb not in self.depbuilders:
-                self._depbuilders.append(lb)
+                self.depbuilders.append(lb)
+                lb.is_dependent = True
                 LibBuilderBase._INCLUDE_DIRS_CACHE = None
-        lb.search_deps_recursive(search_files)
 
-    def search_deps_recursive(self, search_files=None):
+        if recursive:
-        if not self._is_dependent:
+            lb.search_deps_recursive(search_files)
-            self._is_dependent = True
-            self.process_dependencies()
-
-            if self.lib_ldf_mode.startswith("deep"):
-                search_files = self.get_search_files()
-
-        # when LDF is disabled
-        if self.lib_ldf_mode == "off":
-            return
-
-        lib_inc_map = {}
-        for inc in self._get_found_includes(search_files):
-            for lb in self.env.GetLibBuilders():
-                if inc.get_abspath() in lb:
-                    if lb not in lib_inc_map:
-                        lib_inc_map[lb] = []
-                    lib_inc_map[lb].append(inc.get_abspath())
-                    break
-
-        for lb, lb_search_files in lib_inc_map.items():
-            self.depend_recursive(lb, lb_search_files)
 
     def build(self):
         libs = []
-        for lb in self._depbuilders:
+        for lb in self.depbuilders:
             libs.extend(lb.build())
             # copy shared information to self env
             for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
@@ -458,9 +462,9 @@ class LibBuilderBase(object):
         for lb in self._circular_deps:
             self.env.PrependUnique(CPPPATH=lb.get_include_dirs())
 
-        if self._is_built:
+        if self.is_built:
             return libs
-        self._is_built = True
+        self.is_built = True
 
         self.env.PrependUnique(CPPPATH=self.get_include_dirs())
 
@@ -632,7 +636,7 @@ class MbedLibBuilder(LibBuilderBase):
 
     def process_extra_options(self):
         self._process_mbed_lib_confs()
-        return super(MbedLibBuilder, self).process_extra_options()
+        return super().process_extra_options()
 
     def _process_mbed_lib_confs(self):
         mbed_lib_paths = [
@@ -851,7 +855,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
     def __init__(self, env, *args, **kwargs):
         # backup original value, will be reset in base.__init__
         project_src_filter = env.get("SRC_FILTER")
-        super(ProjectAsLibBuilder, self).__init__(env, *args, **kwargs)
+        super().__init__(env, *args, **kwargs)
         self.env["SRC_FILTER"] = project_src_filter
 
     @property
@@ -877,7 +881,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
         # project files
         items = LibBuilderBase.get_search_files(self)
         # test files
-        if "__test" in COMMAND_LINE_TARGETS:
+        if "test" in self.env.GetBuildType():
             items.extend(
                 [
                     os.path.join("$PROJECT_TEST_DIR", item)
@@ -901,13 +905,19 @@ class ProjectAsLibBuilder(LibBuilderBase):
         # pylint: disable=no-member
         return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self)
 
+    @property
+    def build_flags(self):
+        # pylint: disable=no-member
+        return self.env.get("SRC_BUILD_FLAGS") or LibBuilderBase.build_flags.fget(self)
+
     @property
     def dependencies(self):
         return self.env.GetProjectOption("lib_deps", [])
 
     def process_extra_options(self):
-        # skip for project, options are already processed
+        with fs.cd(self.path):
-        pass
+            self.env.ProcessFlags(self.build_flags)
+            self.env.ProcessUnFlags(self.build_unflags)
 
     def install_dependencies(self):
         def _is_builtin(spec):
@@ -947,6 +957,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
             DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None)
 
     def process_dependencies(self):  # pylint: disable=too-many-branches
+        found_lbs = []
         for spec in self.dependencies:
             found = False
             for storage_dir in self.env.GetLibSourceDirs():
@@ -960,7 +971,8 @@ class ProjectAsLibBuilder(LibBuilderBase):
                     if pkg.path != lb.path:
                         continue
                     if lb not in self.depbuilders:
-                        self.depend_recursive(lb)
+                        self.depend_on(lb, recursive=False)
+                        found_lbs.append(lb)
                     found = True
                     break
             if found:
@@ -972,12 +984,16 @@ class ProjectAsLibBuilder(LibBuilderBase):
                 if lb.name != spec:
                     continue
                 if lb not in self.depbuilders:
-                    self.depend_recursive(lb)
+                    self.depend_on(lb)
                 found = True
                 break
 
+        # process library dependencies
+        for lb in found_lbs:
+            lb.search_deps_recursive()
+
     def build(self):
-        self._is_built = True  # do not build Project now
+        self.is_built = True  # do not build Project now
         result = LibBuilderBase.build(self)
         self.env.PrependUnique(CPPPATH=self.get_include_dirs())
         return result
@@ -1015,7 +1031,7 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
     if DefaultEnvironment().get("__PIO_LIB_BUILDERS", None) is not None:
         return sorted(
             DefaultEnvironment()["__PIO_LIB_BUILDERS"],
-            key=lambda lb: 0 if lb.dependent else 1,
+            key=lambda lb: 0 if lb.is_dependent else 1,
         )
 
     DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=[])
@@ -1029,7 +1045,11 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             continue
         for item in sorted(os.listdir(storage_dir)):
             lib_dir = os.path.join(storage_dir, item)
-            if item == "__cores__" or not os.path.isdir(lib_dir):
+            if item == "__cores__":
+                continue
+            if LibraryPackageManager.is_symlink(lib_dir):
+                lib_dir, _ = LibraryPackageManager.resolve_symlink(lib_dir)
+            if not lib_dir or not os.path.isdir(lib_dir):
                 continue
             try:
                 lb = LibBuilderFactory.new(env, lib_dir)
@@ -1061,9 +1081,21 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
 
 
 def ConfigureProjectLibBuilder(env):
+    _pm_storage = {}
+
+    def _get_lib_license(pkg):
+        storage_dir = os.path.dirname(os.path.dirname(pkg.path))
+        if storage_dir not in _pm_storage:
+            _pm_storage[storage_dir] = LibraryPackageManager(storage_dir)
+        try:
+            return (_pm_storage[storage_dir].load_manifest(pkg) or {}).get("license")
+        except MissingPackageManifestError:
+            pass
+        return None
+
     def _correct_found_libs(lib_builders):
         # build full dependency graph
-        found_lbs = [lb for lb in lib_builders if lb.dependent]
+        found_lbs = [lb for lb in lib_builders if lb.is_dependent]
         for lb in lib_builders:
             if lb in found_lbs:
                 lb.search_deps_recursive(lb.get_search_files())
@@ -1075,18 +1107,20 @@ def ConfigureProjectLibBuilder(env):
     def _print_deps_tree(root, level=0):
         margin = "|   " * (level)
         for lb in root.depbuilders:
-            title = "<%s>" % lb.name
+            title = lb.name
             pkg = PackageItem(lb.path)
             if pkg.metadata:
-                title += " %s" % pkg.metadata.version
+                title += " @ %s" % pkg.metadata.version
             elif lb.version:
-                title += " %s" % lb.version
+                title += " @ %s" % lb.version
             click.echo("%s|-- %s" % (margin, title), nl=False)
             if int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                click.echo(
+                    " (License: %s, " % (_get_lib_license(pkg) or "Unknown"), nl=False
+                )
                 if pkg.metadata and pkg.metadata.spec.external:
-                    click.echo(" [%s]" % pkg.metadata.spec.url, nl=False)
+                    click.echo("URI: %s, " % pkg.metadata.spec.uri, nl=False)
-                click.echo(" (", nl=False)
+                click.echo("Path: %s" % lb.path, nl=False)
-                click.echo(lb.path, nl=False)
                 click.echo(")", nl=False)
             click.echo("")
             if lb.depbuilders:

platformio/builder/tools/piomisc.py

@@ -14,244 +14,15 @@
 
 from __future__ import absolute_import
 
-import atexit
-import glob
-import io
 import os
-import re
 import sys
-import tempfile
-
-import click
 
 from platformio import fs, util
-from platformio.compat import get_filesystem_encoding, get_locale_encoding
-from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import exec_command
 
 
-class InoToCPPConverter(object):
-
-    PROTOTYPE_RE = re.compile(
-        r"""^(
-        (?:template\<.*\>\s*)?      # template
-        ([a-z_\d\&]+\*?\s+){1,2}    # return type
-        ([a-z_\d]+\s*)              # name of prototype
-        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
-        )\s*(\{|;)                  # must end with `{` or `;`
-        """,
-        re.X | re.M | re.I,
-    )
-    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
-    PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
-
-    def __init__(self, env):
-        self.env = env
-        self._main_ino = None
-        self._safe_encoding = None
-
-    def read_safe_contents(self, path):
-        error_reported = False
-        for encoding in (
-            "utf-8",
-            None,
-            get_filesystem_encoding(),
-            get_locale_encoding(),
-            "latin-1",
-        ):
-            try:
-                with io.open(path, encoding=encoding) as fp:
-                    contents = fp.read()
-                    self._safe_encoding = encoding
-                    return contents
-            except UnicodeDecodeError:
-                if not error_reported:
-                    error_reported = True
-                    click.secho(
-                        "Unicode decode error has occurred, please remove invalid "
-                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
-                        % path,
-                        fg="yellow",
-                        err=True,
-                    )
-        return ""
-
-    def write_safe_contents(self, path, contents):
-        with io.open(
-            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
-        ) as fp:
-            return fp.write(contents)
-
-    def is_main_node(self, contents):
-        return self.DETECTMAIN_RE.search(contents)
-
-    def convert(self, nodes):
-        contents = self.merge(nodes)
-        if not contents:
-            return None
-        return self.process(contents)
-
-    def merge(self, nodes):
-        assert nodes
-        lines = []
-        for node in nodes:
-            contents = self.read_safe_contents(node.get_path())
-            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
-            if self.is_main_node(contents):
-                lines = _lines + lines
-                self._main_ino = node.get_path()
-            else:
-                lines.extend(_lines)
-
-        if not self._main_ino:
-            self._main_ino = nodes[0].get_path()
-
-        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
-
-    def process(self, contents):
-        out_file = self._main_ino + ".cpp"
-        assert self._gcc_preprocess(contents, out_file)
-        contents = self.read_safe_contents(out_file)
-        contents = self._join_multiline_strings(contents)
-        self.write_safe_contents(out_file, self.append_prototypes(contents))
-        return out_file
-
-    def _gcc_preprocess(self, contents, out_file):
-        tmp_path = tempfile.mkstemp()[1]
-        self.write_safe_contents(tmp_path, contents)
-        self.env.Execute(
-            self.env.VerboseAction(
-                '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
-                    out_file, tmp_path
-                ),
-                "Converting " + os.path.basename(out_file[:-4]),
-            )
-        )
-        atexit.register(_delete_file, tmp_path)
-        return os.path.isfile(out_file)
-
-    def _join_multiline_strings(self, contents):
-        if "\\\n" not in contents:
-            return contents
-        newlines = []
-        linenum = 0
-        stropen = False
-        for line in contents.split("\n"):
-            _linenum = self._parse_preproc_line_num(line)
-            if _linenum is not None:
-                linenum = _linenum
-            else:
-                linenum += 1
-
-            if line.endswith("\\"):
-                if line.startswith('"'):
-                    stropen = True
-                    newlines.append(line[:-1])
-                    continue
-                if stropen:
-                    newlines[len(newlines) - 1] += line[:-1]
-                    continue
-            elif stropen and line.endswith(('",', '";')):
-                newlines[len(newlines) - 1] += line
-                stropen = False
-                newlines.append(
-                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
-                )
-                continue
-
-            newlines.append(line)
-
-        return "\n".join(newlines)
-
-    @staticmethod
-    def _parse_preproc_line_num(line):
-        if not line.startswith("#"):
-            return None
-        tokens = line.split(" ", 3)
-        if len(tokens) > 2 and tokens[1].isdigit():
-            return int(tokens[1])
-        return None
-
-    def _parse_prototypes(self, contents):
-        prototypes = []
-        reserved_keywords = set(["if", "else", "while"])
-        for match in self.PROTOTYPE_RE.finditer(contents):
-            if (
-                set([match.group(2).strip(), match.group(3).strip()])
-                & reserved_keywords
-            ):
-                continue
-            prototypes.append(match)
-        return prototypes
-
-    def _get_total_lines(self, contents):
-        total = 0
-        if contents.endswith("\n"):
-            contents = contents[:-1]
-        for line in contents.split("\n")[::-1]:
-            linenum = self._parse_preproc_line_num(line)
-            if linenum is not None:
-                return total + linenum
-            total += 1
-        return total
-
-    def append_prototypes(self, contents):
-        prototypes = self._parse_prototypes(contents) or []
-
-        # skip already declared prototypes
-        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
-        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
-
-        if not prototypes:
-            return contents
-
-        prototype_names = set(m.group(3).strip() for m in prototypes)
-        split_pos = prototypes[0].start()
-        match_ptrs = re.search(
-            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
-            contents[:split_pos],
-            re.M,
-        )
-        if match_ptrs:
-            split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
-
-        result = []
-        result.append(contents[:split_pos].strip())
-        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
-        result.append(
-            '#line %d "%s"'
-            % (
-                self._get_total_lines(contents[:split_pos]),
-                self._main_ino.replace("\\", "/"),
-            )
-        )
-        result.append(contents[split_pos:].strip())
-        return "\n".join(result)
-
-
-def ConvertInoToCpp(env):
-    src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
-    ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
-        os.path.join(src_dir, "*.pde")
-    )
-    if not ino_nodes:
-        return
-    c = InoToCPPConverter(env)
-    out_file = c.convert(ino_nodes)
-
-    atexit.register(_delete_file, out_file)
-
-
-def _delete_file(path):
-    try:
-        if os.path.isfile(path):
-            os.remove(path)
-    except:  # pylint: disable=bare-except
-        pass
-
-
 @util.memoized()
-def _get_compiler_type(env):
+def GetCompilerType(env):
     if env.subst("$CC").endswith("-gcc"):
         return "gcc"
     try:
@@ -270,10 +41,6 @@ def _get_compiler_type(env):
     return None
 
 
-def GetCompilerType(env):
-    return _get_compiler_type(env)
-
-
 def GetActualLDScript(env):
     def _lookup_in_ldpath(script):
         for d in env.get("LIBPATH", []):
@@ -319,7 +86,7 @@ def GetActualLDScript(env):
     env.Exit(1)
 
 
-def ConfigureDebugFlags(env):
+def ConfigureDebugTarget(env):
     def _cleanup_debug_flags(scope):
         if scope not in env:
             return
@@ -350,22 +117,6 @@ def ConfigureDebugFlags(env):
         env.AppendUnique(ASFLAGS=optimization_flags, LINKFLAGS=optimization_flags)
 
 
-def ConfigureTestTarget(env):
-    env.Append(
-        CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
-        CPPPATH=[os.path.join("$BUILD_DIR", "UnityTestLib")],
-    )
-    unitylib = env.BuildLibrary(
-        os.path.join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity")
-    )
-    env.Prepend(LIBS=[unitylib])
-
-    src_filter = ["+<*.cpp>", "+<*.c>"]
-    if "PIOTEST_RUNNING_NAME" in env:
-        src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], os.path.sep))
-    env.Replace(PIOTEST_SRC_FILTER=src_filter)
-
-
 def GetExtraScripts(env, scope):
     items = []
     for item in env.GetProjectOption("extra_scripts", []):
@@ -376,18 +127,17 @@ def GetExtraScripts(env, scope):
     if not items:
         return items
     with fs.cd(env.subst("$PROJECT_DIR")):
-        return [os.path.abspath(item) for item in items]
+        return [os.path.abspath(env.subst(item)) for item in items]
+
+
+def generate(env):
+    env.AddMethod(GetCompilerType)
+    env.AddMethod(GetActualLDScript)
+    env.AddMethod(ConfigureDebugTarget)
+    env.AddMethod(GetExtraScripts)
+    # bakward-compatibility with Zephyr build script
+    env.AddMethod(ConfigureDebugTarget, "ConfigureDebugFlags")
 
 
 def exists(_):
     return True
-
-
-def generate(env):
-    env.AddMethod(ConvertInoToCpp)
-    env.AddMethod(GetCompilerType)
-    env.AddMethod(GetActualLDScript)
-    env.AddMethod(ConfigureDebugFlags)
-    env.AddMethod(ConfigureTestTarget)
-    env.AddMethod(GetExtraScripts)
-    return env

platformio/builder/tools/pioplatform.py

@@ -19,6 +19,7 @@ import sys
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
+from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import fs, util
 from platformio.compat import IS_MACOS, IS_WINDOWS
@@ -32,16 +33,17 @@ from platformio.project.config import ProjectOptions
 
 
 @util.memoized()
-def PioPlatform(env):
+def _PioPlatform():
-    variables = env.GetProjectOptions(as_dict=True)
+    env = DefaultEnvironment()
-    if "framework" in variables:
-        # support PIO Core 3.0 dev/platforms
-        variables["pioframework"] = variables["framework"]
     p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"]))
-    p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
+    p.configure_project_packages(env["PIOENV"], COMMAND_LINE_TARGETS)
     return p
 
 
+def PioPlatform(_):
+    return _PioPlatform()
+
+
 def BoardConfig(env, board=None):
     with fs.cd(env.subst("$PROJECT_DIR")):
         try:
@@ -160,7 +162,7 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
             and pkg_metadata
             and pkg_metadata.spec.external
         ):
-            data.append("(%s)" % pkg_metadata.spec.url)
+            data.append("(%s)" % pkg_metadata.spec.uri)
         if board_config:
             data.extend([">", board_config.get("name")])
         return data
@@ -213,7 +215,7 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
         data = []
         for item in platform.dump_used_packages():
             original_version = get_original_version(item["version"])
-            info = "%s %s" % (item["name"], item["version"])
+            info = "%s @ %s" % (item["name"], item["version"])
             extra = []
             if original_version:
                 extra.append(original_version)

platformio/builder/tools/pioproject.py

@@ -14,7 +14,8 @@
 
 from __future__ import absolute_import
 
-from platformio.project.config import MISSING, ProjectConfig, ProjectOptions
+from platformio.compat import MISSING
+from platformio.project.config import ProjectConfig, ProjectOptions
 
 
 def GetProjectConfig(env):

platformio/builder/tools/piotest.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import os
+
+from platformio.builder.tools import platformio as piotool
+from platformio.test.result import TestSuite
+from platformio.test.runners.factory import TestRunnerFactory
+
+
+def ConfigureTestTarget(env):
+    env.Append(
+        CPPDEFINES=["UNIT_TEST", "PIO_UNIT_TESTING"],
+        PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piotool.SRC_BUILD_EXT],
+    )
+    env.Prepend(CPPPATH=["$PROJECT_TEST_DIR"])
+
+    if "PIOTEST_RUNNING_NAME" in env:
+        test_name = env["PIOTEST_RUNNING_NAME"]
+        while True:
+            test_name = os.path.dirname(test_name)  # parent dir
+            # skip nested tests (user's side issue?)
+            if not test_name or os.path.basename(test_name).startswith("test_"):
+                break
+            env.Prepend(
+                PIOTEST_SRC_FILTER=[
+                    f"+<{test_name}{os.path.sep}*.{ext}>"
+                    for ext in piotool.SRC_BUILD_EXT
+                ],
+                CPPPATH=[os.path.join("$PROJECT_TEST_DIR", test_name)],
+            )
+
+        env.Prepend(
+            PIOTEST_SRC_FILTER=[f"+<$PIOTEST_RUNNING_NAME{os.path.sep}>"],
+            CPPPATH=[os.path.join("$PROJECT_TEST_DIR", "$PIOTEST_RUNNING_NAME")],
+        )
+
+    test_runner = TestRunnerFactory.new(
+        TestSuite(env["PIOENV"], env.get("PIOTEST_RUNNING_NAME", "*")),
+        env.GetProjectConfig(),
+    )
+    test_runner.configure_build_env(env)
+
+
+def generate(env):
+    env.AddMethod(ConfigureTestTarget)
+
+
+def exists(_):
+    return True

platformio/builder/tools/pioupload.py

@@ -12,25 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=unused-argument
+
 from __future__ import absolute_import
 
+import os
 import re
 import sys
-from fnmatch import fnmatch
-from os import environ
-from os.path import isfile, join
 from shutil import copyfile
 from time import sleep
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from serial import Serial, SerialException
 
-from platformio import exception, fs, util
+from platformio import exception, fs
-from platformio.compat import IS_WINDOWS
+from platformio.device.finder import find_mbed_disk, find_serial_port, is_pattern_port
+from platformio.device.list import list_serial_ports
 from platformio.proc import exec_command
 
-# pylint: disable=unused-argument
-
 
 def FlushSerialBuffer(env, port):
     s = Serial(env.subst(port))
@@ -62,7 +61,7 @@ def WaitForNewSerialPort(env, before):
     elapsed = 0
     before = [p["port"] for p in before]
     while elapsed < 5 and new_port is None:
-        now = [p["port"] for p in util.get_serial_ports()]
+        now = [p["port"] for p in list_serial_ports()]
         for p in now:
             if p not in before:
                 new_port = p
@@ -97,67 +96,28 @@ def WaitForNewSerialPort(env, before):
 
 def AutodetectUploadPort(*args, **kwargs):
     env = args[0]
-
+    initial_port = env.subst("$UPLOAD_PORT")
-    def _get_pattern():
+    upload_protocol = env.subst("$UPLOAD_PROTOCOL")
-        if "UPLOAD_PORT" not in env:
+    if initial_port and not is_pattern_port(initial_port):
-            return None
+        print(env.subst("Using manually specified: $UPLOAD_PORT"))
-        if set(["*", "?", "[", "]"]) & set(env["UPLOAD_PORT"]):
-            return env["UPLOAD_PORT"]
-        return None
-
-    def _is_match_pattern(port):
-        pattern = _get_pattern()
-        if not pattern:
-            return True
-        return fnmatch(port, pattern)
-
-    def _look_for_mbed_disk():
-        msdlabels = ("mbed", "nucleo", "frdm", "microbit")
-        for item in util.get_logical_devices():
-            if item["path"].startswith("/net") or not _is_match_pattern(item["path"]):
-                continue
-            mbed_pages = [join(item["path"], n) for n in ("mbed.htm", "mbed.html")]
-            if any(isfile(p) for p in mbed_pages):
-                return item["path"]
-            if item["name"] and any(l in item["name"].lower() for l in msdlabels):
-                return item["path"]
-        return None
-
-    def _look_for_serial_port():
-        port = None
-        board_hwids = []
-        upload_protocol = env.subst("$UPLOAD_PROTOCOL")
-        if "BOARD" in env and "build.hwids" in env.BoardConfig():
-            board_hwids = env.BoardConfig().get("build.hwids")
-        for item in util.get_serial_ports(filter_hwid=True):
-            if not _is_match_pattern(item["port"]):
-                continue
-            port = item["port"]
-            if upload_protocol.startswith("blackmagic"):
-                if IS_WINDOWS and port.startswith("COM") and len(port) > 4:
-                    port = "\\\\.\\%s" % port
-                if "GDB" in item["description"]:
-                    return port
-            for hwid in board_hwids:
-                hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                if hwid_str in item["hwid"]:
-                    return port
-        return port
-
-    if "UPLOAD_PORT" in env and not _get_pattern():
-        print(env.subst("Use manually specified: $UPLOAD_PORT"))
         return
 
-    if env.subst("$UPLOAD_PROTOCOL") == "mbed" or (
+    if upload_protocol == "mbed" or (
-        "mbed" in env.subst("$PIOFRAMEWORK") and not env.subst("$UPLOAD_PROTOCOL")
+        "mbed" in env.subst("$PIOFRAMEWORK") and not upload_protocol
     ):
-        env.Replace(UPLOAD_PORT=_look_for_mbed_disk())
+        env.Replace(UPLOAD_PORT=find_mbed_disk(initial_port))
     else:
         try:
             fs.ensure_udev_rules()
         except exception.InvalidUdevRules as e:
             sys.stderr.write("\n%s\n\n" % e)
-        env.Replace(UPLOAD_PORT=_look_for_serial_port())
+        env.Replace(
+            UPLOAD_PORT=find_serial_port(
+                initial_port=initial_port,
+                board_config=env.BoardConfig() if "BOARD" in env else None,
+                upload_protocol=upload_protocol,
+            )
+        )
 
     if env.subst("$UPLOAD_PORT"):
         print(env.subst("Auto-detected: $UPLOAD_PORT"))
@@ -175,10 +135,12 @@ def UploadToDisk(_, target, source, env):
     assert "UPLOAD_PORT" in env
     progname = env.subst("$PROGNAME")
     for ext in ("bin", "hex"):
-        fpath = join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
+        fpath = os.path.join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
-        if not isfile(fpath):
+        if not os.path.isfile(fpath):
             continue
-        copyfile(fpath, join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)))
+        copyfile(
+            fpath, os.path.join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext))
+        )
     print(
         "Firmware has been successfully uploaded.\n"
         "(Some boards may require manual hard reset)"
@@ -211,7 +173,7 @@ def CheckUploadSize(_, target, source, env):
         if not isinstance(cmd, list):
             cmd = cmd.split()
         cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
-        sysenv = environ.copy()
+        sysenv = os.environ.copy()
         sysenv["PATH"] = str(env["ENV"]["PATH"])
         result = exec_command(env.subst(cmd), env=sysenv)
         if result["returncode"] != 0:

platformio/builder/tools/platformio.py

@@ -47,14 +47,16 @@ def scons_patched_match_splitext(path, suffixes=None):
 
 
 def GetBuildType(env):
-    return (
+    modes = []
-        "debug"
+    if (
-        if (
+        set(["__debug", "sizedata"])  # sizedata = for memory inspection
-            set(["__debug", "sizedata"]) & set(COMMAND_LINE_TARGETS)
+        & set(COMMAND_LINE_TARGETS)
-            or env.GetProjectOption("build_type") == "debug"
+        or env.GetProjectOption("build_type") == "debug"
-        )
+    ):
-        else "release"
+        modes.append("debug")
-    )
+    if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test":
+        modes.append("test")
+    return "+".join(modes or ["release"])
 
 
 def BuildProgram(env):
@@ -113,10 +115,6 @@ def ProcessProgramDeps(env):
 
     env.PrintConfiguration()
 
-    # fix ASM handling under non case-sensitive OS
-    if not Util.case_sensitive_suffixes(".s", ".S"):
-        env.Replace(AS="$CC", ASCOM="$ASPPCOM")
-
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
         env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
@@ -127,14 +125,20 @@ def ProcessProgramDeps(env):
     # process framework scripts
     env.BuildFrameworks(env.get("PIOFRAMEWORK"))
 
-    if env.GetBuildType() == "debug":
+    if "debug" in env.GetBuildType():
-        env.ConfigureDebugFlags()
+        env.ConfigureDebugTarget()
+    if "test" in env.GetBuildType():
+        env.ConfigureTestTarget()
 
     # remove specified flags
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
-    if "__test" in COMMAND_LINE_TARGETS:
+    if "compiledb" in COMMAND_LINE_TARGETS and env.get(
-        env.ConfigureTestTarget()
+        "COMPILATIONDB_INCLUDE_TOOLCHAIN"
+    ):
+        for scope, includes in env.DumpIntegrationIncludes().items():
+            if scope in ("toolchain",):
+                env.Append(CPPPATH=includes)
 
 
 def ProcessProjectDeps(env):
@@ -158,12 +162,11 @@ def ProcessProjectDeps(env):
     # extra build flags from `platformio.ini`
     projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
 
-    is_test = "__test" in COMMAND_LINE_TARGETS
+    if "test" in env.GetBuildType():
-    if is_test:
         projenv.BuildSources(
             "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
         )
-    if not is_test or env.GetProjectOption("test_build_project_src"):
+    if "test" not in env.GetBuildType() or env.GetProjectOption("test_build_src"):
         projenv.BuildSources(
             "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER")
         )

platformio/clients/account.py

@@ -40,7 +40,7 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
     SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
 
     def __init__(self):
-        super(AccountClient, self).__init__(__accounts_api__)
+        super().__init__(__accounts_api__)
 
     @staticmethod
     def get_refresh_token():
@@ -63,7 +63,7 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
 
     def fetch_json_data(self, *args, **kwargs):
         try:
-            return super(AccountClient, self).fetch_json_data(*args, **kwargs)
+            return super().fetch_json_data(*args, **kwargs)
         except HTTPClientError as exc:
             raise AccountError(exc) from exc
 

platformio/clients/http.py

@@ -16,6 +16,7 @@ import json
 import math
 import os
 import socket
+from urllib.parse import urljoin
 
 import requests.adapters
 from requests.packages.urllib3.util.retry import Retry  # pylint:disable=import-error
@@ -24,15 +25,10 @@ from platformio import __check_internet_hosts__, __default_requests_timeout__, a
 from platformio.cache import ContentCache, cleanup_content_cache
 from platformio.exception import PlatformioException, UserSideException
 
-try:
-    from urllib.parse import urljoin
-except ImportError:
-    from urlparse import urljoin
-
 
 class HTTPClientError(PlatformioException):
     def __init__(self, message, response=None):
-        super(HTTPClientError, self).__init__()
+        super().__init__()
         self.message = message
         self.response = response
 
@@ -51,16 +47,14 @@ class InternetIsOffline(UserSideException):
 
 class EndpointSession(requests.Session):
     def __init__(self, base_url, *args, **kwargs):
-        super(EndpointSession, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.base_url = base_url
 
     def request(  # pylint: disable=signature-differs,arguments-differ
         self, method, url, *args, **kwargs
     ):
         # print(self.base_url, method, url, args, kwargs)
-        return super(EndpointSession, self).request(
+        return super().request(method, urljoin(self.base_url, url), *args, **kwargs)
-            method, urljoin(self.base_url, url), *args, **kwargs
-        )
 
 
 class EndpointSessionIterator(object):
@@ -79,10 +73,6 @@ class EndpointSessionIterator(object):
     def __iter__(self):  # pylint: disable=non-iterator-returned
         return self
 
-    def next(self):
-        """For Python 2 compatibility"""
-        return self.__next__()
-
     def __next__(self):
         base_url = next(self.endpoints_iter)
         session = EndpointSession(base_url)
@@ -149,7 +139,7 @@ class HTTPClient(object):
                     raise HTTPClientError(str(e))
 
     def fetch_json_data(self, method, path, **kwargs):
-        if method != "get":
+        if method not in ("get", "head", "options"):
             cleanup_content_cache("http")
         cache_valid = kwargs.pop("x_cache_valid") if "x_cache_valid" in kwargs else None
         if not cache_valid:

platformio/clients/registry.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio import __registry_api__, fs
+from platformio import __registry_mirror_hosts__, fs
 from platformio.clients.account import AccountClient, AccountError
 from platformio.clients.http import HTTPClient, HTTPClientError
 
@@ -21,7 +21,8 @@ from platformio.clients.http import HTTPClient, HTTPClientError
 
 class RegistryClient(HTTPClient):
     def __init__(self):
-        super(RegistryClient, self).__init__(__registry_api__)
+        endpoints = [f"https://api.{host}" for host in __registry_mirror_hosts__]
+        super().__init__(endpoints)
 
     @staticmethod
     def allowed_private_packages():
@@ -102,14 +103,14 @@ class RegistryClient(HTTPClient):
             "get",
             "/v3/resources",
             params={"owner": owner} if owner else None,
+            x_cache_valid="1h",
             x_with_authorization=True,
         )
 
-    def list_packages(self, query=None, filters=None, page=None):
+    def list_packages(self, query=None, qualifiers=None, page=None, sort=None):
-        assert query or filters
         search_query = []
-        if filters:
+        if qualifiers:
-            valid_filters = (
+            valid_qualifiers = (
                 "authors",
                 "keywords",
                 "frameworks",
@@ -120,8 +121,8 @@ class RegistryClient(HTTPClient):
                 "owners",
                 "types",
             )
-            assert set(filters.keys()) <= set(valid_filters)
+            assert set(qualifiers.keys()) <= set(valid_qualifiers)
-            for name, values in filters.items():
+            for name, values in qualifiers.items():
                 for value in set(
                     values if isinstance(values, (list, tuple)) else [values]
                 ):
@@ -131,6 +132,8 @@ class RegistryClient(HTTPClient):
         params = dict(query=" ".join(search_query))
         if page:
             params["page"] = int(page)
+        if sort:
+            params["sort"] = sort
         return self.fetch_json_data(
             "get",
             "/v3/search",

platformio/commands/__init__.py

@@ -22,7 +22,7 @@ class PlatformioCLI(click.MultiCommand):
     leftover_args = []
 
     def __init__(self, *args, **kwargs):
-        super(PlatformioCLI, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._pio_cmds_dir = os.path.dirname(__file__)
 
     @staticmethod
@@ -41,7 +41,7 @@ class PlatformioCLI(click.MultiCommand):
         PlatformioCLI.leftover_args = ctx.args
         if hasattr(ctx, "protected_args"):
             PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
-        return super(PlatformioCLI, self).invoke(ctx)
+        return super().invoke(ctx)
 
     def list_commands(self, ctx):
         cmds = []
@@ -74,7 +74,13 @@ class PlatformioCLI(click.MultiCommand):
     def _handle_obsolate_command(name):
         # pylint: disable=import-outside-toplevel
         if name == "init":
-            from platformio.commands.project import project_init
+            from platformio.project.commands.init import project_init_cmd
+
+            return project_init_cmd
+
+        if name == "package":
+            from platformio.commands.pkg import cli
+
+            return cli
 
-            return project_init
         raise AttributeError()

platformio/commands/account.py

@@ -14,13 +14,13 @@
 
 # pylint: disable=unused-argument
 
-import datetime
 import json
 import re
 
 import click
 from tabulate import tabulate
 
+from platformio import util
 from platformio.clients.account import AccountClient, AccountNotAuthorized
 
 
@@ -244,12 +244,9 @@ def print_packages(packages):
         data = []
         expire = "-"
         if "subscription" in package:
-            expire = datetime.datetime.strptime(
+            expire = util.parse_datetime(
-                (
+                package["subscription"].get("end_at")
-                    package["subscription"].get("end_at")
+                or package["subscription"].get("next_bill_at")
-                    or package["subscription"].get("next_bill_at")
-                ),
-                "%Y-%m-%dT%H:%M:%SZ",
             ).strftime("%Y-%m-%d")
         data.append(("Expire:", expire))
         services = []
@@ -274,21 +271,17 @@ def print_subscriptions(subscriptions):
         click.secho(subscription.get("product_name"), bold=True)
         click.echo("-" * len(subscription.get("product_name")))
         data = [("State:", subscription.get("status"))]
-        begin_at = datetime.datetime.strptime(
+        begin_at = util.parse_datetime(subscription.get("begin_at")).strftime("%c")
-            subscription.get("begin_at"), "%Y-%m-%dT%H:%M:%SZ"
-        ).strftime("%Y-%m-%d %H:%M:%S")
         data.append(("Start date:", begin_at or "-"))
         end_at = subscription.get("end_at")
         if end_at:
-            end_at = datetime.datetime.strptime(
+            end_at = util.parse_datetime(subscription.get("end_at")).strftime("%c")
-                subscription.get("end_at"), "%Y-%m-%dT%H:%M:%SZ"
-            ).strftime("%Y-%m-%d %H:%M:%S")
         data.append(("End date:", end_at or "-"))
         next_bill_at = subscription.get("next_bill_at")
         if next_bill_at:
-            next_bill_at = datetime.datetime.strptime(
+            next_bill_at = util.parse_datetime(
-                subscription.get("next_bill_at"), "%Y-%m-%dT%H:%M:%SZ"
+                subscription.get("next_bill_at")
-            ).strftime("%Y-%m-%d %H:%M:%S")
+            ).strftime("%c")
         data.append(("Next payment:", next_bill_at or "-"))
         data.append(
             ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")

platformio/commands/boards.py

@@ -22,7 +22,7 @@ from platformio import fs
 from platformio.package.manager.platform import PlatformPackageManager
 
 
-@click.command("boards", short_help="Embedded board explorer")
+@click.command("boards", short_help="Board Explorer")
 @click.argument("query", required=False)
 @click.option("--installed", is_flag=True)
 @click.option("--json-output", is_flag=True)

platformio/commands/check/command.py

@@ -32,7 +32,7 @@ from platformio.project.config import ProjectConfig
 from platformio.project.helpers import find_project_dir_above, get_project_dir
 
 
-@click.command("check", short_help="Static code analysis")
+@click.command("check", short_help="Static Code Analysis")
 @click.option("-e", "--environment", multiple=True)
 @click.option(
     "-d",
@@ -118,6 +118,7 @@ def cli(
                 if silent
                 else severity or config.get("env:" + envname, "check_severity"),
                 skip_packages=skip_packages or env_options.get("check_skip_packages"),
+                platform_packages=env_options.get("platform_packages"),
             )
 
             for tool in config.get("env:" + envname, "check_tool"):
@@ -166,7 +167,10 @@ def cli(
         if json_output:
             click.echo(json.dumps(results_to_json(results)))
         elif not silent:
-            print_check_summary(results)
+            print_check_summary(results, verbose=verbose)
+
+    # Reset custom project config
+    app.set_session_var("custom_project_conf", None)
 
     command_failed = any(r.get("succeeded") is False for r in results)
     if command_failed:
@@ -267,7 +271,7 @@ def print_defects_stats(results):
     click.echo()
 
 
-def print_check_summary(results):
+def print_check_summary(results, verbose=False):
     click.echo()
 
     tabular_data = []
@@ -284,6 +288,8 @@ def print_check_summary(results):
             status_str = click.style("FAILED", fg="red")
         elif result.get("succeeded") is None:
             status_str = "IGNORED"
+            if not verbose:
+                continue
         else:
             succeeded_nums += 1
             status_str = click.style("PASSED", fg="green")

platformio/commands/check/defect.py

@@ -34,7 +34,7 @@ class DefectItem(object):
         severity,
         category,
         message,
-        file="unknown",
+        file=None,
         line=0,
         column=0,
         id=None,
@@ -50,7 +50,7 @@ class DefectItem(object):
         self.callstack = callstack
         self.cwe = cwe
         self.id = id
-        self.file = file
+        self.file = file or "unknown"
         if file.lower().startswith(get_project_dir().lower()):
             self.file = os.path.relpath(file, get_project_dir())
 

platformio/commands/check/tools/base.py

@@ -20,7 +20,9 @@ import click
 
 from platformio import fs, proc
 from platformio.commands.check.defect import DefectItem
-from platformio.project.helpers import load_project_ide_data
+from platformio.package.manager.core import get_core_package_dir
+from platformio.package.meta import PackageSpec
+from platformio.project.helpers import load_build_metadata
 
 
 class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
@@ -55,7 +57,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         ]
 
     def _load_cpp_data(self, project_dir):
-        data = load_project_ide_data(project_dir, self.envname)
+        data = load_build_metadata(project_dir, self.envname)
         if not data:
             return
         self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", ""))
@@ -66,6 +68,13 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         self.cxx_path = data.get("cxx_path")
         self.toolchain_defines = self._get_toolchain_defines()
 
+    def get_tool_dir(self, pkg_name):
+        for spec in self.options["platform_packages"] or []:
+            spec = PackageSpec(spec)
+            if spec.name == pkg_name:
+                return get_core_package_dir(pkg_name, spec=spec)
+        return get_core_package_dir(pkg_name)
+
     def get_flags(self, tool):
         result = []
         flags = self.options.get("flags") or []

platformio/commands/check/tools/clangtidy.py

@@ -17,11 +17,10 @@ from os.path import join
 
 from platformio.commands.check.defect import DefectItem
 from platformio.commands.check.tools.base import CheckToolBase
-from platformio.package.manager.core import get_core_package_dir
 
 
 class ClangtidyCheckTool(CheckToolBase):
-    def tool_output_filter(self, line):
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
         if not self.options.get("verbose") and "[clang-diagnostic-error]" in line:
             return ""
 
@@ -34,7 +33,7 @@ class ClangtidyCheckTool(CheckToolBase):
 
         return ""
 
-    def parse_defect(self, raw_line):
+    def parse_defect(self, raw_line):  # pylint: disable=arguments-differ
         match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line)
         if not match:
             return raw_line
@@ -56,11 +55,13 @@ class ClangtidyCheckTool(CheckToolBase):
         return cmd_result["returncode"] < 2
 
     def configure_command(self):
-        tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy")
+        tool_path = join(self.get_tool_dir("tool-clangtidy"), "clang-tidy")
 
         cmd = [tool_path, "--quiet"]
         flags = self.get_flags("clangtidy")
-        if not self.is_flag_set("--checks", flags):
+        if not (
+            self.is_flag_set("--checks", flags) or self.is_flag_set("--config", flags)
+        ):
             cmd.append("--checks=*")
 
         project_files = self.get_project_target_files(self.options["patterns"])

platformio/commands/check/tools/cppcheck.py

@@ -19,11 +19,11 @@ import click
 from platformio import proc
 from platformio.commands.check.defect import DefectItem
 from platformio.commands.check.tools.base import CheckToolBase
-from platformio.package.manager.core import get_core_package_dir
 
 
 class CppcheckCheckTool(CheckToolBase):
     def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self._field_delimiter = "<&PIO&>"
         self._buffer = ""
         self.defect_fields = [
@@ -36,9 +36,8 @@ class CppcheckCheckTool(CheckToolBase):
             "cwe",
             "id",
         ]
-        super(CppcheckCheckTool, self).__init__(*args, **kwargs)
 
-    def tool_output_filter(self, line):
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
         if (
             not self.options.get("verbose")
             and "--suppress=unmatchedSuppression:" in line
@@ -50,13 +49,14 @@ class CppcheckCheckTool(CheckToolBase):
             for msg in (
                 "No C or C++ source files found",
                 "unrecognized command line option",
+                "there was an internal error",
             )
         ):
             self._bad_input = True
 
         return line
 
-    def parse_defect(self, raw_line):
+    def parse_defect(self, raw_line):  # pylint: disable=arguments-differ
         if self._field_delimiter not in raw_line:
             return None
 
@@ -103,7 +103,7 @@ class CppcheckCheckTool(CheckToolBase):
         return DefectItem(**args)
 
     def configure_command(self, language, src_file):  # pylint: disable=arguments-differ
-        tool_path = os.path.join(get_core_package_dir("tool-cppcheck"), "cppcheck")
+        tool_path = os.path.join(self.get_tool_dir("tool-cppcheck"), "cppcheck")
 
         cmd = [
             tool_path,
@@ -208,7 +208,7 @@ class CppcheckCheckTool(CheckToolBase):
         return self._create_tmp_file("\n".join(result))
 
     def clean_up(self):
-        super(CppcheckCheckTool, self).clean_up()
+        super().clean_up()
 
         # delete temporary dump files generated by addons
         if not self.is_flag_set("--addon", self.get_flags("cppcheck")):

platformio/commands/check/tools/pvsstudio.py

@@ -23,22 +23,21 @@ from platformio import proc
 from platformio.commands.check.defect import DefectItem
 from platformio.commands.check.tools.base import CheckToolBase
 from platformio.compat import IS_WINDOWS
-from platformio.package.manager.core import get_core_package_dir
 
 
 class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-attributes
     def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self._tmp_dir = tempfile.mkdtemp(prefix="piocheck")
         self._tmp_preprocessed_file = self._generate_tmp_file_path() + ".i"
         self._tmp_output_file = self._generate_tmp_file_path() + ".pvs"
         self._tmp_cfg_file = self._generate_tmp_file_path() + ".cfg"
         self._tmp_cmd_file = self._generate_tmp_file_path() + ".cmd"
         self.tool_path = os.path.join(
-            get_core_package_dir("tool-pvs-studio"),
+            self.get_tool_dir("tool-pvs-studio"),
             "x64" if IS_WINDOWS else "bin",
             "pvs-studio",
         )
-        super(PvsStudioCheckTool, self).__init__(*args, **kwargs)
 
         with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp:
             fp.write(
@@ -53,8 +52,14 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
                 )
             )
 
-    def tool_output_filter(self, line):
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
-        if "license was not entered" in line.lower():
+        if any(
+            err_msg in line.lower()
+            for err_msg in (
+                "license was not entered",
+                "license information is incorrect",
+            )
+        ):
             self._bad_input = True
         return line
 
@@ -70,7 +75,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
 
     def _demangle_report(self, output_file):
         converter_tool = os.path.join(
-            get_core_package_dir("tool-pvs-studio"),
+            self.get_tool_dir("tool-pvs-studio"),
             "HtmlGenerator" if IS_WINDOWS else os.path.join("bin", "plog-converter"),
         )
 
@@ -194,7 +199,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
             '"%s"' % self._tmp_preprocessed_file,
         ]
         cmd.extend([f for f in flags if f])
-        cmd.extend(["-D%s" % d for d in self.cpp_defines])
+        cmd.extend(['"-D%s"' % d.replace('"', '\\"') for d in self.cpp_defines])
         cmd.append('@"%s"' % self._tmp_cmd_file)
 
         # Explicitly specify C++ as the language used in .ino files
@@ -209,7 +214,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
             self._bad_input = True
 
     def clean_up(self):
-        super(PvsStudioCheckTool, self).clean_up()
+        super().clean_up()
         if os.path.isdir(self._tmp_dir):
             shutil.rmtree(self._tmp_dir)
 

platformio/commands/ci.py

@@ -20,10 +20,9 @@ import tempfile
 import click
 
 from platformio import app, fs
-from platformio.commands.project import project_init as cmd_project_init
-from platformio.commands.project import validate_boards
 from platformio.commands.run.command import cli as cmd_run
 from platformio.exception import CIBuildEnvsEmpty
+from platformio.project.commands.init import project_init_cmd, validate_boards
 from platformio.project.config import ProjectConfig
 
 
@@ -44,7 +43,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
         raise click.BadParameter("Found invalid path: %s" % invalid_path)
 
 
-@click.command("ci", short_help="Continuous integration")
+@click.command("ci", short_help="Continuous Integration")
 @click.argument("src", nargs=-1, callback=validate_path)
 @click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY")
 @click.option("--exclude", multiple=True)
@@ -109,7 +108,7 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
 
         # initialise project
         ctx.invoke(
-            cmd_project_init,
+            project_init_cmd,
             project_dir=build_dir,
             board=board,
             project_option=project_option,

platformio/commands/debug.py

@@ -12,169 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=too-many-arguments, too-many-locals
+# pylint: disable=unused-import
-# pylint: disable=too-many-branches, too-many-statements
 
-import asyncio
+from platformio.debug.command import debug_cmd as cli
-import os
-import signal
-import subprocess
-
-import click
-
-from platformio import app, exception, fs, proc
-from platformio.commands.platform import init_platform
-from platformio.compat import IS_WINDOWS
-from platformio.debug import helpers
-from platformio.debug.config.factory import DebugConfigFactory
-from platformio.debug.exception import DebugInvalidOptionsError
-from platformio.debug.process.gdb import GDBClientProcess
-from platformio.project.config import ProjectConfig
-from platformio.project.exception import ProjectEnvsNotAvailableError
-from platformio.project.helpers import is_platformio_project
-from platformio.project.options import ProjectOptions
-
-
-@click.command(
-    "debug",
-    context_settings=dict(ignore_unknown_options=True),
-    short_help="Unified debugger",
-)
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
-)
-@click.option(
-    "-c",
-    "--project-conf",
-    type=click.Path(
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
-)
-@click.option("--environment", "-e", metavar="<environment>")
-@click.option("--load-mode", type=ProjectOptions["env.debug_load_mode"].type)
-@click.option("--verbose", "-v", is_flag=True)
-@click.option("--interface", type=click.Choice(["gdb"]))
-@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
-@click.pass_context
-def cli(
-    ctx,
-    project_dir,
-    project_conf,
-    environment,
-    load_mode,
-    verbose,
-    interface,
-    __unprocessed,
-):
-    app.set_session_var("custom_project_conf", project_conf)
-
-    # use env variables from Eclipse or CLion
-    for name in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"):
-        if is_platformio_project(project_dir):
-            break
-        if os.getenv(name):
-            project_dir = os.getenv(name)
-
-    with fs.cd(project_dir):
-        project_config = ProjectConfig.get_instance(project_conf)
-    project_config.validate(envs=[environment] if environment else None)
-    env_name = environment or helpers.get_default_debug_env(project_config)
-
-    if not interface:
-        return helpers.predebug_project(
-            ctx, project_dir, project_config, env_name, False, verbose
-        )
-
-    env_options = project_config.items(env=env_name, as_dict=True)
-    if "platform" not in env_options:
-        raise ProjectEnvsNotAvailableError()
-
-    with fs.cd(project_dir):
-        debug_config = DebugConfigFactory.new(
-            init_platform(env_options["platform"]), project_config, env_name
-        )
-
-    if "--version" in __unprocessed:
-        return subprocess.run(
-            [debug_config.client_executable_path, "--version"], check=True
-        )
-
-    try:
-        fs.ensure_udev_rules()
-    except exception.InvalidUdevRules as e:
-        click.echo(
-            helpers.escape_gdbmi_stream("~", str(e) + "\n")
-            if helpers.is_gdbmi_mode()
-            else str(e) + "\n",
-            nl=False,
-        )
-
-    rebuild_prog = False
-    preload = debug_config.load_cmds == ["preload"]
-    load_mode = load_mode or debug_config.load_mode
-    if load_mode == "always":
-        rebuild_prog = preload or not helpers.has_debug_symbols(
-            debug_config.program_path
-        )
-    elif load_mode == "modified":
-        rebuild_prog = helpers.is_prog_obsolete(
-            debug_config.program_path
-        ) or not helpers.has_debug_symbols(debug_config.program_path)
-
-    if not (debug_config.program_path and os.path.isfile(debug_config.program_path)):
-        rebuild_prog = True
-
-    if preload or (not rebuild_prog and load_mode != "always"):
-        # don't load firmware through debug server
-        debug_config.load_cmds = []
-
-    if rebuild_prog:
-        if helpers.is_gdbmi_mode():
-            click.echo(
-                helpers.escape_gdbmi_stream(
-                    "~", "Preparing firmware for debugging...\n"
-                ),
-                nl=False,
-            )
-            stream = helpers.GDBMIConsoleStream()
-            with proc.capture_std_streams(stream):
-                helpers.predebug_project(
-                    ctx, project_dir, project_config, env_name, preload, verbose
-                )
-            stream.close()
-        else:
-            click.echo("Preparing firmware for debugging...")
-            helpers.predebug_project(
-                ctx, project_dir, project_config, env_name, preload, verbose
-            )
-
-        # save SHA sum of newly created prog
-        if load_mode == "modified":
-            helpers.is_prog_obsolete(debug_config.program_path)
-
-    if not os.path.isfile(debug_config.program_path):
-        raise DebugInvalidOptionsError("Program/firmware is missed")
-
-    loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop()
-    asyncio.set_event_loop(loop)
-
-    with fs.cd(project_dir):
-        client = GDBClientProcess(project_dir, debug_config)
-        coro = client.run(__unprocessed)
-        try:
-            signal.signal(signal.SIGINT, signal.SIG_IGN)
-            loop.run_until_complete(coro)
-            if IS_WINDOWS:
-                # an issue with `asyncio` executor and STIDIN,
-                # it cannot be closed gracefully
-                proc.force_exit()
-        finally:
-            del client
-            loop.close()
-
-    return True

platformio/commands/device/__init__.py

@@ -12,4 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio.commands.device.filters.base import DeviceMonitorFilter
+# pylint: disable=unused-import
+from platformio.device.filters.base import (
+    DeviceMonitorFilterBase as DeviceMonitorFilter,
+)

platformio/commands/device/command.py

@@ -12,225 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
-import os
-import sys
-from fnmatch import fnmatch
-
 import click
-from serial.tools import miniterm
 
-from platformio import exception, fs, util
+from platformio.device.commands.list import device_list_cmd
-from platformio.commands.device import helpers as device_helpers
+from platformio.device.commands.monitor import device_monitor_cmd
-from platformio.platform.factory import PlatformFactory
-from platformio.project.exception import NotPlatformIOProjectError
 
 
-@click.group(short_help="Device manager & serial/socket monitor")
+@click.group(
+    "device",
+    commands=[
+        device_list_cmd,
+        device_monitor_cmd,
+    ],
+    short_help="Device manager & Serial/Socket monitor",
+)
 def cli():
     pass
-
-
-@cli.command("list", short_help="List devices")
-@click.option("--serial", is_flag=True, help="List serial ports, default")
-@click.option("--logical", is_flag=True, help="List logical devices")
-@click.option("--mdns", is_flag=True, help="List multicast DNS services")
-@click.option("--json-output", is_flag=True)
-def device_list(  # pylint: disable=too-many-branches
-    serial, logical, mdns, json_output
-):
-    if not logical and not mdns:
-        serial = True
-    data = {}
-    if serial:
-        data["serial"] = util.get_serial_ports()
-    if logical:
-        data["logical"] = util.get_logical_devices()
-    if mdns:
-        data["mdns"] = util.get_mdns_services()
-
-    single_key = list(data)[0] if len(list(data)) == 1 else None
-
-    if json_output:
-        return click.echo(json.dumps(data[single_key] if single_key else data))
-
-    titles = {
-        "serial": "Serial Ports",
-        "logical": "Logical Devices",
-        "mdns": "Multicast DNS Services",
-    }
-
-    for key, value in data.items():
-        if not single_key:
-            click.secho(titles[key], bold=True)
-            click.echo("=" * len(titles[key]))
-
-        if key == "serial":
-            for item in value:
-                click.secho(item["port"], fg="cyan")
-                click.echo("-" * len(item["port"]))
-                click.echo("Hardware ID: %s" % item["hwid"])
-                click.echo("Description: %s" % item["description"])
-                click.echo("")
-
-        if key == "logical":
-            for item in value:
-                click.secho(item["path"], fg="cyan")
-                click.echo("-" * len(item["path"]))
-                click.echo("Name: %s" % item["name"])
-                click.echo("")
-
-        if key == "mdns":
-            for item in value:
-                click.secho(item["name"], fg="cyan")
-                click.echo("-" * len(item["name"]))
-                click.echo("Type: %s" % item["type"])
-                click.echo("IP: %s" % item["ip"])
-                click.echo("Port: %s" % item["port"])
-                if item["properties"]:
-                    click.echo(
-                        "Properties: %s"
-                        % (
-                            "; ".join(
-                                [
-                                    "%s=%s" % (k, v)
-                                    for k, v in item["properties"].items()
-                                ]
-                            )
-                        )
-                    )
-                click.echo("")
-
-        if single_key:
-            click.echo("")
-
-    return True
-
-
-@cli.command("monitor", short_help="Monitor device (Serial)")
-@click.option("--port", "-p", help="Port, a number or a device name")
-@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
-@click.option(
-    "--parity",
-    default="N",
-    type=click.Choice(["N", "E", "O", "S", "M"]),
-    help="Set parity, default=N",
-)
-@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
-@click.option(
-    "--xonxoff", is_flag=True, help="Enable software flow control, default=Off"
-)
-@click.option(
-    "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state"
-)
-@click.option(
-    "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state"
-)
-@click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
-@click.option(
-    "--encoding",
-    default="UTF-8",
-    help="Set the encoding for the serial port (e.g. hexlify, "
-    "Latin1, UTF-8), default: UTF-8",
-)
-@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations")
-@click.option(
-    "--eol",
-    default="CRLF",
-    type=click.Choice(["CR", "LF", "CRLF"]),
-    help="End of line mode, default=CRLF",
-)
-@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations")
-@click.option(
-    "--exit-char",
-    type=int,
-    default=3,
-    help="ASCII code of special character that is used to exit "
-    "the application, default=3 (Ctrl+C)",
-)
-@click.option(
-    "--menu-char",
-    type=int,
-    default=20,
-    help="ASCII code of special character that is used to "
-    "control miniterm (menu), default=20 (DEC)",
-)
-@click.option(
-    "--quiet",
-    is_flag=True,
-    help="Diagnostics: suppress non-error messages, default=Off",
-)
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
-)
-@click.option(
-    "-e",
-    "--environment",
-    help="Load configuration from `platformio.ini` and specified environment",
-)
-def device_monitor(**kwargs):  # pylint: disable=too-many-branches
-    project_options = {}
-    platform = None
-    try:
-        with fs.cd(kwargs["project_dir"]):
-            project_options = device_helpers.get_project_options(kwargs["environment"])
-            kwargs = device_helpers.apply_project_monitor_options(
-                kwargs, project_options
-            )
-            if "platform" in project_options:
-                platform = PlatformFactory.new(project_options["platform"])
-    except NotPlatformIOProjectError:
-        pass
-
-    with fs.cd(kwargs["project_dir"]):
-        device_helpers.register_filters(platform=platform, options=kwargs)
-
-    if not kwargs["port"]:
-        ports = util.get_serial_ports(filter_hwid=True)
-        if len(ports) == 1:
-            kwargs["port"] = ports[0]["port"]
-        elif "platform" in project_options and "board" in project_options:
-            board_hwids = device_helpers.get_board_hwids(
-                kwargs["project_dir"],
-                platform,
-                project_options["board"],
-            )
-            for item in ports:
-                for hwid in board_hwids:
-                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                    if hwid_str in item["hwid"]:
-                        kwargs["port"] = item["port"]
-                        break
-                if kwargs["port"]:
-                    break
-    elif kwargs["port"] and (set(["*", "?", "[", "]"]) & set(kwargs["port"])):
-        for item in util.get_serial_ports():
-            if fnmatch(item["port"], kwargs["port"]):
-                kwargs["port"] = item["port"]
-                break
-
-    # override system argv with patched options
-    sys.argv = ["monitor"] + device_helpers.options_to_argv(
-        kwargs,
-        project_options,
-        ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"),
-    )
-
-    if not kwargs["quiet"]:
-        click.echo(
-            "--- Available filters and text transformations: %s"
-            % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys()))
-        )
-        click.echo("--- More details at https://bit.ly/pio-monitor-filters")
-    try:
-        miniterm.main(
-            default_port=kwargs["port"],
-            default_baudrate=kwargs["baud"] or 9600,
-            default_rts=kwargs["rts"],
-            default_dtr=kwargs["dtr"],
-        )
-    except Exception as e:
-        raise exception.MinitermException(e)

platformio/commands/device/filters/base.py

@@ -1,43 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from serial.tools import miniterm
-
-from platformio.project.config import ProjectConfig
-
-
-class DeviceMonitorFilter(miniterm.Transform):
-    def __init__(self, options=None):
-        """Called by PlatformIO to pass context"""
-        miniterm.Transform.__init__(self)
-
-        self.options = options or {}
-        self.project_dir = self.options.get("project_dir")
-        self.environment = self.options.get("environment")
-
-        self.config = ProjectConfig.get_instance()
-        if not self.environment:
-            default_envs = self.config.default_envs()
-            if default_envs:
-                self.environment = default_envs[0]
-            elif self.config.envs():
-                self.environment = self.config.envs()[0]
-
-    def __call__(self):
-        """Called by the miniterm library when the filter is actually used"""
-        return self
-
-    @property
-    def NAME(self):
-        raise NotImplementedError("Please declare NAME attribute for the filter class")

platformio/commands/home/helpers.py

@@ -26,7 +26,7 @@ class AsyncSession(requests.Session):
     async def request(  # pylint: disable=signature-differs,invalid-overridden-method
         self, *args, **kwargs
     ):
-        func = super(AsyncSession, self).request
+        func = super().request
         return await run_in_threadpool(func, *args, **kwargs)
 
 

platformio/commands/home/rpc/handlers/os.py

@@ -22,10 +22,11 @@ from functools import cmp_to_key
 
 import click
 
-from platformio import __default_requests_timeout__, fs, util
+from platformio import __default_requests_timeout__, fs
 from platformio.cache import ContentCache
 from platformio.clients.http import ensure_internet_on
 from platformio.commands.home import helpers
+from platformio.device.list import list_logical_devices
 
 
 class OSRPC:
@@ -154,7 +155,7 @@ class OSRPC:
     @staticmethod
     def get_logical_devices():
         items = []
-        for item in util.get_logical_devices():
+        for item in list_logical_devices():
             if item["name"]:
                 item["name"] = item["name"]
             items.append(item)

platformio/commands/home/rpc/handlers/piocore.py

@@ -14,10 +14,11 @@
 
 from __future__ import absolute_import
 
+import io
 import json
 import os
 import sys
-from io import StringIO
+import threading
 
 import click
 from ajsonrpc.core import JSONRPC20DispatchException
@@ -27,27 +28,22 @@ from platformio import __main__, __version__, fs, proc
 from platformio.commands.home import helpers
 from platformio.compat import get_locale_encoding, is_bytes
 
-try:
-    from thread import get_ident as thread_get_ident
-except ImportError:
-    from threading import get_ident as thread_get_ident
-
 
 class MultiThreadingStdStream(object):
     def __init__(self, parent_stream):
-        self._buffers = {thread_get_ident(): parent_stream}
+        self._buffers = {threading.get_ident(): parent_stream}
 
     def __getattr__(self, name):
-        thread_id = thread_get_ident()
+        thread_id = threading.get_ident()
         self._ensure_thread_buffer(thread_id)
         return getattr(self._buffers[thread_id], name)
 
     def _ensure_thread_buffer(self, thread_id):
         if thread_id not in self._buffers:
-            self._buffers[thread_id] = StringIO()
+            self._buffers[thread_id] = io.StringIO()
 
     def write(self, value):
-        thread_id = thread_get_ident()
+        thread_id = threading.get_ident()
         self._ensure_thread_buffer(thread_id)
         return self._buffers[thread_id].write(
             value.decode() if is_bytes(value) else value

platformio/commands/lib/command.py

@@ -15,8 +15,10 @@
 # pylint: disable=too-many-branches, too-many-locals
 
 import json
+import logging
 import os
 import time
+from urllib.parse import quote
 
 import click
 from tabulate import tabulate
@@ -31,11 +33,6 @@ from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import get_project_dir, is_platformio_project
 
-try:
-    from urllib.parse import quote
-except ImportError:
-    from urllib import quote
-
 CTX_META_INPUT_DIRS_KEY = __name__ + ".input_dirs"
 CTX_META_PROJECT_ENVIRONMENTS_KEY = __name__ + ".project_environments"
 CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs"
@@ -46,7 +43,7 @@ def get_project_global_lib_dir():
     return ProjectConfig.get_instance().get("platformio", "globallib_dir")
 
 
-@click.group(short_help="Library manager")
+@click.group(short_help="Library manager", hidden=True)
 @click.option(
     "-d",
     "--storage-dir",
@@ -152,16 +149,16 @@ def lib_install(  # pylint: disable=too-many-arguments,unused-argument
         if not silent and (libraries or storage_dir in storage_libdeps):
             print_storage_header(storage_dirs, storage_dir)
         lm = LibraryPackageManager(storage_dir)
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
 
         if libraries:
             installed_pkgs = {
-                library: lm.install(library, silent=silent, force=force)
+                library: lm.install(library, force=force) for library in libraries
-                for library in libraries
             }
 
         elif storage_dir in storage_libdeps:
             for library in storage_libdeps[storage_dir]:
-                lm.install(library, silent=silent, force=force)
+                lm.install(library, force=force)
 
     if save and installed_pkgs:
         _save_deps(ctx, installed_pkgs)
@@ -212,9 +209,8 @@ def lib_uninstall(ctx, libraries, save, silent):
     for storage_dir in storage_dirs:
         print_storage_header(storage_dirs, storage_dir)
         lm = LibraryPackageManager(storage_dir)
-        uninstalled_pkgs = {
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
-            library: lm.uninstall(library, silent=silent) for library in libraries
+        uninstalled_pkgs = {library: lm.uninstall(library) for library in libraries}
-        }
 
     if save and uninstalled_pkgs:
         _save_deps(ctx, uninstalled_pkgs, action="remove")
@@ -237,14 +233,20 @@ def lib_uninstall(ctx, libraries, save, silent):
 def lib_update(  # pylint: disable=too-many-arguments
     ctx, libraries, only_check, dry_run, silent, json_output
 ):
-    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     only_check = dry_run or only_check
+    if only_check and not json_output:
+        raise exception.UserSideException(
+            "This command is deprecated, please use `pio pkg outdated` instead"
+        )
+
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     json_result = {}
     for storage_dir in storage_dirs:
         if not json_output:
             print_storage_header(storage_dirs, storage_dir)
         lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, [])
         lm = LibraryPackageManager(storage_dir)
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
         _libraries = libraries or lib_deps or lm.get_installed()
 
         if only_check and json_output:
@@ -277,9 +279,7 @@ def lib_update(  # pylint: disable=too-many-arguments
                     None if isinstance(library, PackageItem) else PackageSpec(library)
                 )
                 try:
-                    lm.update(
+                    lm.update(library, to_spec=to_spec)
-                        library, to_spec=to_spec, only_check=only_check, silent=silent
-                    )
                 except UnknownPackageError as e:
                     if library not in lib_deps:
                         raise e
@@ -438,7 +438,8 @@ def lib_builtin(storage, json_output):
 @click.option("--json-output", is_flag=True)
 def lib_show(library, json_output):
     lm = LibraryPackageManager()
-    lib_id = lm.reveal_registry_package_id(library, silent=json_output)
+    lm.set_log_level(logging.ERROR if json_output else logging.DEBUG)
+    lib_id = lm.reveal_registry_package_id(library)
     regclient = lm.get_registry_client_instance()
     lib = regclient.fetch_json_data(
         "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h"
@@ -457,7 +458,7 @@ def lib_show(library, json_output):
         "Version: %s, released %s"
         % (
             lib["version"]["name"],
-            time.strftime("%c", util.parse_date(lib["version"]["released"])),
+            util.parse_datetime(lib["version"]["released"]).strftime("%c"),
         )
     )
     click.echo("Manifest: %s" % lib["confurl"])
@@ -465,9 +466,9 @@ def lib_show(library, json_output):
         if key not in lib or not lib[key]:
             continue
         if isinstance(lib[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(lib[key])))
+            click.echo("%s: %s" % (key.capitalize(), ", ".join(lib[key])))
         else:
-            click.echo("%s: %s" % (key.title(), lib[key]))
+            click.echo("%s: %s" % (key.capitalize(), lib[key]))
 
     blocks = []
 
@@ -499,7 +500,7 @@ def lib_show(library, json_output):
             "Versions",
             [
                 "%s, released %s"
-                % (v["name"], time.strftime("%c", util.parse_date(v["released"])))
+                % (v["name"], util.parse_datetime(v["released"]).strftime("%c"))
                 for v in lib["versions"]
             ],
         )
@@ -529,7 +530,7 @@ def lib_show(library, json_output):
 @click.argument("config_url")
 def lib_register(config_url):  # pylint: disable=unused-argument
     raise exception.UserSideException(
-        "This command is deprecated. Please use `pio package publish` command."
+        "This command is deprecated. Please use `pio pkg publish` command."
     )
 
 
@@ -546,7 +547,7 @@ def lib_stats(json_output):
         tabular_data = [
             (
                 click.style(item["name"], fg="cyan"),
-                time.strftime("%c", util.parse_date(item["date"])),
+                util.parse_datetime(item["date"]).strftime("%c"),
                 "https://platformio.org/lib/show/%s/%s"
                 % (item["id"], quote(item["name"])),
             )
@@ -621,9 +622,9 @@ def print_lib_item(item):
         if key not in item or not item[key]:
             continue
         if isinstance(item[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
+            click.echo("%s: %s" % (key.capitalize(), ", ".join(item[key])))
         else:
-            click.echo("%s: %s" % (key.title(), item[key]))
+            click.echo("%s: %s" % (key.capitalize(), item[key]))
 
     for key in ("frameworks", "platforms"):
         if key not in item:

platformio/commands/lib/helpers.py

@@ -14,6 +14,7 @@
 
 import os
 
+from platformio import util
 from platformio.compat import ci_strings_are_equal
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.meta import PackageSpec
@@ -22,6 +23,7 @@ from platformio.project.config import ProjectConfig
 from platformio.project.exception import InvalidProjectConfError
 
 
+@util.memoized(expire="60s")
 def get_builtin_libs(storage_names=None):
     # pylint: disable=import-outside-toplevel
     from platformio.package.manager.library import LibraryPackageManager
@@ -45,8 +47,8 @@ def get_builtin_libs(storage_names=None):
     return items
 
 
-def is_builtin_lib(name, storages=None):
+def is_builtin_lib(name):
-    for storage in storages or get_builtin_libs():
+    for storage in get_builtin_libs():
         for lib in storage["items"]:
             if lib.get("name") == name:
                 return True

platformio/commands/pkg.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.package.commands.exec import package_exec_cmd
+from platformio.package.commands.install import package_install_cmd
+from platformio.package.commands.list import package_list_cmd
+from platformio.package.commands.outdated import package_outdated_cmd
+from platformio.package.commands.pack import package_pack_cmd
+from platformio.package.commands.publish import package_publish_cmd
+from platformio.package.commands.search import package_search_cmd
+from platformio.package.commands.show import package_show_cmd
+from platformio.package.commands.uninstall import package_uninstall_cmd
+from platformio.package.commands.unpublish import package_unpublish_cmd
+from platformio.package.commands.update import package_update_cmd
+
+
+@click.group(
+    "pkg",
+    commands=[
+        package_exec_cmd,
+        package_install_cmd,
+        package_list_cmd,
+        package_outdated_cmd,
+        package_pack_cmd,
+        package_publish_cmd,
+        package_search_cmd,
+        package_show_cmd,
+        package_uninstall_cmd,
+        package_unpublish_cmd,
+        package_update_cmd,
+    ],
+    short_help="Unified Package Manager",
+)
+def cli():
+    pass

platformio/commands/platform.py

@@ -13,12 +13,14 @@
 # limitations under the License.
 
 import json
+import logging
 import os
 
 import click
 
-from platformio.cache import cleanup_content_cache
 from platformio.commands.boards import print_boards
+from platformio.exception import UserSideException
+from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.meta import PackageItem, PackageSpec
 from platformio.package.version import get_original_version
@@ -26,7 +28,7 @@ from platformio.platform.exception import UnknownPlatform
 from platformio.platform.factory import PlatformFactory
 
 
-@click.group(short_help="Platform manager")
+@click.group(short_help="Platform manager", hidden=True)
 def cli():
     pass
 
@@ -178,7 +180,7 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     is_flag=True,
     help="Reinstall/redownload dev/platform and its packages if exist",
 )
-def platform_install(  # pylint: disable=too-many-arguments
+def platform_install(  # pylint: disable=too-many-arguments,too-many-locals
     platforms,
     with_package,
     without_package,
@@ -187,37 +189,50 @@ def platform_install(  # pylint: disable=too-many-arguments
     silent,
     force,
 ):
-    return _platform_install(
+    def _find_pkg_names(p, candidates):
-        platforms,
+        result = []
-        with_package,
+        for candidate in candidates:
-        without_package,
+            found = False
-        skip_default_package,
+            # lookup by package types
-        with_all_packages,
+            for _name, _opts in p.packages.items():
-        silent,
+                if _opts.get("type") == candidate:
-        force,
+                    result.append(_name)
-    )
+                    found = True
+            if (
+                p.frameworks
+                and candidate.startswith("framework-")
+                and candidate[10:] in p.frameworks
+            ):
+                result.append(p.frameworks[candidate[10:]]["package"])
+                found = True
+            if not found:
+                result.append(candidate)
+        return result
 
-
-def _platform_install(  # pylint: disable=too-many-arguments
-    platforms,
-    with_package=None,
-    without_package=None,
-    skip_default_package=False,
-    with_all_packages=False,
-    silent=False,
-    force=False,
-):
     pm = PlatformPackageManager()
+    pm.set_log_level(logging.WARN if silent else logging.DEBUG)
     for platform in platforms:
-        pkg = pm.install(
+        if with_package or without_package or with_all_packages:
-            spec=platform,
+            pkg = pm.install(platform, skip_dependencies=True)
-            with_packages=with_package or [],
+            p = PlatformFactory.new(pkg)
-            without_packages=without_package or [],
+            if with_all_packages:
-            skip_default_package=skip_default_package,
+                with_package = list(p.packages)
-            with_all_packages=with_all_packages,
+            with_package = set(_find_pkg_names(p, with_package or []))
-            silent=silent,
+            without_package = set(_find_pkg_names(p, without_package or []))
-            force=force,
+            upkgs = with_package | without_package
-        )
+            ppkgs = set(p.packages)
+            if not upkgs.issubset(ppkgs):
+                raise UnknownPackageError(", ".join(upkgs - ppkgs))
+            for name, options in p.packages.items():
+                if name in without_package:
+                    continue
+                if name in with_package or not (
+                    skip_default_package or options.get("optional", False)
+                ):
+                    p.pm.install(p.get_package_spec(name), force=force)
+        else:
+            pkg = pm.install(platform, skip_dependencies=skip_default_package)
+
         if pkg and not silent:
             click.secho(
                 "The platform '%s' has been successfully installed!\n"
@@ -231,6 +246,7 @@ def _platform_install(  # pylint: disable=too-many-arguments
 @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]")
 def platform_uninstall(platforms):
     pm = PlatformPackageManager()
+    pm.set_log_level(logging.DEBUG)
     for platform in platforms:
         if pm.uninstall(platform):
             click.secho(
@@ -256,9 +272,15 @@ def platform_uninstall(platforms):
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option("--json-output", is_flag=True)
 def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
-    platforms, only_packages, only_check, dry_run, silent, json_output
+    platforms, only_check, dry_run, silent, json_output, **_
 ):
+    if only_check and not json_output:
+        raise UserSideException(
+            "This command is deprecated, please use `pio pkg outdated` instead"
+        )
+
     pm = PlatformPackageManager()
+    pm.set_log_level(logging.WARN if silent else logging.DEBUG)
     platforms = platforms or pm.get_installed()
     only_check = dry_run or only_check
 
@@ -290,9 +312,6 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
             result.append(data)
         return click.echo(json.dumps(result))
 
-    # cleanup cached board and platform lists
-    cleanup_content_cache("http")
-
     for platform in platforms:
         click.echo(
             "Platform %s"
@@ -304,9 +323,7 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
             )
         )
         click.echo("--------")
-        pm.update(
+        pm.update(platform)
-            platform, only_packages=only_packages, only_check=only_check, silent=silent
-        )
         click.echo()
 
     return True
@@ -317,15 +334,6 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
 #
 
 
-def init_platform(name, skip_default_package=True, auto_install=True):
-    try:
-        return PlatformFactory.new(name)
-    except UnknownPlatform:
-        if auto_install:
-            _platform_install([name], skip_default_package=skip_default_package)
-        return PlatformFactory.new(name)
-
-
 def _print_platforms(platforms):
     for platform in platforms:
         click.echo(

platformio/commands/project.py

@@ -12,429 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,line-too-long
-
-import json
-import os
-
 import click
-from tabulate import tabulate
 
-from platformio import fs
+from platformio.project.commands.config import project_config_cmd
-from platformio.commands.platform import platform_install as cli_platform_install
+from platformio.project.commands.init import project_init_cmd
-from platformio.package.manager.platform import PlatformPackageManager
+from platformio.project.commands.metadata import project_metadata_cmd
-from platformio.platform.exception import UnknownBoard
-from platformio.project.config import ProjectConfig
-from platformio.project.exception import NotPlatformIOProjectError
-from platformio.project.generator import ProjectGenerator
-from platformio.project.helpers import is_platformio_project, load_project_ide_data
 
 
-@click.group(short_help="Project manager")
+@click.group(
+    "project",
+    commands=[
+        project_config_cmd,
+        project_init_cmd,
+        project_metadata_cmd,
+    ],
+    short_help="Project Manager",
+)
 def cli():
     pass
-
-
-@cli.command("config", short_help="Show computed configuration")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
-)
-@click.option("--json-output", is_flag=True)
-def project_config(project_dir, json_output):
-    if not is_platformio_project(project_dir):
-        raise NotPlatformIOProjectError(project_dir)
-    with fs.cd(project_dir):
-        config = ProjectConfig.get_instance()
-    if json_output:
-        return click.echo(config.to_json())
-    click.echo(
-        "Computed project configuration for %s" % click.style(project_dir, fg="cyan")
-    )
-    for section, options in config.as_tuple():
-        click.secho(section, fg="cyan")
-        click.echo("-" * len(section))
-        click.echo(
-            tabulate(
-                [
-                    (name, "=", "\n".join(value) if isinstance(value, list) else value)
-                    for name, value in options
-                ],
-                tablefmt="plain",
-            )
-        )
-        click.echo()
-    return None
-
-
-@cli.command("data", short_help="Dump data intended for IDE extensions/plugins")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
-)
-@click.option("-e", "--environment", multiple=True)
-@click.option("--json-output", is_flag=True)
-def project_data(project_dir, environment, json_output):
-    if not is_platformio_project(project_dir):
-        raise NotPlatformIOProjectError(project_dir)
-    with fs.cd(project_dir):
-        config = ProjectConfig.get_instance()
-        config.validate(environment)
-        environment = list(environment or config.envs())
-
-    if json_output:
-        return click.echo(json.dumps(load_project_ide_data(project_dir, environment)))
-
-    for envname in environment:
-        click.echo("Environment: " + click.style(envname, fg="cyan", bold=True))
-        click.echo("=" * (13 + len(envname)))
-        click.echo(
-            tabulate(
-                [
-                    (click.style(name, bold=True), "=", json.dumps(value, indent=2))
-                    for name, value in load_project_ide_data(
-                        project_dir, envname
-                    ).items()
-                ],
-                tablefmt="plain",
-            )
-        )
-        click.echo()
-
-    return None
-
-
-def validate_boards(ctx, param, value):  # pylint: disable=W0613
-    pm = PlatformPackageManager()
-    for id_ in value:
-        try:
-            pm.board_config(id_)
-        except UnknownBoard:
-            raise click.BadParameter(
-                "`%s`. Please search for board ID using `platformio boards` "
-                "command" % id_
-            )
-    return value
-
-
-@cli.command("init", short_help="Initialize a project or update existing")
-@click.option(
-    "--project-dir",
-    "-d",
-    default=os.getcwd,
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
-)
-@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
-@click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides()))
-@click.option("-e", "--environment", help="Update using existing environment")
-@click.option("-O", "--project-option", multiple=True)
-@click.option("--env-prefix", default="")
-@click.option("-s", "--silent", is_flag=True)
-@click.pass_context
-def project_init(
-    ctx,  # pylint: disable=R0913
-    project_dir,
-    board,
-    ide,
-    environment,
-    project_option,
-    env_prefix,
-    silent,
-):
-    if not silent:
-        if project_dir == os.getcwd():
-            click.secho("\nThe current working directory ", fg="yellow", nl=False)
-            try:
-                click.secho(project_dir, fg="cyan", nl=False)
-            except UnicodeEncodeError:
-                click.secho(json.dumps(project_dir), fg="cyan", nl=False)
-            click.secho(" will be used for the project.", fg="yellow")
-            click.echo("")
-
-        click.echo("The next files/directories have been created in ", nl=False)
-        try:
-            click.secho(project_dir, fg="cyan")
-        except UnicodeEncodeError:
-            click.secho(json.dumps(project_dir), fg="cyan")
-        click.echo(
-            "%s - Put project header files here" % click.style("include", fg="cyan")
-        )
-        click.echo(
-            "%s - Put here project specific (private) libraries"
-            % click.style("lib", fg="cyan")
-        )
-        click.echo("%s - Put project source files here" % click.style("src", fg="cyan"))
-        click.echo(
-            "%s - Project Configuration File" % click.style("platformio.ini", fg="cyan")
-        )
-
-    is_new_project = not is_platformio_project(project_dir)
-    if is_new_project:
-        init_base_project(project_dir)
-
-    if environment:
-        update_project_env(project_dir, environment, project_option)
-    elif board:
-        update_board_envs(
-            ctx, project_dir, board, project_option, env_prefix, ide is not None
-        )
-
-    if ide:
-        with fs.cd(project_dir):
-            config = ProjectConfig.get_instance(
-                os.path.join(project_dir, "platformio.ini")
-            )
-        config.validate()
-        ProjectGenerator(config, environment, ide, board).generate()
-
-    if is_new_project:
-        init_cvs_ignore(project_dir)
-
-    if silent:
-        return
-
-    if ide:
-        click.secho(
-            "\nProject has been successfully %s including configuration files "
-            "for `%s` IDE." % ("initialized" if is_new_project else "updated", ide),
-            fg="green",
-        )
-    else:
-        click.secho(
-            "\nProject has been successfully %s! Useful commands:\n"
-            "`pio run` - process/build project from the current directory\n"
-            "`pio run --target upload` or `pio run -t upload` "
-            "- upload firmware to a target\n"
-            "`pio run --target clean` - clean project (remove compiled files)"
-            "\n`pio run --help` - additional information"
-            % ("initialized" if is_new_project else "updated"),
-            fg="green",
-        )
-
-
-def init_base_project(project_dir):
-    with fs.cd(project_dir):
-        config = ProjectConfig()
-        config.save()
-        dir_to_readme = [
-            (config.get("platformio", "src_dir"), None),
-            (config.get("platformio", "include_dir"), init_include_readme),
-            (config.get("platformio", "lib_dir"), init_lib_readme),
-            (config.get("platformio", "test_dir"), init_test_readme),
-        ]
-        for (path, cb) in dir_to_readme:
-            if os.path.isdir(path):
-                continue
-            os.makedirs(path)
-            if cb:
-                cb(path)
-
-
-def init_include_readme(include_dir):
-    with open(os.path.join(include_dir, "README"), mode="w", encoding="utf8") as fp:
-        fp.write(
-            """
-This directory is intended for project header files.
-
-A header file is a file containing C declarations and macro definitions
-to be shared between several project source files. You request the use of a
-header file in your project source file (C, C++, etc) located in `src` folder
-by including it, with the C preprocessing directive `#include'.
-
-```src/main.c
-
-#include "header.h"
-
-int main (void)
-{
- ...
-}
-```
-
-Including a header file produces the same results as copying the header file
-into each source file that needs it. Such copying would be time-consuming
-and error-prone. With a header file, the related declarations appear
-in only one place. If they need to be changed, they can be changed in one
-place, and programs that include the header file will automatically use the
-new version when next recompiled. The header file eliminates the labor of
-finding and changing all the copies as well as the risk that a failure to
-find one copy will result in inconsistencies within a program.
-
-In C, the usual convention is to give header files names that end with `.h'.
-It is most portable to use only letters, digits, dashes, and underscores in
-header file names, and at most one dot.
-
-Read more about using header files in official GCC documentation:
-
-* Include Syntax
-* Include Operation
-* Once-Only Headers
-* Computed Includes
-
-https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html
-""",
-        )
-
-
-def init_lib_readme(lib_dir):
-    with open(os.path.join(lib_dir, "README"), mode="w", encoding="utf8") as fp:
-        fp.write(
-            """
-This directory is intended for project specific (private) libraries.
-PlatformIO will compile them to static libraries and link into executable file.
-
-The source code of each library should be placed in a an own separate directory
-("lib/your_library_name/[here are source files]").
-
-For example, see a structure of the following two libraries `Foo` and `Bar`:
-
-|--lib
-|  |
-|  |--Bar
-|  |  |--docs
-|  |  |--examples
-|  |  |--src
-|  |     |- Bar.c
-|  |     |- Bar.h
-|  |  |- library.json (optional, custom build options, etc) https://docs.platformio.org/page/librarymanager/config.html
-|  |
-|  |--Foo
-|  |  |- Foo.c
-|  |  |- Foo.h
-|  |
-|  |- README --> THIS FILE
-|
-|- platformio.ini
-|--src
-   |- main.c
-
-and a contents of `src/main.c`:
-```
-#include <Foo.h>
-#include <Bar.h>
-
-int main (void)
-{
-  ...
-}
-
-```
-
-PlatformIO Library Dependency Finder will find automatically dependent
-libraries scanning project source files.
-
-More information about PlatformIO Library Dependency Finder
-- https://docs.platformio.org/page/librarymanager/ldf.html
-""",
-        )
-
-
-def init_test_readme(test_dir):
-    with open(os.path.join(test_dir, "README"), mode="w", encoding="utf8") as fp:
-        fp.write(
-            """
-This directory is intended for PlatformIO Unit Testing and project tests.
-
-Unit Testing is a software testing method by which individual units of
-source code, sets of one or more MCU program modules together with associated
-control data, usage procedures, and operating procedures, are tested to
-determine whether they are fit for use. Unit testing finds problems early
-in the development cycle.
-
-More information about PlatformIO Unit Testing:
-- https://docs.platformio.org/page/plus/unit-testing.html
-""",
-        )
-
-
-def init_cvs_ignore(project_dir):
-    conf_path = os.path.join(project_dir, ".gitignore")
-    if os.path.isfile(conf_path):
-        return
-    with open(conf_path, mode="w", encoding="utf8") as fp:
-        fp.write(".pio\n")
-
-
-def update_board_envs(
-    ctx, project_dir, board_ids, project_option, env_prefix, force_download
-):
-    config = ProjectConfig(
-        os.path.join(project_dir, "platformio.ini"), parse_extra=False
-    )
-    used_boards = []
-    for section in config.sections():
-        cond = [section.startswith("env:"), config.has_option(section, "board")]
-        if all(cond):
-            used_boards.append(config.get(section, "board"))
-
-    pm = PlatformPackageManager()
-    used_platforms = []
-    modified = False
-    for id_ in board_ids:
-        board_config = pm.board_config(id_)
-        used_platforms.append(board_config["platform"])
-        if id_ in used_boards:
-            continue
-        used_boards.append(id_)
-        modified = True
-
-        envopts = {"platform": board_config["platform"], "board": id_}
-        # find default framework for board
-        frameworks = board_config.get("frameworks")
-        if frameworks:
-            envopts["framework"] = frameworks[0]
-
-        for item in project_option:
-            if "=" not in item:
-                continue
-            _name, _value = item.split("=", 1)
-            envopts[_name.strip()] = _value.strip()
-
-        section = "env:%s%s" % (env_prefix, id_)
-        config.add_section(section)
-
-        for option, value in envopts.items():
-            config.set(section, option, value)
-
-    if force_download and used_platforms:
-        _install_dependent_platforms(ctx, used_platforms)
-
-    if modified:
-        config.save()
-
-
-def _install_dependent_platforms(ctx, platforms):
-    installed_platforms = [
-        pkg.metadata.name for pkg in PlatformPackageManager().get_installed()
-    ]
-    if set(platforms) <= set(installed_platforms):
-        return
-    ctx.invoke(
-        cli_platform_install, platforms=list(set(platforms) - set(installed_platforms))
-    )
-
-
-def update_project_env(project_dir, environment, project_option):
-    if not project_option:
-        return
-    config = ProjectConfig(
-        os.path.join(project_dir, "platformio.ini"), parse_extra=False
-    )
-
-    section = "env:%s" % environment
-    if not config.has_section(section):
-        config.add_section(section)
-
-    for item in project_option:
-        if "=" not in item:
-            continue
-        _name, _value = item.split("=", 1)
-        config.set(section, _name.strip(), _value.strip())
-
-    config.save()

platformio/commands/remote/ac/psync.py

@@ -25,7 +25,7 @@ class ProjectSyncAsyncCmd(AsyncCommandBase):
     def __init__(self, *args, **kwargs):
         self.psync = None
         self._upstream = None
-        super(ProjectSyncAsyncCmd, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def start(self):
         project_dir = os.path.join(

platformio/commands/remote/client/agent_service.py

@@ -17,11 +17,12 @@ import os
 from twisted.logger import LogLevel  # pylint: disable=import-error
 from twisted.spread import pb  # pylint: disable=import-error
 
-from platformio import proc, util
+from platformio import proc
 from platformio.commands.remote.ac.process import ProcessAsyncCmd
 from platformio.commands.remote.ac.psync import ProjectSyncAsyncCmd
 from platformio.commands.remote.ac.serial import SerialPortAsyncCmd
 from platformio.commands.remote.client.base import RemoteClientBase
+from platformio.device.list import list_serial_ports
 from platformio.project.config import ProjectConfig
 from platformio.project.exception import NotPlatformIOProjectError
 
@@ -84,11 +85,11 @@ class RemoteAgentService(RemoteClientBase):
         return (self.id, ac.id)
 
     def _process_cmd_device_list(self, _):
-        return (self.name, util.get_serialports())
+        return (self.name, list_serial_ports())
 
     def _process_cmd_device_monitor(self, options):
         if not options["port"]:
-            for item in util.get_serialports():
+            for item in list_serial_ports():
                 if "VID:PID" in item["hwid"]:
                     options["port"] = item["port"]
                     break

platformio/commands/remote/command.py

@@ -24,15 +24,20 @@ from time import sleep
 import click
 
 from platformio import fs, proc
-from platformio.commands.device import helpers as device_helpers
-from platformio.commands.device.command import device_monitor as cmd_device_monitor
 from platformio.commands.run.command import cli as cmd_run
-from platformio.commands.test.command import cli as cmd_test
+from platformio.device.commands.monitor import (
+    apply_project_monitor_options,
+    device_monitor_cmd,
+    get_project_options,
+    project_options_to_monitor_argv,
+)
 from platformio.package.manager.core import inject_contrib_pysite
 from platformio.project.exception import NotPlatformIOProjectError
+from platformio.project.options import ProjectOptions
+from platformio.test.command import test_cmd
 
 
-@click.group("remote", short_help="Remote development")
+@click.group("remote", short_help="Remote Development")
 @click.option("-a", "--agent", multiple=True)
 @click.pass_context
 def cli(ctx, agent):
@@ -163,7 +168,20 @@ def remote_run(
 
 @cli.command("test", short_help="Remote Unit Testing")
 @click.option("--environment", "-e", multiple=True, metavar="<environment>")
-@click.option("--ignore", "-i", multiple=True, metavar="<pattern>")
+@click.option(
+    "--filter",
+    "-f",
+    multiple=True,
+    metavar="<pattern>",
+    help="Filter tests by a pattern",
+)
+@click.option(
+    "--ignore",
+    "-i",
+    multiple=True,
+    metavar="<pattern>",
+    help="Ignore tests by a pattern",
+)
 @click.option("--upload-port")
 @click.option("--test-port")
 @click.option(
@@ -180,10 +198,11 @@ def remote_run(
 @click.option("--verbose", "-v", is_flag=True)
 @click.pass_obj
 @click.pass_context
-def remote_test(
+def remote_test(  # pylint: disable=redefined-builtin
     ctx,
     agents,
     environment,
+    filter,
     ignore,
     upload_port,
     test_port,
@@ -201,6 +220,7 @@ def remote_test(
         agents,
         dict(
             environment=environment,
+            filter=filter,
             ignore=ignore,
             upload_port=upload_port,
             test_port=test_port,
@@ -217,8 +237,9 @@ def remote_test(
     click.secho("Building project locally", bold=True)
 
     ctx.invoke(
-        cmd_test,
+        test_cmd,
         environment=environment,
+        filter=filter,
         ignore=ignore,
         project_dir=project_dir,
         without_uploading=True,
@@ -249,7 +270,12 @@ def device_list(agents, json_output):
 
 @remote_device.command("monitor", short_help="Monitor remote device")
 @click.option("--port", "-p", help="Port, a number or a device name")
-@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
+@click.option(
+    "--baud",
+    "-b",
+    type=int,
+    help="Set baud rate, default=%d" % ProjectOptions["env.monitor_speed"].default,
+)
 @click.option(
     "--parity",
     default="N",
@@ -328,19 +354,19 @@ def device_monitor(ctx, agents, **kwargs):
     project_options = {}
     try:
         with fs.cd(kwargs["project_dir"]):
-            project_options = device_helpers.get_project_options(kwargs["environment"])
+            project_options = get_project_options(kwargs["environment"])
-        kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options)
+        kwargs = apply_project_monitor_options(kwargs, project_options)
     except NotPlatformIOProjectError:
         pass
 
-    kwargs["baud"] = kwargs["baud"] or 9600
+    kwargs["baud"] = kwargs["baud"] or ProjectOptions["env.monitor_speed"].default
 
     def _tx_target(sock_dir):
         subcmd_argv = ["remote"]
         for agent in agents:
             subcmd_argv.extend(["--agent", agent])
         subcmd_argv.extend(["device", "monitor"])
-        subcmd_argv.extend(device_helpers.options_to_argv(kwargs, project_options))
+        subcmd_argv.extend(project_options_to_monitor_argv(kwargs, project_options))
         subcmd_argv.extend(["--sock", sock_dir])
         subprocess.call([proc.where_is_program("platformio")] + subcmd_argv)
 
@@ -355,7 +381,7 @@ def device_monitor(ctx, agents, **kwargs):
             return
         with open(sock_file, encoding="utf8") as fp:
             kwargs["port"] = fp.read()
-        ctx.invoke(cmd_device_monitor, **kwargs)
+        ctx.invoke(device_monitor_cmd, **kwargs)
         t.join(2)
     finally:
         fs.rmtree(sock_dir)

platformio/commands/remote/factory/ssl.py

@@ -23,7 +23,7 @@ class SSLContextFactory(ssl.ClientContextFactory):
         self.certificate_verified = False
 
     def getContext(self):
-        ctx = super(SSLContextFactory, self).getContext()
+        ctx = super().getContext()
         ctx.set_verify(
             SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, self.verifyHostname
         )

platformio/commands/run/command.py

@@ -22,12 +22,12 @@ import click
 from tabulate import tabulate
 
 from platformio import app, exception, fs, util
-from platformio.commands.device.command import device_monitor as cmd_device_monitor
 from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libdeps
 from platformio.commands.run.processor import EnvironmentProcessor
-from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING
+from platformio.device.commands.monitor import device_monitor_cmd
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import find_project_dir_above, load_project_ide_data
+from platformio.project.helpers import find_project_dir_above, load_build_metadata
+from platformio.test.runners.base import CTX_META_TEST_IS_RUNNING
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
 
@@ -66,10 +66,17 @@ except NotImplementedError:
         "Default is a number of CPUs in a system (N=%d)" % DEFAULT_JOB_NUMS
     ),
 )
-@click.option("-s", "--silent", is_flag=True)
+@click.option(
-@click.option("-v", "--verbose", is_flag=True)
+    "-a",
+    "--program-arg",
+    "program_args",
+    multiple=True,
+    help="A program argument (multiple are allowed)",
+)
 @click.option("--disable-auto-clean", is_flag=True)
 @click.option("--list-targets", is_flag=True)
+@click.option("-s", "--silent", is_flag=True)
+@click.option("-v", "--verbose", is_flag=True)
 @click.pass_context
 def cli(
     ctx,
@@ -79,10 +86,11 @@ def cli(
     project_dir,
     project_conf,
     jobs,
-    silent,
+    program_args,
-    verbose,
     disable_auto_clean,
     list_targets,
+    silent,
+    verbose,
 ):
     app.set_session_var("custom_project_conf", project_conf)
 
@@ -92,6 +100,7 @@ def cli(
 
     is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta
 
+    results = []
     with fs.cd(project_dir):
         config = ProjectConfig.get_instance(project_conf)
         config.validate(environment)
@@ -114,7 +123,6 @@ def cli(
         handle_legacy_libdeps(project_dir, config)
 
         default_envs = config.default_envs()
-        results = []
         for env in config.envs():
             skipenv = any(
                 [
@@ -138,21 +146,25 @@ def cli(
                     environment,
                     target,
                     upload_port,
+                    jobs,
+                    program_args,
+                    is_test_running,
                     silent,
                     verbose,
-                    jobs,
-                    is_test_running,
                 )
             )
 
-        command_failed = any(r.get("succeeded") is False for r in results)
+    command_failed = any(r.get("succeeded") is False for r in results)
 
-        if not is_test_running and (command_failed or not silent) and len(results) > 1:
+    if not is_test_running and (command_failed or not silent) and len(results) > 1:
-            print_processing_summary(results, verbose)
+        print_processing_summary(results, verbose)
 
-        if command_failed:
+    # Reset custom project config
-            raise exception.ReturnErrorCode(1)
+    app.set_session_var("custom_project_conf", None)
-        return True
+
+    if command_failed:
+        raise exception.ReturnErrorCode(1)
+    return True
 
 
 def process_env(
@@ -162,16 +174,25 @@ def process_env(
     environments,
     targets,
     upload_port,
+    jobs,
+    program_args,
+    is_test_running,
     silent,
     verbose,
-    jobs,
-    is_test_running,
 ):
     if not is_test_running and not silent:
         print_processing_header(name, config, verbose)
 
     ep = EnvironmentProcessor(
-        ctx, name, config, targets, upload_port, silent, verbose, jobs
+        ctx,
+        name,
+        config,
+        targets,
+        upload_port,
+        jobs,
+        program_args,
+        silent,
+        verbose,
     )
     result = {"env": name, "duration": time(), "succeeded": ep.process()}
     result["duration"] = time() - result["duration"]
@@ -186,7 +207,7 @@ def process_env(
         and "nobuild" not in ep.get_build_targets()
     ):
         ctx.invoke(
-            cmd_device_monitor, environment=environments[0] if environments else None
+            device_monitor_cmd, environment=environments[0] if environments else None
         )
 
     return result
@@ -273,7 +294,7 @@ def print_processing_summary(results, verbose=False):
 
 def print_target_list(envs):
     tabular_data = []
-    for env, data in load_project_ide_data(os.getcwd(), envs).items():
+    for env, data in load_build_metadata(os.getcwd(), envs).items():
         tabular_data.extend(
             sorted(
                 [

platformio/commands/run/processor.py

@@ -12,29 +12,44 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio.commands.platform import init_platform
+from platformio.package.commands.install import install_project_env_dependencies
-from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME
+from platformio.platform.factory import PlatformFactory
 from platformio.project.exception import UndefinedEnvPlatformError
+from platformio.test.runners.base import CTX_META_TEST_RUNNING_NAME
 
 # pylint: disable=too-many-instance-attributes
 
 
 class EnvironmentProcessor(object):
     def __init__(  # pylint: disable=too-many-arguments
-        self, cmd_ctx, name, config, targets, upload_port, silent, verbose, jobs
+        self,
+        cmd_ctx,
+        name,
+        config,
+        targets,
+        upload_port,
+        jobs,
+        program_args,
+        silent,
+        verbose,
     ):
         self.cmd_ctx = cmd_ctx
         self.name = name
         self.config = config
         self.targets = [str(t) for t in targets]
         self.upload_port = upload_port
+        self.jobs = jobs
+        self.program_args = program_args
         self.silent = silent
         self.verbose = verbose
-        self.jobs = jobs
         self.options = config.items(env=name, as_dict=True)
 
     def get_build_variables(self):
-        variables = {"pioenv": self.name, "project_config": self.config.path}
+        variables = dict(
+            pioenv=self.name,
+            project_config=self.config.path,
+            program_args=self.program_args,
+        )
 
         if CTX_META_TEST_RUNNING_NAME in self.cmd_ctx.meta:
             variables["piotest_running_name"] = self.cmd_ctx.meta[
@@ -64,7 +79,16 @@ class EnvironmentProcessor(object):
         if "monitor" in build_targets:
             build_targets.remove("monitor")
 
-        result = init_platform(self.options["platform"]).run(
+        if "clean" not in build_targets:
+            install_project_env_dependencies(
+                self.name,
+                {
+                    "project_targets": build_targets,
+                    "piotest_running_name": build_vars.get("piotest_running_name"),
+                },
+            )
+
+        result = PlatformFactory.new(self.options["platform"], autoinstall=True).run(
             build_vars, build_targets, self.silent, self.verbose, self.jobs
         )
         return result["returncode"] == 0

platformio/commands/test.py

@@ -12,19 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
+# pylint: disable=unused-import
 
-from platformio import exception
+from platformio.test.command import test_cmd as cli
-
-
-def get_test_names(config):
-    test_dir = config.get("platformio", "test_dir")
-    if not os.path.isdir(test_dir):
-        raise exception.TestDirNotExists(test_dir)
-    names = []
-    for item in sorted(os.listdir(test_dir)):
-        if os.path.isdir(os.path.join(test_dir, item)):
-            names.append(item)
-    if not names:
-        names = ["*"]
-    return names

platformio/commands/test/command.py

@@ -1,266 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
-
-import fnmatch
-import os
-import shutil
-from time import time
-
-import click
-from tabulate import tabulate
-
-from platformio import app, exception, fs, util
-from platformio.commands.platform import init_platform
-from platformio.commands.test.embedded import EmbeddedTestProcessor
-from platformio.commands.test.helpers import get_test_names
-from platformio.commands.test.native import NativeTestProcessor
-from platformio.project.config import ProjectConfig
-
-
-@click.command("test", short_help="Unit testing")
-@click.option("--environment", "-e", multiple=True, metavar="<environment>")
-@click.option(
-    "--filter",
-    "-f",
-    multiple=True,
-    metavar="<pattern>",
-    help="Filter tests by a pattern",
-)
-@click.option(
-    "--ignore",
-    "-i",
-    multiple=True,
-    metavar="<pattern>",
-    help="Ignore tests by a pattern",
-)
-@click.option("--upload-port")
-@click.option("--test-port")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
-)
-@click.option(
-    "-c",
-    "--project-conf",
-    type=click.Path(
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
-)
-@click.option("--without-building", is_flag=True)
-@click.option("--without-uploading", is_flag=True)
-@click.option("--without-testing", is_flag=True)
-@click.option("--no-reset", is_flag=True)
-@click.option(
-    "--monitor-rts",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial RTS line state for Serial Monitor",
-)
-@click.option(
-    "--monitor-dtr",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial DTR line state for Serial Monitor",
-)
-@click.option("--verbose", "-v", is_flag=True)
-@click.pass_context
-def cli(  # pylint: disable=redefined-builtin
-    ctx,
-    environment,
-    ignore,
-    filter,
-    upload_port,
-    test_port,
-    project_dir,
-    project_conf,
-    without_building,
-    without_uploading,
-    without_testing,
-    no_reset,
-    monitor_rts,
-    monitor_dtr,
-    verbose,
-):
-    app.set_session_var("custom_project_conf", project_conf)
-
-    with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(project_conf)
-        config.validate(envs=environment)
-        test_names = get_test_names(config)
-
-        if not verbose:
-            click.echo("Verbose mode can be enabled via `-v, --verbose` option")
-        click.secho("Collected %d items" % len(test_names), bold=True)
-
-        results = []
-        default_envs = config.default_envs()
-        for testname in test_names:
-
-            for envname in config.envs():
-                section = "env:%s" % envname
-
-                # filter and ignore patterns
-                patterns = dict(filter=list(filter), ignore=list(ignore))
-                for key in patterns:
-                    patterns[key].extend(config.get(section, "test_%s" % key, []))
-
-                skip_conditions = [
-                    environment and envname not in environment,
-                    not environment and default_envs and envname not in default_envs,
-                    testname != "*"
-                    and patterns["filter"]
-                    and not any(
-                        fnmatch.fnmatch(testname, p) for p in patterns["filter"]
-                    ),
-                    testname != "*"
-                    and any(fnmatch.fnmatch(testname, p) for p in patterns["ignore"]),
-                ]
-                if any(skip_conditions):
-                    results.append({"env": envname, "test": testname})
-                    continue
-
-                click.echo()
-                print_processing_header(testname, envname)
-
-                cls = (
-                    EmbeddedTestProcessor
-                    if config.get(section, "platform")
-                    and init_platform(config.get(section, "platform")).is_embedded()
-                    else NativeTestProcessor
-                )
-                tp = cls(
-                    ctx,
-                    testname,
-                    envname,
-                    dict(
-                        project_config=config,
-                        project_dir=project_dir,
-                        upload_port=upload_port,
-                        test_port=test_port,
-                        without_building=without_building,
-                        without_uploading=without_uploading,
-                        without_testing=without_testing,
-                        no_reset=no_reset,
-                        monitor_rts=monitor_rts,
-                        monitor_dtr=monitor_dtr,
-                        verbose=verbose,
-                        silent=not verbose,
-                    ),
-                )
-                result = {
-                    "env": envname,
-                    "test": testname,
-                    "duration": time(),
-                    "succeeded": tp.process(),
-                }
-                result["duration"] = time() - result["duration"]
-                results.append(result)
-
-                print_processing_footer(result)
-
-    if without_testing:
-        return
-
-    print_testing_summary(results, verbose)
-
-    command_failed = any(r.get("succeeded") is False for r in results)
-    if command_failed:
-        raise exception.ReturnErrorCode(1)
-
-
-def print_processing_header(test, env):
-    click.echo(
-        "Processing %s in %s environment"
-        % (
-            click.style(test, fg="yellow", bold=True),
-            click.style(env, fg="cyan", bold=True),
-        )
-    )
-    terminal_width, _ = shutil.get_terminal_size()
-    click.secho("-" * terminal_width, bold=True)
-
-
-def print_processing_footer(result):
-    is_failed = not result.get("succeeded")
-    util.print_labeled_bar(
-        "[%s] Took %.2f seconds"
-        % (
-            (
-                click.style("FAILED", fg="red", bold=True)
-                if is_failed
-                else click.style("PASSED", fg="green", bold=True)
-            ),
-            result["duration"],
-        ),
-        is_error=is_failed,
-    )
-
-
-def print_testing_summary(results, verbose=False):
-    click.echo()
-
-    tabular_data = []
-    succeeded_nums = 0
-    failed_nums = 0
-    duration = 0
-
-    for result in results:
-        duration += result.get("duration", 0)
-        if result.get("succeeded") is False:
-            failed_nums += 1
-            status_str = click.style("FAILED", fg="red")
-        elif result.get("succeeded") is None:
-            if not verbose:
-                continue
-            status_str = "IGNORED"
-        else:
-            succeeded_nums += 1
-            status_str = click.style("PASSED", fg="green")
-
-        tabular_data.append(
-            (
-                result["test"],
-                click.style(result["env"], fg="cyan"),
-                status_str,
-                util.humanize_duration_time(result.get("duration")),
-            )
-        )
-
-    click.echo(
-        tabulate(
-            tabular_data,
-            headers=[
-                click.style(s, bold=True)
-                for s in ("Test", "Environment", "Status", "Duration")
-            ],
-        ),
-        err=failed_nums,
-    )
-
-    util.print_labeled_bar(
-        "%s%d succeeded in %s"
-        % (
-            "%d failed, " % failed_nums if failed_nums else "",
-            succeeded_nums,
-            util.humanize_duration_time(duration),
-        ),
-        is_error=failed_nums,
-        fg="red" if failed_nums else "green",
-    )

platformio/commands/test/embedded.py

@@ -1,150 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from time import sleep
-
-import click
-import serial
-
-from platformio import exception, util
-from platformio.commands.test.processor import TestProcessorBase
-from platformio.platform.factory import PlatformFactory
-
-
-class EmbeddedTestProcessor(TestProcessorBase):
-
-    SERIAL_TIMEOUT = 600
-
-    def process(self):
-        if not self.options["without_building"]:
-            self.print_progress("Building...")
-            target = ["__test"]
-            if self.options["without_uploading"]:
-                target.append("checkprogsize")
-            if not self.build_or_upload(target):
-                return False
-
-        if not self.options["without_uploading"]:
-            self.print_progress("Uploading...")
-            target = ["upload"]
-            if self.options["without_building"]:
-                target.append("nobuild")
-            else:
-                target.append("__test")
-            if not self.build_or_upload(target):
-                return False
-
-        if self.options["without_testing"]:
-            return True
-
-        self.print_progress("Testing...")
-        return self.run()
-
-    def run(self):
-        click.echo(
-            "If you don't see any output for the first 10 secs, "
-            "please reset board (press reset button)"
-        )
-        click.echo()
-
-        try:
-            ser = serial.Serial(
-                baudrate=self.get_baudrate(), timeout=self.SERIAL_TIMEOUT
-            )
-            ser.port = self.get_test_port()
-            ser.rts = self.options["monitor_rts"]
-            ser.dtr = self.options["monitor_dtr"]
-            ser.open()
-        except serial.SerialException as e:
-            click.secho(str(e), fg="red", err=True)
-            return False
-
-        if not self.options["no_reset"]:
-            ser.flushInput()
-            ser.setDTR(False)
-            ser.setRTS(False)
-            sleep(0.1)
-            ser.setDTR(True)
-            ser.setRTS(True)
-            sleep(0.1)
-
-        while True:
-            line = ser.readline().strip()
-
-            # fix non-ascii output from device
-            for i, c in enumerate(line[::-1]):
-                if not isinstance(c, int):
-                    c = ord(c)
-                if c > 127:
-                    line = line[-i:]
-                    break
-
-            if not line:
-                continue
-            if isinstance(line, bytes):
-                line = line.decode("utf8", "ignore")
-            self.on_run_out(line)
-            if all(l in line for l in ("Tests", "Failures", "Ignored")):
-                break
-        ser.close()
-        return not self._run_failed
-
-    def get_test_port(self):
-        # if test port is specified manually or in config
-        if self.options.get("test_port"):
-            return self.options.get("test_port")
-        if self.env_options.get("test_port"):
-            return self.env_options.get("test_port")
-
-        assert set(["platform", "board"]) & set(self.env_options.keys())
-        p = PlatformFactory.new(self.env_options["platform"])
-        board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", [])
-        port = None
-        elapsed = 0
-        while elapsed < 5 and not port:
-            for item in util.get_serialports():
-                port = item["port"]
-                for hwid in board_hwids:
-                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                    if hwid_str in item["hwid"] and self.is_serial_port_ready(port):
-                        return port
-
-            if port and not self.is_serial_port_ready(port):
-                port = None
-
-            if not port:
-                sleep(0.25)
-                elapsed += 0.25
-
-        if not port:
-            raise exception.PlatformioException(
-                "Please specify `test_port` for environment or use "
-                "global `--test-port` option."
-            )
-        return port
-
-    @staticmethod
-    def is_serial_port_ready(port, timeout=3):
-        if not port:
-            return False
-        elapsed = 0
-        while elapsed < timeout:
-            try:
-                serial.Serial(port, timeout=1).close()
-                return True
-            except:  # pylint: disable=bare-except
-                pass
-            sleep(1)
-            elapsed += 1
-        return False

platformio/commands/test/native.py

@@ -1,41 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from os.path import join
-
-from platformio import proc
-from platformio.commands.test.processor import TestProcessorBase
-from platformio.proc import LineBufferedAsyncPipe
-
-
-class NativeTestProcessor(TestProcessorBase):
-    def process(self):
-        if not self.options["without_building"]:
-            self.print_progress("Building...")
-            if not self.build_or_upload(["__test"]):
-                return False
-        if self.options["without_testing"]:
-            return None
-        self.print_progress("Testing...")
-        return self.run()
-
-    def run(self):
-        build_dir = self.options["project_config"].get("platformio", "build_dir")
-        result = proc.exec_command(
-            [join(build_dir, self.env_name, "program")],
-            stdout=LineBufferedAsyncPipe(self.on_run_out),
-            stderr=LineBufferedAsyncPipe(self.on_run_out),
-        )
-        assert "returncode" in result
-        return result["returncode"] == 0 and not self._run_failed

platformio/commands/test/processor.py

@@ -1,230 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import atexit
-from os import listdir, remove
-from os.path import isdir, isfile, join
-from string import Template
-
-import click
-
-from platformio import exception
-
-TRANSPORT_OPTIONS = {
-    "arduino": {
-        "include": "#include <Arduino.h>",
-        "object": "",
-        "putchar": "Serial.write(c);",
-        "flush": "Serial.flush();",
-        "begin": "Serial.begin($baudrate);",
-        "end": "Serial.end();",
-        "language": "cpp",
-    },
-    "mbed": {
-        "include": "#include <mbed.h>",
-        "object": (
-            "#if MBED_MAJOR_VERSION == 6\nUnbufferedSerial pc(USBTX, USBRX);\n"
-            "#else\nRawSerial pc(USBTX, USBRX);\n#endif"
-        ),
-        "putchar": (
-            "#if MBED_MAJOR_VERSION == 6\npc.write(&c, 1);\n"
-            "#else\npc.putc(c);\n#endif"
-        ),
-        "flush": "",
-        "begin": "pc.baud($baudrate);",
-        "end": "",
-        "language": "cpp",
-    },
-    "espidf": {
-        "include": "#include <stdio.h>",
-        "object": "",
-        "putchar": "putchar(c);",
-        "flush": "fflush(stdout);",
-        "begin": "",
-        "end": "",
-    },
-    "zephyr": {
-        "include": "#include <sys/printk.h>",
-        "object": "",
-        "putchar": 'printk("%c", c);',
-        "flush": "",
-        "begin": "",
-        "end": "",
-    },
-    "native": {
-        "include": "#include <stdio.h>",
-        "object": "",
-        "putchar": "putchar(c);",
-        "flush": "fflush(stdout);",
-        "begin": "",
-        "end": "",
-    },
-    "custom": {
-        "include": '#include "unittest_transport.h"',
-        "object": "",
-        "putchar": "unittest_uart_putchar(c);",
-        "flush": "unittest_uart_flush();",
-        "begin": "unittest_uart_begin();",
-        "end": "unittest_uart_end();",
-        "language": "cpp",
-    },
-}
-
-CTX_META_TEST_IS_RUNNING = __name__ + ".test_running"
-CTX_META_TEST_RUNNING_NAME = __name__ + ".test_running_name"
-
-
-class TestProcessorBase(object):
-
-    DEFAULT_BAUDRATE = 115200
-
-    def __init__(self, cmd_ctx, testname, envname, options):
-        self.cmd_ctx = cmd_ctx
-        self.cmd_ctx.meta[CTX_META_TEST_IS_RUNNING] = True
-        self.test_name = testname
-        self.options = options
-        self.env_name = envname
-        self.env_options = options["project_config"].items(env=envname, as_dict=True)
-        self._run_failed = False
-        self._output_file_generated = False
-
-    def get_transport(self):
-        transport = None
-        if self.env_options.get("platform") == "native":
-            transport = "native"
-        elif "framework" in self.env_options:
-            transport = self.env_options.get("framework")[0]
-        if "test_transport" in self.env_options:
-            transport = self.env_options["test_transport"]
-        if transport not in TRANSPORT_OPTIONS:
-            raise exception.PlatformioException(
-                "Unknown Unit Test transport `%s`. Please check a documentation how "
-                "to create an own 'Test Transport':\n"
-                "- https://docs.platformio.org/page/plus/unit-testing.html" % transport
-            )
-        return transport.lower()
-
-    def get_baudrate(self):
-        return int(self.env_options.get("test_speed", self.DEFAULT_BAUDRATE))
-
-    def print_progress(self, text):
-        click.secho(text, bold=self.options.get("verbose"))
-
-    def build_or_upload(self, target):
-        if not self._output_file_generated:
-            self.generate_output_file(
-                self.options["project_config"].get("platformio", "test_dir")
-            )
-            self._output_file_generated = True
-
-        if self.test_name != "*":
-            self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name
-
-        try:
-            # pylint: disable=import-outside-toplevel
-            from platformio.commands.run.command import cli as cmd_run
-
-            return self.cmd_ctx.invoke(
-                cmd_run,
-                project_dir=self.options["project_dir"],
-                project_conf=self.options["project_config"].path,
-                upload_port=self.options.get("upload_port"),
-                verbose=self.options["verbose"],
-                silent=self.options.get("silent"),
-                environment=[self.env_name],
-                disable_auto_clean="nobuild" in target,
-                target=target,
-            )
-        except exception.ReturnErrorCode:
-            return False
-
-    def process(self):
-        raise NotImplementedError
-
-    def run(self):
-        raise NotImplementedError
-
-    def on_run_out(self, line):
-        line = line.strip()
-        if line.endswith(":PASS"):
-            click.echo("%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green")))
-        elif ":FAIL" in line:
-            self._run_failed = True
-            click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red")))
-        else:
-            click.echo(line)
-
-    def generate_output_file(self, test_dir):
-        assert isdir(test_dir)
-
-        file_tpl = "\n".join(
-            [
-                "$include",
-                "#include <output_export.h>",
-                "",
-                "$object",
-                "",
-                "#ifdef __GNUC__",
-                "void output_start(unsigned int baudrate __attribute__((unused)))",
-                "#else",
-                "void output_start(unsigned int baudrate)",
-                "#endif",
-                "{",
-                "    $begin",
-                "}",
-                "",
-                "void output_char(int c)",
-                "{",
-                "    $putchar",
-                "}",
-                "",
-                "void output_flush(void)",
-                "{",
-                "    $flush",
-                "}",
-                "",
-                "void output_complete(void)",
-                "{",
-                "   $end",
-                "}",
-            ]
-        )
-
-        tmp_file_prefix = "tmp_pio_test_transport"
-
-        def delete_tmptest_files(test_dir):
-            for item in listdir(test_dir):
-                if item.startswith(tmp_file_prefix) and isfile(join(test_dir, item)):
-                    try:
-                        remove(join(test_dir, item))
-                    except:  # pylint: disable=bare-except
-                        click.secho(
-                            "Warning: Could not remove temporary file '%s'. "
-                            "Please remove it manually." % join(test_dir, item),
-                            fg="yellow",
-                        )
-
-        transport_options = TRANSPORT_OPTIONS[self.get_transport()]
-        tpl = Template(file_tpl).substitute(transport_options)
-        data = Template(tpl).substitute(baudrate=self.get_baudrate())
-
-        delete_tmptest_files(test_dir)
-        tmp_file = join(
-            test_dir,
-            "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")),
-        )
-        with open(tmp_file, mode="w", encoding="utf8") as fp:
-            fp.write(data)
-
-        atexit.register(delete_tmptest_files, test_dir)

platformio/commands/update.py

@@ -14,7 +14,6 @@
 
 import click
 
-from platformio.cache import cleanup_content_cache
 from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
 from platformio.commands.lib.command import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
@@ -23,7 +22,9 @@ from platformio.package.manager.library import LibraryPackageManager
 
 
 @click.command(
-    "update", short_help="Update installed platforms, packages and libraries"
+    "update",
+    short_help="Update installed platforms, packages and libraries",
+    hidden=True,
 )
 @click.option("--core-packages", is_flag=True, help="Update only the core packages")
 @click.option(
@@ -37,12 +38,10 @@ from platformio.package.manager.library import LibraryPackageManager
 )
 @click.pass_context
 def cli(ctx, core_packages, only_check, dry_run):
-    # cleanup lib search results, cached board and platform lists
-    cleanup_content_cache("http")
-
     only_check = dry_run or only_check
 
-    update_core_packages(only_check)
+    if not only_check:
+        update_core_packages()
 
     if core_packages:
         return

platformio/commands/upgrade.py

@@ -22,13 +22,15 @@ import click
 from platformio import VERSION, __version__, app, exception
 from platformio.clients.http import fetch_remote_content
 from platformio.compat import IS_WINDOWS
+from platformio.package.manager.core import update_core_packages
 from platformio.proc import exec_command, get_pythonexe_path
 from platformio.project.helpers import get_project_cache_dir
 
 
-@click.command("upgrade", short_help="Upgrade PlatformIO to the latest version")
+@click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version")
 @click.option("--dev", is_flag=True, help="Use development branch")
 def cli(dev):
+    update_core_packages()
     if not dev and __version__ == get_latest_version():
         return click.secho(
             "You're up-to-date!\nPlatformIO %s is currently the "

platformio/compat.py

@@ -14,25 +14,26 @@
 
 # pylint: disable=unused-import,no-name-in-module
 
+import importlib.util
 import inspect
 import locale
 import sys
 
 from platformio.exception import UserSideException
 
-if sys.version_info >= (3,):
+if sys.version_info >= (3, 7):
-    if sys.version_info >= (3, 7):
+    from asyncio import create_task as aio_create_task
-        from asyncio import create_task as aio_create_task
+    from asyncio import get_running_loop as aio_get_running_loop
-        from asyncio import get_running_loop as aio_get_running_loop
+else:
-    else:
+    from asyncio import ensure_future as aio_create_task
-        from asyncio import ensure_future as aio_create_task
+    from asyncio import get_event_loop as aio_get_running_loop
-        from asyncio import get_event_loop as aio_get_running_loop
 
 
-PY2 = sys.version_info[0] == 2
+PY2 = sys.version_info[0] == 2  # DO NOT REMOVE IT. ESP8266/ESP32 depend on it
 IS_CYGWIN = sys.platform.startswith("cygwin")
 IS_WINDOWS = WINDOWS = sys.platform.startswith("win")
 IS_MACOS = sys.platform.startswith("darwin")
+MISSING = object()
 string_types = (str,)
 
 
@@ -57,8 +58,6 @@ def hashlib_encode_data(data):
 
 
 def load_python_module(name, pathname):
-    import importlib.util  # pylint: disable=import-outside-toplevel
-
     spec = importlib.util.spec_from_file_location(name, pathname)
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)

platformio/debug/command.py

@@ -0,0 +1,203 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-arguments, too-many-locals
+# pylint: disable=too-many-branches, too-many-statements
+
+import asyncio
+import os
+import signal
+import subprocess
+
+import click
+
+from platformio import app, exception, fs, proc
+from platformio.compat import IS_WINDOWS
+from platformio.debug import helpers
+from platformio.debug.config.factory import DebugConfigFactory
+from platformio.debug.exception import DebugInvalidOptionsError
+from platformio.debug.process.gdb import GDBClientProcess
+from platformio.platform.factory import PlatformFactory
+from platformio.project.config import ProjectConfig
+from platformio.project.exception import ProjectEnvsNotAvailableError
+from platformio.project.helpers import is_platformio_project
+from platformio.project.options import ProjectOptions
+
+
+@click.command(
+    "debug",
+    context_settings=dict(ignore_unknown_options=True),
+    short_help="Unified Debugger",
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(
+        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
+    ),
+)
+@click.option(
+    "-c",
+    "--project-conf",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
+    ),
+)
+@click.option("--environment", "-e", metavar="<environment>")
+@click.option("--load-mode", type=ProjectOptions["env.debug_load_mode"].type)
+@click.option("--verbose", "-v", is_flag=True)
+@click.option("--interface", type=click.Choice(["gdb"]))
+@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
+@click.pass_context
+def debug_cmd(
+    ctx,
+    project_dir,
+    project_conf,
+    environment,
+    load_mode,
+    verbose,
+    interface,
+    __unprocessed,
+):
+    app.set_session_var("custom_project_conf", project_conf)
+
+    # use env variables from Eclipse or CLion
+    for name in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"):
+        if is_platformio_project(project_dir):
+            break
+        if os.getenv(name):
+            project_dir = os.getenv(name)
+
+    with fs.cd(project_dir):
+        return _debug_in_project_dir(
+            ctx,
+            project_dir,
+            project_conf,
+            environment,
+            load_mode,
+            verbose,
+            interface,
+            __unprocessed,
+        )
+
+
+def _debug_in_project_dir(
+    ctx,
+    project_dir,
+    project_conf,
+    environment,
+    load_mode,
+    verbose,
+    interface,
+    __unprocessed,
+):
+    project_config = ProjectConfig.get_instance(project_conf)
+    project_config.validate(envs=[environment] if environment else None)
+    env_name = environment or helpers.get_default_debug_env(project_config)
+
+    if not interface:
+        return helpers.predebug_project(
+            ctx, project_dir, project_config, env_name, False, verbose
+        )
+
+    env_options = project_config.items(env=env_name, as_dict=True)
+    if "platform" not in env_options:
+        raise ProjectEnvsNotAvailableError()
+
+    debug_config = DebugConfigFactory.new(
+        PlatformFactory.new(env_options["platform"], autoinstall=True),
+        project_config,
+        env_name,
+    )
+
+    if "--version" in __unprocessed:
+        return subprocess.run(
+            [debug_config.client_executable_path, "--version"], check=True
+        )
+
+    try:
+        fs.ensure_udev_rules()
+    except exception.InvalidUdevRules as e:
+        click.echo(
+            helpers.escape_gdbmi_stream("~", str(e) + "\n")
+            if helpers.is_gdbmi_mode()
+            else str(e) + "\n",
+            nl=False,
+        )
+
+    rebuild_prog = False
+    preload = debug_config.load_cmds == ["preload"]
+    load_mode = load_mode or debug_config.load_mode
+    if load_mode == "always":
+        rebuild_prog = preload or not helpers.has_debug_symbols(
+            debug_config.program_path
+        )
+    elif load_mode == "modified":
+        rebuild_prog = helpers.is_prog_obsolete(
+            debug_config.program_path
+        ) or not helpers.has_debug_symbols(debug_config.program_path)
+
+    if not (debug_config.program_path and os.path.isfile(debug_config.program_path)):
+        rebuild_prog = True
+
+    if preload or (not rebuild_prog and load_mode != "always"):
+        # don't load firmware through debug server
+        debug_config.load_cmds = []
+
+    if rebuild_prog:
+        if helpers.is_gdbmi_mode():
+            click.echo(
+                helpers.escape_gdbmi_stream(
+                    "~", "Preparing firmware for debugging...\n"
+                ),
+                nl=False,
+            )
+            stream = helpers.GDBMIConsoleStream()
+            with proc.capture_std_streams(stream):
+                helpers.predebug_project(
+                    ctx, project_dir, project_config, env_name, preload, verbose
+                )
+            stream.close()
+        else:
+            click.echo("Preparing firmware for debugging...")
+            helpers.predebug_project(
+                ctx, project_dir, project_config, env_name, preload, verbose
+            )
+
+        # save SHA sum of newly created prog
+        if load_mode == "modified":
+            helpers.is_prog_obsolete(debug_config.program_path)
+
+    if not os.path.isfile(debug_config.program_path):
+        raise DebugInvalidOptionsError("Program/firmware is missed")
+
+    loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop()
+    asyncio.set_event_loop(loop)
+
+    client = GDBClientProcess(project_dir, debug_config)
+    coro = client.run(__unprocessed)
+    try:
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+        loop.run_until_complete(coro)
+        if IS_WINDOWS:
+            client.close()
+            # an issue with `asyncio` executor and STIDIN,
+            # it cannot be closed gracefully
+            proc.force_exit()
+    finally:
+        client.close()
+        loop.close()
+
+    return True

platformio/debug/config/base.py

@@ -20,7 +20,7 @@ from platformio.compat import string_types
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.debug.helpers import reveal_debug_port
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import load_project_ide_data
+from platformio.project.helpers import load_build_metadata
 from platformio.project.options import ProjectOptions
 
 
@@ -147,7 +147,7 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
         )
 
     def _load_build_data(self):
-        data = load_project_ide_data(os.getcwd(), self.env_name, cache=True)
+        data = load_build_metadata(os.getcwd(), self.env_name, cache=True)
         if data:
             return data
         raise DebugInvalidOptionsError("Could not load a build configuration")
@@ -186,11 +186,7 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
                 else None
             )
             if server_package and not server_package_dir:
-                self.platform.install_packages(
+                self.platform.install_package(server_package)
-                    with_packages=[server_package],
-                    skip_default_package=True,
-                    silent=True,
-                )
                 server_package_dir = self.platform.get_package_dir(server_package)
             result.update(
                 dict(

platformio/debug/config/factory.py

@@ -23,7 +23,7 @@ class DebugConfigFactory(object):
     @staticmethod
     def get_clsname(name):
         name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I)
-        return "%s%sDebugConfig" % (name.upper()[0], name.lower()[1:])
+        return "%sDebugConfig" % name.lower().capitalize()
 
     @classmethod
     def new(cls, platform, project_config, env_name):

platformio/debug/config/generic.py

@@ -34,5 +34,5 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(GenericDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":3333"

platformio/debug/config/jlink.py

@@ -38,11 +38,9 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(JlinkDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":2331"
 
     @property
     def server_ready_pattern(self):
-        return super(JlinkDebugConfig, self).server_ready_pattern or (
+        return super().server_ready_pattern or ("Waiting for GDB connection")
-            "Waiting for GDB connection"
-        )

platformio/debug/config/mspdebug.py

@@ -32,5 +32,5 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(MspdebugDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":2000"

platformio/debug/config/qemu.py

@@ -33,5 +33,5 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(QemuDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":1234"

platformio/debug/config/renode.py

@@ -35,11 +35,11 @@ monitor start
 """
 
     def __init__(self, *args, **kwargs):
-        super(RenodeDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":3333"
 
     @property
     def server_ready_pattern(self):
-        return super(RenodeDebugConfig, self).server_ready_pattern or (
+        return super().server_ready_pattern or (
             "GDB server with all CPUs started on port"
         )

platformio/debug/helpers.py

@@ -12,22 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import re
 import sys
 import time
 from fnmatch import fnmatch
 from hashlib import sha1
 from io import BytesIO
-from os.path import isfile
 
-from platformio import util
 from platformio.commands import PlatformioCLI
 from platformio.commands.run.command import cli as cmd_run
 from platformio.commands.run.command import print_processing_header
-from platformio.commands.test.helpers import get_test_names
-from platformio.commands.test.processor import TestProcessorBase
 from platformio.compat import IS_WINDOWS, is_bytes
 from platformio.debug.exception import DebugInvalidOptionsError
+from platformio.device.list import list_serial_ports
+from platformio.test.helpers import list_test_names
+from platformio.test.result import TestSuite
+from platformio.test.runners.base import TestRunnerOptions
+from platformio.test.runners.factory import TestRunnerFactory
 
 
 class GDBMIConsoleStream(BytesIO):  # pylint: disable=too-few-public-methods
@@ -80,27 +82,25 @@ def predebug_project(
 ):  # pylint: disable=too-many-arguments
     debug_testname = project_config.get("env:" + env_name, "debug_test")
     if debug_testname:
-        test_names = get_test_names(project_config)
+        test_names = list_test_names(project_config)
         if debug_testname not in test_names:
             raise DebugInvalidOptionsError(
                 "Unknown test name `%s`. Valid names are `%s`"
                 % (debug_testname, ", ".join(test_names))
             )
         print_processing_header(env_name, project_config, verbose)
-        tp = TestProcessorBase(
+        test_runner = TestRunnerFactory.new(
-            ctx,
+            TestSuite(env_name, debug_testname),
-            debug_testname,
+            project_config,
-            env_name,
+            TestRunnerOptions(
-            dict(
+                verbose=verbose,
-                project_config=project_config,
-                project_dir=project_dir,
                 without_building=False,
-                without_uploading=True,
+                without_debugging=False,
+                without_uploading=not preload,
                 without_testing=True,
-                verbose=False,
             ),
         )
-        tp.build_or_upload(["__debug", "__test"] + (["upload"] if preload else []))
+        test_runner.start(ctx)
     else:
         ctx.invoke(
             cmd_run,
@@ -116,7 +116,7 @@ def predebug_project(
 
 
 def has_debug_symbols(prog_path):
-    if not isfile(prog_path):
+    if not os.path.isfile(prog_path):
         return False
     matched = {
         b".debug_info": False,
@@ -142,7 +142,7 @@ def has_debug_symbols(prog_path):
 
 def is_prog_obsolete(prog_path):
     prog_hash_path = prog_path + ".sha1"
-    if not isfile(prog_path):
+    if not os.path.isfile(prog_path):
         return True
     shasum = sha1()
     with open(prog_path, "rb") as fp:
@@ -153,7 +153,7 @@ def is_prog_obsolete(prog_path):
             shasum.update(data)
     new_digest = shasum.hexdigest()
     old_digest = None
-    if isfile(prog_hash_path):
+    if os.path.isfile(prog_hash_path):
         with open(prog_hash_path, encoding="utf8") as fp:
             old_digest = fp.read()
     if new_digest == old_digest:
@@ -178,7 +178,7 @@ def reveal_debug_port(env_debug_port, tool_name, tool_settings):
         return fnmatch(port, pattern)
 
     def _look_for_serial_port(hwids):
-        for item in util.get_serialports(filter_hwid=True):
+        for item in list_serial_ports(filter_hwid=True):
             if not _is_match_pattern(item["port"]):
                 continue
             port = item["port"]

platformio/debug/process/client.py

@@ -27,7 +27,7 @@ from platformio.project.helpers import get_project_cache_dir
 
 class DebugClientProcess(DebugBaseProcess):
     def __init__(self, project_dir, debug_config):
-        super(DebugClientProcess, self).__init__()
+        super().__init__()
         self.project_dir = project_dir
         self.debug_config = debug_config
 
@@ -55,7 +55,7 @@ class DebugClientProcess(DebugBaseProcess):
             self.debug_config.port = await self._server_process.run()
 
     def connection_made(self, transport):
-        super(DebugClientProcess, self).connection_made(transport)
+        super().connection_made(transport)
         self._lock_session(transport.get_pid())
         # Disable SIGINT and allow GDB's Ctrl+C interrupt
         signal.signal(signal.SIGINT, lambda *args, **kwargs: None)
@@ -64,7 +64,15 @@ class DebugClientProcess(DebugBaseProcess):
     def process_exited(self):
         if self._server_process:
             self._server_process.terminate()
-        super(DebugClientProcess, self).process_exited()
+        super().process_exited()
+
+    def close(self):
+        self._unlock_session()
+        if self.working_dir and os.path.isdir(self.working_dir):
+            fs.rmtree(self.working_dir)
+
+    def __del__(self):
+        self.close()
 
     def _kill_previous_session(self):
         assert self._session_id
@@ -94,8 +102,3 @@ class DebugClientProcess(DebugBaseProcess):
             return
         with ContentCache() as cc:
             cc.delete(self._session_id)
-
-    def __del__(self):
-        self._unlock_session()
-        if self.working_dir and os.path.isdir(self.working_dir):
-            fs.rmtree(self.working_dir)

platformio/debug/process/gdb.py

@@ -29,12 +29,12 @@ class GDBClientProcess(DebugClientProcess):
     INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed"
 
     def __init__(self, *args, **kwargs):
-        super(GDBClientProcess, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._target_is_running = False
         self._errors_buffer = b""
 
     async def run(self, extra_args):  # pylint: disable=arguments-differ
-        await super(GDBClientProcess, self).run()
+        await super().run()
 
         self.generate_init_script(os.path.join(self.working_dir, self.PIO_SRC_NAME))
         gdb_path = self.debug_config.client_executable_path or "gdb"
@@ -109,7 +109,7 @@ class GDBClientProcess(DebugClientProcess):
             fp.write("\n".join(self.debug_config.reveal_patterns(commands)))
 
     def stdin_data_received(self, data):
-        super(GDBClientProcess, self).stdin_data_received(data)
+        super().stdin_data_received(data)
         if b"-exec-run" in data:
             if self._target_is_running:
                 token, _ = data.split(b"-", 1)
@@ -127,7 +127,7 @@ class GDBClientProcess(DebugClientProcess):
         self.transport.get_pipe_transport(0).write(data)
 
     def stdout_data_received(self, data):
-        super(GDBClientProcess, self).stdout_data_received(data)
+        super().stdout_data_received(data)
         self._handle_error(data)
         # go to init break automatically
         if self.INIT_COMPLETED_BANNER.encode() in data:
@@ -170,7 +170,7 @@ class GDBClientProcess(DebugClientProcess):
         self._target_is_running = True
 
     def stderr_data_received(self, data):
-        super(GDBClientProcess, self).stderr_data_received(data)
+        super().stderr_data_received(data)
         self._handle_error(data)
 
     def _handle_error(self, data):

platformio/debug/process/server.py

@@ -30,7 +30,7 @@ class DebugServerProcess(DebugBaseProcess):
     STD_BUFFER_SIZE = 1024
 
     def __init__(self, debug_config):
-        super(DebugServerProcess, self).__init__()
+        super().__init__()
         self.debug_config = debug_config
         self._ready = False
         self._std_buffer = {"out": b"", "err": b""}
@@ -134,7 +134,7 @@ class DebugServerProcess(DebugBaseProcess):
         return self._ready
 
     def stdout_data_received(self, data):
-        super(DebugServerProcess, self).stdout_data_received(
+        super().stdout_data_received(
             escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data
         )
         self._std_buffer["out"] += data
@@ -142,7 +142,7 @@ class DebugServerProcess(DebugBaseProcess):
         self._std_buffer["out"] = self._std_buffer["out"][-1 * self.STD_BUFFER_SIZE :]
 
     def stderr_data_received(self, data):
-        super(DebugServerProcess, self).stderr_data_received(data)
+        super().stderr_data_received(data)
         self._std_buffer["err"] += data
         self._check_ready_by_pattern(self._std_buffer["err"])
         self._std_buffer["err"] = self._std_buffer["err"][-1 * self.STD_BUFFER_SIZE :]

platformio/device/commands/list.py

@@ -0,0 +1,99 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+
+from platformio.device.list import (
+    list_logical_devices,
+    list_mdns_services,
+    list_serial_ports,
+)
+
+
+@click.command("list", short_help="List devices")
+@click.option("--serial", is_flag=True, help="List serial ports, default")
+@click.option("--logical", is_flag=True, help="List logical devices")
+@click.option("--mdns", is_flag=True, help="List multicast DNS services")
+@click.option("--json-output", is_flag=True)
+def device_list_cmd(  # pylint: disable=too-many-branches
+    serial, logical, mdns, json_output
+):
+    if not logical and not mdns:
+        serial = True
+    data = {}
+    if serial:
+        data["serial"] = list_serial_ports()
+    if logical:
+        data["logical"] = list_logical_devices()
+    if mdns:
+        data["mdns"] = list_mdns_services()
+
+    single_key = list(data)[0] if len(list(data)) == 1 else None
+
+    if json_output:
+        return click.echo(json.dumps(data[single_key] if single_key else data))
+
+    titles = {
+        "serial": "Serial Ports",
+        "logical": "Logical Devices",
+        "mdns": "Multicast DNS Services",
+    }
+
+    for key, value in data.items():
+        if not single_key:
+            click.secho(titles[key], bold=True)
+            click.echo("=" * len(titles[key]))
+
+        if key == "serial":
+            for item in value:
+                click.secho(item["port"], fg="cyan")
+                click.echo("-" * len(item["port"]))
+                click.echo("Hardware ID: %s" % item["hwid"])
+                click.echo("Description: %s" % item["description"])
+                click.echo("")
+
+        if key == "logical":
+            for item in value:
+                click.secho(item["path"], fg="cyan")
+                click.echo("-" * len(item["path"]))
+                click.echo("Name: %s" % item["name"])
+                click.echo("")
+
+        if key == "mdns":
+            for item in value:
+                click.secho(item["name"], fg="cyan")
+                click.echo("-" * len(item["name"]))
+                click.echo("Type: %s" % item["type"])
+                click.echo("IP: %s" % item["ip"])
+                click.echo("Port: %s" % item["port"])
+                if item["properties"]:
+                    click.echo(
+                        "Properties: %s"
+                        % (
+                            "; ".join(
+                                [
+                                    "%s=%s" % (k, v)
+                                    for k, v in item["properties"].items()
+                                ]
+                            )
+                        )
+                    )
+                click.echo("")
+
+        if single_key:
+            click.echo("")
+
+    return True

platformio/device/commands/monitor.py

@@ -0,0 +1,184 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+import click
+from serial.tools import miniterm
+
+from platformio import exception, fs
+from platformio.device.filters.base import register_filters
+from platformio.device.finder import find_serial_port
+from platformio.platform.factory import PlatformFactory
+from platformio.project.config import ProjectConfig
+from platformio.project.exception import NotPlatformIOProjectError
+from platformio.project.options import ProjectOptions
+
+
+@click.command("monitor", short_help="Monitor device (Serial/Socket)")
+@click.option("--port", "-p", help="Port, a number or a device name")
+@click.option(
+    "--baud",
+    "-b",
+    type=int,
+    help="Set baud rate, default=%d" % ProjectOptions["env.monitor_speed"].default,
+)
+@click.option(
+    "--parity",
+    default="N",
+    type=click.Choice(["N", "E", "O", "S", "M"]),
+    help="Set parity, default=N",
+)
+@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
+@click.option(
+    "--xonxoff", is_flag=True, help="Enable software flow control, default=Off"
+)
+@click.option(
+    "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state"
+)
+@click.option(
+    "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state"
+)
+@click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
+@click.option(
+    "--encoding",
+    default="UTF-8",
+    help="Set the encoding for the serial port (e.g. hexlify, "
+    "Latin1, UTF-8), default: UTF-8",
+)
+@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations")
+@click.option(
+    "--eol",
+    default="CRLF",
+    type=click.Choice(["CR", "LF", "CRLF"]),
+    help="End of line mode, default=CRLF",
+)
+@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations")
+@click.option(
+    "--exit-char",
+    type=int,
+    default=3,
+    help="ASCII code of special character that is used to exit "
+    "the application, default=3 (Ctrl+C)",
+)
+@click.option(
+    "--menu-char",
+    type=int,
+    default=20,
+    help="ASCII code of special character that is used to "
+    "control miniterm (menu), default=20 (DEC)",
+)
+@click.option(
+    "--quiet",
+    is_flag=True,
+    help="Diagnostics: suppress non-error messages, default=Off",
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+)
+@click.option(
+    "-e",
+    "--environment",
+    help="Load configuration from `platformio.ini` and specified environment",
+)
+def device_monitor_cmd(**kwargs):  # pylint: disable=too-many-branches
+    project_options = {}
+    platform = None
+    with fs.cd(kwargs["project_dir"]):
+        try:
+            project_options = get_project_options(kwargs["environment"])
+            kwargs = apply_project_monitor_options(kwargs, project_options)
+            if "platform" in project_options:
+                platform = PlatformFactory.new(project_options["platform"])
+        except NotPlatformIOProjectError:
+            pass
+        register_filters(platform=platform, options=kwargs)
+        kwargs["port"] = find_serial_port(
+            initial_port=kwargs["port"],
+            board_config=platform.board_config(project_options.get("board"))
+            if platform and project_options.get("board")
+            else None,
+            upload_protocol=project_options.get("upload_port"),
+        )
+
+    # override system argv with patched options
+    sys.argv = ["monitor"] + project_options_to_monitor_argv(
+        kwargs,
+        project_options,
+        ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"),
+    )
+
+    if not kwargs["quiet"]:
+        click.echo(
+            "--- Available filters and text transformations: %s"
+            % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys()))
+        )
+        click.echo("--- More details at https://bit.ly/pio-monitor-filters")
+    try:
+        miniterm.main(
+            default_port=kwargs["port"],
+            default_baudrate=kwargs["baud"]
+            or ProjectOptions["env.monitor_speed"].default,
+            default_rts=kwargs["rts"],
+            default_dtr=kwargs["dtr"],
+        )
+    except Exception as e:
+        raise exception.MinitermException(e)
+
+
+def get_project_options(environment=None):
+    config = ProjectConfig.get_instance()
+    config.validate(envs=[environment] if environment else None)
+    environment = environment or config.get_default_env()
+    return config.items(env=environment, as_dict=True)
+
+
+def apply_project_monitor_options(cli_options, project_options):
+    for k in ("port", "speed", "rts", "dtr"):
+        k2 = "monitor_%s" % k
+        if k == "speed":
+            k = "baud"
+        if cli_options[k] is None and k2 in project_options:
+            cli_options[k] = project_options[k2]
+            if k != "port":
+                cli_options[k] = int(cli_options[k])
+    return cli_options
+
+
+def project_options_to_monitor_argv(cli_options, project_options, ignore=None):
+    confmon_flags = project_options.get("monitor_flags", [])
+    result = confmon_flags[::]
+
+    for f in project_options.get("monitor_filters", []):
+        result.extend(["--filter", f])
+
+    for k, v in cli_options.items():
+        if v is None or (ignore and k in ignore):
+            continue
+        k = "--" + k.replace("_", "-")
+        if k in confmon_flags:
+            continue
+        if isinstance(v, bool):
+            if v:
+                result.append(k)
+        elif isinstance(v, tuple):
+            for i in v:
+                result.extend([k, i])
+        else:
+            result.extend([k, str(v)])
+    return result

platformio/device/filters/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/device/filters/base.py

@@ -18,91 +18,35 @@ import os
 from serial.tools import miniterm
 
 from platformio import fs
-from platformio.commands.device import DeviceMonitorFilter
 from platformio.compat import get_object_members, load_python_module
 from platformio.package.manager.tool import ToolPackageManager
 from platformio.project.config import ProjectConfig
 
 
-def apply_project_monitor_options(cli_options, project_options):
+class DeviceMonitorFilterBase(miniterm.Transform):
-    for k in ("port", "speed", "rts", "dtr"):
+    def __init__(self, options=None):
-        k2 = "monitor_%s" % k
+        """Called by PlatformIO to pass context"""
-        if k == "speed":
+        miniterm.Transform.__init__(self)
-            k = "baud"
-        if cli_options[k] is None and k2 in project_options:
-            cli_options[k] = project_options[k2]
-            if k != "port":
-                cli_options[k] = int(cli_options[k])
-    return cli_options
 
+        self.options = options or {}
+        self.project_dir = self.options.get("project_dir")
+        self.environment = self.options.get("environment")
 
-def options_to_argv(cli_options, project_options, ignore=None):
+        self.config = ProjectConfig.get_instance()
-    confmon_flags = project_options.get("monitor_flags", [])
+        if not self.environment:
-    result = confmon_flags[::]
+            default_envs = self.config.default_envs()
+            if default_envs:
+                self.environment = default_envs[0]
+            elif self.config.envs():
+                self.environment = self.config.envs()[0]
 
-    for f in project_options.get("monitor_filters", []):
+    def __call__(self):
-        result.extend(["--filter", f])
+        """Called by the miniterm library when the filter is actually used"""
+        return self
 
-    for k, v in cli_options.items():
+    @property
-        if v is None or (ignore and k in ignore):
+    def NAME(self):
-            continue
+        raise NotImplementedError("Please declare NAME attribute for the filter class")
-        k = "--" + k.replace("_", "-")
-        if k in confmon_flags:
-            continue
-        if isinstance(v, bool):
-            if v:
-                result.append(k)
-        elif isinstance(v, tuple):
-            for i in v:
-                result.extend([k, i])
-        else:
-            result.extend([k, str(v)])
-    return result
-
-
-def get_project_options(environment=None):
-    config = ProjectConfig.get_instance()
-    config.validate(envs=[environment] if environment else None)
-    if not environment:
-        default_envs = config.default_envs()
-        if default_envs:
-            environment = default_envs[0]
-        else:
-            environment = config.envs()[0]
-    return config.items(env=environment, as_dict=True)
-
-
-def get_board_hwids(project_dir, platform, board):
-    with fs.cd(project_dir):
-        return platform.board_config(board).get("build.hwids", [])
-
-
-def load_monitor_filter(path, options=None):
-    name = os.path.basename(path)
-    name = name[: name.find(".")]
-    module = load_python_module("platformio.commands.device.filters.%s" % name, path)
-    for cls in get_object_members(module).values():
-        if (
-            not inspect.isclass(cls)
-            or not issubclass(cls, DeviceMonitorFilter)
-            or cls == DeviceMonitorFilter
-        ):
-            continue
-        obj = cls(options)
-        miniterm.TRANSFORMATIONS[obj.NAME] = obj
-    return True
-
-
-def load_monitor_filters(monitor_dir, prefix=None, options=None):
-    if not os.path.isdir(monitor_dir):
-        return
-    for name in os.listdir(monitor_dir):
-        if (prefix and not name.startswith(prefix)) or not name.endswith(".py"):
-            continue
-        path = os.path.join(monitor_dir, name)
-        if not os.path.isfile(path):
-            continue
-        load_monitor_filter(path, options)
 
 
 def register_filters(platform=None, options=None):
@@ -130,3 +74,31 @@ def register_filters(platform=None, options=None):
         os.path.join(fs.get_source_dir(), "commands", "device", "filters"),
         options=options,
     )
+
+
+def load_monitor_filters(monitor_dir, prefix=None, options=None):
+    if not os.path.isdir(monitor_dir):
+        return
+    for name in os.listdir(monitor_dir):
+        if (prefix and not name.startswith(prefix)) or not name.endswith(".py"):
+            continue
+        path = os.path.join(monitor_dir, name)
+        if not os.path.isfile(path):
+            continue
+        load_monitor_filter(path, options)
+
+
+def load_monitor_filter(path, options=None):
+    name = os.path.basename(path)
+    name = name[: name.find(".")]
+    module = load_python_module("platformio.device.filters.%s" % name, path)
+    for cls in get_object_members(module).values():
+        if (
+            not inspect.isclass(cls)
+            or not issubclass(cls, DeviceMonitorFilterBase)
+            or cls == DeviceMonitorFilterBase
+        ):
+            continue
+        obj = cls(options)
+        miniterm.TRANSFORMATIONS[obj.NAME] = obj
+    return True

platformio/device/filters/hexlify.py

@@ -14,14 +14,14 @@
 
 import serial
 
-from platformio.commands.device import DeviceMonitorFilter
+from platformio.device.filters.base import DeviceMonitorFilterBase
 
 
-class Hexlify(DeviceMonitorFilter):
+class Hexlify(DeviceMonitorFilterBase):
     NAME = "hexlify"
 
     def __init__(self, *args, **kwargs):
-        super(Hexlify, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._counter = 0
 
     def rx(self, text):

platformio/device/filters/log2file.py

@@ -16,14 +16,14 @@ import io
 import os.path
 from datetime import datetime
 
-from platformio.commands.device import DeviceMonitorFilter
+from platformio.device.filters.base import DeviceMonitorFilterBase
 
 
-class LogToFile(DeviceMonitorFilter):
+class LogToFile(DeviceMonitorFilterBase):
     NAME = "log2file"
 
     def __init__(self, *args, **kwargs):
-        super(LogToFile, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._log_fp = None
 
     def __call__(self):

platformio/device/filters/send_on_enter.py

@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio.commands.device import DeviceMonitorFilter
+from platformio.device.filters.base import DeviceMonitorFilterBase
 
 
-class SendOnEnter(DeviceMonitorFilter):
+class SendOnEnter(DeviceMonitorFilterBase):
     NAME = "send_on_enter"
 
     def __init__(self, *args, **kwargs):
-        super(SendOnEnter, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._buffer = ""
 
         if self.options.get("eol") == "CR":

platformio/device/filters/time.py

@@ -14,14 +14,14 @@
 
 from datetime import datetime
 
-from platformio.commands.device import DeviceMonitorFilter
+from platformio.device.filters.base import DeviceMonitorFilterBase
 
 
-class Timestamp(DeviceMonitorFilter):
+class Timestamp(DeviceMonitorFilterBase):
     NAME = "time"
 
     def __init__(self, *args, **kwargs):
-        super(Timestamp, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._line_started = False
 
     def rx(self, text):

platformio/device/finder.py

@@ -0,0 +1,111 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from fnmatch import fnmatch
+
+import serial
+
+from platformio.compat import IS_WINDOWS
+from platformio.device.list import list_logical_devices, list_serial_ports
+
+
+def is_pattern_port(port):
+    if not port:
+        return False
+    return set(["*", "?", "[", "]"]) & set(port)
+
+
+def match_serial_port(pattern):
+    for item in list_serial_ports():
+        if fnmatch(item["port"], pattern):
+            return item["port"]
+    return None
+
+
+def is_serial_port_ready(port, timeout=1):
+    try:
+        serial.Serial(port, timeout=timeout).close()
+        return True
+    except:  # pylint: disable=bare-except
+        pass
+    return False
+
+
+def find_serial_port(
+    initial_port, board_config=None, upload_protocol=None, ensure_ready=False
+):
+    if initial_port:
+        if not is_pattern_port(initial_port):
+            return initial_port
+        return match_serial_port(initial_port)
+    port = None
+    if upload_protocol and upload_protocol.startswith("blackmagic"):
+        port = find_blackmagic_serial_port()
+    if not port and board_config:
+        port = find_board_serial_port(board_config)
+    if port:
+        return port
+
+    # pick the last PID:VID USB device
+    usb_port = None
+    for item in list_serial_ports():
+        if ensure_ready and not is_serial_port_ready(item["port"]):
+            continue
+        port = item["port"]
+        if "VID:PID" in item["hwid"]:
+            usb_port = port
+    return usb_port or port
+
+
+def find_blackmagic_serial_port():
+    for item in list_serial_ports():
+        port = item["port"]
+        if IS_WINDOWS and port.startswith("COM") and len(port) > 4:
+            port = "\\\\.\\%s" % port
+        if "GDB" in item["description"]:
+            return port
+    return None
+
+
+def find_board_serial_port(board_config):
+    board_hwids = board_config.get("build.hwids", [])
+    if not board_hwids:
+        return None
+    for item in list_serial_ports(filter_hwid=True):
+        port = item["port"]
+        for hwid in board_hwids:
+            hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
+            if hwid_str in item["hwid"]:
+                return port
+    return None
+
+
+def find_mbed_disk(initial_port):
+    msdlabels = ("mbed", "nucleo", "frdm", "microbit")
+    for item in list_logical_devices():
+        if item["path"].startswith("/net"):
+            continue
+        if (
+            initial_port
+            and is_pattern_port(initial_port)
+            and not fnmatch(item["path"], initial_port)
+        ):
+            continue
+        mbed_pages = [os.path.join(item["path"], n) for n in ("mbed.htm", "mbed.html")]
+        if any(os.path.isfile(p) for p in mbed_pages):
+            return item["path"]
+        if item["name"] and any(l in item["name"].lower() for l in msdlabels):
+            return item["path"]
+    return None

platformio/device/list.py

@@ -0,0 +1,154 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+import time
+from glob import glob
+
+import zeroconf
+
+from platformio import __version__, exception, proc
+from platformio.compat import IS_MACOS, IS_WINDOWS
+
+
+def list_serial_ports(filter_hwid=False):
+    try:
+        # pylint: disable=import-outside-toplevel
+        from serial.tools.list_ports import comports
+    except ImportError:
+        raise exception.GetSerialPortsError(os.name)
+
+    result = []
+    for p, d, h in comports():
+        if not p:
+            continue
+        if not filter_hwid or "VID:PID" in h:
+            result.append({"port": p, "description": d, "hwid": h})
+
+    if filter_hwid:
+        return result
+
+    # fix for PySerial
+    if not result and IS_MACOS:
+        for p in glob("/dev/tty.*"):
+            result.append({"port": p, "description": "n/a", "hwid": "n/a"})
+    return result
+
+
+def list_logical_devices():
+    items = []
+    if IS_WINDOWS:
+        try:
+            result = proc.exec_command(
+                ["wmic", "logicaldisk", "get", "name,VolumeName"]
+            ).get("out", "")
+            devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?")
+            for line in result.split("\n"):
+                match = devicenamere.match(line.strip())
+                if not match:
+                    continue
+                items.append({"path": match.group(1) + "\\", "name": match.group(2)})
+            return items
+        except WindowsError:  # pylint: disable=undefined-variable
+            pass
+        # try "fsutil"
+        result = proc.exec_command(["fsutil", "fsinfo", "drives"]).get("out", "")
+        for device in re.findall(r"[A-Z]:\\", result):
+            items.append({"path": device, "name": None})
+        return items
+
+    result = proc.exec_command(["df"]).get("out")
+    devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I)
+    for line in result.split("\n"):
+        match = devicenamere.match(line.strip())
+        if not match:
+            continue
+        items.append({"path": match.group(1), "name": os.path.basename(match.group(1))})
+    return items
+
+
+def list_mdns_services():
+    class mDNSListener(object):
+        def __init__(self):
+            self._zc = zeroconf.Zeroconf(interfaces=zeroconf.InterfaceChoice.All)
+            self._found_types = []
+            self._found_services = []
+
+        def __enter__(self):
+            zeroconf.ServiceBrowser(
+                self._zc,
+                [
+                    "_http._tcp.local.",
+                    "_hap._tcp.local.",
+                    "_services._dns-sd._udp.local.",
+                ],
+                self,
+            )
+            return self
+
+        def __exit__(self, etype, value, traceback):
+            self._zc.close()
+
+        def add_service(self, zc, type_, name):
+            try:
+                assert zeroconf.service_type_name(name)
+                assert str(name)
+            except (AssertionError, UnicodeError, zeroconf.BadTypeInNameException):
+                return
+            if name not in self._found_types:
+                self._found_types.append(name)
+                zeroconf.ServiceBrowser(self._zc, name, self)
+            if type_ in self._found_types:
+                s = zc.get_service_info(type_, name)
+                if s:
+                    self._found_services.append(s)
+
+        def remove_service(self, zc, type_, name):
+            pass
+
+        def update_service(self, zc, type_, name):
+            pass
+
+        def get_services(self):
+            return self._found_services
+
+    items = []
+    with mDNSListener() as mdns:
+        time.sleep(3)
+        for service in mdns.get_services():
+            properties = None
+            if service.properties:
+                try:
+                    properties = {
+                        k.decode("utf8"): v.decode("utf8")
+                        if isinstance(v, bytes)
+                        else v
+                        for k, v in service.properties.items()
+                    }
+                    json.dumps(properties)
+                except UnicodeDecodeError:
+                    properties = None
+
+            items.append(
+                {
+                    "type": service.type,
+                    "name": service.name,
+                    "ip": ", ".join(service.parsed_addresses()),
+                    "port": service.port,
+                    "properties": properties,
+                }
+            )
+    return items

platformio/exception.py

@@ -22,7 +22,7 @@ class PlatformioException(Exception):
             # pylint: disable=not-an-iterable
             return self.MESSAGE.format(*self.args)
 
-        return super(PlatformioException, self).__str__()
+        return super().__str__()
 
 
 class ReturnErrorCode(PlatformioException):
@@ -48,7 +48,7 @@ class AbortedByUser(UserSideException):
 #
 
 
-class InvalidUdevRules(PlatformioException):
+class InvalidUdevRules(UserSideException):
     pass
 
 
@@ -135,14 +135,3 @@ class CygwinEnvDetected(PlatformioException):
         "PlatformIO does not work within Cygwin environment. "
         "Use native Terminal instead."
     )
-
-
-class TestDirNotExists(UserSideException):
-
-    MESSAGE = (
-        "A test folder '{0}' does not exist.\nPlease create 'test' "
-        "directory in project's root and put a test set.\n"
-        "More details about Unit "
-        "Testing: https://docs.platformio.org/page/plus/"
-        "unit-testing.html"
-    )

platformio/fs.py

@@ -146,33 +146,40 @@ def path_endswith_ext(path, extensions):
 
 
 def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True):
-    def _append_build_item(items, item, src_dir):
+    def _add_candidate(items, item, src_dir):
         if not src_exts or path_endswith_ext(item, src_exts):
             items.add(os.path.relpath(item, src_dir))
 
+    def _find_candidates(pattern):
+        candidates = set()
+        for item in glob.glob(
+            os.path.join(glob.escape(src_dir), pattern), recursive=True
+        ):
+            if not os.path.isdir(item):
+                _add_candidate(candidates, item, src_dir)
+                continue
+            for root, dirs, files in os.walk(item, followlinks=followlinks):
+                for d in dirs if not followlinks else []:
+                    if os.path.islink(os.path.join(root, d)):
+                        _add_candidate(candidates, os.path.join(root, d), src_dir)
+                for f in files:
+                    _add_candidate(candidates, os.path.join(root, f), src_dir)
+        return candidates
+
     src_filter = src_filter or ""
     if isinstance(src_filter, (list, tuple)):
         src_filter = " ".join(src_filter)
 
-    matches = set()
+    result = set()
     # correct fs directory separator
     src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep)
     for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter):
-        items = set()
+        candidates = _find_candidates(pattern)
-        for item in glob.glob(
-            os.path.join(glob.escape(src_dir), pattern), recursive=True
-        ):
-            if os.path.isdir(item):
-                for root, _, files in os.walk(item, followlinks=followlinks):
-                    for f in files:
-                        _append_build_item(items, os.path.join(root, f), src_dir)
-            else:
-                _append_build_item(items, item, src_dir)
         if action == "+":
-            matches |= items
+            result |= candidates
         else:
-            matches -= items
+            result -= candidates
-    return sorted(list(matches))
+    return sorted(list(result))
 
 
 def to_unix_path(path):

platformio/maintenance.py

@@ -23,18 +23,13 @@ from platformio import __version__, app, exception, fs, telemetry
 from platformio.cache import cleanup_content_cache
 from platformio.clients import http
 from platformio.commands import PlatformioCLI
-from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
-from platformio.commands.lib.command import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.commands.system.prune import calculate_unnecessary_system_data
 from platformio.commands.upgrade import get_latest_version
 from platformio.package.manager.core import update_core_packages
-from platformio.package.manager.library import LibraryPackageManager
-from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.manager.tool import ToolPackageManager
 from platformio.package.meta import PackageSpec
 from platformio.package.version import pepver_to_semver
-from platformio.platform.factory import PlatformFactory
 
 
 def on_platformio_start(ctx, force, caller):
@@ -54,8 +49,6 @@ def on_platformio_end(ctx, result):  # pylint: disable=unused-argument
 
     try:
         check_platformio_upgrade()
-        check_internal_updates(ctx, "platforms")
-        check_internal_updates(ctx, "libraries")
         check_prune_system()
     except (
         http.HTTPClientError,
@@ -157,13 +150,10 @@ def after_upgrade(ctx):
         return
     else:
         click.secho("Please wait while upgrading PlatformIO...", fg="yellow")
-        try:
-            cleanup_content_cache("http")
-        except:  # pylint: disable=bare-except
-            pass
 
         # Update PlatformIO's Core packages
-        update_core_packages(silent=True)
+        cleanup_content_cache("http")
+        update_core_packages()
 
         u = Upgrader(last_version, __version__)
         if u.run(ctx):
@@ -212,18 +202,21 @@ def after_upgrade(ctx):
 
 
 def check_platformio_upgrade():
-    last_check = app.get_state_item("last_check", {})
     interval = int(app.get_setting("check_platformio_interval")) * 3600 * 24
-    if (time() - interval) < last_check.get("platformio_upgrade", 0):
+    check_state = app.get_state_item("last_check", {})
+    last_checked_time = check_state.get("platformio_upgrade", 0)
+    if (time() - interval) < last_checked_time:
         return
 
-    last_check["platformio_upgrade"] = int(time())
+    check_state["platformio_upgrade"] = int(time())
-    app.set_state_item("last_check", last_check)
+    app.set_state_item("last_check", check_state)
+    if not last_checked_time:
+        return
 
     http.ensure_internet_on(raise_exception=True)
 
-    # Update PlatformIO's Core packages
+    # Update PlatformIO Core packages
-    update_core_packages(silent=True)
+    update_core_packages()
 
     latest_version = get_latest_version()
     if pepver_to_semver(latest_version) <= pepver_to_semver(__version__):
@@ -239,10 +232,7 @@ def check_platformio_upgrade():
         fg="yellow",
         nl=False,
     )
-    if os.getenv("PLATFORMIO_IDE"):
+    if os.path.join("Cellar", "platformio") in fs.get_source_dir():
-        click.secho("PlatformIO IDE Menu: Upgrade PlatformIO", fg="cyan", nl=False)
-        click.secho("`.", fg="yellow")
-    elif os.path.join("Cellar", "platformio") in fs.get_source_dir():
         click.secho("brew update && brew upgrade", fg="cyan", nl=False)
         click.secho("` command.", fg="yellow")
     else:
@@ -256,86 +246,19 @@ def check_platformio_upgrade():
     click.echo("")
 
 
-def check_internal_updates(ctx, what):  # pylint: disable=too-many-branches
-    last_check = app.get_state_item("last_check", {})
-    interval = int(app.get_setting("check_%s_interval" % what)) * 3600 * 24
-    if (time() - interval) < last_check.get(what + "_update", 0):
-        return
-
-    last_check[what + "_update"] = int(time())
-    app.set_state_item("last_check", last_check)
-
-    http.ensure_internet_on(raise_exception=True)
-
-    outdated_items = []
-    pm = PlatformPackageManager() if what == "platforms" else LibraryPackageManager()
-    for pkg in pm.get_installed():
-        if pkg.metadata.name in outdated_items:
-            continue
-        conds = [
-            pm.outdated(pkg).is_outdated(),
-            what == "platforms" and PlatformFactory.new(pkg).are_outdated_packages(),
-        ]
-        if any(conds):
-            outdated_items.append(pkg.metadata.name)
-
-    if not outdated_items:
-        return
-
-    terminal_width, _ = shutil.get_terminal_size()
-
-    click.echo("")
-    click.echo("*" * terminal_width)
-    click.secho(
-        "There are the new updates for %s (%s)" % (what, ", ".join(outdated_items)),
-        fg="yellow",
-    )
-
-    if not app.get_setting("auto_update_" + what):
-        click.secho("Please update them via ", fg="yellow", nl=False)
-        click.secho(
-            "`platformio %s update`"
-            % ("lib --global" if what == "libraries" else "platform"),
-            fg="cyan",
-            nl=False,
-        )
-        click.secho(" command.\n", fg="yellow")
-        click.secho(
-            "If you want to manually check for the new versions "
-            "without updating, please use ",
-            fg="yellow",
-            nl=False,
-        )
-        click.secho(
-            "`platformio %s update --dry-run`"
-            % ("lib --global" if what == "libraries" else "platform"),
-            fg="cyan",
-            nl=False,
-        )
-        click.secho(" command.", fg="yellow")
-    else:
-        click.secho("Please wait while updating %s ..." % what, fg="yellow")
-        if what == "platforms":
-            ctx.invoke(cmd_platform_update, platforms=outdated_items)
-        elif what == "libraries":
-            ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [pm.package_dir]
-            ctx.invoke(cmd_lib_update, libraries=outdated_items)
-        click.echo()
-
-        telemetry.send_event(category="Auto", action="Update", label=what.title())
-
-    click.echo("*" * terminal_width)
-    click.echo("")
-
-
 def check_prune_system():
-    last_check = app.get_state_item("last_check", {})
     interval = 30 * 3600 * 24  # 1 time per month
-    if (time() - interval) < last_check.get("prune_system", 0):
+    check_state = app.get_state_item("last_check", {})
+    last_checked_time = check_state.get("prune_system", 0)
+    if (time() - interval) < last_checked_time:
+        return
+
+    check_state["prune_system"] = int(time())
+    app.set_state_item("last_check", check_state)
+
+    if not last_checked_time:
         return
 
-    last_check["prune_system"] = int(time())
-    app.set_state_item("last_check", last_check)
     threshold_mb = int(app.get_setting("check_prune_system_threshold") or 0)
     if threshold_mb <= 0:
         return

platformio/package/commands/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/package/commands/exec.py

@@ -0,0 +1,105 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+import click
+
+from platformio.compat import IS_MACOS, IS_WINDOWS
+from platformio.exception import ReturnErrorCode, UserSideException
+from platformio.package.manager.tool import ToolPackageManager
+from platformio.proc import get_pythonexe_path
+
+
+@click.command("exec", short_help="Run command from package tool")
+@click.option("-p", "--package", metavar="SPECIFICATION")
+@click.option("-c", "--call", metavar="<cmd> [args...]")
+@click.argument("args", nargs=-1, type=click.UNPROCESSED)
+@click.pass_obj
+def package_exec_cmd(obj, package, call, args):
+    if not call and not args:
+        raise click.BadArgumentUsage("Please provide command name")
+    pkg = None
+    if package:
+        pm = ToolPackageManager()
+        pkg = pm.get_package(package)
+        if not pkg:
+            pkg = pm.install(package)
+    else:
+        executable = args[0] if args else call.split(" ")[0]
+        pkg = find_pkg_by_executable(executable)
+        if not pkg:
+            raise UserSideException(
+                "Could not find a package with '%s' executable file" % executable
+            )
+
+    click.echo(
+        "Using %s package"
+        % click.style("%s@%s" % (pkg.metadata.name, pkg.metadata.version), fg="cyan")
+    )
+
+    inject_pkg_to_environ(pkg)
+    os.environ["PIO_PYTHON_EXE"] = get_pythonexe_path()
+    # inject current python interpreter on Windows
+    if IS_WINDOWS and args and args[0].endswith(".py"):
+        args = [os.environ["PIO_PYTHON_EXE"]] + list(args)
+    result = None
+    try:
+        run_options = dict(shell=call is not None, env=os.environ)
+        force_click_stream = (obj or {}).get("force_click_stream")
+        if force_click_stream:
+            run_options.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        result = subprocess.run(  # pylint: disable=subprocess-run-check
+            call or args, **run_options
+        )
+        if force_click_stream:
+            click.echo(result.stdout.decode().strip(), err=result.returncode != 0)
+    except Exception as exc:
+        raise UserSideException(exc)
+
+    if result and result.returncode != 0:
+        raise ReturnErrorCode(result.returncode)
+
+
+def find_pkg_by_executable(executable):
+    exes = [executable]
+    if IS_WINDOWS and not executable.endswith(".exe"):
+        exes.append(f"{executable}.exe")
+    for pkg in ToolPackageManager().get_installed():
+        for exe in exes:
+            if os.path.exists(os.path.join(pkg.path, exe)) or os.path.exists(
+                os.path.join(pkg.path, "bin", exe)
+            ):
+                return pkg
+    return None
+
+
+def inject_pkg_to_environ(pkg):
+    bin_dir = os.path.join(pkg.path, "bin")
+    lib_dir = os.path.join(pkg.path, "lib")
+
+    paths = [bin_dir, pkg.path] if os.path.isdir(bin_dir) else [pkg.path]
+    if os.environ.get("PATH"):
+        paths.append(os.environ.get("PATH"))
+    os.environ["PATH"] = os.pathsep.join(paths)
+
+    if IS_WINDOWS or not os.path.isdir(lib_dir) or "toolchain" in pkg.metadata.name:
+        return
+
+    lib_path_key = "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH"
+    lib_paths = [lib_dir]
+    if os.environ.get(lib_path_key):
+        lib_paths.append(os.environ.get(lib_path_key))
+    os.environ[lib_path_key] = os.pathsep.join(lib_paths)

platformio/package/commands/install.py

@@ -0,0 +1,316 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from pathlib import Path
+
+import click
+
+from platformio import fs
+from platformio.package.exception import UnknownPackageError
+from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.package.manager.tool import ToolPackageManager
+from platformio.package.meta import PackageSpec
+from platformio.project.config import ProjectConfig
+from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies
+from platformio.test.result import TestSuite
+from platformio.test.runners.factory import TestRunnerFactory
+
+
+@click.command(
+    "install", short_help="Install the project dependencies or custom packages"
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+)
+@click.option("-e", "--environment", "environments", multiple=True)
+@click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True)
+@click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True)
+@click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True)
+@click.option(
+    "--no-save",
+    is_flag=True,
+    help="Prevent saving specified packages to `platformio.ini`",
+)
+@click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies")
+@click.option("-g", "--global", is_flag=True, help="Install package globally")
+@click.option(
+    "--storage-dir",
+    default=None,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    help="Custom Package Manager storage for global packages",
+)
+@click.option("-f", "--force", is_flag=True, help="Reinstall package if it exists")
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
+def package_install_cmd(**options):
+    if options.get("global"):
+        install_global_dependencies(options)
+    else:
+        install_project_dependencies(options)
+
+
+def install_global_dependencies(options):
+    pm = PlatformPackageManager(options.get("storage_dir"))
+    tm = ToolPackageManager(options.get("storage_dir"))
+    lm = LibraryPackageManager(options.get("storage_dir"))
+    for obj in (pm, tm, lm):
+        obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG)
+    for spec in options.get("platforms"):
+        pm.install(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+    for spec in options.get("tools"):
+        tm.install(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+    for spec in options.get("libraries", []):
+        lm.install(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+
+
+def install_project_dependencies(options):
+    environments = options["environments"]
+    with fs.cd(options["project_dir"]):
+        config = ProjectConfig.get_instance()
+        config.validate(environments)
+        for env in config.envs():
+            if environments and env not in environments:
+                continue
+            if not options.get("silent"):
+                click.echo(
+                    "Resolving %s environment packages..." % click.style(env, fg="cyan")
+                )
+            already_up_to_date = not install_project_env_dependencies(env, options)
+            if not options.get("silent") and already_up_to_date:
+                click.secho("Already up-to-date.", fg="green")
+
+
+def install_project_env_dependencies(project_env, options=None):
+    """Used in `pio run` -> Processor"""
+    options = options or {}
+    installed_conds = []
+    # custom platforms
+    if options.get("platforms"):
+        installed_conds.append(
+            _install_project_env_custom_platforms(project_env, options)
+        )
+    # custom tools
+    if options.get("tools"):
+        installed_conds.append(_install_project_env_custom_tools(project_env, options))
+    # custom ibraries
+    if options.get("libraries"):
+        installed_conds.append(
+            _install_project_env_custom_libraries(project_env, options)
+        )
+    # declared dependencies
+    if not installed_conds:
+        installed_conds = [
+            _install_project_env_platform(project_env, options),
+            _install_project_env_libraries(project_env, options),
+        ]
+    return any(installed_conds)
+
+
+def _install_project_env_platform(project_env, options):
+    config = ProjectConfig.get_instance()
+    pm = PlatformPackageManager()
+    if options.get("silent"):
+        pm.set_log_level(logging.WARN)
+    spec = config.get(f"env:{project_env}", "platform")
+    if not spec:
+        return False
+    already_up_to_date = not options.get("force")
+    if not pm.get_package(spec):
+        already_up_to_date = False
+    PlatformPackageManager().install(
+        spec,
+        project_env=project_env,
+        project_targets=options.get("project_targets"),
+        skip_dependencies=options.get("skip_dependencies"),
+        force=options.get("force"),
+    )
+    return not already_up_to_date
+
+
+def _install_project_env_custom_platforms(project_env, options):
+    already_up_to_date = not options.get("force")
+    pm = PlatformPackageManager()
+    if not options.get("silent"):
+        pm.set_log_level(logging.DEBUG)
+    for spec in options.get("platforms"):
+        if not pm.get_package(spec):
+            already_up_to_date = False
+        pm.install(
+            spec,
+            project_env=project_env,
+            project_targets=options.get("project_targets"),
+            skip_dependencies=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+    return not already_up_to_date
+
+
+def _install_project_env_custom_tools(project_env, options):
+    already_up_to_date = not options.get("force")
+    tm = ToolPackageManager()
+    if not options.get("silent"):
+        tm.set_log_level(logging.DEBUG)
+    specs_to_save = []
+    for tool in options.get("tools"):
+        spec = PackageSpec(tool)
+        if not tm.get_package(spec):
+            already_up_to_date = False
+        pkg = tm.install(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+        specs_to_save.append(pkg_to_save_spec(pkg, spec))
+    if not options.get("no_save") and specs_to_save:
+        save_project_dependencies(
+            os.getcwd(),
+            specs_to_save,
+            scope="platform_packages",
+            action="add",
+            environments=[project_env],
+        )
+    return not already_up_to_date
+
+
+def _install_project_env_libraries(project_env, options):
+    _uninstall_project_unused_libdeps(project_env, options)
+    already_up_to_date = not options.get("force")
+    config = ProjectConfig.get_instance()
+    env_lm = LibraryPackageManager(
+        os.path.join(config.get("platformio", "libdeps_dir"), project_env)
+    )
+    private_lm = LibraryPackageManager(
+        os.path.join(config.get("platformio", "lib_dir"))
+    )
+    if options.get("silent"):
+        env_lm.set_log_level(logging.WARN)
+        private_lm.set_log_level(logging.WARN)
+
+    lib_deps = config.get(f"env:{project_env}", "lib_deps")
+    if "__test" in options.get("project_targets", []):
+        test_runner = TestRunnerFactory.new(
+            TestSuite(project_env, options.get("piotest_running_name", "*")), config
+        )
+        lib_deps.extend(test_runner.EXTRA_LIB_DEPS or [])
+
+    for library in lib_deps:
+        spec = PackageSpec(library)
+        # skip built-in dependencies
+        if not spec.external and not spec.owner:
+            continue
+        if not env_lm.get_package(spec):
+            already_up_to_date = False
+        env_lm.install(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+
+    # install dependencies from the private libraries
+    for pkg in private_lm.get_installed():
+        _install_project_private_library_deps(pkg, private_lm, env_lm, options)
+
+    return not already_up_to_date
+
+
+def _uninstall_project_unused_libdeps(project_env, options):
+    config = ProjectConfig.get_instance()
+    lib_deps = set(config.get(f"env:{project_env}", "lib_deps"))
+    if not lib_deps:
+        return
+    storage_dir = Path(config.get("platformio", "libdeps_dir"), project_env)
+    integrity_dat = storage_dir / "integrity.dat"
+    if integrity_dat.is_file():
+        prev_lib_deps = set(
+            integrity_dat.read_text(encoding="utf-8").strip().split("\n")
+        )
+        if lib_deps == prev_lib_deps:
+            return
+        lm = LibraryPackageManager(str(storage_dir))
+        if options.get("silent"):
+            lm.set_log_level(logging.WARN)
+        else:
+            click.secho("Removing unused dependencies...")
+        for spec in set(prev_lib_deps) - set(lib_deps):
+            try:
+                lm.uninstall(spec)
+            except UnknownPackageError:
+                pass
+    storage_dir.mkdir(parents=True, exist_ok=True)
+    integrity_dat.write_text("\n".join(lib_deps), encoding="utf-8")
+
+
+def _install_project_private_library_deps(private_pkg, private_lm, env_lm, options):
+    for dependency in private_lm.get_pkg_dependencies(private_pkg) or []:
+        spec = private_lm.dependency_to_spec(dependency)
+        # skip built-in dependencies
+        if not spec.external and not spec.owner:
+            continue
+        pkg = private_lm.get_package(spec)
+        if not pkg and not env_lm.get_package(spec):
+            pkg = env_lm.install(
+                spec,
+                skip_dependencies=True,
+                force=options.get("force"),
+            )
+        if not pkg:
+            continue
+        _install_project_private_library_deps(pkg, private_lm, env_lm, options)
+
+
+def _install_project_env_custom_libraries(project_env, options):
+    already_up_to_date = not options.get("force")
+    config = ProjectConfig.get_instance()
+    lm = LibraryPackageManager(
+        os.path.join(config.get("platformio", "libdeps_dir"), project_env)
+    )
+    if not options.get("silent"):
+        lm.set_log_level(logging.DEBUG)
+    specs_to_save = []
+    for library in options.get("libraries") or []:
+        spec = PackageSpec(library)
+        if not lm.get_package(spec):
+            already_up_to_date = False
+        pkg = lm.install(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+            force=options.get("force"),
+        )
+        specs_to_save.append(pkg_to_save_spec(pkg, spec))
+    if not options.get("no_save") and specs_to_save:
+        save_project_dependencies(
+            os.getcwd(),
+            specs_to_save,
+            scope="lib_deps",
+            action="add",
+            environments=[project_env],
+        )
+    return not already_up_to_date

platformio/package/commands/list.py

@@ -0,0 +1,221 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import List
+
+import click
+
+from platformio import fs
+from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.package.manager.tool import ToolPackageManager
+from platformio.package.meta import PackageItem, PackageSpec
+from platformio.platform.factory import PlatformFactory
+from platformio.project.config import ProjectConfig
+
+
+@click.command("list", short_help="List installed packages")
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+)
+@click.option("-e", "--environment", "environments", multiple=True)
+@click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True)
+@click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True)
+@click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True)
+@click.option("-g", "--global", is_flag=True, help="List globally installed packages")
+@click.option(
+    "--storage-dir",
+    default=None,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    help="Custom Package Manager storage for global packages",
+)
+@click.option("--only-platforms", is_flag=True, help="List only platform packages")
+@click.option("--only-tools", is_flag=True, help="List only tool packages")
+@click.option("--only-libraries", is_flag=True, help="List only library packages")
+@click.option("-v", "--verbose", is_flag=True)
+def package_list_cmd(**options):
+    if options.get("global"):
+        list_global_packages(options)
+    else:
+        list_project_packages(options)
+
+
+def humanize_package(pkg, spec=None, verbose=False):
+    if spec and not isinstance(spec, PackageSpec):
+        spec = PackageSpec(spec)
+    data = [
+        click.style("{name} @ {version}".format(**pkg.metadata.as_dict()), fg="cyan")
+    ]
+    extra_data = ["required: %s" % (spec.humanize() if spec else "Any")]
+    if verbose:
+        extra_data.append(pkg.path)
+    data.append("(%s)" % ", ".join(extra_data))
+    return " ".join(data)
+
+
+def print_dependency_tree(pm, specs=None, filter_specs=None, level=0, verbose=False):
+    filtered_pkgs = [
+        pm.get_package(spec) for spec in filter_specs or [] if pm.get_package(spec)
+    ]
+    candidates = {}
+    if specs:
+        for spec in specs:
+            pkg = pm.get_package(spec)
+            if not pkg:
+                continue
+            candidates[pkg.path] = (pkg, spec)
+    else:
+        candidates = {pkg.path: (pkg, pkg.metadata.spec) for pkg in pm.get_installed()}
+    if not candidates:
+        return
+    candidates = sorted(candidates.values(), key=lambda item: item[0].metadata.name)
+    for index, (pkg, spec) in enumerate(candidates):
+        if filtered_pkgs and not _pkg_tree_contains(pm, pkg, filtered_pkgs):
+            continue
+        dependencies = pm.get_pkg_dependencies(pkg)
+        click.echo(
+            "%s%s %s"
+            % (
+                "│   " * level,
+                "├──" if index < len(candidates) - 1 else "└──",
+                humanize_package(
+                    pkg,
+                    spec=spec,
+                    verbose=verbose,
+                ),
+            )
+        )
+        if dependencies:
+            print_dependency_tree(
+                pm,
+                specs=[pm.dependency_to_spec(item) for item in dependencies],
+                filter_specs=filter_specs,
+                level=level + 1,
+                verbose=verbose,
+            )
+
+
+def _pkg_tree_contains(pm, root: PackageItem, children: List[PackageItem]):
+    if root in children:
+        return True
+    for dependency in pm.get_pkg_dependencies(root) or []:
+        pkg = pm.get_package(pm.dependency_to_spec(dependency))
+        if pkg and _pkg_tree_contains(pm, pkg, children):
+            return True
+    return False
+
+
+def list_global_packages(options):
+    data = [
+        ("platforms", PlatformPackageManager(options.get("storage_dir"))),
+        ("tools", ToolPackageManager(options.get("storage_dir"))),
+        ("libraries", LibraryPackageManager(options.get("storage_dir"))),
+    ]
+    only_packages = any(
+        options.get(type_) or options.get(f"only_{type_}") for (type_, _) in data
+    )
+    for (type_, pm) in data:
+        skip_conds = [
+            only_packages
+            and not options.get(type_)
+            and not options.get(f"only_{type_}"),
+            not pm.get_installed(),
+        ]
+        if any(skip_conds):
+            continue
+        click.secho(type_.capitalize(), bold=True)
+        print_dependency_tree(
+            pm, filter_specs=options.get(type_), verbose=options.get("verbose")
+        )
+        click.echo()
+
+
+def list_project_packages(options):
+    environments = options["environments"]
+    only_packages = any(
+        options.get(type_) or options.get(f"only_{type_}")
+        for type_ in ("platforms", "tools", "libraries")
+    )
+    only_platform_packages = any(
+        options.get(type_) or options.get(f"only_{type_}")
+        for type_ in ("platforms", "tools")
+    )
+    only_library_packages = options.get("libraries") or options.get("only_libraries")
+
+    with fs.cd(options["project_dir"]):
+        config = ProjectConfig.get_instance()
+        config.validate(environments)
+        for env in config.envs():
+            if environments and env not in environments:
+                continue
+            click.echo(
+                "Resolving %s environment packages..." % click.style(env, fg="cyan")
+            )
+            found = False
+            if not only_packages or only_platform_packages:
+                _found = print_project_env_platform_packages(env, options)
+                found = found or _found
+            if not only_packages or only_library_packages:
+                _found = print_project_env_library_packages(env, options)
+                found = found or _found
+            if not found:
+                click.echo("No packages")
+            if (not environments and len(config.envs()) > 1) or len(environments) > 1:
+                click.echo()
+
+
+def print_project_env_platform_packages(project_env, options):
+    config = ProjectConfig.get_instance()
+    platform = config.get(f"env:{project_env}", "platform")
+    if not platform:
+        return None
+    pkg = PlatformPackageManager().get_package(platform)
+    if not pkg:
+        return None
+    click.echo(
+        "Platform %s"
+        % (humanize_package(pkg, platform, verbose=options.get("verbose")))
+    )
+    p = PlatformFactory.new(pkg)
+    if project_env:
+        p.configure_project_packages(project_env)
+    print_dependency_tree(
+        p.pm,
+        specs=[p.get_package_spec(name) for name in p.packages],
+        filter_specs=options.get("tools"),
+    )
+    click.echo()
+    return True
+
+
+def print_project_env_library_packages(project_env, options):
+    config = ProjectConfig.get_instance()
+    lib_deps = config.get(f"env:{project_env}", "lib_deps")
+    lm = LibraryPackageManager(
+        os.path.join(config.get("platformio", "libdeps_dir"), project_env)
+    )
+    if not lib_deps or not lm.get_installed():
+        return None
+    click.echo("Libraries")
+    print_dependency_tree(
+        lm,
+        lib_deps,
+        filter_specs=options.get("libraries"),
+        verbose=options.get("verbose"),
+    )
+    return True

platformio/package/commands/outdated.py

@@ -0,0 +1,220 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import click
+from tabulate import tabulate
+
+from platformio import fs
+from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.package.meta import PackageSpec
+from platformio.platform.factory import PlatformFactory
+from platformio.project.config import ProjectConfig
+
+
+class OutdatedCandidate:
+    def __init__(self, pm, pkg, spec, envs=None):
+        self.pm = pm
+        self.pkg = pkg
+        self.spec = spec
+        self.envs = envs or []
+        self.outdated = None
+        if not isinstance(self.envs, list):
+            self.envs = [self.envs]
+
+    def __eq__(self, other):
+        return all(
+            [
+                self.pm.package_dir == other.pm.package_dir,
+                self.pkg == other.pkg,
+                self.spec == other.spec,
+            ]
+        )
+
+    def check(self):
+        self.outdated = self.pm.outdated(self.pkg, self.spec)
+
+    def is_outdated(self):
+        if not self.outdated:
+            self.check()
+        return self.outdated.is_outdated(allow_incompatible=self.pm.pkg_type != "tool")
+
+
+@click.command("outdated", short_help="Check for outdated packages")
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+)
+@click.option("-e", "--environment", "environments", multiple=True)
+def package_outdated_cmd(project_dir, environments):
+    candidates = fetch_outdated_candidates(
+        project_dir, environments, with_progress=True
+    )
+    print_outdated_candidates(candidates)
+
+
+def print_outdated_candidates(candidates):
+    if not candidates:
+        click.secho("Everything is up-to-date!", fg="green")
+        return
+    tabulate_data = [
+        (
+            click.style(
+                candidate.pkg.metadata.name,
+                fg=get_candidate_update_color(candidate.outdated),
+            ),
+            candidate.outdated.current,
+            candidate.outdated.wanted,
+            click.style(candidate.outdated.latest, fg="cyan"),
+            candidate.pm.pkg_type.capitalize(),
+            ", ".join(set(candidate.envs)),
+        )
+        for candidate in candidates
+    ]
+    click.echo()
+    click.secho("Semantic Versioning color legend:", bold=True)
+    click.echo(
+        tabulate(
+            [
+                (
+                    click.style("<Major Update>", fg="red"),
+                    "backward-incompatible updates",
+                ),
+                (
+                    click.style("<Minor Update>", fg="yellow"),
+                    "backward-compatible features",
+                ),
+                (
+                    click.style("<Patch Update>", fg="green"),
+                    "backward-compatible bug fixes",
+                ),
+            ],
+            tablefmt="plain",
+        )
+    )
+    click.echo()
+    click.echo(
+        tabulate(
+            tabulate_data,
+            headers=["Package", "Current", "Wanted", "Latest", "Type", "Environments"],
+        )
+    )
+
+
+def get_candidate_update_color(outdated):
+    if outdated.update_increment_type == outdated.UPDATE_INCREMENT_MAJOR:
+        return "red"
+    if outdated.update_increment_type == outdated.UPDATE_INCREMENT_MINOR:
+        return "yellow"
+    if outdated.update_increment_type == outdated.UPDATE_INCREMENT_PATCH:
+        return "green"
+    return None
+
+
+def fetch_outdated_candidates(project_dir, environments, with_progress=False):
+    candidates = []
+
+    def _add_candidate(data):
+        new_candidate = OutdatedCandidate(
+            data["pm"], data["pkg"], data["spec"], data["env"]
+        )
+        for candidate in candidates:
+            if candidate == new_candidate:
+                candidate.envs.append(data["env"])
+                return
+        candidates.append(new_candidate)
+
+    with fs.cd(project_dir):
+        config = ProjectConfig.get_instance()
+        config.validate(environments)
+
+        # platforms
+        for item in find_platform_candidates(config, environments):
+            _add_candidate(item)
+            # platform package dependencies
+            for dep_item in find_platform_dependency_candidates(item):
+                _add_candidate(dep_item)
+
+        # libraries
+        for item in find_library_candidates(config, environments):
+            _add_candidate(item)
+
+    result = []
+    if not with_progress:
+        for candidate in candidates:
+            if candidate.is_outdated():
+                result.append(candidate)
+        return result
+
+    with click.progressbar(candidates, label="Checking") as pb:
+        for candidate in pb:
+            if candidate.is_outdated():
+                result.append(candidate)
+    return result
+
+
+def find_platform_candidates(config, environments):
+    result = []
+    pm = PlatformPackageManager()
+    for env in config.envs():
+        platform = config.get(f"env:{env}", "platform")
+        if not platform or (environments and env not in environments):
+            continue
+        spec = PackageSpec(platform)
+        pkg = pm.get_package(spec)
+        if not pkg:
+            continue
+        result.append(dict(env=env, pm=pm, pkg=pkg, spec=spec))
+    return result
+
+
+def find_platform_dependency_candidates(platform_candidate):
+    result = []
+    p = PlatformFactory.new(platform_candidate["spec"])
+    p.configure_project_packages(platform_candidate["env"])
+    for pkg in p.get_installed_packages():
+        result.append(
+            dict(
+                env=platform_candidate["env"],
+                pm=p.pm,
+                pkg=pkg,
+                spec=p.get_package_spec(pkg.metadata.name),
+            )
+        )
+    return sorted(result, key=lambda item: item["pkg"].metadata.name)
+
+
+def find_library_candidates(config, environments):
+    result = []
+    for env in config.envs():
+        if environments and env not in environments:
+            continue
+        package_dir = os.path.join(config.get("platformio", "libdeps_dir") or "", env)
+        lib_deps = [
+            item for item in config.get(f"env:{env}", "lib_deps", []) if "/" in item
+        ]
+        if not os.path.isdir(package_dir) or not lib_deps:
+            continue
+        pm = LibraryPackageManager(package_dir)
+        for lib in lib_deps:
+            spec = PackageSpec(lib)
+            pkg = pm.get_package(spec)
+            if not pkg:
+                continue
+            result.append(dict(env=env, pm=pm, pkg=pkg, spec=spec))
+    return sorted(result, key=lambda item: item["pkg"].metadata.name)

platformio/package/commands/pack.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import click
+
+from platformio.package.manifest.parser import ManifestParserFactory
+from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError
+from platformio.package.pack import PackagePacker
+
+
+@click.command("pack", short_help="Create a tarball from a package")
+@click.argument(
+    "package",
+    default=os.getcwd,
+    metavar="<source directory, tar.gz or zip>",
+    type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True),
+)
+@click.option(
+    "-o", "--output", help="A destination path (folder or a full path to file)"
+)
+def package_pack_cmd(package, output):
+    p = PackagePacker(package)
+    archive_path = p.pack(output)
+    # validate manifest
+    try:
+        ManifestSchema().load_manifest(
+            ManifestParserFactory.new_from_archive(archive_path).as_dict()
+        )
+    except ManifestValidationError as e:
+        os.remove(archive_path)
+        raise e
+    click.secho('Wrote a tarball to "%s"' % archive_path, fg="green")

platformio/package/commands/publish.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import tarfile
 import tempfile
 from datetime import datetime
 
@@ -24,8 +25,8 @@ from platformio.clients.account import AccountClient
 from platformio.clients.registry import RegistryClient
 from platformio.exception import UserSideException
 from platformio.package.manifest.parser import ManifestParserFactory
-from platformio.package.manifest.schema import ManifestSchema, ManifestValidationError
+from platformio.package.manifest.schema import ManifestSchema
-from platformio.package.meta import PackageSpec, PackageType
+from platformio.package.meta import PackageType
 from platformio.package.pack import PackagePacker
 from platformio.package.unpack import FileUnpacker, TARArchiver
 
@@ -40,10 +41,119 @@ def validate_datetime(ctx, param, value):  # pylint: disable=unused-argument
     return value
 
 
-def load_manifest_from_archive(path):
+@click.command("publish", short_help="Publish a package to the registry")
-    return ManifestSchema().load_manifest(
+@click.argument(
-        ManifestParserFactory.new_from_archive(path).as_dict()
+    "package",
+    default=os.getcwd,
+    metavar="<source directory, tar.gz or zip>",
+    type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True),
+)
+@click.option(
+    "--owner",
+    help="PIO Account username (can be organization username). "
+    "Default is set to a username of the authorized PIO Account",
+)
+@click.option(
+    "--type",
+    "type_",
+    type=click.Choice(list(PackageType.items().values())),
+    help="Custom package type",
+)
+@click.option(
+    "--released-at",
+    callback=validate_datetime,
+    help="Custom release date and time in the next format (UTC): 2014-06-13 17:08:52",
+)
+@click.option("--private", is_flag=True, help="Restricted access (not a public)")
+@click.option(
+    "--notify/--no-notify",
+    default=True,
+    help="Notify by email when package is processed",
+)
+@click.option(
+    "--non-interactive",
+    is_flag=True,
+    help="Do not show interactive prompt",
+)
+def package_publish_cmd(  # pylint: disable=too-many-arguments, too-many-locals
+    package, owner, type_, released_at, private, notify, non_interactive
+):
+    click.secho("Preparing a package...", fg="cyan")
+    owner = owner or AccountClient().get_logged_username()
+    do_not_pack = (
+        not os.path.isdir(package)
+        and isinstance(FileUnpacker.new_archiver(package), TARArchiver)
+        and PackageType.from_archive(package)
     )
+    archive_path = None
+    with tempfile.TemporaryDirectory() as tmp_dir:  # pylint: disable=no-member
+        # publish .tar.gz instantly without repacking
+        if do_not_pack:
+            archive_path = package
+        else:
+            with fs.cd(tmp_dir):
+                p = PackagePacker(package)
+                archive_path = p.pack()
+
+        type_ = type_ or PackageType.from_archive(archive_path)
+        manifest = ManifestSchema().load_manifest(
+            ManifestParserFactory.new_from_archive(archive_path).as_dict()
+        )
+        name = manifest.get("name")
+        version = manifest.get("version")
+        data = [
+            ("Type:", type_),
+            ("Owner:", owner),
+            ("Name:", name),
+            ("Version:", version),
+            ("Size:", fs.humanize_file_size(os.path.getsize(archive_path))),
+        ]
+        if manifest.get("system"):
+            data.insert(len(data) - 1, ("System:", ", ".join(manifest.get("system"))))
+        click.echo(tabulate(data, tablefmt="plain"))
+
+        # check files containing non-ascii chars
+        check_archive_file_names(archive_path)
+
+        # look for duplicates
+        check_package_duplicates(owner, type_, name, version, manifest.get("system"))
+
+        if not non_interactive:
+            click.confirm(
+                "Are you sure you want to publish the %s %s to the registry?\n"
+                % (
+                    type_,
+                    click.style(
+                        "%s/%s@%s" % (owner, name, version),
+                        fg="cyan",
+                    ),
+                ),
+                abort=True,
+            )
+
+        click.secho(
+            "The package publishing may take some time depending "
+            "on your Internet connection and the package size.",
+            fg="yellow",
+        )
+        click.echo("Publishing...")
+        response = RegistryClient().publish_package(
+            owner, type_, archive_path, released_at, private, notify
+        )
+        if not do_not_pack:
+            os.remove(archive_path)
+        click.secho(response.get("message"), fg="green")
+
+
+def check_archive_file_names(archive_path):
+    with tarfile.open(archive_path, mode="r:gz") as tf:
+        for name in tf.getnames():
+            if not name.isascii():
+                click.secho(
+                    f"Warning! The `{name}` file contains non-ASCII chars and can "
+                    "lead to the unpacking issues on a user machine",
+                    fg="yellow",
+                )
 
 
 def check_package_duplicates(
@@ -52,7 +162,7 @@ def check_package_duplicates(
     found = False
     items = (
         RegistryClient()
-        .list_packages(filters=dict(types=[type], names=[name]))
+        .list_packages(qualifiers=dict(types=[type], names=[name]))
         .get("items")
     )
     if not items:
@@ -86,147 +196,3 @@ def check_package_duplicates(
             fg="yellow",
         )
     return True
-
-
-@click.group("package", short_help="Package manager")
-def cli():
-    pass
-
-
-@cli.command("pack", short_help="Create a tarball from a package")
-@click.argument(
-    "package",
-    required=True,
-    default=os.getcwd,
-    metavar="<source directory, tar.gz or zip>",
-)
-@click.option(
-    "-o", "--output", help="A destination path (folder or a full path to file)"
-)
-def package_pack(package, output):
-    p = PackagePacker(package)
-    archive_path = p.pack(output)
-    # validate manifest
-    try:
-        load_manifest_from_archive(archive_path)
-    except ManifestValidationError as e:
-        os.remove(archive_path)
-        raise e
-    click.secho('Wrote a tarball to "%s"' % archive_path, fg="green")
-
-
-@cli.command("publish", short_help="Publish a package to the registry")
-@click.argument(
-    "package",
-    required=True,
-    default=os.getcwd,
-    metavar="<source directory, tar.gz or zip>",
-)
-@click.option(
-    "--owner",
-    help="PIO Account username (can be organization username). "
-    "Default is set to a username of the authorized PIO Account",
-)
-@click.option(
-    "--released-at",
-    callback=validate_datetime,
-    help="Custom release date and time in the next format (UTC): 2014-06-13 17:08:52",
-)
-@click.option("--private", is_flag=True, help="Restricted access (not a public)")
-@click.option(
-    "--notify/--no-notify",
-    default=True,
-    help="Notify by email when package is processed",
-)
-@click.option(
-    "--non-interactive",
-    is_flag=True,
-    help="Do not show interactive prompt",
-)
-def package_publish(  # pylint: disable=too-many-arguments, too-many-locals
-    package, owner, released_at, private, notify, non_interactive
-):
-    click.secho("Preparing a package...", fg="cyan")
-    owner = owner or AccountClient().get_logged_username()
-    do_not_pack = not os.path.isdir(package) and isinstance(
-        FileUnpacker.new_archiver(package), TARArchiver
-    )
-    archive_path = None
-    with tempfile.TemporaryDirectory() as tmp_dir:  # pylint: disable=no-member
-        # publish .tar.gz instantly without repacking
-        if do_not_pack:
-            archive_path = package
-        else:
-            with fs.cd(tmp_dir):
-                p = PackagePacker(package)
-                archive_path = p.pack()
-
-        type_ = PackageType.from_archive(archive_path)
-        manifest = load_manifest_from_archive(archive_path)
-        name = manifest.get("name")
-        version = manifest.get("version")
-        data = [
-            ("Type:", type_),
-            ("Owner:", owner),
-            ("Name:", name),
-            ("Version:", version),
-        ]
-        if manifest.get("system"):
-            data.insert(len(data) - 1, ("System:", ", ".join(manifest.get("system"))))
-        click.echo(tabulate(data, tablefmt="plain"))
-
-        # look for duplicates
-        check_package_duplicates(owner, type_, name, version, manifest.get("system"))
-
-        if not non_interactive:
-            click.confirm(
-                "Are you sure you want to publish the %s %s to the registry?\n"
-                % (
-                    type_,
-                    click.style(
-                        "%s/%s@%s" % (owner, name, version),
-                        fg="cyan",
-                    ),
-                ),
-                abort=True,
-            )
-
-        click.secho(
-            "The package publishing may take some time depending "
-            "on your Internet connection and the package size.",
-            fg="yellow",
-        )
-        click.echo("Publishing...")
-        response = RegistryClient().publish_package(
-            owner, type_, archive_path, released_at, private, notify
-        )
-        if not do_not_pack:
-            os.remove(archive_path)
-        click.secho(response.get("message"), fg="green")
-
-
-@cli.command("unpublish", short_help="Remove a pushed package from the registry")
-@click.argument(
-    "package", required=True, metavar="[<organization>/]<pkgname>[@<version>]"
-)
-@click.option(
-    "--type",
-    type=click.Choice(list(PackageType.items().values())),
-    default="library",
-    help="Package type, default is set to `library`",
-)
-@click.option(
-    "--undo",
-    is_flag=True,
-    help="Undo a remove, putting a version back into the registry",
-)
-def package_unpublish(package, type, undo):  # pylint: disable=redefined-builtin
-    spec = PackageSpec(package)
-    response = RegistryClient().unpublish_package(
-        owner=spec.owner or AccountClient().get_logged_username(),
-        type=type,
-        name=spec.name,
-        version=str(spec.requirements),
-        undo=undo,
-    )
-    click.secho(response.get("message"), fg="green")

platformio/package/commands/search.py

@@ -0,0 +1,77 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import click
+
+from platformio import util
+from platformio.clients.registry import RegistryClient
+
+
+@click.command("search", short_help="Search for packages")
+@click.argument("query")
+@click.option("-p", "--page", type=click.IntRange(min=1))
+@click.option(
+    "-s",
+    "--sort",
+    type=click.Choice(["relevance", "popularity", "trending", "added", "updated"]),
+)
+def package_search_cmd(query, page, sort):
+    client = RegistryClient()
+    result = client.list_packages(query, page=page, sort=sort)
+    if not result["total"]:
+        click.secho("Nothing has been found by your request", fg="yellow")
+        click.echo(
+            "Try a less-specific search or use truncation (or wildcard) operator *"
+        )
+        return
+    print_search_result(result)
+
+
+def print_search_result(result):
+    click.echo(
+        "Found %d packages (page %d of %d)"
+        % (
+            result["total"],
+            result["page"],
+            math.ceil(result["total"] / result["limit"]),
+        )
+    )
+    for item in result["items"]:
+        click.echo()
+        print_search_item(item)
+
+
+def print_search_item(item):
+    click.echo(
+        "%s/%s"
+        % (
+            click.style(item["owner"]["username"], fg="cyan"),
+            click.style(item["name"], fg="cyan", bold=True),
+        )
+    )
+    click.echo(
+        "%s • %s • Published on %s"
+        % (
+            item["type"].capitalize()
+            if item["tier"] == "community"
+            else click.style(
+                ("%s %s" % (item["tier"], item["type"])).title(), bold=True
+            ),
+            item["version"]["name"],
+            util.parse_datetime(item["version"]["released_at"]).strftime("%c"),
+        )
+    )
+    click.echo(item["description"])

platformio/package/commands/show.py

@@ -0,0 +1,148 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from urllib.parse import quote
+
+import click
+from tabulate import tabulate
+
+from platformio import fs, util
+from platformio.clients.registry import RegistryClient
+from platformio.exception import UserSideException
+from platformio.package.manager._registry import PackageManagerRegistryMixin
+from platformio.package.meta import PackageSpec, PackageType
+
+
+@click.command("show", short_help="Show package information")
+@click.argument("spec", metavar="[<owner>/]<pkg>[@<version>]")
+@click.option(
+    "-t",
+    "--type",
+    "pkg_type",
+    type=click.Choice(list(PackageType.items().values())),
+    help="Package type",
+)
+def package_show_cmd(spec, pkg_type):
+    spec = PackageSpec(spec)
+    data = fetch_package_data(spec, pkg_type)
+    if not data:
+        raise UserSideException(
+            "Could not find '%s' package in the PlatormIO Registry" % spec.humanize()
+        )
+
+    click.echo()
+    click.echo(
+        "%s/%s"
+        % (
+            click.style(data["owner"]["username"], fg="cyan"),
+            click.style(data["name"], fg="cyan", bold=True),
+        )
+    )
+    click.echo(
+        "%s • %s • %s • Published on %s"
+        % (
+            data["type"].capitalize(),
+            data["version"]["name"],
+            "Private" if data.get("private") else "Public",
+            util.parse_datetime(data["version"]["released_at"]).strftime("%c"),
+        )
+    )
+
+    click.echo()
+    type_plural = "libraries" if data["type"] == "library" else (data["type"] + "s")
+    click.secho(
+        "https://registry.platformio.org/%s/%s/%s"
+        % (type_plural, data["owner"]["username"], quote(data["name"])),
+        fg="blue",
+    )
+
+    #  Description
+    click.echo()
+    click.echo(data["description"])
+
+    # Extra info
+    click.echo()
+    fields = [
+        ("homepage", "Homepage"),
+        ("repository_url", "Repository"),
+        ("license", "License"),
+        ("popularity_rank", "Popularity"),
+        ("stars_count", "Stars"),
+        ("examples_count", "Examples"),
+        ("version.unpacked_size", "Installed Size"),
+        ("dependents_count", "Used By"),
+        ("dependencies_count", "Dependencies"),
+        ("platforms", "Compatible Platforms"),
+        ("frameworks", "Compatible Frameworks"),
+        ("keywords", "Keywords"),
+    ]
+    extra = []
+    for key, title in fields:
+        if "." in key:
+            k1, k2 = key.split(".")
+            value = data.get(k1, {}).get(k2)
+        else:
+            value = data.get(key)
+        if not value:
+            continue
+        if isinstance(value, list):
+            value = ", ".join(value)
+        elif key.endswith("_size"):
+            value = fs.humanize_file_size(value)
+        extra.append((title, value))
+    click.echo(tabulate(extra))
+
+    # Versions
+    click.echo("")
+    table = tabulate(
+        [
+            (
+                version["name"],
+                fs.humanize_file_size(max(f["size"] for f in version["files"])),
+                util.parse_datetime(version["released_at"]),
+            )
+            for version in data["versions"]
+        ],
+        headers=["Version", "Size", "Published"],
+    )
+    click.echo(table)
+    click.echo("")
+
+
+def fetch_package_data(spec, pkg_type=None):
+    assert isinstance(spec, PackageSpec)
+    client = RegistryClient()
+    if pkg_type and spec.owner and spec.name:
+        return client.get_package(
+            pkg_type, spec.owner, spec.name, version=spec.requirements
+        )
+    qualifiers = dict(names=spec.name.lower())
+    if pkg_type:
+        qualifiers["types"] = pkg_type
+    if spec.owner:
+        qualifiers["owners"] = spec.owner.lower()
+    packages = client.list_packages(qualifiers=qualifiers)["items"]
+    if not packages:
+        return None
+    if len(packages) > 1:
+        PackageManagerRegistryMixin.print_multi_package_issue(
+            click.echo, packages, spec
+        )
+        return None
+    return client.get_package(
+        packages[0]["type"],
+        packages[0]["owner"]["username"],
+        packages[0]["name"],
+        version=spec.requirements,
+    )

platformio/package/commands/uninstall.py

@@ -0,0 +1,239 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+import click
+
+from platformio import fs
+from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.package.manager.tool import ToolPackageManager
+from platformio.package.meta import PackageSpec
+from platformio.project.config import ProjectConfig
+from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies
+
+
+@click.command(
+    "uninstall", short_help="Uninstall the project dependencies or custom packages"
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+)
+@click.option("-e", "--environment", "environments", multiple=True)
+@click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True)
+@click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True)
+@click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True)
+@click.option(
+    "--no-save",
+    is_flag=True,
+    help="Prevent removing specified packages from `platformio.ini`",
+)
+@click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies")
+@click.option("-g", "--global", is_flag=True, help="Uninstall global packages")
+@click.option(
+    "--storage-dir",
+    default=None,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    help="Custom Package Manager storage for global packages",
+)
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
+def package_uninstall_cmd(**options):
+    if options.get("global"):
+        uninstall_global_dependencies(options)
+    else:
+        uninstall_project_dependencies(options)
+
+
+def uninstall_global_dependencies(options):
+    pm = PlatformPackageManager(options.get("storage_dir"))
+    tm = ToolPackageManager(options.get("storage_dir"))
+    lm = LibraryPackageManager(options.get("storage_dir"))
+    for obj in (pm, tm, lm):
+        obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG)
+    for spec in options.get("platforms"):
+        pm.uninstall(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+    for spec in options.get("tools"):
+        tm.uninstall(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+    for spec in options.get("libraries", []):
+        lm.uninstall(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+
+
+def uninstall_project_dependencies(options):
+    environments = options["environments"]
+    with fs.cd(options["project_dir"]):
+        config = ProjectConfig.get_instance()
+        config.validate(environments)
+        for env in config.envs():
+            if environments and env not in environments:
+                continue
+            if not options["silent"]:
+                click.echo(
+                    "Resolving %s environment packages..." % click.style(env, fg="cyan")
+                )
+            already_up_to_date = not uninstall_project_env_dependencies(env, options)
+            if not options["silent"] and already_up_to_date:
+                click.secho("Already up-to-date.", fg="green")
+
+
+def uninstall_project_env_dependencies(project_env, options=None):
+    options = options or {}
+    uninstalled_conds = []
+    # custom platforms
+    if options.get("platforms"):
+        uninstalled_conds.append(
+            _uninstall_project_env_custom_platforms(project_env, options)
+        )
+    # custom tools
+    if options.get("tools"):
+        uninstalled_conds.append(
+            _uninstall_project_env_custom_tools(project_env, options)
+        )
+    # custom ibraries
+    if options.get("libraries"):
+        uninstalled_conds.append(
+            _uninstall_project_env_custom_libraries(project_env, options)
+        )
+    # declared dependencies
+    if not uninstalled_conds:
+        uninstalled_conds = [
+            _uninstall_project_env_platform(project_env, options),
+            _uninstall_project_env_libraries(project_env, options),
+        ]
+    return any(uninstalled_conds)
+
+
+def _uninstall_project_env_platform(project_env, options):
+    config = ProjectConfig.get_instance()
+    pm = PlatformPackageManager()
+    if options.get("silent"):
+        pm.set_log_level(logging.WARN)
+    spec = config.get(f"env:{project_env}", "platform")
+    if not spec:
+        return None
+    already_up_to_date = True
+    if not pm.get_package(spec):
+        return None
+    PlatformPackageManager().uninstall(
+        spec,
+        project_env=project_env,
+        skip_dependencies=options.get("skip_dependencies"),
+    )
+    return not already_up_to_date
+
+
+def _uninstall_project_env_custom_platforms(project_env, options):
+    already_up_to_date = True
+    pm = PlatformPackageManager()
+    if not options.get("silent"):
+        pm.set_log_level(logging.DEBUG)
+    for spec in options.get("platforms"):
+        if pm.get_package(spec):
+            already_up_to_date = False
+        pm.uninstall(
+            spec,
+            project_env=project_env,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+    return not already_up_to_date
+
+
+def _uninstall_project_env_custom_tools(project_env, options):
+    already_up_to_date = True
+    tm = ToolPackageManager()
+    if not options.get("silent"):
+        tm.set_log_level(logging.DEBUG)
+    specs_to_save = []
+    for tool in options.get("tools"):
+        spec = PackageSpec(tool)
+        if tm.get_package(spec):
+            already_up_to_date = False
+        pkg = tm.uninstall(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+        specs_to_save.append(pkg_to_save_spec(pkg, spec))
+    if not options.get("no_save") and specs_to_save:
+        save_project_dependencies(
+            os.getcwd(),
+            specs_to_save,
+            scope="platform_packages",
+            action="remove",
+            environments=[project_env],
+        )
+    return not already_up_to_date
+
+
+def _uninstall_project_env_libraries(project_env, options):
+    already_up_to_date = True
+    config = ProjectConfig.get_instance()
+    lm = LibraryPackageManager(
+        os.path.join(config.get("platformio", "libdeps_dir"), project_env)
+    )
+    if options.get("silent"):
+        lm.set_log_level(logging.WARN)
+    for library in config.get(f"env:{project_env}", "lib_deps"):
+        spec = PackageSpec(library)
+        # skip built-in dependencies
+        if not spec.external and not spec.owner:
+            continue
+        if lm.get_package(spec):
+            already_up_to_date = False
+            lm.uninstall(
+                spec,
+                skip_dependencies=options.get("skip_dependencies"),
+            )
+    return not already_up_to_date
+
+
+def _uninstall_project_env_custom_libraries(project_env, options):
+    already_up_to_date = True
+    config = ProjectConfig.get_instance()
+    lm = LibraryPackageManager(
+        os.path.join(config.get("platformio", "libdeps_dir"), project_env)
+    )
+    if not options.get("silent"):
+        lm.set_log_level(logging.DEBUG)
+    specs_to_save = []
+    for library in options.get("libraries") or []:
+        spec = PackageSpec(library)
+        if lm.get_package(spec):
+            already_up_to_date = False
+        pkg = lm.uninstall(
+            spec,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+        specs_to_save.append(pkg_to_save_spec(pkg, spec))
+    if not options.get("no_save") and specs_to_save:
+        save_project_dependencies(
+            os.getcwd(),
+            specs_to_save,
+            scope="lib_deps",
+            action="remove",
+            environments=[project_env],
+        )
+    return not already_up_to_date

platformio/package/commands/unpublish.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.clients.account import AccountClient
+from platformio.clients.registry import RegistryClient
+from platformio.package.meta import PackageSpec, PackageType
+
+
+@click.command("unpublish", short_help="Remove a pushed package from the registry")
+@click.argument(
+    "package", required=True, metavar="[<organization>/]<pkgname>[@<version>]"
+)
+@click.option(
+    "--type",
+    type=click.Choice(list(PackageType.items().values())),
+    default="library",
+    help="Package type, default is set to `library`",
+)
+@click.option(
+    "--undo",
+    is_flag=True,
+    help="Undo a remove, putting a version back into the registry",
+)
+def package_unpublish_cmd(package, type, undo):  # pylint: disable=redefined-builtin
+    spec = PackageSpec(package)
+    response = RegistryClient().unpublish_package(
+        owner=spec.owner or AccountClient().get_logged_username(),
+        type=type,
+        name=spec.name,
+        version=str(spec.requirements),
+        undo=undo,
+    )
+    click.secho(response.get("message"), fg="green")

platformio/package/commands/update.py

@@ -0,0 +1,252 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+import click
+
+from platformio import fs
+from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.package.manager.tool import ToolPackageManager
+from platformio.package.meta import PackageSpec
+from platformio.project.config import ProjectConfig
+from platformio.project.savedeps import pkg_to_save_spec, save_project_dependencies
+
+
+@click.command(
+    "update", short_help="Update the project dependencies or custom packages"
+)
+@click.option(
+    "-d",
+    "--project-dir",
+    default=os.getcwd,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+)
+@click.option("-e", "--environment", "environments", multiple=True)
+@click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True)
+@click.option("-t", "--tool", "tools", metavar="SPECIFICATION", multiple=True)
+@click.option("-l", "--library", "libraries", metavar="SPECIFICATION", multiple=True)
+@click.option(
+    "--no-save",
+    is_flag=True,
+    help="Prevent saving specified packages to `platformio.ini`",
+)
+@click.option("--skip-dependencies", is_flag=True, help="Skip package dependencies")
+@click.option("-g", "--global", is_flag=True, help="Update global packages")
+@click.option(
+    "--storage-dir",
+    default=None,
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+    help="Custom Package Manager storage for global packages",
+)
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
+def package_update_cmd(**options):
+    if options.get("global"):
+        update_global_dependencies(options)
+    else:
+        update_project_dependencies(options)
+
+
+def update_global_dependencies(options):
+    pm = PlatformPackageManager(options.get("storage_dir"))
+    tm = ToolPackageManager(options.get("storage_dir"))
+    lm = LibraryPackageManager(options.get("storage_dir"))
+    for obj in (pm, tm, lm):
+        obj.set_log_level(logging.WARN if options.get("silent") else logging.DEBUG)
+    for spec in options.get("platforms"):
+        pm.update(
+            from_spec=spec,
+            to_spec=spec,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+    for spec in options.get("tools"):
+        tm.update(
+            from_spec=spec,
+            to_spec=spec,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+    for spec in options.get("libraries", []):
+        lm.update(
+            from_spec=spec,
+            to_spec=spec,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+
+
+def update_project_dependencies(options):
+    environments = options["environments"]
+    with fs.cd(options["project_dir"]):
+        config = ProjectConfig.get_instance()
+        config.validate(environments)
+        for env in config.envs():
+            if environments and env not in environments:
+                continue
+            if not options["silent"]:
+                click.echo(
+                    "Resolving %s environment packages..." % click.style(env, fg="cyan")
+                )
+            already_up_to_date = not update_project_env_dependencies(env, options)
+            if not options["silent"] and already_up_to_date:
+                click.secho("Already up-to-date.", fg="green")
+
+
+def update_project_env_dependencies(project_env, options=None):
+    options = options or {}
+    updated_conds = []
+    # custom platforms
+    if options.get("platforms"):
+        updated_conds.append(_update_project_env_custom_platforms(project_env, options))
+    # custom tools
+    if options.get("tools"):
+        updated_conds.append(_update_project_env_custom_tools(project_env, options))
+    # custom ibraries
+    if options.get("libraries"):
+        updated_conds.append(_update_project_env_custom_libraries(project_env, options))
+    # declared dependencies
+    if not updated_conds:
+        updated_conds = [
+            _update_project_env_platform(project_env, options),
+            _update_project_env_libraries(project_env, options),
+        ]
+    return any(updated_conds)
+
+
+def _update_project_env_platform(project_env, options):
+    config = ProjectConfig.get_instance()
+    pm = PlatformPackageManager()
+    if options.get("silent"):
+        pm.set_log_level(logging.WARN)
+    spec = config.get(f"env:{project_env}", "platform")
+    if not spec:
+        return None
+    cur_pkg = pm.get_package(spec)
+    if not cur_pkg:
+        return None
+    new_pkg = PlatformPackageManager().update(
+        cur_pkg,
+        to_spec=spec,
+        project_env=project_env,
+        skip_dependencies=options.get("skip_dependencies"),
+    )
+    return cur_pkg != new_pkg
+
+
+def _update_project_env_custom_platforms(project_env, options):
+    already_up_to_date = True
+    pm = PlatformPackageManager()
+    if not options.get("silent"):
+        pm.set_log_level(logging.DEBUG)
+    for spec in options.get("platforms"):
+        cur_pkg = pm.get_package(spec)
+        new_pkg = pm.update(
+            cur_pkg,
+            to_spec=spec,
+            project_env=project_env,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+        if cur_pkg != new_pkg:
+            already_up_to_date = False
+    return not already_up_to_date
+
+
+def _update_project_env_custom_tools(project_env, options):
+    already_up_to_date = True
+    tm = ToolPackageManager()
+    if not options.get("silent"):
+        tm.set_log_level(logging.DEBUG)
+
+    specs_to_save = []
+    for tool in options.get("tools"):
+        spec = PackageSpec(tool)
+        cur_pkg = tm.get_package(spec)
+        new_pkg = tm.update(
+            cur_pkg,
+            to_spec=spec,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+        if cur_pkg != new_pkg:
+            already_up_to_date = False
+        specs_to_save.append(pkg_to_save_spec(new_pkg, spec))
+
+    if not options.get("no_save") and specs_to_save:
+        save_project_dependencies(
+            os.getcwd(),
+            specs_to_save,
+            scope="platform_packages",
+            action="add",
+            environments=[project_env],
+        )
+
+    return not already_up_to_date
+
+
+def _update_project_env_libraries(project_env, options):
+    already_up_to_date = True
+    config = ProjectConfig.get_instance()
+    lm = LibraryPackageManager(
+        os.path.join(config.get("platformio", "libdeps_dir"), project_env)
+    )
+    if options.get("silent"):
+        lm.set_log_level(logging.WARN)
+    for library in config.get(f"env:{project_env}", "lib_deps"):
+        spec = PackageSpec(library)
+        # skip built-in dependencies
+        if not spec.external and not spec.owner:
+            continue
+        cur_pkg = lm.get_package(spec)
+        if cur_pkg:
+            new_pkg = lm.update(
+                cur_pkg,
+                to_spec=spec,
+                skip_dependencies=options.get("skip_dependencies"),
+            )
+            if cur_pkg != new_pkg:
+                already_up_to_date = False
+    return not already_up_to_date
+
+
+def _update_project_env_custom_libraries(project_env, options):
+    already_up_to_date = True
+    config = ProjectConfig.get_instance()
+    lm = LibraryPackageManager(
+        os.path.join(config.get("platformio", "libdeps_dir"), project_env)
+    )
+    if not options.get("silent"):
+        lm.set_log_level(logging.DEBUG)
+
+    specs_to_save = []
+    for library in options.get("libraries") or []:
+        spec = PackageSpec(library)
+        cur_pkg = lm.get_package(spec)
+        new_pkg = lm.update(
+            cur_pkg,
+            to_spec=spec,
+            skip_dependencies=options.get("skip_dependencies"),
+        )
+        if cur_pkg != new_pkg:
+            already_up_to_date = False
+        specs_to_save.append(pkg_to_save_spec(new_pkg, spec))
+
+    if not options.get("no_save") and specs_to_save:
+        save_project_dependencies(
+            os.getcwd(),
+            specs_to_save,
+            scope="lib_deps",
+            action="add",
+            environments=[project_env],
+        )
+
+    return not already_up_to_date

platformio/package/exception.py

@@ -34,7 +34,7 @@ class ManifestParserError(ManifestException):
 
 class ManifestValidationError(ManifestException):
     def __init__(self, messages, data, valid_data):
-        super(ManifestValidationError, self).__init__()
+        super().__init__()
         self.messages = messages
         self.data = data
         self.valid_data = valid_data

platformio/package/lockfile.py

@@ -69,8 +69,11 @@ class LockFile(object):
             if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL:
                 fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
             elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT:
-                msvcrt.locking(self._fp.fileno(), msvcrt.LK_NBLCK, 1)
+                msvcrt.locking(  # pylint: disable=used-before-assignment
-        except IOError:
+                    self._fp.fileno(), msvcrt.LK_NBLCK, 1
+                )
+        except (BlockingIOError, IOError):
+            self._fp.close()
             self._fp = None
             raise LockFileExists
         return True