Merge branch 'release/v6.0.2'

* Fix filter argument used in remote agent

* Linting

* Renamed reserved variable name to filter_

.github/workflows/core.yml

@@ -14,15 +14,19 @@ jobs:
             python-version: "3.6"
           - os: windows-latest
             python-version: "3.10"
+
     runs-on: ${{ matrix.os }}
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: "recursive"
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -31,11 +35,8 @@ jobs:
       - name: Python Lint
         run: |
           tox -e lint
+
       - name: Integration Tests
-        env:
-          TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }}
-          TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }}
-          TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }}
         run: |
           tox -e testcore
 

.github/workflows/deployment.yml

@@ -0,0 +1,42 @@
+name: Deployment
+
+on:
+  push:
+    branches:
+      - master
+      - "release/**"
+
+jobs:
+  deployment:
+    runs-on: ubuntu-latest
+    environment: production
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+
+      - name: Deployment Tests
+        env:
+          TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }}
+          TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }}
+          TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }}
+        run: |
+          tox -e testcore
+
+      - name: Publish package to PyPI
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/examples.yml

@@ -2,22 +2,28 @@ name: Examples
 
 on: [push, pull_request]
 
+
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04, windows-latest, macos-latest]
-        python-version: [3.7]
+        os: [ubuntu-latest, windows-latest, macos-latest]
     runs-on: ${{ matrix.os }}
+    env:
+      PIO_INSTALL_DEVPLATFORM_OWNERNAMES: "platformio"
+      PIO_INSTALL_DEVPLATFORM_NAMES: "aceinna_imu,atmelavr,atmelmegaavr,atmelsam,espressif32,espressif8266,nordicnrf52,raspberrypi,ststm32,teensy"
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: "recursive"
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.9"
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -25,24 +31,15 @@ jobs:
 
       - name: Run on Linux
         if: startsWith(matrix.os, 'ubuntu')
-        env:
-          PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio"
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,intel_mcs51"
         run: |
-          # ChipKIT issue: install 32-bit support for GCC PIC32
-          sudo apt-get install libc6-i386
           # Free space
           sudo apt clean
           docker rmi $(docker image ls -aq)
           df -h
-          # Run
           tox -e testexamples
 
       - name: Run on macOS
         if: startsWith(matrix.os, 'macos')
-        env:
-          PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio"
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,microchippic32,lattice_ice40,gd32v"
         run: |
           df -h
           tox -e testexamples
@@ -52,8 +49,6 @@ jobs:
         env:
           PLATFORMIO_CORE_DIR: C:/pio
           PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH
-          PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio"
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap"
         run: |
           tox -e testexamples
 

.github/workflows/projects.yml

@@ -0,0 +1,69 @@
+name: Projects
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        project:
+          - marlin:
+            repository: "MarlinFirmware/Marlin"
+            folder: "Marlin"
+            config_dir: "Marlin"
+            env_name: "mega2560"
+          - esphome:
+            repository: "esphome/esphome"
+            folder: "esphome"
+            config_dir: "esphome"
+            env_name: "esp32-arduino"
+          - smartknob:
+            repository: "scottbez1/smartknob"
+            folder: "smartknob"
+            config_dir: "smartknob/firmware"
+            env_name: "view"
+          - espurna:
+            repository: "xoseperez/espurna"
+            folder: "espurna"
+            config_dir: "espurna/code"
+            env_name: "nodemcu-lolin"
+          - OpenMQTTGateway:
+            repository: "1technophile/OpenMQTTGateway"
+            folder: "OpenMQTTGateway"
+            config_dir: "OpenMQTTGateway"
+            env_name: "esp32-m5atom"
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        exclude:
+        - os: windows-latest
+          project: {"esphome": "", "repository": "esphome/esphome", "folder": "esphome", "config_dir": "esphome", "env_name": "esp32-arduino"}
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.9
+
+      - name: Install PlatformIO
+        run: pip install -U .
+
+      - name: Check out ${{ matrix.project.repository }}
+        uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
+          repository: ${{ matrix.project.repository }}
+          path: ${{ matrix.project.folder }}
+
+      - name: Install ESPHome dependencies
+        # Requires esptool package as it's used in a custom prescript
+        if: ${{ contains(matrix.project.repository, 'esphome') }}
+        run: pip install esptool==3.*
+
+      - name: Compile ${{ matrix.project.repository }}
+        run: pio run -d ${{ matrix.project.config_dir }} -e ${{ matrix.project.env_name }}
+

.gitignore

@@ -1,6 +1,6 @@
 *.egg-info
 *.pyc
-.pioenvs
+__pycache__
 .tox
 docs/_build
 dist

HISTORY.rst

@@ -1,301 +1,133 @@
 Release Notes
 =============
 
+.. |PIOCONF| replace:: `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ configuration file
+.. |LDF| replace:: `LDF <https://docs.platformio.org/en/latest/librarymanager/ldf.html>`__
+.. |INTERPOLATION| replace:: `Interpolation of Values <https://docs.platformio.org/en/latest/projectconf/interpolation.html>`__
+.. |UNITTESTING| replace:: `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`__
+
+.. _release_notes_6:
+
+PlatformIO Core 6
+-----------------
+
+**A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.**
+
+6.0.2 (2022-06-01)
+~~~~~~~~~~~~~~~~~~
+
+* Control |UNITTESTING| verbosity with a new multilevel `pio test -v <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-v>`__ command option (`issue #4276 <https://github.com/platformio/platformio-core/issues/4276>`_)
+* Follow symbolic links during searching for the unit test suites (`issue #4288 <https://github.com/platformio/platformio-core/issues/4288>`_)
+* Show a warning when testing an empty project without a test suite (`issue #4278 <https://github.com/platformio/platformio-core/issues/4278>`_)
+* Improved support for `Asking for input (prompts) <https://docs.platformio.org/en/latest/scripting/examples/asking_for_input.html>`_
+* Fixed an issue when the `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__ option was applied outside the project scope (`issue #4277 <https://github.com/platformio/platformio-core/issues/4277>`_)
+* Fixed an issue with debugging assembly files without preprocessor (".s")
+
+6.0.1 (2022-05-17)
+~~~~~~~~~~~~~~~~~~
+
+* Improved support for the renamed configuration options (`issue #4270 <https://github.com/platformio/platformio-core/issues/4270>`_)
+* Fixed an issue when calling the built-in `pio device monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#filters>`__ filters
+* Fixed an issue when using |INTERPOLATION| and merging str+int options (`issue #4271 <https://github.com/platformio/platformio-core/issues/4271>`_)
+
+6.0.0 (2022-05-16)
+~~~~~~~~~~~~~~~~~~
+
+Please check the `Migration guide from 5.x to 6.0 <https://docs.platformio.org/en/latest/core/migration.html>`__.
+
+* **Package Management**
+
+  - New unified Package Management CLI (``pio pkg``):
+
+    * `pio pkg exec <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_exec.html>`_ - run command from package tool (`issue #4163 <https://github.com/platformio/platformio-core/issues/4163>`_)
+    * `pio pkg install <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html>`_ - install the project dependencies or custom packages
+    * `pio pkg list <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_list.html>`__ - list installed packages
+    * `pio pkg outdated <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_outdated.html>`__ - check for project outdated packages
+    * `pio pkg search <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_search.html>`__ - search for packages
+    * `pio pkg show <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_show.html>`__ - show package information
+    * `pio pkg uninstall <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_uninstall.html>`_ - uninstall the project dependencies or custom packages
+    * `pio pkg update <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_update.html>`__ - update the project dependencies or custom packages
+
+  - Package Manifest
+
+    * Added support for `"scripts" <https://docs.platformio.org/en/latest/librarymanager/config.html#scripts>`__ (`issue #485 <https://github.com/platformio/platformio-core/issues/485>`_)
+    * Added support for `multi-licensed <https://docs.platformio.org/en/latest/librarymanager/config.html#license>`__ packages using SPDX Expressions (`issue #4037 <https://github.com/platformio/platformio-core/issues/4037>`_)
+    * Added support for `"dependencies" <https://docs.platformio.org/en/latest/librarymanager/config.html#dependencies>`__ declared in a "tool" package manifest
+
+  - Added support for `symbolic links <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html#local-folder>`__ allowing pointing the local source folder to the Package Manager (`issue #3348 <https://github.com/platformio/platformio-core/issues/3348>`_)
+  - Automatically install dependencies of the local (private) project libraries (`issue #2910 <https://github.com/platformio/platformio-core/issues/2910>`_)
+  - Improved detection of a package type from the tarball archive (`issue #3828 <https://github.com/platformio/platformio-core/issues/3828>`_)
+  - Ignore files according to the patterns declared in ".gitignore" when using the `pio package pack <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_pack.html>`__ command (`issue #4188 <https://github.com/platformio/platformio-core/issues/4188>`_)
+  - Dropped automatic updates of global libraries and development platforms (`issue #4179 <https://github.com/platformio/platformio-core/issues/4179>`_)
+  - Dropped support for the "pythonPackages" field in "platform.json" manifest in favor of `Extra Python Dependencies <https://docs.platformio.org/en/latest/scripting/examples/extra_python_packages.html>`__
+  - Fixed an issue when manually removed dependencies from the |PIOCONF| were not uninstalled from the storage (`issue #3076 <https://github.com/platformio/platformio-core/issues/3076>`_)
+
+* **Unit Testing**
+
+  - Refactored from scratch |UNITTESTING| solution and its documentation
+  - New: `Test Hierarchy <https://docs.platformio.org/en/latest/advanced/unit-testing/structure.html>`_ (`issue #4135 <https://github.com/platformio/platformio-core/issues/4135>`_)
+  - New: `Doctest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/doctest.html>`__ testing framework (`issue #4240 <https://github.com/platformio/platformio-core/issues/4240>`_)
+  - New: `GoogleTest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/googletest.html>`__ testing and mocking framework (`issue #3572 <https://github.com/platformio/platformio-core/issues/3572>`_)
+  - New: `Semihosting <https://docs.platformio.org/en/latest/advanced/unit-testing/semihosting.html>`__ (`issue #3516 <https://github.com/platformio/platformio-core/issues/3516>`_)
+  - New: Hardware `Simulators <https://docs.platformio.org/en/latest/advanced/unit-testing/simulators/index.html>`__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions)
+  - New: ``test`` `build configuration <https://docs.platformio.org/en/latest/projectconf/build_configurations.html>`__
+  - Added support for a `custom testing framework <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/index.html>`_
+  - Added support for a custom `testing command <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-testing-command>`__
+  - Added support for a `custom Unity library <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/examples/custom_unity_library.html>`__ (`issue #3980 <https://github.com/platformio/platformio-core/issues/3980>`_)
+  - Added support for the ``socket://`` and ``rfc2217://`` protocols using `test_port <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-port>`__ option (`issue #4229 <https://github.com/platformio/platformio-core/issues/4229>`_)
+  - List available project tests with a new `pio test --list-tests <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-list-tests>`__ option
+  - Pass extra arguments to the testing program with a new `pio test --program-arg <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-a>`__ option (`issue #3132 <https://github.com/platformio/platformio-core/issues/3132>`_)
+  - Generate reports in JUnit and JSON formats using the `pio test <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html>`__ command (`issue #2891 <https://github.com/platformio/platformio-core/issues/2891>`_)
+  - Provide more information when the native program crashed on a host (errored with a non-zero return code) (`issue #3429 <https://github.com/platformio/platformio-core/issues/3429>`_)
+  - Improved automatic detection of a testing serial port (`issue #4076 <https://github.com/platformio/platformio-core/issues/4076>`_)
+  - Fixed an issue when command line parameters (``--ignore``, ``--filter``) do not override values defined in the |PIOCONF| (`issue #3845 <https://github.com/platformio/platformio-core/issues/3845>`_)
+  - Renamed the "test_build_project_src" project configuration option to the `test_build_src <https://docs.platformio.org/en/latest//projectconf/section_env_test.html#test-build-src>`__
+  - Removed the "test_transport" option in favor of the `Custom "unity_config.h" <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/unity.html>`_
+
+* **Static Code Analysis**
+
+  - Updated analysis tools:
+
+    * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.7 with various checker improvements and fixed false positives
+    * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.18 with improved and updated semantic analysis system
+
+  - Added support for the custom `Clang-Tidy <https://docs.platformio.org/en/latest/plus/check-tools/clang-tidy.html>`__ configuration file (`issue #4186 <https://github.com/platformio/platformio-core/issues/4186>`_)
+  - Added ability to override a tool version using the `platform_packages <https://docs.platformio.org/en/latest/projectconf/section_env_platform.html#platform-packages>`__ option (`issue #3798 <https://github.com/platformio/platformio-core/issues/3798>`_)
+  - Fixed an issue with improper handling of defects that don't specify a source file (`issue #4237 <https://github.com/platformio/platformio-core/issues/4237>`_)
+
+* **Build System**
+
+  - Show project dependency licenses when building in the verbose mode
+  - Fixed an issue when |LDF| ignores the project `lib_deps <https://docs.platformio.org/en/latest/projectconf/section_env_library.html#lib-deps>`__ while resolving library dependencies (`issue #3598 <https://github.com/platformio/platformio-core/issues/3598>`_)
+  - Fixed an issue with calling an extra script located outside a project (`issue #4220 <https://github.com/platformio/platformio-core/issues/4220>`_)
+  - Fixed an issue when GCC preprocessor was applied to the ".s" assembly files on case-sensitive OS such as Window OS (`issue #3917 <https://github.com/platformio/platformio-core/issues/3917>`_)
+  - Fixed an issue when |LDF| ignores `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__ in the "deep+" mode (`issue #4253 <https://github.com/platformio/platformio-core/issues/4253>`_)
+
+* **Integration**
+
+  - Added a new build variable (``COMPILATIONDB_INCLUDE_TOOLCHAIN``) to include toolchain paths in the compilation database (`issue #3735 <https://github.com/platformio/platformio-core/issues/3735>`_)
+  - Changed a default path for compilation database `compile_commands.json <https://docs.platformio.org/en/latest/integration/compile_commands.html>`__ to the project root
+  - Enhanced integration for Qt Creator (`issue #3046 <https://github.com/platformio/platformio-core/issues/3046>`_)
+
+* **Project Configuration**
+
+  - Extended |INTERPOLATION| with ``${this}`` pattern (`issue #3953 <https://github.com/platformio/platformio-core/issues/3953>`_)
+  - Embed environment name of the current section in the |PIOCONF| using ``${this.__env__}`` pattern
+  - Renamed the "src_build_flags" project configuration option to the `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__
+  - Renamed the "src_filter" project configuration option to the `build_src_filter <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-filter>`__
+
+* **Miscellaneous**
+
+  - Pass extra arguments to the `native <https://docs.platformio.org/en/latest/platforms/native.html>`__ program with a new `pio run --program-arg <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html#cmdoption-pio-run-a>`__ option (`issue #4246 <https://github.com/platformio/platformio-core/issues/4246>`_)
+  - Improved PIO Remote setup on credit-card sized computers (Raspberry Pi, BeagleBon, etc) (`issue #3865 <https://github.com/platformio/platformio-core/issues/3865>`_)
+  - Finally removed all tracks to the Python 2.7, the Python 3.6 is the minimum supported version.
+
 .. _release_notes_5:
 
 PlatformIO Core 5
 -----------------
 
-**A professional collaborative platform for embedded development**
-
-5.2.5 (2022-02-10)
-~~~~~~~~~~~~~~~~~~
-
-- Improved support for private packages in `PlatformIO Registry <https://registry.platformio.org/>`__
-- Improved checking of available Internet connection for IPv6-only workstations (`pull #4151 <https://github.com/platformio/platformio-core/pull/4151>`_)
-- Better detecting of default PlatformIO project directory on Linux OS (`pull #4158 <https://github.com/platformio/platformio-core/pull/4158>`_)
-- Respect disabling debugging server from "platformio.ini" passing an empty value to the `debug_server <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-server>`__ option
-- Fixed a "module 'asyncio' has no attribute 'run'" error when launching PIO Home using Python 3.6 (`issue #4169 <https://github.com/platformio/platformio-core/issues/4169>`_)
-
-5.2.4 (2021-12-15)
-~~~~~~~~~~~~~~~~~~
-
-- Added support for a new ``headers`` field in `library.json <https://docs.platformio.org/en/latest/librarymanager/config.html>`__ (declare a list of header files that can be included in a project source files using ``#include <...>`` directive)
-- Improved tab completion support for Bash, ZSH, and Fish shells (`issue #4114 <https://github.com/platformio/platformio-core/issues/4114>`_)
-- Improved support for projects located on a network share (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_, `issue #3926 <https://github.com/platformio/platformio-core/issues/3926>`_, `issue #4099 <https://github.com/platformio/platformio-core/issues/4099>`_)
-- Improved PIO Remote setup on credit-card sized computers (Raspberry Pi, BeagleBon, etc) (`issue #3865 <https://github.com/platformio/platformio-core/issues/3865>`_)
-- Upgraded build engine to the SCons 4.3 (`release notes <https://github.com/SCons/scons/blob/rel_4.3.0/CHANGES.txt>`__)
-- Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 <https://github.com/platformio/platformio-core/issues/4102>`_)
-- Fixed an issue with the NetBeans project generator when the path to PlatformIO contains a space (`issue #4096 <https://github.com/platformio/platformio-core/issues/4096>`_)
-- Fixed an issue when the system environment variable does not override a project configuration option (`issue #4125 <https://github.com/platformio/platformio-core/issues/4125>`_)
-- Fixed an issue when referencing ``*_dir`` option from a custom project configuration environment (`issue #4110 <https://github.com/platformio/platformio-core/issues/4110>`_)
-- Fixed an issue with the CLion template that generated a broken CMake file if user's home directory contained an unescaped backslash (`issue #4071 <https://github.com/platformio/platformio-core/issues/4071>`_)
-- Fixed an issue with wrong detecting Windows architecture when Python 32bit is used (`issue #4134 <https://github.com/platformio/platformio-core/issues/4134>`_)
-
-5.2.3 (2021-11-05)
-~~~~~~~~~~~~~~~~~~
-
-- Automatically synchronize active projects between IDE and `PlatformIO Home <https://docs.platformio.org/en/latest/home/index.html>`__
-- Added support for custom `device monitor filters <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#filters>`__ (`issue #3924 <https://github.com/platformio/platformio-core/issues/3924>`_)
-- Show human-readable message when infinite recursion is detected while processing `Interpolation of Values <https://docs.platformio.org/en/latest/projectconf/interpolation.html>`__ (`issue #3883 <https://github.com/platformio/platformio-core/issues/3883>`_)
-- Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ configuration file (`issue #3934 <https://github.com/platformio/platformio-core/issues/3934>`_)
-- Ignore resolving of SCons variables (e.g., ``${(SOURCE.get_abspath())}``) when preprocessing interpolations (`issue #3933 <https://github.com/platformio/platformio-core/issues/3933>`_)
-- Added "inc" as a sign that it's the root of the library (`issue #4093 <https://github.com/platformio/platformio-core/issues/4093>`_)
-- Fixed an issue when the ``$PROJECT_DIR`` variable was not properly replaced in the `debug_server <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-server>`__ option (`issue #4086 <https://github.com/platformio/platformio-core/issues/4086>`_)
-- Fixed an issue when `PIO Remote <https://docs.platformio.org/en/latest/plus/pio-remote.html>`__ device monitor crashes on the first keypress (`issue #3832 <https://github.com/platformio/platformio-core/issues/3832>`_)
-- Fixed "Do not know how to make File target 'debug'" issue when debugging project using `CLion IDE <https://docs.platformio.org/en/latest/integration/ide/clion.html>`__ (`pull #4089 <https://github.com/platformio/platformio-core/issues/4089>`_)
-- Fixed "UnicodeEncodeError" when a build output contains non-ASCII characters (`issue #3971 <https://github.com/platformio/platformio-core/issues/3971>`_)
-- Fixed an issue when VSCode's debugger does not the honor default environment (`issue #4098 <https://github.com/platformio/platformio-core/issues/4098>`_)
-
-5.2.2 (2021-10-20)
-~~~~~~~~~~~~~~~~~~
-
-- Override debugging firmware loading mode using ``--load-mode`` option for `pio debug <https://docs.platformio.org/en/latest/core/userguide/cmd_debug.html>`__ command
-- Added support for CLion IDE 2021.3 (`pull #4085 <https://github.com/platformio/platformio-core/issues/4085>`_)
-- Removed debugging "legacy Click" message from CLI (`issue #4083 <https://github.com/platformio/platformio-core/issues/4083>`_)
-- Fixed a "TypeError: sequence item 1: expected str instance, list found" issue when extending configuration option in `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ with the multi-line default value (`issue #4082 <https://github.com/platformio/platformio-core/issues/4082>`_)
-
-5.2.1 (2021-10-11)
-~~~~~~~~~~~~~~~~~~
-
-- Clean a build environment and installed library dependencies using a new ``cleanall`` target (`issue #4062 <https://github.com/platformio/platformio-core/issues/4062>`_)
-- Override a default library builder via a new ``builder`` field in a ``build`` group of `library.json <https://docs.platformio.org/en/latest/librarymanager/config.html#build>`__ manifest (`issue #3957 <https://github.com/platformio/platformio-core/issues/3957>`_)
-- Updated `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.6 with new checks, increased reliability of advanced addons (MISRA/CERT) and various improvements
-- Handle the "test" folder as a part of CLion project (`issue #4005 <https://github.com/platformio/platformio-core/issues/4005>`_)
-- Improved handling of a library root based on "Conan" or "CMake" build systems (`issue #3887 <https://github.com/platformio/platformio-core/issues/3887>`_)
-- Fixed a "KeyError: Invalid board option 'build.cpu'" when using a precompiled library with a board that does not have a CPU field in the manifest (`issue #4056 <https://github.com/platformio/platformio-core/issues/4056>`_)
-- Fixed a "FileExist" error when the `platformio ci <https://docs.platformio.org/en/latest/userguide/cmd_ci.html>`__ command is used in pair with the ``--keep-build-dir`` option (`issue #4011 <https://github.com/platformio/platformio-core/issues/4011>`_)
-- Fixed an issue with draft values of C++ language standards that broke static analysis via Cppcheck (`issue #3944 <https://github.com/platformio/platformio-core/issues/3944>`_)
-
-5.2.0 (2021-09-13)
-~~~~~~~~~~~~~~~~~~
-
-* **PlatformIO Debugging**
-
-  - Boosted `PlatformIO Debugging <https://docs.platformio.org/en/latest/plus/debugging.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
-  - `Debug unit tests <https://docs.platformio.org/en/latest/plus/debugging.html#debug-unit-tests>`__ created with `PlatformIO Unit Testing <https://docs.platformio.org/en/latest/plus/unit-testing.html>`__ solution  (`issue #948 <https://github.com/platformio/platformio-core/issues/948>`_)
-  - Debug native (desktop) applications on a host machine (`issue #980 <https://github.com/platformio/platformio-core/issues/980>`_)
-  - Support debugging on Windows using Windows CMD/CLI (`pio debug <https://docs.platformio.org/en/latest/core/userguide/cmd_debug.html>`__) (`issue #3793 <https://github.com/platformio/platformio-core/issues/3793>`_)
-  - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-server-ready-pattern>`__ option
-  - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 <https://github.com/platformio/platformio-core/issues/3756>`_)
-
-* **Package Management**
-
-  - Improved a package publishing process:
-
-    * Show package details
-    * Check for conflicting names in the PlatformIO Trusted Registry
-    * Check for duplicates and used version
-    * Validate package manifest
-
-  - Added a new option ``--non-interactive`` to `pio package publish <https://docs.platformio.org/en/latest/core/userguide/package/cmd_publish.html>`__ command
-
-* **Build System**
-
-  - Process "precompiled" and "ldflags" properties of the "library.properties" manifest (`issue #3994 <https://github.com/platformio/platformio-core/issues/3994>`_)
-  - Upgraded build engine to the SCons 4.2 (`release notes <https://github.com/SCons/scons/blob/rel_4.2.0/CHANGES.txt>`__)
-  - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 <https://github.com/platformio/platformio-core/issues/3906>`_)
-  - Fixed an issue when PlatformIO archives a library that does not contain C/C++ source files (`issue #4019 <https://github.com/platformio/platformio-core/issues/4019>`_)
-
-* **Static Code Analysis**
-
-  - Updated analysis tools:
-
-    * `Clang-Tidy <https://docs.platformio.org/en/latest/plus/check-tools/clang-tidy.html>`__ v12.0.1 with new modules and extended checks list
-    * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.5.0 with improved code analysis and MISRA improvements
-    * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.14 with support for intermodular analysis, improved MISRA support and new diagnostics
-
-* **Miscellaneous**
-
-  - Ensure that a serial port is ready before running unit tests on a remote target (`issue #3742 <https://github.com/platformio/platformio-core/issues/3742>`_)
-  - Fixed an error "Unknown development platform" when running unit tests on a clean machine (`issue #3901 <https://github.com/platformio/platformio-core/issues/3901>`_)
-  - Fixed an issue when "main.cpp" was generated for a new project for 8-bit development platforms (`issue #3872 <https://github.com/platformio/platformio-core/issues/3872>`_)
-
-5.1.1 (2021-03-17)
-~~~~~~~~~~~~~~~~~~
-
-* Fixed a "The command line is too long" issue with a linking process on Windows (`issue #3827 <https://github.com/platformio/platformio-core/issues/3827>`_)
-* Fixed an issue with `device monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html>`__ when the "send_on_enter" filter didn't send EOL chars (`issue #3787 <https://github.com/platformio/platformio-core/issues/3787>`_)
-* Fixed an issue with silent mode when unwanted data is printed to stdout (`issue #3837 <https://github.com/platformio/platformio-core/issues/3837>`_)
-* Fixed an issue when code inspection fails with "Bad JSON" (`issue #3790 <https://github.com/platformio/platformio-core/issues/3790>`_)
-* Fixed an issue with overriding user-specified debugging configuration information in VSCode (`issue #3824 <https://github.com/platformio/platformio-core/issues/3824>`_)
-
-5.1.0 (2021-01-28)
-~~~~~~~~~~~~~~~~~~
-
-* **PlatformIO Home**
-
-  - Boosted `PlatformIO Home <https://docs.platformio.org/en/latest/home/index.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
-  - Added a new ``--session-id`` option to `pio home <https://docs.platformio.org/en/latest/core/userguide/cmd_home.html>`__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 <https://github.com/platformio/platformio-core/issues/3397>`_)
-
-* **Build System**
-
-  - Upgraded build engine to the SCons 4.1 (`release notes <https://scons.org/scons-410-is-available.html>`_)
-  - Refactored a workaround for a maximum command line character limitation (`issue #3792 <https://github.com/platformio/platformio-core/issues/3792>`_)
-  - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
-
-* **Package Management**
-
-  - New options for `pio system prune <https://docs.platformio.org/en/latest/core/userguide/system/cmd_prune.html>`__ command:
-
-    + ``--dry-run`` option to show data that will be removed
-    + ``--core-packages`` option to remove unnecessary core packages
-    + ``--platform-packages`` option to remove unnecessary development platform packages (`issue #923 <https://github.com/platformio/platformio-core/issues/923>`_)
-
-  - Added new `check_prune_system_threshold <https://docs.platformio.org/en/latest/core/userguide/cmd_settings.html#check-prune-system-threshold>`__ setting
-  - Disabled automatic removal of unnecessary development platform packages (`issue #3708 <https://github.com/platformio/platformio-core/issues/3708>`_, `issue #3770 <https://github.com/platformio/platformio-core/issues/3770>`_)
-  - Fixed an issue when unnecessary packages were removed in  ``update --dry-run`` mode (`issue #3809 <https://github.com/platformio/platformio-core/issues/3809>`_)
-  - Fixed a "ValueError: Invalid simple block" when uninstalling a package with a custom name and external source (`issue #3816 <https://github.com/platformio/platformio-core/issues/3816>`_)
-
-* **Debugging**
-
-  - Configure a custom debug adapter speed using a new `debug_speed <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-speed>`__ option (`issue #3799 <https://github.com/platformio/platformio-core/issues/3799>`_)
-  - Handle debugging server's "ready_pattern" in "stderr" output
-
-* **Miscellaneous**
-
-  - Improved listing of `multicast DNS services <https://docs.platformio.org/en/latest/core/userguide/device/cmd_list.html>`_
-  - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 <https://github.com/platformio/platformio-core/issues/3804>`_)
-  - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 <https://github.com/platformio/platformio-core/issues/3808>`_)
-  - Fixed an issue when `pio device monitor --eol <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#cmdoption-pio-device-monitor-eol>`__ and "send_on_enter" filter do not work properly (`issue #3787 <https://github.com/platformio/platformio-core/issues/3787>`_)
-
-5.0.4 (2020-12-30)
-~~~~~~~~~~~~~~~~~~
-
-- Added "Core" suffix when showing PlatformIO Core version using ``pio --version`` command
-- Improved ".ccls" configuration file for Emacs, Vim, and Sublime Text integrations
-- Updated analysis tools:
-
-  * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.3 with improved C++ parser and several new MISRA rules
-  * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.11 with new diagnostics and updated mass suppression mechanism
-
-- Show a warning message about deprecated support for Python 2 and Python 3.5
-- Do not provide "intelliSenseMode" option when generating configuration for VSCode C/C++ extension
-- Fixed a "git-sh-setup: file not found" error when installing project dependencies from Git VCS (`issue #3740 <https://github.com/platformio/platformio-core/issues/3740>`_)
-- Fixed an issue with package publishing on Windows when Unix permissions are not preserved (`issue #3776 <https://github.com/platformio/platformio-core/issues/3776>`_)
-
-5.0.3 (2020-11-12)
-~~~~~~~~~~~~~~~~~~
-
-- Added an error selector for `Sublime Text <https://docs.platformio.org/en/latest/integration/ide/sublimetext.html>`__ build runner (`issue #3733 <https://github.com/platformio/platformio-core/issues/3733>`_)
-- Generate a working "projectEnvName" for PlatformIO IDE's debugger for VSCode
-- Force VSCode's intelliSenseMode to "gcc-x64" when GCC toolchain is used
-- Print ignored test suites and environments in the test summary report only in verbose mode (`issue #3726 <https://github.com/platformio/platformio-core/issues/3726>`_)
-- Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 <https://github.com/platformio/platformio-core/issues/3662>`_)
-- Fixed an issue when `pio package pack <https://docs.platformio.org/en/latest/core/userguide/package/cmd_pack.html>`__ ignores some folders (`issue #3730 <https://github.com/platformio/platformio-core/issues/3730>`_)
-
-5.0.2 (2020-10-30)
-~~~~~~~~~~~~~~~~~~
-
-- Initialize a new project or update the existing passing working environment name and its options (`issue #3686 <https://github.com/platformio/platformio-core/issues/3686>`_)
-- Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry (`issue #3700 <https://github.com/platformio/platformio-core/issues/3700>`_)
-- Improved "core.call" RPC for PlatformIO Home (`issue #3671 <https://github.com/platformio/platformio-core/issues/3671>`_)
-- Fixed a "PermissionError: [WinError 5]" on Windows when an external repository is used with `lib_deps <https://docs.platformio.org/en/latest/projectconf/section_env_library.html#lib-deps>`__ option (`issue #3664 <https://github.com/platformio/platformio-core/issues/3664>`_)
-- Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 <https://github.com/platformio/platformio-core/issues/3666>`_)
-- Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 <https://github.com/platformio/platformio-core/issues/3669>`_)
-- Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses (`issue #3677 <https://github.com/platformio/platformio-core/issues/3677>`_)
-- Fixed an issue when `pio package publish <https://docs.platformio.org/en/latest/core/userguide/package/cmd_publish.html>`__ command removes original archive after submitting to the registry (`issue #3716 <https://github.com/platformio/platformio-core/issues/3716>`_)
-- Fixed an issue when multiple `pio lib install <https://docs.platformio.org/en/latest/core/userguide/lib/cmd_install.html>`__ command with the same local library results in duplicates in ``lib_deps`` (`issue #3715 <https://github.com/platformio/platformio-core/issues/3715>`_)
-- Fixed an issue with a "wrong" timestamp in device monitor output using `"time" filter <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#filters>`__ (`issue #3712 <https://github.com/platformio/platformio-core/issues/3712>`_)
-
-5.0.1 (2020-09-10)
-~~~~~~~~~~~~~~~~~~
-
-- Added support for "owner" requirement when declaring ``dependencies`` using `library.json <https://docs.platformio.org/en/latest/librarymanager/config.html#dependencies>`__
-- Fixed an issue when using a custom git/ssh package with `platform_packages <https://docs.platformio.org/en/latest/projectconf/section_env_platform.html#platform-packages>`__ option (`issue #3624 <https://github.com/platformio/platformio-core/issues/3624>`_)
-- Fixed an issue with "ImportError: cannot import name '_get_backend' from 'cryptography.hazmat.backends'" when using `Remote Development <https://docs.platformio.org/en/latest/plus/pio-remote.html>`__ on RaspberryPi device (`issue #3652 <https://github.com/platformio/platformio-core/issues/3652>`_)
-- Fixed an issue when `pio package unpublish <https://docs.platformio.org/en/latest/core/userguide/package/cmd_unpublish.html>`__ command crashes (`issue #3660 <https://github.com/platformio/platformio-core/issues/3660>`_)
-- Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 <https://github.com/platformio/platformio-core/issues/3662>`_)
-- Fixed an issue with incorrect value for C++ language standard in IDE projects when an in-progress language standard is used (`issue #3653 <https://github.com/platformio/platformio-core/issues/3653>`_)
-- Fixed an issue with "Invalid simple block (semantic_version)" from library dependency that refs to an external source (repository, ZIP/Tar archives) (`issue #3658 <https://github.com/platformio/platformio-core/issues/3658>`_)
-- Fixed an issue when can not remove update or remove external dev-platform using PlatformIO Home (`issue #3663 <https://github.com/platformio/platformio-core/issues/3663>`_)
-
-5.0.0 (2020-09-03)
-~~~~~~~~~~~~~~~~~~
-
-Please check `Migration guide from 4.x to 5.0 <https://docs.platformio.org/en/latest/core/migration.html>`__.
-
-* Integration with the new **PlatformIO Trusted Registry**
-
-  - Enterprise-grade package storage with high availability (multi replicas)
-  - Secure, fast, and reliable global content delivery network (CDN)
-  - Universal support for all packages:
-
-    * Libraries
-    * Development platforms
-    * Toolchains
-
-  - Built-in fine-grained access control (role-based, teams, organizations)
-  - New CLI commands:
-
-    * `pio package <https://docs.platformio.org/en/latest/core/userguide/package/index.html>`__ – manage packages in the registry
-    * `pio access <https://docs.platformio.org/en/latest/core/userguide/access/index.html>`__ – manage package access for users, teams, and maintainers
-
-* Integration with the new **Account Management System**
-
-  - `Manage organizations <https://docs.platformio.org/en/latest/core/userguide/org/index.html>`__
-  - `Manage teams and team memberships <https://docs.platformio.org/en/latest/core/userguide/team/index.html>`__
-
-* New **Package Management System**
-
-  - Integrated PlatformIO Core with the new PlatformIO Registry
-  - Support for owner-based dependency declaration (resolves name conflicts) (`issue #1824 <https://github.com/platformio/platformio-core/issues/1824>`_)
-  - Automatically save dependencies to `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ when installing using PlatformIO CLI (`issue #2964 <https://github.com/platformio/platformio-core/issues/2964>`_)
-  - Follow SemVer complaint version constraints when checking library updates `issue #1281 <https://github.com/platformio/platformio-core/issues/1281>`_)
-  - Dropped support for "packageRepositories" section in "platform.json" manifest (please publish packages directly to the registry)
-
-* **Build System**
-
-  - Upgraded build engine to the `SCons 4.0 - a next-generation software construction tool <https://scons.org/>`__
-
-    * `Configuration files are Python scripts <https://docs.platformio.org/en/latest/projectconf/advanced_scripting.html>`__ – use the power of a real programming language to solve build problems
-    * Built-in reliable and automatic dependency analysis
-    * Improved support for parallel builds
-    * Ability to `share built files in a cache <https://docs.platformio.org/en/latest/projectconf/section_platformio.html#projectconf-pio-build-cache-dir>`__ to speed up multiple builds
-
-  - New `Custom Targets <https://docs.platformio.org/en/latest/projectconf/advanced_scripting.html#custom-targets>`__
-
-    * Pre/Post processing based on dependent sources (another target, source file, etc.)
-    * Command launcher with own arguments
-    * Launch command with custom options declared in `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__
-    * Python callback as a target (use the power of Python interpreter and PlatformIO Build API)
-    * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html#cmdoption-platformio-run-list-targets>`__ command (`issue #3544 <https://github.com/platformio/platformio-core/issues/3544>`_)
-
-  - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 <https://github.com/platformio/platformio-core/issues/3570>`_)
-  - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) <https://docs.platformio.org/en/latest/librarymanager/ldf.html#dependency-finder-mode>`__ for Arduino library when "library.property" has "depends" field (`issue #3607 <https://github.com/platformio/platformio-core/issues/3607>`_)
-  - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 <https://github.com/platformio/platformio-core/issues/3531>`_)
-  - Fixed an issue with the ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 <https://github.com/platformio/platformio-core/issues/3542>`_)
-
-* **Project Management**
-
-  - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci <https://docs.platformio.org/en/latest/core/userguide/cmd_ci.html>`__, `src_filter <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#src-filter>`__, `check_patterns <https://docs.platformio.org/en/latest/projectconf/section_env_check.html#check-patterns>`__, `library.json > srcFilter <https://docs.platformio.org/en/latest/librarymanager/config.html#srcfilter>`__). Python 3.5+ is required
-  - Added a new ``-e, --environment`` option to `pio project init <https://docs.platformio.org/en/latest/core/userguide/project/cmd_init.html#cmdoption-platformio-project-init-e>`__ command that helps to update a PlatformIO project using the existing environment
-  - Dump build system data intended for IDE extensions/plugins using a new `pio project data <https://docs.platformio.org/en/latest/core/userguide/project/cmd_data.html>`__ command
-  - Do not generate ".travis.yml" for a new project, let the user have a choice
-
-* **Unit Testing**
-
-  - Updated PIO Unit Testing support for Mbed framework and added compatibility with Mbed OS 6
-  - Fixed an issue when running multiple test environments (`issue #3523 <https://github.com/platformio/platformio-core/issues/3523>`_)
-  - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 <https://github.com/platformio/platformio-core/issues/3583>`_)
-
-* **Static Code Analysis**
-
-  - Updated analysis tools:
-
-    * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.1 with a new "soundy" analysis option and improved code parser
-    * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.09 with a new file list analysis mode and an extended list of analysis diagnostics
-
-  - Added Cppcheck package for ARM-based single-board computers (`issue #3559 <https://github.com/platformio/platformio-core/issues/3559>`_)
-  - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 <https://github.com/platformio/platformio-core/issues/3631>`_)
-
-* **Miscellaneous**
-
-  - Display system-wide information using a new `pio system info <https://docs.platformio.org/en/latest/core/userguide/system/cmd_info.html>`__ command (`issue #3521 <https://github.com/platformio/platformio-core/issues/3521>`_)
-  - Remove unused data using a new `pio system prune <https://docs.platformio.org/en/latest/core/userguide/system/cmd_prune.html>`__ command (`issue #3522 <https://github.com/platformio/platformio-core/issues/3522>`_)
-  - Show ignored project environments only in the verbose mode (`issue #3641 <https://github.com/platformio/platformio-core/issues/3641>`_)
-  - Do not escape compiler arguments in VSCode template on Windows
-  - Drop support for Python 2 and 3.5
+See `PlatformIO Core 5.0 history <https://github.com/platformio/platformio-core/blob/v5.2.5/HISTORY.rst>`__.
 
 .. _release_notes_4:
 

Makefile

@@ -11,7 +11,7 @@ format:
 	black ./tests
 
 test:
-	py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
+	py.test --verbose --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
 
 before-commit: isort format lint
 

README.rst

@@ -1,21 +1,25 @@
+.. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg
+    :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md
+    :alt:  SWUbanner
+
 PlatformIO Core
 ===============
 
 .. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg
-    :target: https://docs.platformio.org/page/core/index.html
+    :target: https://docs.platformio.org/en/latest/core/index.html
     :alt:  CI Build for PlatformIO Core
-.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
-    :target: https://github.com/platformio/platformio-examples
-    :alt:  CI Build for dev-platform examples
 .. image:: https://github.com/platformio/platformio-core/workflows/Docs/badge.svg
     :target: https://docs.platformio.org?utm_source=github&utm_medium=core
     :alt:  CI Build for Docs
+.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
+    :target: https://github.com/platformio/platformio-examples
+    :alt:  CI Build for dev-platform examples
+.. image:: https://github.com/platformio/platformio-core/workflows/Projects/badge.svg
+    :target: https://docs.platformio.org/en/latest/tutorials/index.html#projects
+    :alt:  CI Build for the Community Projects
 .. image:: https://img.shields.io/pypi/v/platformio.svg
     :target: https://pypi.python.org/pypi/platformio/
     :alt: Latest Version
-.. image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
-    :target: https://pypi.python.org/pypi/platformio/
-    :alt:  License
 .. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg
    :alt: PlatformIO Labs
    :target: https://piolabs.com/?utm_source=github&utm_medium=core
@@ -36,7 +40,7 @@ PlatformIO Core
 .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png
     :target: https://platformio.org?utm_source=github&utm_medium=core
 
-`PlatformIO <https://platformio.org?utm_source=github&utm_medium=core>`_ is a professional collaborative platform for embedded development
+`PlatformIO <https://platformio.org>`_ is a professional collaborative platform for embedded development.
 
 **A place where Developers and Teams have true Freedom! No more vendor lock-in!**
 
@@ -49,24 +53,24 @@ PlatformIO Core
 Get Started
 -----------
 
-* `What is PlatformIO? <https://docs.platformio.org/page/what-is-platformio.html?utm_source=github&utm_medium=core>`_
+* `What is PlatformIO? <https://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
 * `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_
-* `PlatformIO Core (CLI) <https://docs.platformio.org/page/core.html?utm_source=github&utm_medium=core>`_
+* `PlatformIO Core (CLI) <https://docs.platformio.org/en/latest/core.html?utm_source=github&utm_medium=core>`_
 * `Project Examples <https://github.com/platformio/platformio-examples?utm_source=github&utm_medium=core>`__
 
 Solutions
 ---------
 
-* `Library Management <https://docs.platformio.org/page/librarymanager/index.html?utm_source=github&utm_medium=core>`_
-* `Desktop IDEs Integration <https://docs.platformio.org/page/ide.html?utm_source=github&utm_medium=core>`_
-* `Continuous Integration <https://docs.platformio.org/page/ci/index.html?utm_source=github&utm_medium=core>`_
+* `Library Management <https://docs.platformio.org/en/latest/librarymanager/index.html?utm_source=github&utm_medium=core>`_
+* `Desktop IDEs Integration <https://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core>`_
+* `Continuous Integration <https://docs.platformio.org/en/latest/ci/index.html?utm_source=github&utm_medium=core>`_
 
 **Advanced**
 
-* `Debugging <https://docs.platformio.org/page/plus/debugging.html?utm_source=github&utm_medium=core>`_
-* `Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
-* `Static Code Analysis <https://docs.platformio.org/page/plus/pio-check.html?utm_source=github&utm_medium=core>`_
-* `Remote Development <https://docs.platformio.org/page/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
+* `Debugging <https://docs.platformio.org/en/latest/plus/debugging.html?utm_source=github&utm_medium=core>`_
+* `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html?utm_source=github&utm_medium=core>`_
+* `Static Code Analysis <https://docs.platformio.org/en/latest/plus/pio-check.html?utm_source=github&utm_medium=core>`_
+* `Remote Development <https://docs.platformio.org/en/latest/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
 
 Registry
 --------
@@ -86,7 +90,7 @@ Telemetry / Privacy Policy
 Share minimal diagnostics and usage information to help us make PlatformIO better.
 It is enabled by default. For more information see:
 
-* `Telemetry Setting <https://docs.platformio.org/page/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
+* `Telemetry Setting <https://docs.platformio.org/en/latest/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
 
 License
 -------

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (5, 2, 5)
+VERSION = (6, 0, 2)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -38,9 +38,9 @@ __license__ = "Apache Software License"
 __copyright__ = "Copyright 2014-present PlatformIO Labs"
 
 __accounts_api__ = "https://api.accounts.platformio.org"
-__registry_api__ = [
-    "https://api.registry.platformio.org",
-    "https://api.registry.ns1.platformio.org",
+__registry_mirror_hosts__ = [
+    "registry.platformio.org",
+    "registry.nm1.platformio.org",
 ]
 __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
 
@@ -49,16 +49,14 @@ __default_requests_timeout__ = (10, None)  # (connect, read)
 __core_packages__ = {
     "contrib-piohome": "~3.4.1",
     "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
-    "tool-unity": "~1.20500.0",
     "tool-scons": "~4.40300.0",
-    "tool-cppcheck": "~1.260.0",
+    "tool-cppcheck": "~1.270.0",
     "tool-clangtidy": "~1.120001.0",
-    "tool-pvs-studio": "~7.14.0",
+    "tool-pvs-studio": "~7.18.0",
 }
 
 __check_internet_hosts__ = [
     "185.199.110.153",  # Github.com
     "88.198.170.159",  # platformio.org
     "github.com",
-    "platformio.org",
-]
+] + __registry_mirror_hosts__

platformio/__main__.py

@@ -12,16 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=import-outside-toplevel
-
 import os
 import sys
 from traceback import format_exc
 
 import click
 
-from platformio import __version__, exception
-from platformio.commands import PlatformioCLI
+from platformio import __version__, exception, maintenance
+from platformio.cli import PlatformioCLI
 from platformio.compat import IS_CYGWIN, ensure_python3
 
 
@@ -29,7 +27,7 @@ from platformio.compat import IS_CYGWIN, ensure_python3
     cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])
 )
 @click.version_option(__version__, prog_name="PlatformIO Core")
-@click.option("--force", "-f", is_flag=True, help="DEPRECATE")
+@click.option("--force", "-f", is_flag=True, help="DEPRECATED", hidden=True)
 @click.option("--caller", "-c", help="Caller ID (service)")
 @click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters")
 @click.pass_context
@@ -55,29 +53,12 @@ def cli(ctx, force, caller, no_ansi):
     except:  # pylint: disable=bare-except
         pass
 
-    from platformio import maintenance
-
     maintenance.on_platformio_start(ctx, force, caller)
 
 
-try:
-
-    @cli.result_callback()
-    @click.pass_context
-    def process_result(ctx, result, *_, **__):
-        _process_result(ctx, result)
-
-except (AttributeError, TypeError):  # legacy support for CLick > 8.0.1
-
-    @cli.resultcallback()
-    @click.pass_context
-    def process_result(ctx, result, *_, **__):
-        _process_result(ctx, result)
-
-
-def _process_result(ctx, result):
-    from platformio import maintenance
-
+@cli.result_callback()
+@click.pass_context
+def process_result(ctx, result, *_, **__):
     maintenance.on_platformio_end(ctx, result)
 
 
@@ -124,10 +105,7 @@ def main(argv=None):
             exit_code = int(e.code)
     except Exception as e:  # pylint: disable=broad-except
         if not isinstance(e, exception.ReturnErrorCode):
-            if sys.version_info.major != 2:
-                from platformio import maintenance
-
-                maintenance.on_platformio_exception(e)
+            maintenance.on_platformio_exception(e)
             error_str = "Error: "
             if isinstance(e, exception.PlatformioException):
                 error_str += str(e)
@@ -142,7 +120,7 @@ An unexpected error occurred. Further steps:
   `pip install -U platformio` command
 
 * Try to find answer in FAQ Troubleshooting section
-  https://docs.platformio.org/page/faq.html
+  https://docs.platformio.org/page/faq/index.html
 
 * Report this problem to the developers
   https://github.com/platformio/platformio-core/issues

platformio/account/cli.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.commands.destroy import account_destroy_cmd
+from platformio.account.commands.forgot import account_forgot_cmd
+from platformio.account.commands.login import account_login_cmd
+from platformio.account.commands.logout import account_logout_cmd
+from platformio.account.commands.password import account_password_cmd
+from platformio.account.commands.register import account_register_cmd
+from platformio.account.commands.show import account_show_cmd
+from platformio.account.commands.token import account_token_cmd
+from platformio.account.commands.update import account_update_cmd
+
+
+@click.group(
+    "account",
+    commands=[
+        account_destroy_cmd,
+        account_forgot_cmd,
+        account_login_cmd,
+        account_logout_cmd,
+        account_password_cmd,
+        account_register_cmd,
+        account_show_cmd,
+        account_token_cmd,
+        account_update_cmd,
+    ],
+    short_help="Manage PlatformIO account",
+)
+def cli():
+    pass

platformio/account/client.py

@@ -16,8 +16,8 @@ import os
 import time
 
 from platformio import __accounts_api__, app
-from platformio.clients.http import HTTPClient, HTTPClientError
 from platformio.exception import PlatformioException
+from platformio.http import HTTPClient, HTTPClientError
 
 
 class AccountError(PlatformioException):
@@ -40,7 +40,7 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
     SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
 
     def __init__(self):
-        super(AccountClient, self).__init__(__accounts_api__)
+        super().__init__(__accounts_api__)
 
     @staticmethod
     def get_refresh_token():
@@ -63,7 +63,7 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
 
     def fetch_json_data(self, *args, **kwargs):
         try:
-            return super(AccountClient, self).fetch_json_data(*args, **kwargs)
+            return super().fetch_json_data(*args, **kwargs)
         except HTTPClientError as exc:
             raise AccountError(exc) from exc
 

platformio/account/commands/destroy.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient, AccountNotAuthorized
+
+
+@click.command("destroy", short_help="Destroy account")
+def account_destroy_cmd():
+    client = AccountClient()
+    click.confirm(
+        "Are you sure you want to delete the %s user account?\n"
+        "Warning! All linked data will be permanently removed and can not be restored."
+        % client.get_logged_username(),
+        abort=True,
+    )
+    client.destroy_account()
+    try:
+        client.logout()
+    except AccountNotAuthorized:
+        pass
+    click.secho(
+        "User account has been destroyed.",
+        fg="green",
+    )

platformio/account/commands/forgot.py

@@ -12,16 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=unused-argument
+import click
 
-from platformio.commands.update import cli as cmd_update
+from platformio.account.client import AccountClient
 
 
-def test_update(clirunner, validate_cliresult, isolated_pio_core):
-    matches = ("Platform Manager", "Library Manager")
-    result = clirunner.invoke(cmd_update, ["--only-check"])
-    validate_cliresult(result)
-    assert all(m in result.output for m in matches)
-    result = clirunner.invoke(cmd_update)
-    validate_cliresult(result)
-    assert all(m in result.output for m in matches)
+@click.command("forgot", short_help="Forgot password")
+@click.option("--username", prompt="Username or email")
+def account_forgot_cmd(username):
+    client = AccountClient()
+    client.forgot_password(username)
+    click.secho(
+        "If this account is registered, we will send the "
+        "further instructions to your email.",
+        fg="green",
+    )

platformio/account/commands/login.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("login", short_help="Log in to PlatformIO Account")
+@click.option("-u", "--username", prompt="Username or email")
+@click.option("-p", "--password", prompt=True, hide_input=True)
+def account_login_cmd(username, password):
+    client = AccountClient()
+    client.login(username, password)
+    click.secho("Successfully logged in!", fg="green")

platformio/account/commands/logout.py

@@ -12,19 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
+import click
 
-from platformio import exception
+from platformio.account.client import AccountClient
 
 
-def get_test_names(config):
-    test_dir = config.get("platformio", "test_dir")
-    if not os.path.isdir(test_dir):
-        raise exception.TestDirNotExists(test_dir)
-    names = []
-    for item in sorted(os.listdir(test_dir)):
-        if os.path.isdir(os.path.join(test_dir, item)):
-            names.append(item)
-    if not names:
-        names = ["*"]
-    return names
+@click.command("logout", short_help="Log out of PlatformIO Account")
+def account_logout_cmd():
+    client = AccountClient()
+    client.logout()
+    click.secho("Successfully logged out!", fg="green")

platformio/account/commands/password.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("password", short_help="Change password")
+@click.option("--old-password", prompt=True, hide_input=True)
+@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
+def account_password_cmd(old_password, new_password):
+    client = AccountClient()
+    client.change_password(old_password, new_password)
+    click.secho("Password successfully changed!", fg="green")

platformio/account/commands/register.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import (
+    validate_email,
+    validate_password,
+    validate_username,
+)
+
+
+@click.command("register", short_help="Create new PlatformIO Account")
+@click.option(
+    "-u",
+    "--username",
+    prompt=True,
+    callback=lambda _, __, value: validate_username(value),
+)
+@click.option(
+    "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value)
+)
+@click.option(
+    "-p",
+    "--password",
+    prompt=True,
+    hide_input=True,
+    confirmation_prompt=True,
+    callback=lambda _, __, value: validate_password(value),
+)
+@click.option("--firstname", prompt=True)
+@click.option("--lastname", prompt=True)
+def account_register_cmd(username, email, password, firstname, lastname):
+    client = AccountClient()
+    client.registration(username, email, password, firstname, lastname)
+    click.secho(
+        "An account has been successfully created. "
+        "Please check your mail to activate your account and verify your email address.",
+        fg="green",
+    )

platformio/account/commands/show.py

@@ -0,0 +1,116 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio import util
+from platformio.account.client import AccountClient
+
+
+@click.command("show", short_help="PlatformIO Account information")
+@click.option("--offline", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_show_cmd(offline, json_output):
+    client = AccountClient()
+    info = client.get_account_info(offline)
+    if json_output:
+        click.echo(json.dumps(info))
+        return
+    click.echo()
+    if info.get("profile"):
+        print_profile(info["profile"])
+    if info.get("packages"):
+        print_packages(info["packages"])
+    if info.get("subscriptions"):
+        print_subscriptions(info["subscriptions"])
+    click.echo()
+
+
+def print_profile(profile):
+    click.secho("Profile", fg="cyan", bold=True)
+    click.echo("=" * len("Profile"))
+    data = []
+    if profile.get("username"):
+        data.append(("Username:", profile["username"]))
+    if profile.get("email"):
+        data.append(("Email:", profile["email"]))
+    if profile.get("firstname"):
+        data.append(("First name:", profile["firstname"]))
+    if profile.get("lastname"):
+        data.append(("Last name:", profile["lastname"]))
+    click.echo(tabulate(data, tablefmt="plain"))
+
+
+def print_packages(packages):
+    click.echo()
+    click.secho("Packages", fg="cyan")
+    click.echo("=" * len("Packages"))
+    for package in packages:
+        click.echo()
+        click.secho(package.get("name"), bold=True)
+        click.echo("-" * len(package.get("name")))
+        if package.get("description"):
+            click.echo(package.get("description"))
+        data = []
+        expire = "-"
+        if "subscription" in package:
+            expire = util.parse_datetime(
+                package["subscription"].get("end_at")
+                or package["subscription"].get("next_bill_at")
+            ).strftime("%Y-%m-%d")
+        data.append(("Expire:", expire))
+        services = []
+        for key in package:
+            if not key.startswith("service."):
+                continue
+            if isinstance(package[key], dict):
+                services.append(package[key].get("title"))
+            else:
+                services.append(package[key])
+        if services:
+            data.append(("Services:", ", ".join(services)))
+        click.echo(tabulate(data, tablefmt="plain"))
+
+
+def print_subscriptions(subscriptions):
+    click.echo()
+    click.secho("Subscriptions", fg="cyan")
+    click.echo("=" * len("Subscriptions"))
+    for subscription in subscriptions:
+        click.echo()
+        click.secho(subscription.get("product_name"), bold=True)
+        click.echo("-" * len(subscription.get("product_name")))
+        data = [("State:", subscription.get("status"))]
+        begin_at = util.parse_datetime(subscription.get("begin_at")).strftime("%c")
+        data.append(("Start date:", begin_at or "-"))
+        end_at = subscription.get("end_at")
+        if end_at:
+            end_at = util.parse_datetime(subscription.get("end_at")).strftime("%c")
+        data.append(("End date:", end_at or "-"))
+        next_bill_at = subscription.get("next_bill_at")
+        if next_bill_at:
+            next_bill_at = util.parse_datetime(
+                subscription.get("next_bill_at")
+            ).strftime("%c")
+        data.append(("Next payment:", next_bill_at or "-"))
+        data.append(
+            ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")
+        )
+        data.append(
+            ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-")
+        )
+        click.echo(tabulate(data, tablefmt="plain"))

platformio/account/commands/token.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("token", short_help="Get or regenerate Authentication Token")
+@click.option("-p", "--password", prompt=True, hide_input=True)
+@click.option("--regenerate", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_token_cmd(password, regenerate, json_output):
+    client = AccountClient()
+    auth_token = client.auth_token(password, regenerate)
+    if json_output:
+        click.echo(json.dumps({"status": "success", "result": auth_token}))
+        return
+    click.secho("Personal Authentication Token: %s" % auth_token, fg="green")

platformio/account/commands/update.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient, AccountNotAuthorized
+from platformio.account.validate import validate_email, validate_username
+
+
+@click.command("update", short_help="Update profile information")
+@click.option("--current-password", prompt=True, hide_input=True)
+@click.option("--username")
+@click.option("--email")
+@click.option("--firstname")
+@click.option("--lastname")
+def account_update_cmd(current_password, **kwargs):
+    client = AccountClient()
+    profile = client.get_profile()
+    new_profile = profile.copy()
+    if not any(kwargs.values()):
+        for field in profile:
+            new_profile[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=profile[field]
+            )
+            if field == "email":
+                validate_email(new_profile[field])
+            if field == "username":
+                validate_username(new_profile[field])
+    else:
+        new_profile.update({key: value for key, value in kwargs.items() if value})
+    client.update_profile(new_profile, current_password)
+    click.secho("Profile successfully updated!", fg="green")
+    username_changed = new_profile["username"] != profile["username"]
+    email_changed = new_profile["email"] != profile["email"]
+    if not username_changed and not email_changed:
+        return None
+    try:
+        client.logout()
+    except AccountNotAuthorized:
+        pass
+    if email_changed:
+        click.secho(
+            "Please check your mail to verify your new email address and re-login. ",
+            fg="yellow",
+        )
+        return None
+    click.secho("Please re-login.", fg="yellow")
+    return None

platformio/account/org/cli.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.org.commands.add import org_add_cmd
+from platformio.account.org.commands.create import org_create_cmd
+from platformio.account.org.commands.destroy import org_destroy_cmd
+from platformio.account.org.commands.list import org_list_cmd
+from platformio.account.org.commands.remove import org_remove_cmd
+from platformio.account.org.commands.update import org_update_cmd
+
+
+@click.group(
+    "account",
+    commands=[
+        org_add_cmd,
+        org_create_cmd,
+        org_destroy_cmd,
+        org_list_cmd,
+        org_remove_cmd,
+        org_update_cmd,
+    ],
+    short_help="Manage organizations",
+)
+def cli():
+    pass

platformio/account/org/commands/add.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("add", short_help="Add a new owner to organization")
+@click.argument(
+    "orgname",
+)
+@click.argument(
+    "username",
+)
+def org_add_cmd(orgname, username):
+    client = AccountClient()
+    client.add_org_owner(orgname, username)
+    return click.secho(
+        "The new owner `%s` has been successfully added to the `%s` organization."
+        % (username, orgname),
+        fg="green",
+    )

platformio/account/org/commands/create.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_email, validate_orgname
+
+
+@click.command("create", short_help="Create a new organization")
+@click.argument(
+    "orgname",
+    callback=lambda _, __, value: validate_orgname(value),
+)
+@click.option(
+    "--email", callback=lambda _, __, value: validate_email(value) if value else value
+)
+@click.option(
+    "--displayname",
+)
+def org_create_cmd(orgname, email, displayname):
+    client = AccountClient()
+    client.create_org(orgname, email, displayname)
+    return click.secho(
+        "The organization `%s` has been successfully created." % orgname,
+        fg="green",
+    )

platformio/account/org/commands/destroy.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("destroy", short_help="Destroy organization")
+@click.argument("orgname")
+def org_destroy_cmd(orgname):
+    client = AccountClient()
+    click.confirm(
+        "Are you sure you want to delete the `%s` organization account?\n"
+        "Warning! All linked data will be permanently removed and can not be restored."
+        % orgname,
+        abort=True,
+    )
+    client.destroy_org(orgname)
+    return click.secho(
+        "Organization `%s` has been destroyed." % orgname,
+        fg="green",
+    )

platformio/account/org/commands/list.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio.account.client import AccountClient
+
+
+@click.command("list", short_help="List organizations and their members")
+@click.option("--json-output", is_flag=True)
+def org_list_cmd(json_output):
+    client = AccountClient()
+    orgs = client.list_orgs()
+    if json_output:
+        return click.echo(json.dumps(orgs))
+    if not orgs:
+        return click.echo("You do not have any organization")
+    for org in orgs:
+        click.echo()
+        click.secho(org.get("orgname"), fg="cyan")
+        click.echo("-" * len(org.get("orgname")))
+        data = []
+        if org.get("displayname"):
+            data.append(("Display Name:", org.get("displayname")))
+        if org.get("email"):
+            data.append(("Email:", org.get("email")))
+        data.append(
+            (
+                "Owners:",
+                ", ".join((owner.get("username") for owner in org.get("owners"))),
+            )
+        )
+        click.echo(tabulate(data, tablefmt="plain"))
+    return click.echo()

platformio/account/org/commands/remove.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("remove", short_help="Remove an owner from organization")
+@click.argument(
+    "orgname",
+)
+@click.argument(
+    "username",
+)
+def org_remove_cmd(orgname, username):
+    client = AccountClient()
+    client.remove_org_owner(orgname, username)
+    return click.secho(
+        "The `%s` owner has been successfully removed from the `%s` organization."
+        % (username, orgname),
+        fg="green",
+    )

platformio/account/org/commands/update.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_email, validate_orgname
+
+
+@click.command("update", short_help="Update organization")
+@click.argument("cur_orgname")
+@click.option(
+    "--orgname",
+    callback=lambda _, __, value: validate_orgname(value),
+    help="A new orgname",
+)
+@click.option("--email")
+@click.option("--displayname")
+def org_update_cmd(cur_orgname, **kwargs):
+    client = AccountClient()
+    org = client.get_org(cur_orgname)
+    del org["owners"]
+    new_org = org.copy()
+    if not any(kwargs.values()):
+        for field in org:
+            new_org[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=org[field]
+            )
+            if field == "email":
+                validate_email(new_org[field])
+            if field == "orgname":
+                validate_orgname(new_org[field])
+    else:
+        new_org.update(
+            {key.replace("new_", ""): value for key, value in kwargs.items() if value}
+        )
+    client.update_org(cur_orgname, new_org)
+    return click.secho(
+        "The organization `%s` has been successfully updated." % cur_orgname,
+        fg="green",
+    )

platformio/account/team/cli.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.team.commands.add import team_add_cmd
+from platformio.account.team.commands.create import team_create_cmd
+from platformio.account.team.commands.destroy import team_destroy_cmd
+from platformio.account.team.commands.list import team_list_cmd
+from platformio.account.team.commands.remove import team_remove_cmd
+from platformio.account.team.commands.update import team_update_cmd
+
+
+@click.group(
+    "team",
+    commands=[
+        team_add_cmd,
+        team_create_cmd,
+        team_destroy_cmd,
+        team_list_cmd,
+        team_remove_cmd,
+        team_update_cmd,
+    ],
+    short_help="Manage organization teams",
+)
+def cli():
+    pass

platformio/account/team/commands/add.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("add", short_help="Add a new member to team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.argument(
+    "username",
+)
+def team_add_cmd(orgname_teamname, username):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.add_team_member(orgname, teamname, username)
+    return click.secho(
+        "The new member %s has been successfully added to the %s team."
+        % (username, teamname),
+        fg="green",
+    )

platformio/account/team/commands/create.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("create", short_help="Create a new team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(
+        value, teamname_validate=True
+    ),
+)
+@click.option(
+    "--description",
+)
+def team_create_cmd(orgname_teamname, description):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.create_team(orgname, teamname, description)
+    return click.secho(
+        "The team %s has been successfully created." % teamname,
+        fg="green",
+    )

platformio/account/team/commands/destroy.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("destroy", short_help="Destroy a team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+def team_destroy_cmd(orgname_teamname):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    click.confirm(
+        click.style(
+            "Are you sure you want to destroy the %s team?" % teamname, fg="yellow"
+        ),
+        abort=True,
+    )
+    client = AccountClient()
+    client.destroy_team(orgname, teamname)
+    return click.secho(
+        "The team %s has been successfully destroyed." % teamname,
+        fg="green",
+    )

platformio/account/team/commands/list.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio.account.client import AccountClient
+
+
+@click.command("list", short_help="List teams")
+@click.argument("orgname", required=False)
+@click.option("--json-output", is_flag=True)
+def team_list_cmd(orgname, json_output):
+    client = AccountClient()
+    data = {}
+    if not orgname:
+        for item in client.list_orgs():
+            teams = client.list_teams(item.get("orgname"))
+            data[item.get("orgname")] = teams
+    else:
+        teams = client.list_teams(orgname)
+        data[orgname] = teams
+    if json_output:
+        return click.echo(json.dumps(data[orgname] if orgname else data))
+    if not any(data.values()):
+        return click.secho("You do not have any teams.", fg="yellow")
+    for org_name in data:
+        for team in data[org_name]:
+            click.echo()
+            click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan")
+            click.echo("-" * len("%s:%s" % (org_name, team.get("name"))))
+            table_data = []
+            if team.get("description"):
+                table_data.append(("Description:", team.get("description")))
+            table_data.append(
+                (
+                    "Members:",
+                    ", ".join(
+                        (member.get("username") for member in team.get("members"))
+                    )
+                    if team.get("members")
+                    else "-",
+                )
+            )
+            click.echo(tabulate(table_data, tablefmt="plain"))
+    return click.echo()

platformio/account/team/commands/remove.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("remove", short_help="Remove a member from team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.argument("username")
+def team_remove_cmd(orgname_teamname, username):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.remove_team_member(orgname, teamname, username)
+    return click.secho(
+        "The %s member has been successfully removed from the %s team."
+        % (username, teamname),
+        fg="green",
+    )

platformio/account/team/commands/update.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname, validate_teamname
+
+
+@click.command("update", short_help="Update team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.option(
+    "--name",
+    callback=lambda _, __, value: validate_teamname(value),
+    help="A new team name",
+)
+@click.option(
+    "--description",
+)
+def team_update_cmd(orgname_teamname, **kwargs):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    team = client.get_team(orgname, teamname)
+    del team["id"]
+    del team["members"]
+    new_team = team.copy()
+    if not any(kwargs.values()):
+        for field in team:
+            new_team[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=team[field]
+            )
+            if field == "name":
+                validate_teamname(new_team[field])
+    else:
+        new_team.update({key: value for key, value in kwargs.items() if value})
+    client.update_team(orgname, teamname, new_team)
+    return click.secho(
+        "The team %s has been successfully updated." % teamname,
+        fg="green",
+    )

platformio/account/validate.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import click
+
+
+def validate_username(value, field="username"):
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I):
+        raise click.BadParameter(
+            "Invalid %s format. "
+            "%s must contain only alphanumeric characters "
+            "or single hyphens, cannot begin or end with a hyphen, "
+            "and must not be longer than 38 characters."
+            % (field.lower(), field.capitalize())
+        )
+    return value
+
+
+def validate_email(value):
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d_.+-]+@[a-z\d\-]+\.[a-z\d\-.]+$", value, flags=re.I):
+        raise click.BadParameter("Invalid email address")
+    return value
+
+
+def validate_password(value):
+    value = str(value).strip()
+    if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
+        raise click.BadParameter(
+            "Invalid password format. "
+            "Password must contain at least 8 characters"
+            " including a number and a lowercase letter"
+        )
+    return value
+
+
+def validate_orgname(value):
+    return validate_username(value, "Organization name")
+
+
+def validate_orgname_teamname(value, teamname_validate=False):
+    if ":" not in value:
+        raise click.BadParameter(
+            "Please specify organization and team name in the next"
+            " format - orgname:teamname. For example, mycompany:DreamTeam"
+        )
+    teamname = str(value.strip().split(":", 1)[1])
+    if teamname_validate:
+        validate_teamname(teamname)
+    return value
+
+
+def validate_teamname(value):
+    if not value:
+        return value
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I):
+        raise click.BadParameter(
+            "Invalid team name format. "
+            "Team name must only contain alphanumeric characters, "
+            "single hyphens, underscores, spaces. It can not "
+            "begin or end with a hyphen or a underscore and must"
+            " not be longer than 20 characters."
+        )
+    return value

platformio/app.py

@@ -35,24 +35,8 @@ def projects_dir_validate(projects_dir):
 
 
 DEFAULT_SETTINGS = {
-    "auto_update_libraries": {
-        "description": "Automatically update libraries (Yes/No)",
-        "value": False,
-    },
-    "auto_update_platforms": {
-        "description": "Automatically update platforms (Yes/No)",
-        "value": False,
-    },
-    "check_libraries_interval": {
-        "description": "Check for the library updates interval (days)",
-        "value": 7,
-    },
     "check_platformio_interval": {
-        "description": "Check for the new PlatformIO interval (days)",
-        "value": 3,
-    },
-    "check_platforms_interval": {
-        "description": "Check for the platform updates interval (days)",
+        "description": "Check for the new PlatformIO Core interval (days)",
         "value": 7,
     },
     "check_prune_system_threshold": {

platformio/builder/main.py

@@ -44,25 +44,28 @@ clivars.AddVariables(
     ("PIOENV",),
     ("PIOTEST_RUNNING_NAME",),
     ("UPLOAD_PORT",),
+    ("PROGRAM_ARGS",),
 )
 
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
         "ar",
-        "as",
         "cc",
         "c++",
         "link",
+        "pioasm",
         "platformio",
-        "piotarget",
-        "pioplatform",
         "pioproject",
+        "pioplatform",
+        "piotest",
+        "piotarget",
         "piomaxlen",
         "piolib",
         "pioupload",
-        "piomisc",
-        "pioide",
         "piosize",
+        "pioino",
+        "piomisc",
+        "piointegration",
     ],
     toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")],
     variables=clivars,
@@ -72,7 +75,7 @@ DEFAULT_ENV_OPTIONS = dict(
     BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
     BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
     BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
-    COMPILATIONDB_PATH=os.path.join("$BUILD_DIR", "compile_commands.json"),
+    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
     LIBPATH=["$BUILD_DIR"],
     PROGNAME="program",
     PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
@@ -222,13 +225,14 @@ if "envdump" in COMMAND_LINE_TARGETS:
     click.echo(env.Dump())
     env.Exit(0)
 
-if set(["_idedata", "idedata"]) & set(COMMAND_LINE_TARGETS):
+if env.IsIntegrationDump():
+    projenv = None
     try:
         Import("projenv")
     except:  # pylint: disable=bare-except
         projenv = env
-    data = projenv.DumpIDEData(env)
-    # dump to file for the further reading by project.helpers.load_project_ide_data
+    data = projenv.DumpIntegrationData(env)
+    # dump to file for the further reading by project.helpers.load_build_metadata
     with open(
         projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")),
         mode="w",

platformio/builder/tools/pioasm.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import SCons.Tool.asm  # pylint: disable=import-error
+
+#
+# Resolve https://github.com/platformio/platformio-core/issues/3917
+# Avoid forcing .S to bare assembly on Windows OS
+#
+
+if ".S" in SCons.Tool.asm.ASSuffixes:
+    SCons.Tool.asm.ASSuffixes.remove(".S")
+if ".S" not in SCons.Tool.asm.ASPPSuffixes:
+    SCons.Tool.asm.ASPPSuffixes.append(".S")
+
+
+generate = SCons.Tool.asm.generate
+exists = SCons.Tool.asm.exists

platformio/builder/tools/pioino.py

@@ -0,0 +1,259 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import atexit
+import glob
+import io
+import os
+import re
+import tempfile
+
+import click
+
+from platformio.compat import get_filesystem_encoding, get_locale_encoding
+
+
+class InoToCPPConverter(object):
+
+    PROTOTYPE_RE = re.compile(
+        r"""^(
+        (?:template\<.*\>\s*)?      # template
+        ([a-z_\d\&]+\*?\s+){1,2}    # return type
+        ([a-z_\d]+\s*)              # name of prototype
+        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
+        )\s*(\{|;)                  # must end with `{` or `;`
+        """,
+        re.X | re.M | re.I,
+    )
+    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
+    PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
+
+    def __init__(self, env):
+        self.env = env
+        self._main_ino = None
+        self._safe_encoding = None
+
+    def read_safe_contents(self, path):
+        error_reported = False
+        for encoding in (
+            "utf-8",
+            None,
+            get_filesystem_encoding(),
+            get_locale_encoding(),
+            "latin-1",
+        ):
+            try:
+                with io.open(path, encoding=encoding) as fp:
+                    contents = fp.read()
+                    self._safe_encoding = encoding
+                    return contents
+            except UnicodeDecodeError:
+                if not error_reported:
+                    error_reported = True
+                    click.secho(
+                        "Unicode decode error has occurred, please remove invalid "
+                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
+                        % path,
+                        fg="yellow",
+                        err=True,
+                    )
+        return ""
+
+    def write_safe_contents(self, path, contents):
+        with io.open(
+            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
+        ) as fp:
+            return fp.write(contents)
+
+    def is_main_node(self, contents):
+        return self.DETECTMAIN_RE.search(contents)
+
+    def convert(self, nodes):
+        contents = self.merge(nodes)
+        if not contents:
+            return None
+        return self.process(contents)
+
+    def merge(self, nodes):
+        assert nodes
+        lines = []
+        for node in nodes:
+            contents = self.read_safe_contents(node.get_path())
+            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
+            if self.is_main_node(contents):
+                lines = _lines + lines
+                self._main_ino = node.get_path()
+            else:
+                lines.extend(_lines)
+
+        if not self._main_ino:
+            self._main_ino = nodes[0].get_path()
+
+        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
+
+    def process(self, contents):
+        out_file = self._main_ino + ".cpp"
+        assert self._gcc_preprocess(contents, out_file)
+        contents = self.read_safe_contents(out_file)
+        contents = self._join_multiline_strings(contents)
+        self.write_safe_contents(out_file, self.append_prototypes(contents))
+        return out_file
+
+    def _gcc_preprocess(self, contents, out_file):
+        tmp_path = tempfile.mkstemp()[1]
+        self.write_safe_contents(tmp_path, contents)
+        self.env.Execute(
+            self.env.VerboseAction(
+                '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
+                    out_file, tmp_path
+                ),
+                "Converting " + os.path.basename(out_file[:-4]),
+            )
+        )
+        atexit.register(_delete_file, tmp_path)
+        return os.path.isfile(out_file)
+
+    def _join_multiline_strings(self, contents):
+        if "\\\n" not in contents:
+            return contents
+        newlines = []
+        linenum = 0
+        stropen = False
+        for line in contents.split("\n"):
+            _linenum = self._parse_preproc_line_num(line)
+            if _linenum is not None:
+                linenum = _linenum
+            else:
+                linenum += 1
+
+            if line.endswith("\\"):
+                if line.startswith('"'):
+                    stropen = True
+                    newlines.append(line[:-1])
+                    continue
+                if stropen:
+                    newlines[len(newlines) - 1] += line[:-1]
+                    continue
+            elif stropen and line.endswith(('",', '";')):
+                newlines[len(newlines) - 1] += line
+                stropen = False
+                newlines.append(
+                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
+                )
+                continue
+
+            newlines.append(line)
+
+        return "\n".join(newlines)
+
+    @staticmethod
+    def _parse_preproc_line_num(line):
+        if not line.startswith("#"):
+            return None
+        tokens = line.split(" ", 3)
+        if len(tokens) > 2 and tokens[1].isdigit():
+            return int(tokens[1])
+        return None
+
+    def _parse_prototypes(self, contents):
+        prototypes = []
+        reserved_keywords = set(["if", "else", "while"])
+        for match in self.PROTOTYPE_RE.finditer(contents):
+            if (
+                set([match.group(2).strip(), match.group(3).strip()])
+                & reserved_keywords
+            ):
+                continue
+            prototypes.append(match)
+        return prototypes
+
+    def _get_total_lines(self, contents):
+        total = 0
+        if contents.endswith("\n"):
+            contents = contents[:-1]
+        for line in contents.split("\n")[::-1]:
+            linenum = self._parse_preproc_line_num(line)
+            if linenum is not None:
+                return total + linenum
+            total += 1
+        return total
+
+    def append_prototypes(self, contents):
+        prototypes = self._parse_prototypes(contents) or []
+
+        # skip already declared prototypes
+        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
+        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
+
+        if not prototypes:
+            return contents
+
+        prototype_names = set(m.group(3).strip() for m in prototypes)
+        split_pos = prototypes[0].start()
+        match_ptrs = re.search(
+            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
+            contents[:split_pos],
+            re.M,
+        )
+        if match_ptrs:
+            split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
+
+        result = []
+        result.append(contents[:split_pos].strip())
+        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
+        result.append(
+            '#line %d "%s"'
+            % (
+                self._get_total_lines(contents[:split_pos]),
+                self._main_ino.replace("\\", "/"),
+            )
+        )
+        result.append(contents[split_pos:].strip())
+        return "\n".join(result)
+
+
+def FindInoNodes(env):
+    src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
+    return env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
+        os.path.join(src_dir, "*.pde")
+    )
+
+
+def ConvertInoToCpp(env):
+    ino_nodes = env.FindInoNodes()
+    if not ino_nodes:
+        return
+    c = InoToCPPConverter(env)
+    out_file = c.convert(ino_nodes)
+
+    atexit.register(_delete_file, out_file)
+
+
+def _delete_file(path):
+    try:
+        if os.path.isfile(path):
+            os.remove(path)
+    except:  # pylint: disable=bare-except
+        pass
+
+
+def generate(env):
+    env.AddMethod(FindInoNodes)
+    env.AddMethod(ConvertInoToCpp)
+
+
+def exists(_):
+    return True

platformio/builder/tools/piointegration.py

@@ -19,33 +19,37 @@ import os
 
 import SCons.Defaults  # pylint: disable=import-error
 import SCons.Subst  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 
-from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import exec_command, where_is_program
 
 
-def _dump_includes(env):
-    includes = {}
+def IsIntegrationDump(_):
+    return set(["_idedata", "idedata"]) & set(COMMAND_LINE_TARGETS)
 
-    includes["build"] = [
-        env.subst("$PROJECT_INCLUDE_DIR"),
-        env.subst("$PROJECT_SRC_DIR"),
-    ]
-    includes["build"].extend(
+
+def DumpIntegrationIncludes(env):
+    result = dict(build=[], compatlib=[], toolchain=[])
+
+    result["build"].extend(
+        [
+            env.subst("$PROJECT_INCLUDE_DIR"),
+            env.subst("$PROJECT_SRC_DIR"),
+        ]
+    )
+    result["build"].extend(
         [os.path.abspath(env.subst(item)) for item in env.get("CPPPATH", [])]
     )
 
     # installed libs
-    includes["compatlib"] = []
     for lb in env.GetLibBuilders():
-        includes["compatlib"].extend(
+        result["compatlib"].extend(
             [os.path.abspath(inc) for inc in lb.get_include_dirs()]
         )
 
     # includes from toolchains
     p = env.PioPlatform()
-    includes["toolchain"] = []
-    for pkg in p.get_installed_packages():
+    for pkg in p.get_installed_packages(with_optional=False):
         if p.get_package_type(pkg.metadata.name) != "toolchain":
             continue
         toolchain_dir = glob.escape(pkg.path)
@@ -56,25 +60,12 @@ def _dump_includes(env):
             os.path.join(toolchain_dir, "*", "include*"),
         ]
         for g in toolchain_incglobs:
-            includes["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)])
+            result["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)])
 
-    # include Unity framework if there are tests in project
-    includes["unity"] = []
-    auto_install_unity = False
-    test_dir = env.GetProjectConfig().get("platformio", "test_dir")
-    if os.path.isdir(test_dir) and os.listdir(test_dir) != ["README"]:
-        auto_install_unity = True
-    unity_dir = get_core_package_dir(
-        "tool-unity",
-        auto_install=auto_install_unity,
-    )
-    if unity_dir:
-        includes["unity"].append(unity_dir)
-
-    return includes
+    return result
 
 
-def _get_gcc_defines(env):
+def get_gcc_defines(env):
     items = []
     try:
         sysenv = os.environ.copy()
@@ -97,7 +88,7 @@ def _get_gcc_defines(env):
     return items
 
 
-def _dump_defines(env):
+def dump_defines(env):
     defines = []
     # global symbols
     for item in SCons.Defaults.processDefines(env.get("CPPDEFINES", [])):
@@ -122,12 +113,12 @@ def _dump_defines(env):
 
     # built-in GCC marcos
     # if env.GetCompilerType() == "gcc":
-    #     defines.extend(_get_gcc_defines(env))
+    #     defines.extend(get_gcc_defines(env))
 
     return defines
 
 
-def _get_svd_path(env):
+def dump_svd_path(env):
     svd_path = env.GetProjectOption("debug_svd_path")
     if svd_path:
         return os.path.abspath(svd_path)
@@ -154,19 +145,19 @@ def _subst_cmd(env, cmd):
     return " ".join([SCons.Subst.quote_spaces(arg) for arg in args])
 
 
-def DumpIDEData(env, globalenv):
+def DumpIntegrationData(env, globalenv):
     """env here is `projenv`"""
 
     data = {
         "env_name": env["PIOENV"],
         "libsource_dirs": [env.subst(item) for item in env.GetLibSourceDirs()],
-        "defines": _dump_defines(env),
-        "includes": _dump_includes(env),
+        "defines": dump_defines(env),
+        "includes": env.DumpIntegrationIncludes(),
         "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
         "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
         "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
         "prog_path": env.subst("$PROG_PATH"),
-        "svd_path": _get_svd_path(env),
+        "svd_path": dump_svd_path(env),
         "compiler_type": env.GetCompilerType(),
         "targets": globalenv.DumpTargets(),
         "extra": dict(
@@ -176,7 +167,9 @@ def DumpIDEData(env, globalenv):
             ]
         ),
     }
-    data["extra"].update(env.get("IDE_EXTRA_DATA", {}))
+    data["extra"].update(
+        env.get("INTEGRATION_EXTRA_DATA", env.get("IDE_EXTRA_DATA", {}))
+    )
 
     env_ = env.Clone()
     # https://github.com/platformio/platformio-atom-ide/issues/34
@@ -205,5 +198,7 @@ def exists(_):
 
 
 def generate(env):
-    env.AddMethod(DumpIDEData)
+    env.AddMethod(IsIntegrationDump)
+    env.AddMethod(DumpIntegrationIncludes)
+    env.AddMethod(DumpIntegrationData)
     return env

platformio/builder/tools/piolib.py

@@ -27,14 +27,16 @@ import sys
 import click
 import SCons.Scanner  # pylint: disable=import-error
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
-from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import exception, fs, util
 from platformio.builder.tools import platformio as piotool
-from platformio.clients.http import HTTPClientError, InternetIsOffline
 from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
-from platformio.package.exception import UnknownPackageError
+from platformio.http import HTTPClientError, InternetIsOffline
+from platformio.package.exception import (
+    MissingPackageManifestError,
+    UnknownPackageError,
+)
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manifest.parser import (
     ManifestParserError,
@@ -54,20 +56,20 @@ class LibBuilderFactory(object):
             used_frameworks = LibBuilderFactory.get_used_frameworks(env, path)
             common_frameworks = set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks)
             if common_frameworks:
-                clsname = "%sLibBuilder" % list(common_frameworks)[0].title()
+                clsname = "%sLibBuilder" % list(common_frameworks)[0].capitalize()
             elif used_frameworks:
-                clsname = "%sLibBuilder" % used_frameworks[0].title()
+                clsname = "%sLibBuilder" % used_frameworks[0].capitalize()
 
-        obj = getattr(sys.modules[__name__], clsname)(env, path, verbose=verbose)
+        obj = globals()[clsname](env, path, verbose=verbose)
 
         # Handle PlatformIOLibBuilder.manifest.build.builder
         # pylint: disable=protected-access
         if isinstance(obj, PlatformIOLibBuilder) and obj._manifest.get("build", {}).get(
             "builder"
         ):
-            obj = getattr(
-                sys.modules[__name__], obj._manifest.get("build", {}).get("builder")
-            )(env, path, verbose=verbose)
+            obj = globals()[obj._manifest.get("build", {}).get("builder")](
+                env, path, verbose=verbose
+            )
 
         assert isinstance(obj, LibBuilderBase)
         return obj
@@ -109,7 +111,7 @@ class LibBuilderFactory(object):
         return []
 
 
-class LibBuilderBase(object):
+class LibBuilderBase:
 
     CLASSIC_SCANNER = SCons.Scanner.C.CScanner()
     CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner()
@@ -136,9 +138,11 @@ class LibBuilderBase(object):
             )
             self._manifest = {}
 
-        self._is_dependent = False
-        self._is_built = False
-        self._depbuilders = []
+        self.is_dependent = False
+        self.is_built = False
+        self.depbuilders = []
+
+        self._deps_are_processed = False
         self._circular_deps = []
         self._processed_files = []
 
@@ -159,7 +163,12 @@ class LibBuilderBase(object):
             p2 = p2.lower()
         if p1 == p2:
             return True
-        return os.path.commonprefix((p1 + os.path.sep, p2)) == p1 + os.path.sep
+        if os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep:
+            return True
+        # try to resolve paths
+        p1 = os.path.os.path.realpath(p1)
+        p2 = os.path.os.path.realpath(p2)
+        return os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep
 
     @property
     def name(self):
@@ -169,6 +178,11 @@ class LibBuilderBase(object):
     def version(self):
         return self._manifest.get("version")
 
+    @property
+    def dependent(self):
+        """Backward compatibility with ESP-IDF"""
+        return self.is_dependent
+
     @property
     def dependencies(self):
         return self._manifest.get("dependencies")
@@ -225,18 +239,6 @@ class LibBuilderBase(object):
     def extra_script(self):
         return None
 
-    @property
-    def depbuilders(self):
-        return self._depbuilders
-
-    @property
-    def dependent(self):
-        return self._is_dependent
-
-    @property
-    def is_built(self):
-        return self._is_built
-
     @property
     def lib_archive(self):
         return self.env.GetProjectOption("lib_archive")
@@ -296,8 +298,9 @@ class LibBuilderBase(object):
             self.env.ProcessUnFlags(self.build_unflags)
 
     def process_dependencies(self):
-        if not self.dependencies:
+        if not self.dependencies or self._deps_are_processed:
             return
+        self._deps_are_processed = True
         for item in self.dependencies:
             found = False
             for lb in self.env.GetLibBuilders():
@@ -305,7 +308,7 @@ class LibBuilderBase(object):
                     continue
                 found = True
                 if lb not in self.depbuilders:
-                    self.depend_recursive(lb)
+                    self.depend_on(lb)
                 break
 
             if not found and self.verbose:
@@ -400,7 +403,29 @@ class LibBuilderBase(object):
 
         return result
 
-    def depend_recursive(self, lb, search_files=None):
+    def search_deps_recursive(self, search_files=None):
+        self.process_dependencies()
+
+        # when LDF is disabled
+        if self.lib_ldf_mode == "off":
+            return
+
+        if self.lib_ldf_mode.startswith("deep"):
+            search_files = self.get_search_files()
+
+        lib_inc_map = {}
+        for inc in self._get_found_includes(search_files):
+            for lb in self.env.GetLibBuilders():
+                if inc.get_abspath() in lb:
+                    if lb not in lib_inc_map:
+                        lib_inc_map[lb] = []
+                    lib_inc_map[lb].append(inc.get_abspath())
+                    break
+
+        for lb, lb_search_files in lib_inc_map.items():
+            self.depend_on(lb, search_files=lb_search_files)
+
+    def depend_on(self, lb, search_files=None, recursive=True):
         def _already_depends(_lb):
             if self in _lb.depbuilders:
                 return True
@@ -418,49 +443,34 @@ class LibBuilderBase(object):
                         "between `%s` and `%s`\n" % (self.path, lb.path)
                     )
                 self._circular_deps.append(lb)
-            elif lb not in self._depbuilders:
-                self._depbuilders.append(lb)
+            elif lb not in self.depbuilders:
+                self.depbuilders.append(lb)
+                lb.is_dependent = True
                 LibBuilderBase._INCLUDE_DIRS_CACHE = None
-        lb.search_deps_recursive(search_files)
 
-    def search_deps_recursive(self, search_files=None):
-        if not self._is_dependent:
-            self._is_dependent = True
-            self.process_dependencies()
-
-            if self.lib_ldf_mode.startswith("deep"):
-                search_files = self.get_search_files()
-
-        # when LDF is disabled
-        if self.lib_ldf_mode == "off":
-            return
-
-        lib_inc_map = {}
-        for inc in self._get_found_includes(search_files):
-            for lb in self.env.GetLibBuilders():
-                if inc.get_abspath() in lb:
-                    if lb not in lib_inc_map:
-                        lib_inc_map[lb] = []
-                    lib_inc_map[lb].append(inc.get_abspath())
-                    break
-
-        for lb, lb_search_files in lib_inc_map.items():
-            self.depend_recursive(lb, lb_search_files)
+        if recursive:
+            lb.search_deps_recursive(search_files)
 
     def build(self):
         libs = []
-        for lb in self._depbuilders:
+        shared_scopes = ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS")
+        for lb in self.depbuilders:
             libs.extend(lb.build())
             # copy shared information to self env
-            for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
-                self.env.PrependUnique(**{key: lb.env.get(key)})
+            self.env.PrependUnique(
+                **{
+                    scope: lb.env.get(scope)
+                    for scope in shared_scopes
+                    if lb.env.get(scope)
+                }
+            )
 
         for lb in self._circular_deps:
             self.env.PrependUnique(CPPPATH=lb.get_include_dirs())
 
-        if self._is_built:
+        if self.is_built:
             return libs
-        self._is_built = True
+        self.is_built = True
 
         self.env.PrependUnique(CPPPATH=self.get_include_dirs())
 
@@ -468,8 +478,13 @@ class LibBuilderBase(object):
             for lb in self.env.GetLibBuilders():
                 if self == lb or not lb.is_built:
                     continue
-                for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
-                    self.env.PrependUnique(**{key: lb.env.get(key)})
+                self.env.PrependUnique(
+                    **{
+                        scope: lb.env.get(scope)
+                        for scope in shared_scopes
+                        if lb.env.get(scope)
+                    }
+                )
 
         do_not_archive = not self.lib_archive
         if not do_not_archive:
@@ -510,7 +525,7 @@ class ArduinoLibBuilder(LibBuilderBase):
         return os.path.join(self.path, "include")
 
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
+        include_dirs = super().get_include_dirs()
         if os.path.isdir(os.path.join(self.path, "src")):
             return include_dirs
         if os.path.isdir(os.path.join(self.path, "utility")):
@@ -608,7 +623,7 @@ class MbedLibBuilder(LibBuilderBase):
         return LibBuilderBase.src_dir.fget(self)  # pylint: disable=no-member
 
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
+        include_dirs = super().get_include_dirs()
         if self.path not in include_dirs:
             include_dirs.append(self.path)
 
@@ -632,7 +647,7 @@ class MbedLibBuilder(LibBuilderBase):
 
     def process_extra_options(self):
         self._process_mbed_lib_confs()
-        return super(MbedLibBuilder, self).process_extra_options()
+        return super().process_extra_options()
 
     def _process_mbed_lib_confs(self):
         mbed_lib_paths = [
@@ -829,7 +844,7 @@ class PlatformIOLibBuilder(LibBuilderBase):
         return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"])
 
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
+        include_dirs = super().get_include_dirs()
 
         # backwards compatibility with PlatformIO 2.0
         if (
@@ -851,7 +866,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
     def __init__(self, env, *args, **kwargs):
         # backup original value, will be reset in base.__init__
         project_src_filter = env.get("SRC_FILTER")
-        super(ProjectAsLibBuilder, self).__init__(env, *args, **kwargs)
+        super().__init__(env, *args, **kwargs)
         self.env["SRC_FILTER"] = project_src_filter
 
     @property
@@ -868,16 +883,16 @@ class ProjectAsLibBuilder(LibBuilderBase):
         project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
         if os.path.isdir(project_include_dir):
             include_dirs.append(project_include_dir)
-        for include_dir in LibBuilderBase.get_include_dirs(self):
+        for include_dir in super().get_include_dirs():
             if include_dir not in include_dirs:
                 include_dirs.append(include_dir)
         return include_dirs
 
     def get_search_files(self):
         # project files
-        items = LibBuilderBase.get_search_files(self)
+        items = super().get_search_files()
         # test files
-        if "__test" in COMMAND_LINE_TARGETS:
+        if "test" in self.env.GetBuildType():
             items.extend(
                 [
                     os.path.join("$PROJECT_TEST_DIR", item)
@@ -901,13 +916,19 @@ class ProjectAsLibBuilder(LibBuilderBase):
         # pylint: disable=no-member
         return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self)
 
+    @property
+    def build_flags(self):
+        # pylint: disable=no-member
+        return self.env.get("SRC_BUILD_FLAGS") or LibBuilderBase.build_flags.fget(self)
+
     @property
     def dependencies(self):
         return self.env.GetProjectOption("lib_deps", [])
 
     def process_extra_options(self):
-        # skip for project, options are already processed
-        pass
+        with fs.cd(self.path):
+            self.env.ProcessFlags(self.build_flags)
+            self.env.ProcessUnFlags(self.build_unflags)
 
     def install_dependencies(self):
         def _is_builtin(spec):
@@ -947,6 +968,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
             DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None)
 
     def process_dependencies(self):  # pylint: disable=too-many-branches
+        found_lbs = []
         for spec in self.dependencies:
             found = False
             for storage_dir in self.env.GetLibSourceDirs():
@@ -960,7 +982,8 @@ class ProjectAsLibBuilder(LibBuilderBase):
                     if pkg.path != lb.path:
                         continue
                     if lb not in self.depbuilders:
-                        self.depend_recursive(lb)
+                        self.depend_on(lb, recursive=False)
+                        found_lbs.append(lb)
                     found = True
                     break
             if found:
@@ -972,13 +995,17 @@ class ProjectAsLibBuilder(LibBuilderBase):
                 if lb.name != spec:
                     continue
                 if lb not in self.depbuilders:
-                    self.depend_recursive(lb)
+                    self.depend_on(lb)
                 found = True
                 break
 
+        # process library dependencies
+        for lb in found_lbs:
+            lb.search_deps_recursive()
+
     def build(self):
-        self._is_built = True  # do not build Project now
-        result = LibBuilderBase.build(self)
+        self.is_built = True  # do not build Project now
+        result = super().build()
         self.env.PrependUnique(CPPPATH=self.get_include_dirs())
         return result
 
@@ -1011,14 +1038,15 @@ def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)))
     return True
 
 
-def GetLibBuilders(env):  # pylint: disable=too-many-branches
-    if DefaultEnvironment().get("__PIO_LIB_BUILDERS", None) is not None:
+def GetLibBuilders(_):  # pylint: disable=too-many-branches
+    env = DefaultEnvironment()
+    if env.get("__PIO_LIB_BUILDERS", None) is not None:
         return sorted(
-            DefaultEnvironment()["__PIO_LIB_BUILDERS"],
-            key=lambda lb: 0 if lb.dependent else 1,
+            env["__PIO_LIB_BUILDERS"],
+            key=lambda lb: 0 if lb.is_dependent else 1,
         )
 
-    DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=[])
+    env.Replace(__PIO_LIB_BUILDERS=[])
 
     verbose = int(ARGUMENTS.get("PIOVERBOSE", 0))
     found_incompat = False
@@ -1029,7 +1057,11 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             continue
         for item in sorted(os.listdir(storage_dir)):
             lib_dir = os.path.join(storage_dir, item)
-            if item == "__cores__" or not os.path.isdir(lib_dir):
+            if item == "__cores__":
+                continue
+            if LibraryPackageManager.is_symlink(lib_dir):
+                lib_dir, _ = LibraryPackageManager.resolve_symlink(lib_dir)
+            if not lib_dir or not os.path.isdir(lib_dir):
                 continue
             try:
                 lb = LibBuilderFactory.new(env, lib_dir)
@@ -1040,13 +1072,13 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
                     )
                 continue
             if env.IsCompatibleLibBuilder(lb):
-                DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
+                env.Append(__PIO_LIB_BUILDERS=[lb])
             else:
                 found_incompat = True
 
     for lb in env.get("EXTRA_LIB_BUILDERS", []):
         if env.IsCompatibleLibBuilder(lb):
-            DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
+            env.Append(__PIO_LIB_BUILDERS=[lb])
         else:
             found_incompat = True
 
@@ -1057,13 +1089,25 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             "ldf-compat-mode\n"
         )
 
-    return DefaultEnvironment()["__PIO_LIB_BUILDERS"]
+    return env["__PIO_LIB_BUILDERS"]
 
 
 def ConfigureProjectLibBuilder(env):
+    _pm_storage = {}
+
+    def _get_lib_license(pkg):
+        storage_dir = os.path.dirname(os.path.dirname(pkg.path))
+        if storage_dir not in _pm_storage:
+            _pm_storage[storage_dir] = LibraryPackageManager(storage_dir)
+        try:
+            return (_pm_storage[storage_dir].load_manifest(pkg) or {}).get("license")
+        except MissingPackageManifestError:
+            pass
+        return None
+
     def _correct_found_libs(lib_builders):
         # build full dependency graph
-        found_lbs = [lb for lb in lib_builders if lb.dependent]
+        found_lbs = [lb for lb in lib_builders if lb.is_dependent]
         for lb in lib_builders:
             if lb in found_lbs:
                 lb.search_deps_recursive(lb.get_search_files())
@@ -1075,18 +1119,20 @@ def ConfigureProjectLibBuilder(env):
     def _print_deps_tree(root, level=0):
         margin = "|   " * (level)
         for lb in root.depbuilders:
-            title = "<%s>" % lb.name
+            title = lb.name
             pkg = PackageItem(lb.path)
             if pkg.metadata:
-                title += " %s" % pkg.metadata.version
+                title += " @ %s" % pkg.metadata.version
             elif lb.version:
-                title += " %s" % lb.version
+                title += " @ %s" % lb.version
             click.echo("%s|-- %s" % (margin, title), nl=False)
             if int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                click.echo(
+                    " (License: %s, " % (_get_lib_license(pkg) or "Unknown"), nl=False
+                )
                 if pkg.metadata and pkg.metadata.spec.external:
-                    click.echo(" [%s]" % pkg.metadata.spec.url, nl=False)
-                click.echo(" (", nl=False)
-                click.echo(lb.path, nl=False)
+                    click.echo("URI: %s, " % pkg.metadata.spec.uri, nl=False)
+                click.echo("Path: %s" % lb.path, nl=False)
                 click.echo(")", nl=False)
             click.echo("")
             if lb.depbuilders:

platformio/builder/tools/piomisc.py

@@ -14,244 +14,15 @@
 
 from __future__ import absolute_import
 
-import atexit
-import glob
-import io
 import os
-import re
 import sys
-import tempfile
-
-import click
 
 from platformio import fs, util
-from platformio.compat import get_filesystem_encoding, get_locale_encoding
-from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import exec_command
 
 
-class InoToCPPConverter(object):
-
-    PROTOTYPE_RE = re.compile(
-        r"""^(
-        (?:template\<.*\>\s*)?      # template
-        ([a-z_\d\&]+\*?\s+){1,2}    # return type
-        ([a-z_\d]+\s*)              # name of prototype
-        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
-        )\s*(\{|;)                  # must end with `{` or `;`
-        """,
-        re.X | re.M | re.I,
-    )
-    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
-    PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
-
-    def __init__(self, env):
-        self.env = env
-        self._main_ino = None
-        self._safe_encoding = None
-
-    def read_safe_contents(self, path):
-        error_reported = False
-        for encoding in (
-            "utf-8",
-            None,
-            get_filesystem_encoding(),
-            get_locale_encoding(),
-            "latin-1",
-        ):
-            try:
-                with io.open(path, encoding=encoding) as fp:
-                    contents = fp.read()
-                    self._safe_encoding = encoding
-                    return contents
-            except UnicodeDecodeError:
-                if not error_reported:
-                    error_reported = True
-                    click.secho(
-                        "Unicode decode error has occurred, please remove invalid "
-                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
-                        % path,
-                        fg="yellow",
-                        err=True,
-                    )
-        return ""
-
-    def write_safe_contents(self, path, contents):
-        with io.open(
-            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
-        ) as fp:
-            return fp.write(contents)
-
-    def is_main_node(self, contents):
-        return self.DETECTMAIN_RE.search(contents)
-
-    def convert(self, nodes):
-        contents = self.merge(nodes)
-        if not contents:
-            return None
-        return self.process(contents)
-
-    def merge(self, nodes):
-        assert nodes
-        lines = []
-        for node in nodes:
-            contents = self.read_safe_contents(node.get_path())
-            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
-            if self.is_main_node(contents):
-                lines = _lines + lines
-                self._main_ino = node.get_path()
-            else:
-                lines.extend(_lines)
-
-        if not self._main_ino:
-            self._main_ino = nodes[0].get_path()
-
-        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
-
-    def process(self, contents):
-        out_file = self._main_ino + ".cpp"
-        assert self._gcc_preprocess(contents, out_file)
-        contents = self.read_safe_contents(out_file)
-        contents = self._join_multiline_strings(contents)
-        self.write_safe_contents(out_file, self.append_prototypes(contents))
-        return out_file
-
-    def _gcc_preprocess(self, contents, out_file):
-        tmp_path = tempfile.mkstemp()[1]
-        self.write_safe_contents(tmp_path, contents)
-        self.env.Execute(
-            self.env.VerboseAction(
-                '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
-                    out_file, tmp_path
-                ),
-                "Converting " + os.path.basename(out_file[:-4]),
-            )
-        )
-        atexit.register(_delete_file, tmp_path)
-        return os.path.isfile(out_file)
-
-    def _join_multiline_strings(self, contents):
-        if "\\\n" not in contents:
-            return contents
-        newlines = []
-        linenum = 0
-        stropen = False
-        for line in contents.split("\n"):
-            _linenum = self._parse_preproc_line_num(line)
-            if _linenum is not None:
-                linenum = _linenum
-            else:
-                linenum += 1
-
-            if line.endswith("\\"):
-                if line.startswith('"'):
-                    stropen = True
-                    newlines.append(line[:-1])
-                    continue
-                if stropen:
-                    newlines[len(newlines) - 1] += line[:-1]
-                    continue
-            elif stropen and line.endswith(('",', '";')):
-                newlines[len(newlines) - 1] += line
-                stropen = False
-                newlines.append(
-                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
-                )
-                continue
-
-            newlines.append(line)
-
-        return "\n".join(newlines)
-
-    @staticmethod
-    def _parse_preproc_line_num(line):
-        if not line.startswith("#"):
-            return None
-        tokens = line.split(" ", 3)
-        if len(tokens) > 2 and tokens[1].isdigit():
-            return int(tokens[1])
-        return None
-
-    def _parse_prototypes(self, contents):
-        prototypes = []
-        reserved_keywords = set(["if", "else", "while"])
-        for match in self.PROTOTYPE_RE.finditer(contents):
-            if (
-                set([match.group(2).strip(), match.group(3).strip()])
-                & reserved_keywords
-            ):
-                continue
-            prototypes.append(match)
-        return prototypes
-
-    def _get_total_lines(self, contents):
-        total = 0
-        if contents.endswith("\n"):
-            contents = contents[:-1]
-        for line in contents.split("\n")[::-1]:
-            linenum = self._parse_preproc_line_num(line)
-            if linenum is not None:
-                return total + linenum
-            total += 1
-        return total
-
-    def append_prototypes(self, contents):
-        prototypes = self._parse_prototypes(contents) or []
-
-        # skip already declared prototypes
-        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
-        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
-
-        if not prototypes:
-            return contents
-
-        prototype_names = set(m.group(3).strip() for m in prototypes)
-        split_pos = prototypes[0].start()
-        match_ptrs = re.search(
-            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
-            contents[:split_pos],
-            re.M,
-        )
-        if match_ptrs:
-            split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
-
-        result = []
-        result.append(contents[:split_pos].strip())
-        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
-        result.append(
-            '#line %d "%s"'
-            % (
-                self._get_total_lines(contents[:split_pos]),
-                self._main_ino.replace("\\", "/"),
-            )
-        )
-        result.append(contents[split_pos:].strip())
-        return "\n".join(result)
-
-
-def ConvertInoToCpp(env):
-    src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
-    ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
-        os.path.join(src_dir, "*.pde")
-    )
-    if not ino_nodes:
-        return
-    c = InoToCPPConverter(env)
-    out_file = c.convert(ino_nodes)
-
-    atexit.register(_delete_file, out_file)
-
-
-def _delete_file(path):
-    try:
-        if os.path.isfile(path):
-            os.remove(path)
-    except:  # pylint: disable=bare-except
-        pass
-
-
 @util.memoized()
-def _get_compiler_type(env):
+def GetCompilerType(env):
     if env.subst("$CC").endswith("-gcc"):
         return "gcc"
     try:
@@ -270,10 +41,6 @@ def _get_compiler_type(env):
     return None
 
 
-def GetCompilerType(env):
-    return _get_compiler_type(env)
-
-
 def GetActualLDScript(env):
     def _lookup_in_ldpath(script):
         for d in env.get("LIBPATH", []):
@@ -319,7 +86,7 @@ def GetActualLDScript(env):
     env.Exit(1)
 
 
-def ConfigureDebugFlags(env):
+def ConfigureDebugTarget(env):
     def _cleanup_debug_flags(scope):
         if scope not in env:
             return
@@ -347,23 +114,15 @@ def ConfigureDebugFlags(env):
     ]
 
     if optimization_flags:
-        env.AppendUnique(ASFLAGS=optimization_flags, LINKFLAGS=optimization_flags)
-
-
-def ConfigureTestTarget(env):
-    env.Append(
-        CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
-        CPPPATH=[os.path.join("$BUILD_DIR", "UnityTestLib")],
-    )
-    unitylib = env.BuildLibrary(
-        os.path.join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity")
-    )
-    env.Prepend(LIBS=[unitylib])
-
-    src_filter = ["+<*.cpp>", "+<*.c>"]
-    if "PIOTEST_RUNNING_NAME" in env:
-        src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], os.path.sep))
-    env.Replace(PIOTEST_SRC_FILTER=src_filter)
+        env.AppendUnique(
+            ASFLAGS=[
+                # skip -O flags for assembler
+                f
+                for f in optimization_flags
+                if f.startswith("-g")
+            ],
+            LINKFLAGS=optimization_flags,
+        )
 
 
 def GetExtraScripts(env, scope):
@@ -376,18 +135,17 @@ def GetExtraScripts(env, scope):
     if not items:
         return items
     with fs.cd(env.subst("$PROJECT_DIR")):
-        return [os.path.abspath(item) for item in items]
+        return [os.path.abspath(env.subst(item)) for item in items]
+
+
+def generate(env):
+    env.AddMethod(GetCompilerType)
+    env.AddMethod(GetActualLDScript)
+    env.AddMethod(ConfigureDebugTarget)
+    env.AddMethod(GetExtraScripts)
+    # bakward-compatibility with Zephyr build script
+    env.AddMethod(ConfigureDebugTarget, "ConfigureDebugFlags")
 
 
 def exists(_):
     return True
-
-
-def generate(env):
-    env.AddMethod(ConvertInoToCpp)
-    env.AddMethod(GetCompilerType)
-    env.AddMethod(GetActualLDScript)
-    env.AddMethod(ConfigureDebugFlags)
-    env.AddMethod(ConfigureTestTarget)
-    env.AddMethod(GetExtraScripts)
-    return env

platformio/builder/tools/pioplatform.py

@@ -19,6 +19,7 @@ import sys
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
+from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import fs, util
 from platformio.compat import IS_MACOS, IS_WINDOWS
@@ -32,16 +33,17 @@ from platformio.project.config import ProjectOptions
 
 
 @util.memoized()
-def PioPlatform(env):
-    variables = env.GetProjectOptions(as_dict=True)
-    if "framework" in variables:
-        # support PIO Core 3.0 dev/platforms
-        variables["pioframework"] = variables["framework"]
+def _PioPlatform():
+    env = DefaultEnvironment()
     p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"]))
-    p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
+    p.configure_project_packages(env["PIOENV"], COMMAND_LINE_TARGETS)
     return p
 
 
+def PioPlatform(_):
+    return _PioPlatform()
+
+
 def BoardConfig(env, board=None):
     with fs.cd(env.subst("$PROJECT_DIR")):
         try:
@@ -160,7 +162,7 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
             and pkg_metadata
             and pkg_metadata.spec.external
         ):
-            data.append("(%s)" % pkg_metadata.spec.url)
+            data.append("(%s)" % pkg_metadata.spec.uri)
         if board_config:
             data.extend([">", board_config.get("name")])
         return data
@@ -213,7 +215,7 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
         data = []
         for item in platform.dump_used_packages():
             original_version = get_original_version(item["version"])
-            info = "%s %s" % (item["name"], item["version"])
+            info = "%s @ %s" % (item["name"], item["version"])
             extra = []
             if original_version:
                 extra.append(original_version)

platformio/builder/tools/pioproject.py

@@ -14,7 +14,8 @@
 
 from __future__ import absolute_import
 
-from platformio.project.config import MISSING, ProjectConfig, ProjectOptions
+from platformio.compat import MISSING
+from platformio.project.config import ProjectConfig
 
 
 def GetProjectConfig(env):
@@ -30,15 +31,17 @@ def GetProjectOption(env, option, default=MISSING):
 
 
 def LoadProjectOptions(env):
-    for option, value in env.GetProjectOptions():
-        option_meta = ProjectOptions.get("env." + option)
+    config = env.GetProjectConfig()
+    section = "env:" + env["PIOENV"]
+    for option in config.options(section):
+        option_meta = config.find_option_meta(section, option)
         if (
             not option_meta
             or not option_meta.buildenvvar
             or option_meta.buildenvvar in env
         ):
             continue
-        env[option_meta.buildenvvar] = value
+        env[option_meta.buildenvvar] = config.get(section, option)
 
 
 def exists(_):

platformio/builder/tools/piotest.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import os
+
+from platformio.builder.tools import platformio as piotool
+from platformio.test.result import TestSuite
+from platformio.test.runners.factory import TestRunnerFactory
+
+
+def ConfigureTestTarget(env):
+    env.Append(
+        CPPDEFINES=["UNIT_TEST", "PIO_UNIT_TESTING"],
+        PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piotool.SRC_BUILD_EXT],
+    )
+    env.Prepend(CPPPATH=["$PROJECT_TEST_DIR"])
+
+    if "PIOTEST_RUNNING_NAME" in env:
+        test_name = env["PIOTEST_RUNNING_NAME"]
+        while True:
+            test_name = os.path.dirname(test_name)  # parent dir
+            # skip nested tests (user's side issue?)
+            if not test_name or os.path.basename(test_name).startswith("test_"):
+                break
+            env.Prepend(
+                PIOTEST_SRC_FILTER=[
+                    f"+<{test_name}{os.path.sep}*.{ext}>"
+                    for ext in piotool.SRC_BUILD_EXT
+                ],
+                CPPPATH=[os.path.join("$PROJECT_TEST_DIR", test_name)],
+            )
+
+        env.Prepend(
+            PIOTEST_SRC_FILTER=[f"+<$PIOTEST_RUNNING_NAME{os.path.sep}>"],
+            CPPPATH=[os.path.join("$PROJECT_TEST_DIR", "$PIOTEST_RUNNING_NAME")],
+        )
+
+    test_runner = TestRunnerFactory.new(
+        TestSuite(env["PIOENV"], env.get("PIOTEST_RUNNING_NAME", "*")),
+        env.GetProjectConfig(),
+    )
+    test_runner.configure_build_env(env)
+
+
+def generate(env):
+    env.AddMethod(ConfigureTestTarget)
+
+
+def exists(_):
+    return True

platformio/builder/tools/pioupload.py

@@ -12,25 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=unused-argument
+
 from __future__ import absolute_import
 
+import os
 import re
 import sys
-from fnmatch import fnmatch
-from os import environ
-from os.path import isfile, join
 from shutil import copyfile
 from time import sleep
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from serial import Serial, SerialException
 
-from platformio import exception, fs, util
-from platformio.compat import IS_WINDOWS
+from platformio import exception, fs
+from platformio.device.finder import find_mbed_disk, find_serial_port, is_pattern_port
+from platformio.device.list import list_serial_ports
 from platformio.proc import exec_command
 
-# pylint: disable=unused-argument
-
 
 def FlushSerialBuffer(env, port):
     s = Serial(env.subst(port))
@@ -62,7 +61,7 @@ def WaitForNewSerialPort(env, before):
     elapsed = 0
     before = [p["port"] for p in before]
     while elapsed < 5 and new_port is None:
-        now = [p["port"] for p in util.get_serial_ports()]
+        now = [p["port"] for p in list_serial_ports()]
         for p in now:
             if p not in before:
                 new_port = p
@@ -97,67 +96,28 @@ def WaitForNewSerialPort(env, before):
 
 def AutodetectUploadPort(*args, **kwargs):
     env = args[0]
-
-    def _get_pattern():
-        if "UPLOAD_PORT" not in env:
-            return None
-        if set(["*", "?", "[", "]"]) & set(env["UPLOAD_PORT"]):
-            return env["UPLOAD_PORT"]
-        return None
-
-    def _is_match_pattern(port):
-        pattern = _get_pattern()
-        if not pattern:
-            return True
-        return fnmatch(port, pattern)
-
-    def _look_for_mbed_disk():
-        msdlabels = ("mbed", "nucleo", "frdm", "microbit")
-        for item in util.get_logical_devices():
-            if item["path"].startswith("/net") or not _is_match_pattern(item["path"]):
-                continue
-            mbed_pages = [join(item["path"], n) for n in ("mbed.htm", "mbed.html")]
-            if any(isfile(p) for p in mbed_pages):
-                return item["path"]
-            if item["name"] and any(l in item["name"].lower() for l in msdlabels):
-                return item["path"]
-        return None
-
-    def _look_for_serial_port():
-        port = None
-        board_hwids = []
-        upload_protocol = env.subst("$UPLOAD_PROTOCOL")
-        if "BOARD" in env and "build.hwids" in env.BoardConfig():
-            board_hwids = env.BoardConfig().get("build.hwids")
-        for item in util.get_serial_ports(filter_hwid=True):
-            if not _is_match_pattern(item["port"]):
-                continue
-            port = item["port"]
-            if upload_protocol.startswith("blackmagic"):
-                if IS_WINDOWS and port.startswith("COM") and len(port) > 4:
-                    port = "\\\\.\\%s" % port
-                if "GDB" in item["description"]:
-                    return port
-            for hwid in board_hwids:
-                hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                if hwid_str in item["hwid"]:
-                    return port
-        return port
-
-    if "UPLOAD_PORT" in env and not _get_pattern():
-        print(env.subst("Use manually specified: $UPLOAD_PORT"))
+    initial_port = env.subst("$UPLOAD_PORT")
+    upload_protocol = env.subst("$UPLOAD_PROTOCOL")
+    if initial_port and not is_pattern_port(initial_port):
+        print(env.subst("Using manually specified: $UPLOAD_PORT"))
         return
 
-    if env.subst("$UPLOAD_PROTOCOL") == "mbed" or (
-        "mbed" in env.subst("$PIOFRAMEWORK") and not env.subst("$UPLOAD_PROTOCOL")
+    if upload_protocol == "mbed" or (
+        "mbed" in env.subst("$PIOFRAMEWORK") and not upload_protocol
     ):
-        env.Replace(UPLOAD_PORT=_look_for_mbed_disk())
+        env.Replace(UPLOAD_PORT=find_mbed_disk(initial_port))
     else:
         try:
             fs.ensure_udev_rules()
         except exception.InvalidUdevRules as e:
             sys.stderr.write("\n%s\n\n" % e)
-        env.Replace(UPLOAD_PORT=_look_for_serial_port())
+        env.Replace(
+            UPLOAD_PORT=find_serial_port(
+                initial_port=initial_port,
+                board_config=env.BoardConfig() if "BOARD" in env else None,
+                upload_protocol=upload_protocol,
+            )
+        )
 
     if env.subst("$UPLOAD_PORT"):
         print(env.subst("Auto-detected: $UPLOAD_PORT"))
@@ -175,10 +135,12 @@ def UploadToDisk(_, target, source, env):
     assert "UPLOAD_PORT" in env
     progname = env.subst("$PROGNAME")
     for ext in ("bin", "hex"):
-        fpath = join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
-        if not isfile(fpath):
+        fpath = os.path.join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
+        if not os.path.isfile(fpath):
             continue
-        copyfile(fpath, join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)))
+        copyfile(
+            fpath, os.path.join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext))
+        )
     print(
         "Firmware has been successfully uploaded.\n"
         "(Some boards may require manual hard reset)"
@@ -211,7 +173,7 @@ def CheckUploadSize(_, target, source, env):
         if not isinstance(cmd, list):
             cmd = cmd.split()
         cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
-        sysenv = environ.copy()
+        sysenv = os.environ.copy()
         sysenv["PATH"] = str(env["ENV"]["PATH"])
         result = exec_command(env.subst(cmd), env=sysenv)
         if result["returncode"] != 0:

platformio/builder/tools/platformio.py

@@ -47,14 +47,16 @@ def scons_patched_match_splitext(path, suffixes=None):
 
 
 def GetBuildType(env):
-    return (
-        "debug"
-        if (
-            set(["__debug", "sizedata"]) & set(COMMAND_LINE_TARGETS)
-            or env.GetProjectOption("build_type") == "debug"
-        )
-        else "release"
-    )
+    modes = []
+    if (
+        set(["__debug", "sizedata"])  # sizedata = for memory inspection
+        & set(COMMAND_LINE_TARGETS)
+        or env.GetProjectOption("build_type") == "debug"
+    ):
+        modes.append("debug")
+    if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test":
+        modes.append("test")
+    return "+".join(modes or ["release"])
 
 
 def BuildProgram(env):
@@ -113,10 +115,6 @@ def ProcessProgramDeps(env):
 
     env.PrintConfiguration()
 
-    # fix ASM handling under non case-sensitive OS
-    if not Util.case_sensitive_suffixes(".s", ".S"):
-        env.Replace(AS="$CC", ASCOM="$ASPPCOM")
-
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
         env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
@@ -127,18 +125,25 @@ def ProcessProgramDeps(env):
     # process framework scripts
     env.BuildFrameworks(env.get("PIOFRAMEWORK"))
 
-    if env.GetBuildType() == "debug":
-        env.ConfigureDebugFlags()
+    if "debug" in env.GetBuildType():
+        env.ConfigureDebugTarget()
+    if "test" in env.GetBuildType():
+        env.ConfigureTestTarget()
 
     # remove specified flags
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
-    if "__test" in COMMAND_LINE_TARGETS:
-        env.ConfigureTestTarget()
+    if "compiledb" in COMMAND_LINE_TARGETS and env.get(
+        "COMPILATIONDB_INCLUDE_TOOLCHAIN"
+    ):
+        for scope, includes in env.DumpIntegrationIncludes().items():
+            if scope in ("toolchain",):
+                env.Append(CPPPATH=includes)
 
 
 def ProcessProjectDeps(env):
     project_lib_builder = env.ConfigureProjectLibBuilder()
+    projenv = project_lib_builder.env
 
     # prepend project libs to the beginning of list
     env.Prepend(LIBS=project_lib_builder.build())
@@ -151,19 +156,19 @@ def ProcessProjectDeps(env):
         }
     )
 
-    projenv = env.Clone()
-
-    # CPPPATH from dependencies
-    projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH"))
-    # extra build flags from `platformio.ini`
-    projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
-
-    is_test = "__test" in COMMAND_LINE_TARGETS
-    if is_test:
+    if "test" in env.GetBuildType():
+        build_files_before_nums = len(env.get("PIOBUILDFILES", []))
         projenv.BuildSources(
             "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
         )
-    if not is_test or env.GetProjectOption("test_build_project_src"):
+        if len(env.get("PIOBUILDFILES", [])) - build_files_before_nums < 1:
+            sys.stderr.write(
+                "Error: Nothing to build. Please put your test suites "
+                "to the '%s' folder\n" % env.subst("$PROJECT_TEST_DIR")
+            )
+            env.Exit(1)
+
+    if "test" not in env.GetBuildType() or env.GetProjectOption("test_build_src"):
         projenv.BuildSources(
             "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER")
         )
@@ -171,7 +176,7 @@ def ProcessProjectDeps(env):
     if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
         sys.stderr.write(
             "Error: Nothing to build. Please put your source code files "
-            "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
+            "to the '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
         )
         env.Exit(1)
 
@@ -324,25 +329,18 @@ def BuildFrameworks(env, frameworks):
         )
         env.Exit(1)
 
-    board_frameworks = env.BoardConfig().get("frameworks", [])
-    if frameworks == ["platformio"]:
-        if board_frameworks:
-            frameworks.insert(0, board_frameworks[0])
-        else:
-            sys.stderr.write("Error: Please specify `board` in `platformio.ini`\n")
-            env.Exit(1)
-
-    for f in frameworks:
-        if f == "arduino":
-            # Arduino IDE appends .o the end of filename
+    supported_frameworks = env.BoardConfig().get("frameworks", [])
+    for name in frameworks:
+        if name == "arduino":
+            # Arduino IDE appends .o to the end of filename
             Builder.match_splitext = scons_patched_match_splitext
             if "nobuild" not in COMMAND_LINE_TARGETS:
                 env.ConvertInoToCpp()
 
-        if f in board_frameworks:
-            SConscript(env.GetFrameworkScript(f), exports="env")
+        if name in supported_frameworks:
+            SConscript(env.GetFrameworkScript(name), exports="env")
         else:
-            sys.stderr.write("Error: This board doesn't support %s framework!\n" % f)
+            sys.stderr.write("Error: This board doesn't support %s framework!\n" % name)
             env.Exit(1)
 
 

platformio/check/cli.py

@@ -26,13 +26,13 @@ import click
 from tabulate import tabulate
 
 from platformio import app, exception, fs, util
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools import CheckToolFactory
+from platformio.check.defect import DefectItem
+from platformio.check.tools import CheckToolFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import find_project_dir_above, get_project_dir
 
 
-@click.command("check", short_help="Static code analysis")
+@click.command("check", short_help="Static Code Analysis")
 @click.option("-e", "--environment", multiple=True)
 @click.option(
     "-d",
@@ -118,6 +118,7 @@ def cli(
                 if silent
                 else severity or config.get("env:" + envname, "check_severity"),
                 skip_packages=skip_packages or env_options.get("check_skip_packages"),
+                platform_packages=env_options.get("platform_packages"),
             )
 
             for tool in config.get("env:" + envname, "check_tool"):
@@ -166,7 +167,10 @@ def cli(
         if json_output:
             click.echo(json.dumps(results_to_json(results)))
         elif not silent:
-            print_check_summary(results)
+            print_check_summary(results, verbose=verbose)
+
+    # Reset custom project config
+    app.set_session_var("custom_project_conf", None)
 
     command_failed = any(r.get("succeeded") is False for r in results)
     if command_failed:
@@ -267,7 +271,7 @@ def print_defects_stats(results):
     click.echo()
 
 
-def print_check_summary(results):
+def print_check_summary(results, verbose=False):
     click.echo()
 
     tabular_data = []
@@ -284,6 +288,8 @@ def print_check_summary(results):
             status_str = click.style("FAILED", fg="red")
         elif result.get("succeeded") is None:
             status_str = "IGNORED"
+            if not verbose:
+                continue
         else:
             succeeded_nums += 1
             status_str = click.style("PASSED", fg="green")

platformio/check/defect.py

@@ -34,7 +34,7 @@ class DefectItem(object):
         severity,
         category,
         message,
-        file="unknown",
+        file=None,
         line=0,
         column=0,
         id=None,
@@ -50,7 +50,7 @@ class DefectItem(object):
         self.callstack = callstack
         self.cwe = cwe
         self.id = id
-        self.file = file
+        self.file = file or "unknown"
         if file.lower().startswith(get_project_dir().lower()):
             self.file = os.path.relpath(file, get_project_dir())
 

platformio/check/tools/__init__.py

@@ -13,9 +13,9 @@
 # limitations under the License.
 
 from platformio import exception
-from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool
-from platformio.commands.check.tools.cppcheck import CppcheckCheckTool
-from platformio.commands.check.tools.pvsstudio import PvsStudioCheckTool
+from platformio.check.tools.clangtidy import ClangtidyCheckTool
+from platformio.check.tools.cppcheck import CppcheckCheckTool
+from platformio.check.tools.pvsstudio import PvsStudioCheckTool
 
 
 class CheckToolFactory(object):

platformio/check/tools/base.py

@@ -19,8 +19,10 @@ import tempfile
 import click
 
 from platformio import fs, proc
-from platformio.commands.check.defect import DefectItem
-from platformio.project.helpers import load_project_ide_data
+from platformio.check.defect import DefectItem
+from platformio.package.manager.core import get_core_package_dir
+from platformio.package.meta import PackageSpec
+from platformio.project.helpers import load_build_metadata
 
 
 class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
@@ -55,7 +57,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         ]
 
     def _load_cpp_data(self, project_dir):
-        data = load_project_ide_data(project_dir, self.envname)
+        data = load_build_metadata(project_dir, self.envname)
         if not data:
             return
         self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", ""))
@@ -66,6 +68,13 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         self.cxx_path = data.get("cxx_path")
         self.toolchain_defines = self._get_toolchain_defines()
 
+    def get_tool_dir(self, pkg_name):
+        for spec in self.options["platform_packages"] or []:
+            spec = PackageSpec(spec)
+            if spec.name == pkg_name:
+                return get_core_package_dir(pkg_name, spec=spec)
+        return get_core_package_dir(pkg_name)
+
     def get_flags(self, tool):
         result = []
         flags = self.options.get("flags") or []

platformio/check/tools/clangtidy.py

@@ -15,13 +15,12 @@
 import re
 from os.path import join
 
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
-from platformio.package.manager.core import get_core_package_dir
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 
 
 class ClangtidyCheckTool(CheckToolBase):
-    def tool_output_filter(self, line):
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
         if not self.options.get("verbose") and "[clang-diagnostic-error]" in line:
             return ""
 
@@ -34,7 +33,7 @@ class ClangtidyCheckTool(CheckToolBase):
 
         return ""
 
-    def parse_defect(self, raw_line):
+    def parse_defect(self, raw_line):  # pylint: disable=arguments-differ
         match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line)
         if not match:
             return raw_line
@@ -56,11 +55,13 @@ class ClangtidyCheckTool(CheckToolBase):
         return cmd_result["returncode"] < 2
 
     def configure_command(self):
-        tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy")
+        tool_path = join(self.get_tool_dir("tool-clangtidy"), "clang-tidy")
 
         cmd = [tool_path, "--quiet"]
         flags = self.get_flags("clangtidy")
-        if not self.is_flag_set("--checks", flags):
+        if not (
+            self.is_flag_set("--checks", flags) or self.is_flag_set("--config", flags)
+        ):
             cmd.append("--checks=*")
 
         project_files = self.get_project_target_files(self.options["patterns"])

platformio/check/tools/cppcheck.py

@@ -17,13 +17,13 @@ import os
 import click
 
 from platformio import proc
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
-from platformio.package.manager.core import get_core_package_dir
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 
 
 class CppcheckCheckTool(CheckToolBase):
     def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self._field_delimiter = "<&PIO&>"
         self._buffer = ""
         self.defect_fields = [
@@ -36,9 +36,8 @@ class CppcheckCheckTool(CheckToolBase):
             "cwe",
             "id",
         ]
-        super(CppcheckCheckTool, self).__init__(*args, **kwargs)
 
-    def tool_output_filter(self, line):
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
         if (
             not self.options.get("verbose")
             and "--suppress=unmatchedSuppression:" in line
@@ -50,13 +49,14 @@ class CppcheckCheckTool(CheckToolBase):
             for msg in (
                 "No C or C++ source files found",
                 "unrecognized command line option",
+                "there was an internal error",
             )
         ):
             self._bad_input = True
 
         return line
 
-    def parse_defect(self, raw_line):
+    def parse_defect(self, raw_line):  # pylint: disable=arguments-differ
         if self._field_delimiter not in raw_line:
             return None
 
@@ -103,7 +103,7 @@ class CppcheckCheckTool(CheckToolBase):
         return DefectItem(**args)
 
     def configure_command(self, language, src_file):  # pylint: disable=arguments-differ
-        tool_path = os.path.join(get_core_package_dir("tool-cppcheck"), "cppcheck")
+        tool_path = os.path.join(self.get_tool_dir("tool-cppcheck"), "cppcheck")
 
         cmd = [
             tool_path,
@@ -208,7 +208,7 @@ class CppcheckCheckTool(CheckToolBase):
         return self._create_tmp_file("\n".join(result))
 
     def clean_up(self):
-        super(CppcheckCheckTool, self).clean_up()
+        super().clean_up()
 
         # delete temporary dump files generated by addons
         if not self.is_flag_set("--addon", self.get_flags("cppcheck")):

platformio/check/tools/pvsstudio.py

@@ -20,25 +20,24 @@ from xml.etree.ElementTree import fromstring
 import click
 
 from platformio import proc
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 from platformio.compat import IS_WINDOWS
-from platformio.package.manager.core import get_core_package_dir
 
 
 class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-attributes
     def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self._tmp_dir = tempfile.mkdtemp(prefix="piocheck")
         self._tmp_preprocessed_file = self._generate_tmp_file_path() + ".i"
         self._tmp_output_file = self._generate_tmp_file_path() + ".pvs"
         self._tmp_cfg_file = self._generate_tmp_file_path() + ".cfg"
         self._tmp_cmd_file = self._generate_tmp_file_path() + ".cmd"
         self.tool_path = os.path.join(
-            get_core_package_dir("tool-pvs-studio"),
+            self.get_tool_dir("tool-pvs-studio"),
             "x64" if IS_WINDOWS else "bin",
             "pvs-studio",
         )
-        super(PvsStudioCheckTool, self).__init__(*args, **kwargs)
 
         with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp:
             fp.write(
@@ -53,8 +52,14 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
                 )
             )
 
-    def tool_output_filter(self, line):
-        if "license was not entered" in line.lower():
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
+        if any(
+            err_msg in line.lower()
+            for err_msg in (
+                "license was not entered",
+                "license information is incorrect",
+            )
+        ):
             self._bad_input = True
         return line
 
@@ -70,7 +75,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
 
     def _demangle_report(self, output_file):
         converter_tool = os.path.join(
-            get_core_package_dir("tool-pvs-studio"),
+            self.get_tool_dir("tool-pvs-studio"),
             "HtmlGenerator" if IS_WINDOWS else os.path.join("bin", "plog-converter"),
         )
 
@@ -194,7 +199,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
             '"%s"' % self._tmp_preprocessed_file,
         ]
         cmd.extend([f for f in flags if f])
-        cmd.extend(["-D%s" % d for d in self.cpp_defines])
+        cmd.extend(['"-D%s"' % d.replace('"', '\\"') for d in self.cpp_defines])
         cmd.append('@"%s"' % self._tmp_cmd_file)
 
         # Explicitly specify C++ as the language used in .ino files
@@ -209,7 +214,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
             self._bad_input = True
 
     def clean_up(self):
-        super(PvsStudioCheckTool, self).clean_up()
+        super().clean_up()
         if os.path.isdir(self._tmp_dir):
             shutil.rmtree(self._tmp_dir)
 

platformio/cli.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+from pathlib import Path
+
+import click
+
+
+class PlatformioCLI(click.MultiCommand):
+
+    leftover_args = []
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._pio_root_path = Path(__file__).parent
+        self._pio_cmd_aliases = dict(package="pkg")
+
+    def _find_pio_commands(self):
+        def _to_module_path(p):
+            return (
+                "platformio." + ".".join(p.relative_to(self._pio_root_path).parts)[:-3]
+            )
+
+        result = {}
+        for p in self._pio_root_path.rglob("cli.py"):
+            # skip this module
+            if p.parent == self._pio_root_path:
+                continue
+            cmd_name = p.parent.name
+            result[self._pio_cmd_aliases.get(cmd_name, cmd_name)] = _to_module_path(p)
+
+        # find legacy commands
+        for p in (self._pio_root_path / "commands").iterdir():
+            if p.name.startswith("_"):
+                continue
+            if (p / "command.py").is_file():
+                result[p.name] = _to_module_path(p / "command.py")
+            elif p.name.endswith(".py"):
+                result[p.name[:-3]] = _to_module_path(p)
+
+        return result
+
+    @staticmethod
+    def in_silence():
+        args = PlatformioCLI.leftover_args
+        return args and any(
+            [
+                args[0] == "debug" and "--interpreter" in " ".join(args),
+                args[0] == "upgrade",
+                "--json-output" in args,
+                "--version" in args,
+            ]
+        )
+
+    def invoke(self, ctx):
+        PlatformioCLI.leftover_args = ctx.args
+        if hasattr(ctx, "protected_args"):
+            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
+        return super().invoke(ctx)
+
+    def list_commands(self, ctx):
+        return sorted(list(self._find_pio_commands()))
+
+    def get_command(self, ctx, cmd_name):
+        commands = self._find_pio_commands()
+        if cmd_name not in commands:
+            return self._handle_obsolate_command(ctx, cmd_name)
+        module = importlib.import_module(commands[cmd_name])
+        return getattr(module, "cli")
+
+    @staticmethod
+    def _handle_obsolate_command(ctx, cmd_name):
+        # pylint: disable=import-outside-toplevel
+        if cmd_name == "init":
+            from platformio.project.commands.init import project_init_cmd
+
+            return project_init_cmd
+
+        if cmd_name == "package":
+            from platformio.package.cli import cli
+
+            return cli
+
+        raise click.UsageError('No such command "%s"' % cmd_name, ctx)

platformio/commands/__init__.py

@@ -11,70 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import os
-
-import click
-
-
-class PlatformioCLI(click.MultiCommand):
-
-    leftover_args = []
-
-    def __init__(self, *args, **kwargs):
-        super(PlatformioCLI, self).__init__(*args, **kwargs)
-        self._pio_cmds_dir = os.path.dirname(__file__)
-
-    @staticmethod
-    def in_silence():
-        args = PlatformioCLI.leftover_args
-        return args and any(
-            [
-                args[0] == "debug" and "--interpreter" in " ".join(args),
-                args[0] == "upgrade",
-                "--json-output" in args,
-                "--version" in args,
-            ]
-        )
-
-    def invoke(self, ctx):
-        PlatformioCLI.leftover_args = ctx.args
-        if hasattr(ctx, "protected_args"):
-            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
-        return super(PlatformioCLI, self).invoke(ctx)
-
-    def list_commands(self, ctx):
-        cmds = []
-        for cmd_name in os.listdir(self._pio_cmds_dir):
-            if cmd_name.startswith("__init__"):
-                continue
-            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
-                cmds.append(cmd_name)
-            elif cmd_name.endswith(".py"):
-                cmds.append(cmd_name[:-3])
-        cmds.sort()
-        return cmds
-
-    def get_command(self, ctx, cmd_name):
-        mod = None
-        try:
-            mod_path = "platformio.commands." + cmd_name
-            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
-                mod_path = "platformio.commands.%s.command" % cmd_name
-            mod = __import__(mod_path, None, None, ["cli"])
-        except ImportError:
-            try:
-                return self._handle_obsolate_command(cmd_name)
-            except AttributeError:
-                pass
-            raise click.UsageError('No such command "%s"' % cmd_name, ctx)
-        return mod.cli
-
-    @staticmethod
-    def _handle_obsolate_command(name):
-        # pylint: disable=import-outside-toplevel
-        if name == "init":
-            from platformio.commands.project import project_init
-
-            return project_init
-        raise AttributeError()

platformio/commands/access.py

@@ -1,154 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import json
-import re
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.registry import RegistryClient
-from platformio.commands.account import validate_username
-from platformio.commands.team import validate_orgname_teamname
-
-
-def validate_client(value):
-    if ":" in value:
-        validate_orgname_teamname(value)
-    else:
-        validate_username(value)
-    return value
-
-
-@click.group("access", short_help="Manage resource access")
-def cli():
-    pass
-
-
-def validate_urn(value):
-    value = str(value).strip()
-    if not re.match(r"^prn:reg:pkg:(\d+):(\w+)$", value, flags=re.I):
-        raise click.BadParameter("Invalid URN format.")
-    return value
-
-
-@cli.command("public", short_help="Make resource public")
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_public(urn, urn_type):
-    client = RegistryClient()
-    client.update_resource(urn=urn, private=0)
-    return click.secho(
-        "The resource %s has been successfully updated." % urn,
-        fg="green",
-    )
-
-
-@cli.command("private", short_help="Make resource private")
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_private(urn, urn_type):
-    client = RegistryClient()
-    client.update_resource(urn=urn, private=1)
-    return click.secho(
-        "The resource %s has been successfully updated." % urn,
-        fg="green",
-    )
-
-
-@cli.command("grant", short_help="Grant access")
-@click.argument("level", type=click.Choice(["admin", "maintainer", "guest"]))
-@click.argument(
-    "client",
-    metavar="[<ORGNAME:TEAMNAME>|<USERNAME>]",
-    callback=lambda _, __, value: validate_client(value),
-)
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_grant(level, client, urn, urn_type):
-    reg_client = RegistryClient()
-    reg_client.grant_access_for_resource(urn=urn, client=client, level=level)
-    return click.secho(
-        "Access for resource %s has been granted for %s" % (urn, client),
-        fg="green",
-    )
-
-
-@cli.command("revoke", short_help="Revoke access")
-@click.argument(
-    "client",
-    metavar="[ORGNAME:TEAMNAME|USERNAME]",
-    callback=lambda _, __, value: validate_client(value),
-)
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_revoke(client, urn, urn_type):
-    reg_client = RegistryClient()
-    reg_client.revoke_access_from_resource(urn=urn, client=client)
-    return click.secho(
-        "Access for resource %s has been revoked for %s" % (urn, client),
-        fg="green",
-    )
-
-
-@cli.command("list", short_help="List published resources")
-@click.argument("owner", required=False)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-@click.option("--json-output", is_flag=True)
-def access_list(owner, urn_type, json_output):
-    reg_client = RegistryClient()
-    resources = reg_client.list_resources(owner=owner)
-    if json_output:
-        return click.echo(json.dumps(resources))
-    if not resources:
-        return click.secho("You do not have any resources.", fg="yellow")
-    for resource in resources:
-        click.echo()
-        click.secho(resource.get("name"), fg="cyan")
-        click.echo("-" * len(resource.get("name")))
-        table_data = []
-        table_data.append(("URN:", resource.get("urn")))
-        table_data.append(("Owner:", resource.get("owner")))
-        table_data.append(
-            (
-                "Access:",
-                click.style("Private", fg="red")
-                if resource.get("private", False)
-                else "Public",
-            )
-        )
-        table_data.append(
-            (
-                "Access level(s):",
-                ", ".join(
-                    (level.capitalize() for level in resource.get("access_levels"))
-                ),
-            )
-        )
-        click.echo(tabulate(table_data, tablefmt="plain"))
-    return click.echo()

platformio/commands/account.py

@@ -1,299 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import datetime
-import json
-import re
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.account import AccountClient, AccountNotAuthorized
-
-
-@click.group("account", short_help="Manage PlatformIO account")
-def cli():
-    pass
-
-
-def validate_username(value, field="username"):
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I):
-        raise click.BadParameter(
-            "Invalid %s format. "
-            "%s must contain only alphanumeric characters "
-            "or single hyphens, cannot begin or end with a hyphen, "
-            "and must not be longer than 38 characters."
-            % (field.lower(), field.capitalize())
-        )
-    return value
-
-
-def validate_email(value):
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d_.+-]+@[a-z\d\-]+\.[a-z\d\-.]+$", value, flags=re.I):
-        raise click.BadParameter("Invalid email address")
-    return value
-
-
-def validate_password(value):
-    value = str(value).strip()
-    if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
-        raise click.BadParameter(
-            "Invalid password format. "
-            "Password must contain at least 8 characters"
-            " including a number and a lowercase letter"
-        )
-    return value
-
-
-@cli.command("register", short_help="Create new PlatformIO Account")
-@click.option(
-    "-u",
-    "--username",
-    prompt=True,
-    callback=lambda _, __, value: validate_username(value),
-)
-@click.option(
-    "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value)
-)
-@click.option(
-    "-p",
-    "--password",
-    prompt=True,
-    hide_input=True,
-    confirmation_prompt=True,
-    callback=lambda _, __, value: validate_password(value),
-)
-@click.option("--firstname", prompt=True)
-@click.option("--lastname", prompt=True)
-def account_register(username, email, password, firstname, lastname):
-    client = AccountClient()
-    client.registration(username, email, password, firstname, lastname)
-    return click.secho(
-        "An account has been successfully created. "
-        "Please check your mail to activate your account and verify your email address.",
-        fg="green",
-    )
-
-
-@cli.command("login", short_help="Log in to PlatformIO Account")
-@click.option("-u", "--username", prompt="Username or email")
-@click.option("-p", "--password", prompt=True, hide_input=True)
-def account_login(username, password):
-    client = AccountClient()
-    client.login(username, password)
-    return click.secho("Successfully logged in!", fg="green")
-
-
-@cli.command("logout", short_help="Log out of PlatformIO Account")
-def account_logout():
-    client = AccountClient()
-    client.logout()
-    return click.secho("Successfully logged out!", fg="green")
-
-
-@cli.command("password", short_help="Change password")
-@click.option("--old-password", prompt=True, hide_input=True)
-@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
-def account_password(old_password, new_password):
-    client = AccountClient()
-    client.change_password(old_password, new_password)
-    return click.secho("Password successfully changed!", fg="green")
-
-
-@cli.command("token", short_help="Get or regenerate Authentication Token")
-@click.option("-p", "--password", prompt=True, hide_input=True)
-@click.option("--regenerate", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_token(password, regenerate, json_output):
-    client = AccountClient()
-    auth_token = client.auth_token(password, regenerate)
-    if json_output:
-        return click.echo(json.dumps({"status": "success", "result": auth_token}))
-    return click.secho("Personal Authentication Token: %s" % auth_token, fg="green")
-
-
-@cli.command("forgot", short_help="Forgot password")
-@click.option("--username", prompt="Username or email")
-def account_forgot(username):
-    client = AccountClient()
-    client.forgot_password(username)
-    return click.secho(
-        "If this account is registered, we will send the "
-        "further instructions to your email.",
-        fg="green",
-    )
-
-
-@cli.command("update", short_help="Update profile information")
-@click.option("--current-password", prompt=True, hide_input=True)
-@click.option("--username")
-@click.option("--email")
-@click.option("--firstname")
-@click.option("--lastname")
-def account_update(current_password, **kwargs):
-    client = AccountClient()
-    profile = client.get_profile()
-    new_profile = profile.copy()
-    if not any(kwargs.values()):
-        for field in profile:
-            new_profile[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=profile[field]
-            )
-            if field == "email":
-                validate_email(new_profile[field])
-            if field == "username":
-                validate_username(new_profile[field])
-    else:
-        new_profile.update({key: value for key, value in kwargs.items() if value})
-    client.update_profile(new_profile, current_password)
-    click.secho("Profile successfully updated!", fg="green")
-    username_changed = new_profile["username"] != profile["username"]
-    email_changed = new_profile["email"] != profile["email"]
-    if not username_changed and not email_changed:
-        return None
-    try:
-        client.logout()
-    except AccountNotAuthorized:
-        pass
-    if email_changed:
-        return click.secho(
-            "Please check your mail to verify your new email address and re-login. ",
-            fg="yellow",
-        )
-    return click.secho("Please re-login.", fg="yellow")
-
-
-@cli.command("destroy", short_help="Destroy account")
-def account_destroy():
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the %s user account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % client.get_logged_username(),
-        abort=True,
-    )
-    client.destroy_account()
-    try:
-        client.logout()
-    except AccountNotAuthorized:
-        pass
-    return click.secho(
-        "User account has been destroyed.",
-        fg="green",
-    )
-
-
-@cli.command("show", short_help="PlatformIO Account information")
-@click.option("--offline", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_show(offline, json_output):
-    client = AccountClient()
-    info = client.get_account_info(offline)
-    if json_output:
-        return click.echo(json.dumps(info))
-    click.echo()
-    if info.get("profile"):
-        print_profile(info["profile"])
-    if info.get("packages"):
-        print_packages(info["packages"])
-    if info.get("subscriptions"):
-        print_subscriptions(info["subscriptions"])
-    return click.echo()
-
-
-def print_profile(profile):
-    click.secho("Profile", fg="cyan", bold=True)
-    click.echo("=" * len("Profile"))
-    data = []
-    if profile.get("username"):
-        data.append(("Username:", profile["username"]))
-    if profile.get("email"):
-        data.append(("Email:", profile["email"]))
-    if profile.get("firstname"):
-        data.append(("First name:", profile["firstname"]))
-    if profile.get("lastname"):
-        data.append(("Last name:", profile["lastname"]))
-    click.echo(tabulate(data, tablefmt="plain"))
-
-
-def print_packages(packages):
-    click.echo()
-    click.secho("Packages", fg="cyan")
-    click.echo("=" * len("Packages"))
-    for package in packages:
-        click.echo()
-        click.secho(package.get("name"), bold=True)
-        click.echo("-" * len(package.get("name")))
-        if package.get("description"):
-            click.echo(package.get("description"))
-        data = []
-        expire = "-"
-        if "subscription" in package:
-            expire = datetime.datetime.strptime(
-                (
-                    package["subscription"].get("end_at")
-                    or package["subscription"].get("next_bill_at")
-                ),
-                "%Y-%m-%dT%H:%M:%SZ",
-            ).strftime("%Y-%m-%d")
-        data.append(("Expire:", expire))
-        services = []
-        for key in package:
-            if not key.startswith("service."):
-                continue
-            if isinstance(package[key], dict):
-                services.append(package[key].get("title"))
-            else:
-                services.append(package[key])
-        if services:
-            data.append(("Services:", ", ".join(services)))
-        click.echo(tabulate(data, tablefmt="plain"))
-
-
-def print_subscriptions(subscriptions):
-    click.echo()
-    click.secho("Subscriptions", fg="cyan")
-    click.echo("=" * len("Subscriptions"))
-    for subscription in subscriptions:
-        click.echo()
-        click.secho(subscription.get("product_name"), bold=True)
-        click.echo("-" * len(subscription.get("product_name")))
-        data = [("State:", subscription.get("status"))]
-        begin_at = datetime.datetime.strptime(
-            subscription.get("begin_at"), "%Y-%m-%dT%H:%M:%SZ"
-        ).strftime("%Y-%m-%d %H:%M:%S")
-        data.append(("Start date:", begin_at or "-"))
-        end_at = subscription.get("end_at")
-        if end_at:
-            end_at = datetime.datetime.strptime(
-                subscription.get("end_at"), "%Y-%m-%dT%H:%M:%SZ"
-            ).strftime("%Y-%m-%d %H:%M:%S")
-        data.append(("End date:", end_at or "-"))
-        next_bill_at = subscription.get("next_bill_at")
-        if next_bill_at:
-            next_bill_at = datetime.datetime.strptime(
-                subscription.get("next_bill_at"), "%Y-%m-%dT%H:%M:%SZ"
-            ).strftime("%Y-%m-%d %H:%M:%S")
-        data.append(("Next payment:", next_bill_at or "-"))
-        data.append(
-            ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")
-        )
-        data.append(
-            ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-")
-        )
-        click.echo(tabulate(data, tablefmt="plain"))

platformio/commands/boards.py

@@ -22,7 +22,7 @@ from platformio import fs
 from platformio.package.manager.platform import PlatformPackageManager
 
 
-@click.command("boards", short_help="Embedded board explorer")
+@click.command("boards", short_help="Board Explorer")
 @click.argument("query", required=False)
 @click.option("--installed", is_flag=True)
 @click.option("--json-output", is_flag=True)

platformio/commands/ci.py

@@ -20,11 +20,10 @@ import tempfile
 import click
 
 from platformio import app, fs
-from platformio.commands.project import project_init as cmd_project_init
-from platformio.commands.project import validate_boards
-from platformio.commands.run.command import cli as cmd_run
 from platformio.exception import CIBuildEnvsEmpty
+from platformio.project.commands.init import project_init_cmd, validate_boards
 from platformio.project.config import ProjectConfig
+from platformio.run.cli import cli as cmd_run
 
 
 def validate_path(ctx, param, value):  # pylint: disable=unused-argument
@@ -44,7 +43,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
         raise click.BadParameter("Found invalid path: %s" % invalid_path)
 
 
-@click.command("ci", short_help="Continuous integration")
+@click.command("ci", short_help="Continuous Integration")
 @click.argument("src", nargs=-1, callback=validate_path)
 @click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY")
 @click.option("--exclude", multiple=True)
@@ -109,7 +108,7 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
 
         # initialise project
         ctx.invoke(
-            cmd_project_init,
+            project_init_cmd,
             project_dir=build_dir,
             board=board,
             project_option=project_option,

platformio/commands/device/__init__.py

@@ -12,4 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio.commands.device.filters.base import DeviceMonitorFilter
+# pylint: disable=unused-import
+from platformio.device.filters.base import (
+    DeviceMonitorFilterBase as DeviceMonitorFilter,
+)

platformio/commands/device/command.py

@@ -1,236 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import os
-import sys
-from fnmatch import fnmatch
-
-import click
-from serial.tools import miniterm
-
-from platformio import exception, fs, util
-from platformio.commands.device import helpers as device_helpers
-from platformio.platform.factory import PlatformFactory
-from platformio.project.exception import NotPlatformIOProjectError
-
-
-@click.group(short_help="Device manager & serial/socket monitor")
-def cli():
-    pass
-
-
-@cli.command("list", short_help="List devices")
-@click.option("--serial", is_flag=True, help="List serial ports, default")
-@click.option("--logical", is_flag=True, help="List logical devices")
-@click.option("--mdns", is_flag=True, help="List multicast DNS services")
-@click.option("--json-output", is_flag=True)
-def device_list(  # pylint: disable=too-many-branches
-    serial, logical, mdns, json_output
-):
-    if not logical and not mdns:
-        serial = True
-    data = {}
-    if serial:
-        data["serial"] = util.get_serial_ports()
-    if logical:
-        data["logical"] = util.get_logical_devices()
-    if mdns:
-        data["mdns"] = util.get_mdns_services()
-
-    single_key = list(data)[0] if len(list(data)) == 1 else None
-
-    if json_output:
-        return click.echo(json.dumps(data[single_key] if single_key else data))
-
-    titles = {
-        "serial": "Serial Ports",
-        "logical": "Logical Devices",
-        "mdns": "Multicast DNS Services",
-    }
-
-    for key, value in data.items():
-        if not single_key:
-            click.secho(titles[key], bold=True)
-            click.echo("=" * len(titles[key]))
-
-        if key == "serial":
-            for item in value:
-                click.secho(item["port"], fg="cyan")
-                click.echo("-" * len(item["port"]))
-                click.echo("Hardware ID: %s" % item["hwid"])
-                click.echo("Description: %s" % item["description"])
-                click.echo("")
-
-        if key == "logical":
-            for item in value:
-                click.secho(item["path"], fg="cyan")
-                click.echo("-" * len(item["path"]))
-                click.echo("Name: %s" % item["name"])
-                click.echo("")
-
-        if key == "mdns":
-            for item in value:
-                click.secho(item["name"], fg="cyan")
-                click.echo("-" * len(item["name"]))
-                click.echo("Type: %s" % item["type"])
-                click.echo("IP: %s" % item["ip"])
-                click.echo("Port: %s" % item["port"])
-                if item["properties"]:
-                    click.echo(
-                        "Properties: %s"
-                        % (
-                            "; ".join(
-                                [
-                                    "%s=%s" % (k, v)
-                                    for k, v in item["properties"].items()
-                                ]
-                            )
-                        )
-                    )
-                click.echo("")
-
-        if single_key:
-            click.echo("")
-
-    return True
-
-
-@cli.command("monitor", short_help="Monitor device (Serial)")
-@click.option("--port", "-p", help="Port, a number or a device name")
-@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
-@click.option(
-    "--parity",
-    default="N",
-    type=click.Choice(["N", "E", "O", "S", "M"]),
-    help="Set parity, default=N",
-)
-@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
-@click.option(
-    "--xonxoff", is_flag=True, help="Enable software flow control, default=Off"
-)
-@click.option(
-    "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state"
-)
-@click.option(
-    "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state"
-)
-@click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
-@click.option(
-    "--encoding",
-    default="UTF-8",
-    help="Set the encoding for the serial port (e.g. hexlify, "
-    "Latin1, UTF-8), default: UTF-8",
-)
-@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations")
-@click.option(
-    "--eol",
-    default="CRLF",
-    type=click.Choice(["CR", "LF", "CRLF"]),
-    help="End of line mode, default=CRLF",
-)
-@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations")
-@click.option(
-    "--exit-char",
-    type=int,
-    default=3,
-    help="ASCII code of special character that is used to exit "
-    "the application, default=3 (Ctrl+C)",
-)
-@click.option(
-    "--menu-char",
-    type=int,
-    default=20,
-    help="ASCII code of special character that is used to "
-    "control miniterm (menu), default=20 (DEC)",
-)
-@click.option(
-    "--quiet",
-    is_flag=True,
-    help="Diagnostics: suppress non-error messages, default=Off",
-)
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
-)
-@click.option(
-    "-e",
-    "--environment",
-    help="Load configuration from `platformio.ini` and specified environment",
-)
-def device_monitor(**kwargs):  # pylint: disable=too-many-branches
-    project_options = {}
-    platform = None
-    try:
-        with fs.cd(kwargs["project_dir"]):
-            project_options = device_helpers.get_project_options(kwargs["environment"])
-            kwargs = device_helpers.apply_project_monitor_options(
-                kwargs, project_options
-            )
-            if "platform" in project_options:
-                platform = PlatformFactory.new(project_options["platform"])
-    except NotPlatformIOProjectError:
-        pass
-
-    with fs.cd(kwargs["project_dir"]):
-        device_helpers.register_filters(platform=platform, options=kwargs)
-
-    if not kwargs["port"]:
-        ports = util.get_serial_ports(filter_hwid=True)
-        if len(ports) == 1:
-            kwargs["port"] = ports[0]["port"]
-        elif "platform" in project_options and "board" in project_options:
-            board_hwids = device_helpers.get_board_hwids(
-                kwargs["project_dir"],
-                platform,
-                project_options["board"],
-            )
-            for item in ports:
-                for hwid in board_hwids:
-                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                    if hwid_str in item["hwid"]:
-                        kwargs["port"] = item["port"]
-                        break
-                if kwargs["port"]:
-                    break
-    elif kwargs["port"] and (set(["*", "?", "[", "]"]) & set(kwargs["port"])):
-        for item in util.get_serial_ports():
-            if fnmatch(item["port"], kwargs["port"]):
-                kwargs["port"] = item["port"]
-                break
-
-    # override system argv with patched options
-    sys.argv = ["monitor"] + device_helpers.options_to_argv(
-        kwargs,
-        project_options,
-        ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"),
-    )
-
-    if not kwargs["quiet"]:
-        click.echo(
-            "--- Available filters and text transformations: %s"
-            % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys()))
-        )
-        click.echo("--- More details at https://bit.ly/pio-monitor-filters")
-    try:
-        miniterm.main(
-            default_port=kwargs["port"],
-            default_baudrate=kwargs["baud"] or 9600,
-            default_rts=kwargs["rts"],
-            default_dtr=kwargs["dtr"],
-        )
-    except Exception as e:
-        raise exception.MinitermException(e)

platformio/commands/device/filters/base.py

@@ -1,43 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from serial.tools import miniterm
-
-from platformio.project.config import ProjectConfig
-
-
-class DeviceMonitorFilter(miniterm.Transform):
-    def __init__(self, options=None):
-        """Called by PlatformIO to pass context"""
-        miniterm.Transform.__init__(self)
-
-        self.options = options or {}
-        self.project_dir = self.options.get("project_dir")
-        self.environment = self.options.get("environment")
-
-        self.config = ProjectConfig.get_instance()
-        if not self.environment:
-            default_envs = self.config.default_envs()
-            if default_envs:
-                self.environment = default_envs[0]
-            elif self.config.envs():
-                self.environment = self.config.envs()[0]
-
-    def __call__(self):
-        """Called by the miniterm library when the filter is actually used"""
-        return self
-
-    @property
-    def NAME(self):
-        raise NotImplementedError("Please declare NAME attribute for the filter class")

platformio/commands/lib/command.py

@@ -15,14 +15,16 @@
 # pylint: disable=too-many-branches, too-many-locals
 
 import json
+import logging
 import os
 import time
+from urllib.parse import quote
 
 import click
 from tabulate import tabulate
 
 from platformio import exception, fs, util
-from platformio.commands import PlatformioCLI
+from platformio.cli import PlatformioCLI
 from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps
 from platformio.package.exception import NotGlobalLibDir, UnknownPackageError
 from platformio.package.manager.library import LibraryPackageManager
@@ -31,11 +33,6 @@ from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import get_project_dir, is_platformio_project
 
-try:
-    from urllib.parse import quote
-except ImportError:
-    from urllib import quote
-
 CTX_META_INPUT_DIRS_KEY = __name__ + ".input_dirs"
 CTX_META_PROJECT_ENVIRONMENTS_KEY = __name__ + ".project_environments"
 CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs"
@@ -46,7 +43,7 @@ def get_project_global_lib_dir():
     return ProjectConfig.get_instance().get("platformio", "globallib_dir")
 
 
-@click.group(short_help="Library manager")
+@click.group(short_help="Library manager", hidden=True)
 @click.option(
     "-d",
     "--storage-dir",
@@ -71,6 +68,7 @@ def get_project_global_lib_dir():
 )
 @click.pass_context
 def cli(ctx, **options):
+    in_silence = PlatformioCLI.in_silence()
     storage_cmds = ("install", "uninstall", "update", "list")
     # skip commands that don't need storage folder
     if ctx.invoked_subcommand not in storage_cmds or (
@@ -97,7 +95,6 @@ def cli(ctx, **options):
             get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand
         )
 
-    in_silence = PlatformioCLI.in_silence()
     ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options["environment"]
     ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs
     ctx.meta[CTX_META_STORAGE_DIRS_KEY] = []
@@ -144,6 +141,11 @@ def cli(ctx, **options):
 def lib_install(  # pylint: disable=too-many-arguments,unused-argument
     ctx, libraries, save, silent, interactive, force
 ):
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg install` instead.\n",
+        fg="yellow",
+    )
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, [])
 
@@ -152,16 +154,16 @@ def lib_install(  # pylint: disable=too-many-arguments,unused-argument
         if not silent and (libraries or storage_dir in storage_libdeps):
             print_storage_header(storage_dirs, storage_dir)
         lm = LibraryPackageManager(storage_dir)
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
 
         if libraries:
             installed_pkgs = {
-                library: lm.install(library, silent=silent, force=force)
-                for library in libraries
+                library: lm.install(library, force=force) for library in libraries
             }
 
         elif storage_dir in storage_libdeps:
             for library in storage_libdeps[storage_dir]:
-                lm.install(library, silent=silent, force=force)
+                lm.install(library, force=force)
 
     if save and installed_pkgs:
         _save_deps(ctx, installed_pkgs)
@@ -207,14 +209,18 @@ def _save_deps(ctx, pkgs, action="add"):
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.pass_context
 def lib_uninstall(ctx, libraries, save, silent):
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg uninstall` instead.\n",
+        fg="yellow",
+    )
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     uninstalled_pkgs = {}
     for storage_dir in storage_dirs:
         print_storage_header(storage_dirs, storage_dir)
         lm = LibraryPackageManager(storage_dir)
-        uninstalled_pkgs = {
-            library: lm.uninstall(library, silent=silent) for library in libraries
-        }
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
+        uninstalled_pkgs = {library: lm.uninstall(library) for library in libraries}
 
     if save and uninstalled_pkgs:
         _save_deps(ctx, uninstalled_pkgs, action="remove")
@@ -237,14 +243,27 @@ def lib_uninstall(ctx, libraries, save, silent):
 def lib_update(  # pylint: disable=too-many-arguments
     ctx, libraries, only_check, dry_run, silent, json_output
 ):
-    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     only_check = dry_run or only_check
+    if only_check and not json_output:
+        raise exception.UserSideException(
+            "This command is deprecated, please use `pio pkg outdated` instead"
+        )
+
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg update` instead.\n",
+            fg="yellow",
+        )
+
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     json_result = {}
     for storage_dir in storage_dirs:
         if not json_output:
             print_storage_header(storage_dirs, storage_dir)
         lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, [])
         lm = LibraryPackageManager(storage_dir)
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
         _libraries = libraries or lib_deps or lm.get_installed()
 
         if only_check and json_output:
@@ -277,9 +296,7 @@ def lib_update(  # pylint: disable=too-many-arguments
                     None if isinstance(library, PackageItem) else PackageSpec(library)
                 )
                 try:
-                    lm.update(
-                        library, to_spec=to_spec, only_check=only_check, silent=silent
-                    )
+                    lm.update(library, to_spec=to_spec)
                 except UnknownPackageError as e:
                     if library not in lib_deps:
                         raise e
@@ -298,6 +315,12 @@ def lib_update(  # pylint: disable=too-many-arguments
 @click.option("--json-output", is_flag=True)
 @click.pass_context
 def lib_list(ctx, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg list` instead.\n",
+            fg="yellow",
+        )
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     json_result = {}
     for storage_dir in storage_dirs:
@@ -341,6 +364,12 @@ def lib_list(ctx, json_output):
     help="Do not prompt, automatically paginate with delay",
 )
 def lib_search(query, json_output, page, noninteractive, **filters):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg search` instead.\n",
+            fg="yellow",
+        )
     regclient = LibraryPackageManager().get_registry_client_instance()
     if not query:
         query = []
@@ -437,8 +466,15 @@ def lib_builtin(storage, json_output):
 @click.argument("library", metavar="[LIBRARY]")
 @click.option("--json-output", is_flag=True)
 def lib_show(library, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg show` instead.\n",
+            fg="yellow",
+        )
     lm = LibraryPackageManager()
-    lib_id = lm.reveal_registry_package_id(library, silent=json_output)
+    lm.set_log_level(logging.ERROR if json_output else logging.DEBUG)
+    lib_id = lm.reveal_registry_package_id(library)
     regclient = lm.get_registry_client_instance()
     lib = regclient.fetch_json_data(
         "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h"
@@ -457,7 +493,7 @@ def lib_show(library, json_output):
         "Version: %s, released %s"
         % (
             lib["version"]["name"],
-            time.strftime("%c", util.parse_date(lib["version"]["released"])),
+            util.parse_datetime(lib["version"]["released"]).strftime("%c"),
         )
     )
     click.echo("Manifest: %s" % lib["confurl"])
@@ -465,9 +501,9 @@ def lib_show(library, json_output):
         if key not in lib or not lib[key]:
             continue
         if isinstance(lib[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(lib[key])))
+            click.echo("%s: %s" % (key.capitalize(), ", ".join(lib[key])))
         else:
-            click.echo("%s: %s" % (key.title(), lib[key]))
+            click.echo("%s: %s" % (key.capitalize(), lib[key]))
 
     blocks = []
 
@@ -499,7 +535,7 @@ def lib_show(library, json_output):
             "Versions",
             [
                 "%s, released %s"
-                % (v["name"], time.strftime("%c", util.parse_date(v["released"])))
+                % (v["name"], util.parse_datetime(v["released"]).strftime("%c"))
                 for v in lib["versions"]
             ],
         )
@@ -529,7 +565,7 @@ def lib_show(library, json_output):
 @click.argument("config_url")
 def lib_register(config_url):  # pylint: disable=unused-argument
     raise exception.UserSideException(
-        "This command is deprecated. Please use `pio package publish` command."
+        "This command is deprecated. Please use `pio pkg publish` command."
     )
 
 
@@ -546,7 +582,7 @@ def lib_stats(json_output):
         tabular_data = [
             (
                 click.style(item["name"], fg="cyan"),
-                time.strftime("%c", util.parse_date(item["date"])),
+                util.parse_datetime(item["date"]).strftime("%c"),
                 "https://platformio.org/lib/show/%s/%s"
                 % (item["id"], quote(item["name"])),
             )
@@ -621,9 +657,9 @@ def print_lib_item(item):
         if key not in item or not item[key]:
             continue
         if isinstance(item[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
+            click.echo("%s: %s" % (key.capitalize(), ", ".join(item[key])))
         else:
-            click.echo("%s: %s" % (key.title(), item[key]))
+            click.echo("%s: %s" % (key.capitalize(), item[key]))
 
     for key in ("frameworks", "platforms"):
         if key not in item:

platformio/commands/lib/helpers.py

@@ -14,6 +14,7 @@
 
 import os
 
+from platformio import util
 from platformio.compat import ci_strings_are_equal
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.meta import PackageSpec
@@ -22,6 +23,7 @@ from platformio.project.config import ProjectConfig
 from platformio.project.exception import InvalidProjectConfError
 
 
+@util.memoized(expire="60s")
 def get_builtin_libs(storage_names=None):
     # pylint: disable=import-outside-toplevel
     from platformio.package.manager.library import LibraryPackageManager
@@ -45,8 +47,8 @@ def get_builtin_libs(storage_names=None):
     return items
 
 
-def is_builtin_lib(name, storages=None):
-    for storage in storages or get_builtin_libs():
+def is_builtin_lib(name):
+    for storage in get_builtin_libs():
         for lib in storage["items"]:
             if lib.get("name") == name:
                 return True

platformio/commands/org.py

@@ -1,165 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import json
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.account import AccountClient
-from platformio.commands.account import validate_email, validate_username
-
-
-@click.group("org", short_help="Manage organizations")
-def cli():
-    pass
-
-
-def validate_orgname(value):
-    return validate_username(value, "Organization name")
-
-
-@cli.command("create", short_help="Create a new organization")
-@click.argument(
-    "orgname",
-    callback=lambda _, __, value: validate_orgname(value),
-)
-@click.option(
-    "--email", callback=lambda _, __, value: validate_email(value) if value else value
-)
-@click.option(
-    "--displayname",
-)
-def org_create(orgname, email, displayname):
-    client = AccountClient()
-    client.create_org(orgname, email, displayname)
-    return click.secho(
-        "The organization `%s` has been successfully created." % orgname,
-        fg="green",
-    )
-
-
-@cli.command("list", short_help="List organizations and their members")
-@click.option("--json-output", is_flag=True)
-def org_list(json_output):
-    client = AccountClient()
-    orgs = client.list_orgs()
-    if json_output:
-        return click.echo(json.dumps(orgs))
-    if not orgs:
-        return click.echo("You do not have any organization")
-    for org in orgs:
-        click.echo()
-        click.secho(org.get("orgname"), fg="cyan")
-        click.echo("-" * len(org.get("orgname")))
-        data = []
-        if org.get("displayname"):
-            data.append(("Display Name:", org.get("displayname")))
-        if org.get("email"):
-            data.append(("Email:", org.get("email")))
-        data.append(
-            (
-                "Owners:",
-                ", ".join((owner.get("username") for owner in org.get("owners"))),
-            )
-        )
-        click.echo(tabulate(data, tablefmt="plain"))
-    return click.echo()
-
-
-@cli.command("update", short_help="Update organization")
-@click.argument("cur_orgname")
-@click.option(
-    "--orgname",
-    callback=lambda _, __, value: validate_orgname(value),
-    help="A new orgname",
-)
-@click.option("--email")
-@click.option("--displayname")
-def org_update(cur_orgname, **kwargs):
-    client = AccountClient()
-    org = client.get_org(cur_orgname)
-    del org["owners"]
-    new_org = org.copy()
-    if not any(kwargs.values()):
-        for field in org:
-            new_org[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=org[field]
-            )
-            if field == "email":
-                validate_email(new_org[field])
-            if field == "orgname":
-                validate_orgname(new_org[field])
-    else:
-        new_org.update(
-            {key.replace("new_", ""): value for key, value in kwargs.items() if value}
-        )
-    client.update_org(cur_orgname, new_org)
-    return click.secho(
-        "The organization `%s` has been successfully updated." % cur_orgname,
-        fg="green",
-    )
-
-
-@cli.command("destroy", short_help="Destroy organization")
-@click.argument("orgname")
-def account_destroy(orgname):
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the `%s` organization account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % orgname,
-        abort=True,
-    )
-    client.destroy_org(orgname)
-    return click.secho(
-        "Organization `%s` has been destroyed." % orgname,
-        fg="green",
-    )
-
-
-@cli.command("add", short_help="Add a new owner to organization")
-@click.argument(
-    "orgname",
-)
-@click.argument(
-    "username",
-)
-def org_add_owner(orgname, username):
-    client = AccountClient()
-    client.add_org_owner(orgname, username)
-    return click.secho(
-        "The new owner `%s` has been successfully added to the `%s` organization."
-        % (username, orgname),
-        fg="green",
-    )
-
-
-@cli.command("remove", short_help="Remove an owner from organization")
-@click.argument(
-    "orgname",
-)
-@click.argument(
-    "username",
-)
-def org_remove_owner(orgname, username):
-    client = AccountClient()
-    client.remove_org_owner(orgname, username)
-    return click.secho(
-        "The `%s` owner has been successfully removed from the `%s` organization."
-        % (username, orgname),
-        fg="green",
-    )

platformio/commands/platform.py

@@ -13,12 +13,14 @@
 # limitations under the License.
 
 import json
+import logging
 import os
 
 import click
 
-from platformio.cache import cleanup_content_cache
 from platformio.commands.boards import print_boards
+from platformio.exception import UserSideException
+from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.meta import PackageItem, PackageSpec
 from platformio.package.version import get_original_version
@@ -26,7 +28,7 @@ from platformio.platform.exception import UnknownPlatform
 from platformio.platform.factory import PlatformFactory
 
 
-@click.group(short_help="Platform manager")
+@click.group(short_help="Platform manager", hidden=True)
 def cli():
     pass
 
@@ -35,6 +37,12 @@ def cli():
 @click.argument("query", required=False)
 @click.option("--json-output", is_flag=True)
 def platform_search(query, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg search` instead.\n",
+            fg="yellow",
+        )
     platforms = []
     for platform in _get_registry_platforms():
         if query == "all":
@@ -86,6 +94,12 @@ def platform_frameworks(query, json_output):
 @cli.command("list", short_help="List installed development platforms")
 @click.option("--json-output", is_flag=True)
 def platform_list(json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg list` instead.\n",
+            fg="yellow",
+        )
     platforms = []
     pm = PlatformPackageManager()
     for pkg in pm.get_installed():
@@ -104,6 +118,12 @@ def platform_list(json_output):
 @click.argument("platform")
 @click.option("--json-output", is_flag=True)
 def platform_show(platform, json_output):  # pylint: disable=too-many-branches
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg show` instead.\n",
+            fg="yellow",
+        )
     data = _get_platform_data(platform)
     if not data:
         raise UnknownPlatform(platform)
@@ -178,7 +198,7 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     is_flag=True,
     help="Reinstall/redownload dev/platform and its packages if exist",
 )
-def platform_install(  # pylint: disable=too-many-arguments
+def platform_install(  # pylint: disable=too-many-arguments,too-many-locals
     platforms,
     with_package,
     without_package,
@@ -187,37 +207,56 @@ def platform_install(  # pylint: disable=too-many-arguments
     silent,
     force,
 ):
-    return _platform_install(
-        platforms,
-        with_package,
-        without_package,
-        skip_default_package,
-        with_all_packages,
-        silent,
-        force,
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg install` instead.\n",
+        fg="yellow",
     )
 
+    def _find_pkg_names(p, candidates):
+        result = []
+        for candidate in candidates:
+            found = False
+            # lookup by package types
+            for _name, _opts in p.packages.items():
+                if _opts.get("type") == candidate:
+                    result.append(_name)
+                    found = True
+            if (
+                p.frameworks
+                and candidate.startswith("framework-")
+                and candidate[10:] in p.frameworks
+            ):
+                result.append(p.frameworks[candidate[10:]]["package"])
+                found = True
+            if not found:
+                result.append(candidate)
+        return result
 
-def _platform_install(  # pylint: disable=too-many-arguments
-    platforms,
-    with_package=None,
-    without_package=None,
-    skip_default_package=False,
-    with_all_packages=False,
-    silent=False,
-    force=False,
-):
     pm = PlatformPackageManager()
+    pm.set_log_level(logging.WARN if silent else logging.DEBUG)
     for platform in platforms:
-        pkg = pm.install(
-            spec=platform,
-            with_packages=with_package or [],
-            without_packages=without_package or [],
-            skip_default_package=skip_default_package,
-            with_all_packages=with_all_packages,
-            silent=silent,
-            force=force,
-        )
+        if with_package or without_package or with_all_packages:
+            pkg = pm.install(platform, skip_dependencies=True)
+            p = PlatformFactory.new(pkg)
+            if with_all_packages:
+                with_package = list(p.packages)
+            with_package = set(_find_pkg_names(p, with_package or []))
+            without_package = set(_find_pkg_names(p, without_package or []))
+            upkgs = with_package | without_package
+            ppkgs = set(p.packages)
+            if not upkgs.issubset(ppkgs):
+                raise UnknownPackageError(", ".join(upkgs - ppkgs))
+            for name, options in p.packages.items():
+                if name in without_package:
+                    continue
+                if name in with_package or not (
+                    skip_default_package or options.get("optional", False)
+                ):
+                    p.pm.install(p.get_package_spec(name), force=force)
+        else:
+            pkg = pm.install(platform, skip_dependencies=skip_default_package)
+
         if pkg and not silent:
             click.secho(
                 "The platform '%s' has been successfully installed!\n"
@@ -230,7 +269,13 @@ def _platform_install(  # pylint: disable=too-many-arguments
 @cli.command("uninstall", short_help="Uninstall development platform")
 @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]")
 def platform_uninstall(platforms):
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg uninstall` instead.\n",
+        fg="yellow",
+    )
     pm = PlatformPackageManager()
+    pm.set_log_level(logging.DEBUG)
     for platform in platforms:
         if pm.uninstall(platform):
             click.secho(
@@ -256,9 +301,22 @@ def platform_uninstall(platforms):
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option("--json-output", is_flag=True)
 def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
-    platforms, only_packages, only_check, dry_run, silent, json_output
+    platforms, only_check, dry_run, silent, json_output, **_
 ):
+    if only_check and not json_output:
+        raise UserSideException(
+            "This command is deprecated, please use `pio pkg outdated` instead"
+        )
+
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg update` instead.\n",
+            fg="yellow",
+        )
+
     pm = PlatformPackageManager()
+    pm.set_log_level(logging.WARN if silent else logging.DEBUG)
     platforms = platforms or pm.get_installed()
     only_check = dry_run or only_check
 
@@ -290,9 +348,6 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
             result.append(data)
         return click.echo(json.dumps(result))
 
-    # cleanup cached board and platform lists
-    cleanup_content_cache("http")
-
     for platform in platforms:
         click.echo(
             "Platform %s"
@@ -304,9 +359,7 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
             )
         )
         click.echo("--------")
-        pm.update(
-            platform, only_packages=only_packages, only_check=only_check, silent=silent
-        )
+        pm.update(platform)
         click.echo()
 
     return True
@@ -317,15 +370,6 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
 #
 
 
-def init_platform(name, skip_default_package=True, auto_install=True):
-    try:
-        return PlatformFactory.new(name)
-    except UnknownPlatform:
-        if auto_install:
-            _platform_install([name], skip_default_package=skip_default_package)
-        return PlatformFactory.new(name)
-
-
 def _print_platforms(platforms):
     for platform in platforms:
         click.echo(

platformio/commands/system/command.py

@@ -1,191 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import platform
-import sys
-
-import click
-from tabulate import tabulate
-
-from platformio import __version__, compat, fs, proc, util
-from platformio.commands.system.completion import (
-    ShellType,
-    get_completion_install_path,
-    install_completion_code,
-    uninstall_completion_code,
-)
-from platformio.commands.system.prune import (
-    prune_cached_data,
-    prune_core_packages,
-    prune_platform_packages,
-)
-from platformio.package.manager.library import LibraryPackageManager
-from platformio.package.manager.platform import PlatformPackageManager
-from platformio.package.manager.tool import ToolPackageManager
-from platformio.project.config import ProjectConfig
-
-
-@click.group("system", short_help="Miscellaneous system commands")
-def cli():
-    pass
-
-
-@cli.command("info", short_help="Display system-wide information")
-@click.option("--json-output", is_flag=True)
-def system_info(json_output):
-    project_config = ProjectConfig()
-    data = {}
-    data["core_version"] = {"title": "PlatformIO Core", "value": __version__}
-    data["python_version"] = {
-        "title": "Python",
-        "value": "{0}.{1}.{2}-{3}.{4}".format(*list(sys.version_info)),
-    }
-    data["system"] = {"title": "System Type", "value": util.get_systype()}
-    data["platform"] = {"title": "Platform", "value": platform.platform(terse=True)}
-    data["filesystem_encoding"] = {
-        "title": "File System Encoding",
-        "value": compat.get_filesystem_encoding(),
-    }
-    data["locale_encoding"] = {
-        "title": "Locale Encoding",
-        "value": compat.get_locale_encoding(),
-    }
-    data["core_dir"] = {
-        "title": "PlatformIO Core Directory",
-        "value": project_config.get("platformio", "core_dir"),
-    }
-    data["platformio_exe"] = {
-        "title": "PlatformIO Core Executable",
-        "value": proc.where_is_program(
-            "platformio.exe" if compat.IS_WINDOWS else "platformio"
-        ),
-    }
-    data["python_exe"] = {
-        "title": "Python Executable",
-        "value": proc.get_pythonexe_path(),
-    }
-    data["global_lib_nums"] = {
-        "title": "Global Libraries",
-        "value": len(LibraryPackageManager().get_installed()),
-    }
-    data["dev_platform_nums"] = {
-        "title": "Development Platforms",
-        "value": len(PlatformPackageManager().get_installed()),
-    }
-    data["package_tool_nums"] = {
-        "title": "Tools & Toolchains",
-        "value": len(
-            ToolPackageManager(
-                project_config.get("platformio", "packages_dir")
-            ).get_installed()
-        ),
-    }
-
-    click.echo(
-        json.dumps(data)
-        if json_output
-        else tabulate([(item["title"], item["value"]) for item in data.values()])
-    )
-
-
-@cli.command("prune", short_help="Remove unused data")
-@click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation")
-@click.option(
-    "--dry-run", is_flag=True, help="Do not prune, only show data that will be removed"
-)
-@click.option("--cache", is_flag=True, help="Prune only cached data")
-@click.option(
-    "--core-packages", is_flag=True, help="Prune only unnecessary core packages"
-)
-@click.option(
-    "--platform-packages",
-    is_flag=True,
-    help="Prune only unnecessary development platform packages",
-)
-def system_prune(force, dry_run, cache, core_packages, platform_packages):
-    if dry_run:
-        click.secho(
-            "Dry run mode (do not prune, only show data that will be removed)",
-            fg="yellow",
-        )
-        click.echo()
-
-    reclaimed_cache = 0
-    reclaimed_core_packages = 0
-    reclaimed_platform_packages = 0
-    prune_all = not any([cache, core_packages, platform_packages])
-
-    if cache or prune_all:
-        reclaimed_cache = prune_cached_data(force, dry_run)
-        click.echo()
-
-    if core_packages or prune_all:
-        reclaimed_core_packages = prune_core_packages(force, dry_run)
-        click.echo()
-
-    if platform_packages or prune_all:
-        reclaimed_platform_packages = prune_platform_packages(force, dry_run)
-        click.echo()
-
-    click.secho(
-        "Total reclaimed space: %s"
-        % fs.humanize_file_size(
-            reclaimed_cache + reclaimed_core_packages + reclaimed_platform_packages
-        ),
-        fg="green",
-    )
-
-
-@cli.group("completion", short_help="Shell completion support")
-def completion():
-    pass
-
-
-@completion.command("install", short_help="Install shell completion files/code")
-@click.argument("shell", type=click.Choice([t.value for t in ShellType]))
-@click.option(
-    "--path",
-    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
-    help="Custom installation path of the code to be evaluated by the shell. "
-    "The standard installation path is used by default.",
-)
-def completion_install(shell, path):
-    shell = ShellType(shell)
-    path = path or get_completion_install_path(shell)
-    install_completion_code(shell, path)
-    click.echo(
-        "PlatformIO CLI completion has been installed for %s shell to %s \n"
-        "Please restart a current shell session."
-        % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue"))
-    )
-
-
-@completion.command("uninstall", short_help="Uninstall shell completion files/code")
-@click.argument("shell", type=click.Choice([t.value for t in ShellType]))
-@click.option(
-    "--path",
-    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
-    help="Custom installation path of the code to be evaluated by the shell. "
-    "The standard installation path is used by default.",
-)
-def completion_uninstall(shell, path):
-    shell = ShellType(shell)
-    path = path or get_completion_install_path(shell)
-    uninstall_completion_code(shell, path)
-    click.echo(
-        "PlatformIO CLI completion has been uninstalled for %s shell from %s \n"
-        "Please restart a current shell session."
-        % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue"))
-    )

platformio/commands/team.py

@@ -1,212 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import json
-import re
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.account import AccountClient
-
-
-def validate_orgname_teamname(value, teamname_validate=False):
-    if ":" not in value:
-        raise click.BadParameter(
-            "Please specify organization and team name in the next"
-            " format - orgname:teamname. For example, mycompany:DreamTeam"
-        )
-    teamname = str(value.strip().split(":", 1)[1])
-    if teamname_validate:
-        validate_teamname(teamname)
-    return value
-
-
-def validate_teamname(value):
-    if not value:
-        return value
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I):
-        raise click.BadParameter(
-            "Invalid team name format. "
-            "Team name must only contain alphanumeric characters, "
-            "single hyphens, underscores, spaces. It can not "
-            "begin or end with a hyphen or a underscore and must"
-            " not be longer than 20 characters."
-        )
-    return value
-
-
-@click.group("team", short_help="Manage organization teams")
-def cli():
-    pass
-
-
-@cli.command("create", short_help="Create a new team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(
-        value, teamname_validate=True
-    ),
-)
-@click.option(
-    "--description",
-)
-def team_create(orgname_teamname, description):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.create_team(orgname, teamname, description)
-    return click.secho(
-        "The team %s has been successfully created." % teamname,
-        fg="green",
-    )
-
-
-@cli.command("list", short_help="List teams")
-@click.argument("orgname", required=False)
-@click.option("--json-output", is_flag=True)
-def team_list(orgname, json_output):
-    client = AccountClient()
-    data = {}
-    if not orgname:
-        for item in client.list_orgs():
-            teams = client.list_teams(item.get("orgname"))
-            data[item.get("orgname")] = teams
-    else:
-        teams = client.list_teams(orgname)
-        data[orgname] = teams
-    if json_output:
-        return click.echo(json.dumps(data[orgname] if orgname else data))
-    if not any(data.values()):
-        return click.secho("You do not have any teams.", fg="yellow")
-    for org_name in data:
-        for team in data[org_name]:
-            click.echo()
-            click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan")
-            click.echo("-" * len("%s:%s" % (org_name, team.get("name"))))
-            table_data = []
-            if team.get("description"):
-                table_data.append(("Description:", team.get("description")))
-            table_data.append(
-                (
-                    "Members:",
-                    ", ".join(
-                        (member.get("username") for member in team.get("members"))
-                    )
-                    if team.get("members")
-                    else "-",
-                )
-            )
-            click.echo(tabulate(table_data, tablefmt="plain"))
-    return click.echo()
-
-
-@cli.command("update", short_help="Update team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.option(
-    "--name",
-    callback=lambda _, __, value: validate_teamname(value),
-    help="A new team name",
-)
-@click.option(
-    "--description",
-)
-def team_update(orgname_teamname, **kwargs):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    team = client.get_team(orgname, teamname)
-    del team["id"]
-    del team["members"]
-    new_team = team.copy()
-    if not any(kwargs.values()):
-        for field in team:
-            new_team[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=team[field]
-            )
-            if field == "name":
-                validate_teamname(new_team[field])
-    else:
-        new_team.update({key: value for key, value in kwargs.items() if value})
-    client.update_team(orgname, teamname, new_team)
-    return click.secho(
-        "The team %s has been successfully updated." % teamname,
-        fg="green",
-    )
-
-
-@cli.command("destroy", short_help="Destroy a team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-def team_destroy(orgname_teamname):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    click.confirm(
-        click.style(
-            "Are you sure you want to destroy the %s team?" % teamname, fg="yellow"
-        ),
-        abort=True,
-    )
-    client = AccountClient()
-    client.destroy_team(orgname, teamname)
-    return click.secho(
-        "The team %s has been successfully destroyed." % teamname,
-        fg="green",
-    )
-
-
-@cli.command("add", short_help="Add a new member to team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.argument(
-    "username",
-)
-def team_add_member(orgname_teamname, username):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.add_team_member(orgname, teamname, username)
-    return click.secho(
-        "The new member %s has been successfully added to the %s team."
-        % (username, teamname),
-        fg="green",
-    )
-
-
-@cli.command("remove", short_help="Remove a member from team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.argument("username")
-def team_remove_owner(orgname_teamname, username):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.remove_team_member(orgname, teamname, username)
-    return click.secho(
-        "The %s member has been successfully removed from the %s team."
-        % (username, teamname),
-        fg="green",
-    )

platformio/commands/test/command.py

@@ -1,266 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
-
-import fnmatch
-import os
-import shutil
-from time import time
-
-import click
-from tabulate import tabulate
-
-from platformio import app, exception, fs, util
-from platformio.commands.platform import init_platform
-from platformio.commands.test.embedded import EmbeddedTestProcessor
-from platformio.commands.test.helpers import get_test_names
-from platformio.commands.test.native import NativeTestProcessor
-from platformio.project.config import ProjectConfig
-
-
-@click.command("test", short_help="Unit testing")
-@click.option("--environment", "-e", multiple=True, metavar="<environment>")
-@click.option(
-    "--filter",
-    "-f",
-    multiple=True,
-    metavar="<pattern>",
-    help="Filter tests by a pattern",
-)
-@click.option(
-    "--ignore",
-    "-i",
-    multiple=True,
-    metavar="<pattern>",
-    help="Ignore tests by a pattern",
-)
-@click.option("--upload-port")
-@click.option("--test-port")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
-)
-@click.option(
-    "-c",
-    "--project-conf",
-    type=click.Path(
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
-)
-@click.option("--without-building", is_flag=True)
-@click.option("--without-uploading", is_flag=True)
-@click.option("--without-testing", is_flag=True)
-@click.option("--no-reset", is_flag=True)
-@click.option(
-    "--monitor-rts",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial RTS line state for Serial Monitor",
-)
-@click.option(
-    "--monitor-dtr",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial DTR line state for Serial Monitor",
-)
-@click.option("--verbose", "-v", is_flag=True)
-@click.pass_context
-def cli(  # pylint: disable=redefined-builtin
-    ctx,
-    environment,
-    ignore,
-    filter,
-    upload_port,
-    test_port,
-    project_dir,
-    project_conf,
-    without_building,
-    without_uploading,
-    without_testing,
-    no_reset,
-    monitor_rts,
-    monitor_dtr,
-    verbose,
-):
-    app.set_session_var("custom_project_conf", project_conf)
-
-    with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(project_conf)
-        config.validate(envs=environment)
-        test_names = get_test_names(config)
-
-        if not verbose:
-            click.echo("Verbose mode can be enabled via `-v, --verbose` option")
-        click.secho("Collected %d items" % len(test_names), bold=True)
-
-        results = []
-        default_envs = config.default_envs()
-        for testname in test_names:
-
-            for envname in config.envs():
-                section = "env:%s" % envname
-
-                # filter and ignore patterns
-                patterns = dict(filter=list(filter), ignore=list(ignore))
-                for key in patterns:
-                    patterns[key].extend(config.get(section, "test_%s" % key, []))
-
-                skip_conditions = [
-                    environment and envname not in environment,
-                    not environment and default_envs and envname not in default_envs,
-                    testname != "*"
-                    and patterns["filter"]
-                    and not any(
-                        fnmatch.fnmatch(testname, p) for p in patterns["filter"]
-                    ),
-                    testname != "*"
-                    and any(fnmatch.fnmatch(testname, p) for p in patterns["ignore"]),
-                ]
-                if any(skip_conditions):
-                    results.append({"env": envname, "test": testname})
-                    continue
-
-                click.echo()
-                print_processing_header(testname, envname)
-
-                cls = (
-                    EmbeddedTestProcessor
-                    if config.get(section, "platform")
-                    and init_platform(config.get(section, "platform")).is_embedded()
-                    else NativeTestProcessor
-                )
-                tp = cls(
-                    ctx,
-                    testname,
-                    envname,
-                    dict(
-                        project_config=config,
-                        project_dir=project_dir,
-                        upload_port=upload_port,
-                        test_port=test_port,
-                        without_building=without_building,
-                        without_uploading=without_uploading,
-                        without_testing=without_testing,
-                        no_reset=no_reset,
-                        monitor_rts=monitor_rts,
-                        monitor_dtr=monitor_dtr,
-                        verbose=verbose,
-                        silent=not verbose,
-                    ),
-                )
-                result = {
-                    "env": envname,
-                    "test": testname,
-                    "duration": time(),
-                    "succeeded": tp.process(),
-                }
-                result["duration"] = time() - result["duration"]
-                results.append(result)
-
-                print_processing_footer(result)
-
-    if without_testing:
-        return
-
-    print_testing_summary(results, verbose)
-
-    command_failed = any(r.get("succeeded") is False for r in results)
-    if command_failed:
-        raise exception.ReturnErrorCode(1)
-
-
-def print_processing_header(test, env):
-    click.echo(
-        "Processing %s in %s environment"
-        % (
-            click.style(test, fg="yellow", bold=True),
-            click.style(env, fg="cyan", bold=True),
-        )
-    )
-    terminal_width, _ = shutil.get_terminal_size()
-    click.secho("-" * terminal_width, bold=True)
-
-
-def print_processing_footer(result):
-    is_failed = not result.get("succeeded")
-    util.print_labeled_bar(
-        "[%s] Took %.2f seconds"
-        % (
-            (
-                click.style("FAILED", fg="red", bold=True)
-                if is_failed
-                else click.style("PASSED", fg="green", bold=True)
-            ),
-            result["duration"],
-        ),
-        is_error=is_failed,
-    )
-
-
-def print_testing_summary(results, verbose=False):
-    click.echo()
-
-    tabular_data = []
-    succeeded_nums = 0
-    failed_nums = 0
-    duration = 0
-
-    for result in results:
-        duration += result.get("duration", 0)
-        if result.get("succeeded") is False:
-            failed_nums += 1
-            status_str = click.style("FAILED", fg="red")
-        elif result.get("succeeded") is None:
-            if not verbose:
-                continue
-            status_str = "IGNORED"
-        else:
-            succeeded_nums += 1
-            status_str = click.style("PASSED", fg="green")
-
-        tabular_data.append(
-            (
-                result["test"],
-                click.style(result["env"], fg="cyan"),
-                status_str,
-                util.humanize_duration_time(result.get("duration")),
-            )
-        )
-
-    click.echo(
-        tabulate(
-            tabular_data,
-            headers=[
-                click.style(s, bold=True)
-                for s in ("Test", "Environment", "Status", "Duration")
-            ],
-        ),
-        err=failed_nums,
-    )
-
-    util.print_labeled_bar(
-        "%s%d succeeded in %s"
-        % (
-            "%d failed, " % failed_nums if failed_nums else "",
-            succeeded_nums,
-            util.humanize_duration_time(duration),
-        ),
-        is_error=failed_nums,
-        fg="red" if failed_nums else "green",
-    )

platformio/commands/test/embedded.py

@@ -1,150 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from time import sleep
-
-import click
-import serial
-
-from platformio import exception, util
-from platformio.commands.test.processor import TestProcessorBase
-from platformio.platform.factory import PlatformFactory
-
-
-class EmbeddedTestProcessor(TestProcessorBase):
-
-    SERIAL_TIMEOUT = 600
-
-    def process(self):
-        if not self.options["without_building"]:
-            self.print_progress("Building...")
-            target = ["__test"]
-            if self.options["without_uploading"]:
-                target.append("checkprogsize")
-            if not self.build_or_upload(target):
-                return False
-
-        if not self.options["without_uploading"]:
-            self.print_progress("Uploading...")
-            target = ["upload"]
-            if self.options["without_building"]:
-                target.append("nobuild")
-            else:
-                target.append("__test")
-            if not self.build_or_upload(target):
-                return False
-
-        if self.options["without_testing"]:
-            return True
-
-        self.print_progress("Testing...")
-        return self.run()
-
-    def run(self):
-        click.echo(
-            "If you don't see any output for the first 10 secs, "
-            "please reset board (press reset button)"
-        )
-        click.echo()
-
-        try:
-            ser = serial.Serial(
-                baudrate=self.get_baudrate(), timeout=self.SERIAL_TIMEOUT
-            )
-            ser.port = self.get_test_port()
-            ser.rts = self.options["monitor_rts"]
-            ser.dtr = self.options["monitor_dtr"]
-            ser.open()
-        except serial.SerialException as e:
-            click.secho(str(e), fg="red", err=True)
-            return False
-
-        if not self.options["no_reset"]:
-            ser.flushInput()
-            ser.setDTR(False)
-            ser.setRTS(False)
-            sleep(0.1)
-            ser.setDTR(True)
-            ser.setRTS(True)
-            sleep(0.1)
-
-        while True:
-            line = ser.readline().strip()
-
-            # fix non-ascii output from device
-            for i, c in enumerate(line[::-1]):
-                if not isinstance(c, int):
-                    c = ord(c)
-                if c > 127:
-                    line = line[-i:]
-                    break
-
-            if not line:
-                continue
-            if isinstance(line, bytes):
-                line = line.decode("utf8", "ignore")
-            self.on_run_out(line)
-            if all(l in line for l in ("Tests", "Failures", "Ignored")):
-                break
-        ser.close()
-        return not self._run_failed
-
-    def get_test_port(self):
-        # if test port is specified manually or in config
-        if self.options.get("test_port"):
-            return self.options.get("test_port")
-        if self.env_options.get("test_port"):
-            return self.env_options.get("test_port")
-
-        assert set(["platform", "board"]) & set(self.env_options.keys())
-        p = PlatformFactory.new(self.env_options["platform"])
-        board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", [])
-        port = None
-        elapsed = 0
-        while elapsed < 5 and not port:
-            for item in util.get_serialports():
-                port = item["port"]
-                for hwid in board_hwids:
-                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                    if hwid_str in item["hwid"] and self.is_serial_port_ready(port):
-                        return port
-
-            if port and not self.is_serial_port_ready(port):
-                port = None
-
-            if not port:
-                sleep(0.25)
-                elapsed += 0.25
-
-        if not port:
-            raise exception.PlatformioException(
-                "Please specify `test_port` for environment or use "
-                "global `--test-port` option."
-            )
-        return port
-
-    @staticmethod
-    def is_serial_port_ready(port, timeout=3):
-        if not port:
-            return False
-        elapsed = 0
-        while elapsed < timeout:
-            try:
-                serial.Serial(port, timeout=1).close()
-                return True
-            except:  # pylint: disable=bare-except
-                pass
-            sleep(1)
-            elapsed += 1
-        return False

platformio/commands/test/native.py

@@ -1,41 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from os.path import join
-
-from platformio import proc
-from platformio.commands.test.processor import TestProcessorBase
-from platformio.proc import LineBufferedAsyncPipe
-
-
-class NativeTestProcessor(TestProcessorBase):
-    def process(self):
-        if not self.options["without_building"]:
-            self.print_progress("Building...")
-            if not self.build_or_upload(["__test"]):
-                return False
-        if self.options["without_testing"]:
-            return None
-        self.print_progress("Testing...")
-        return self.run()
-
-    def run(self):
-        build_dir = self.options["project_config"].get("platformio", "build_dir")
-        result = proc.exec_command(
-            [join(build_dir, self.env_name, "program")],
-            stdout=LineBufferedAsyncPipe(self.on_run_out),
-            stderr=LineBufferedAsyncPipe(self.on_run_out),
-        )
-        assert "returncode" in result
-        return result["returncode"] == 0 and not self._run_failed

platformio/commands/test/processor.py

@@ -1,230 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import atexit
-from os import listdir, remove
-from os.path import isdir, isfile, join
-from string import Template
-
-import click
-
-from platformio import exception
-
-TRANSPORT_OPTIONS = {
-    "arduino": {
-        "include": "#include <Arduino.h>",
-        "object": "",
-        "putchar": "Serial.write(c);",
-        "flush": "Serial.flush();",
-        "begin": "Serial.begin($baudrate);",
-        "end": "Serial.end();",
-        "language": "cpp",
-    },
-    "mbed": {
-        "include": "#include <mbed.h>",
-        "object": (
-            "#if MBED_MAJOR_VERSION == 6\nUnbufferedSerial pc(USBTX, USBRX);\n"
-            "#else\nRawSerial pc(USBTX, USBRX);\n#endif"
-        ),
-        "putchar": (
-            "#if MBED_MAJOR_VERSION == 6\npc.write(&c, 1);\n"
-            "#else\npc.putc(c);\n#endif"
-        ),
-        "flush": "",
-        "begin": "pc.baud($baudrate);",
-        "end": "",
-        "language": "cpp",
-    },
-    "espidf": {
-        "include": "#include <stdio.h>",
-        "object": "",
-        "putchar": "putchar(c);",
-        "flush": "fflush(stdout);",
-        "begin": "",
-        "end": "",
-    },
-    "zephyr": {
-        "include": "#include <sys/printk.h>",
-        "object": "",
-        "putchar": 'printk("%c", c);',
-        "flush": "",
-        "begin": "",
-        "end": "",
-    },
-    "native": {
-        "include": "#include <stdio.h>",
-        "object": "",
-        "putchar": "putchar(c);",
-        "flush": "fflush(stdout);",
-        "begin": "",
-        "end": "",
-    },
-    "custom": {
-        "include": '#include "unittest_transport.h"',
-        "object": "",
-        "putchar": "unittest_uart_putchar(c);",
-        "flush": "unittest_uart_flush();",
-        "begin": "unittest_uart_begin();",
-        "end": "unittest_uart_end();",
-        "language": "cpp",
-    },
-}
-
-CTX_META_TEST_IS_RUNNING = __name__ + ".test_running"
-CTX_META_TEST_RUNNING_NAME = __name__ + ".test_running_name"
-
-
-class TestProcessorBase(object):
-
-    DEFAULT_BAUDRATE = 115200
-
-    def __init__(self, cmd_ctx, testname, envname, options):
-        self.cmd_ctx = cmd_ctx
-        self.cmd_ctx.meta[CTX_META_TEST_IS_RUNNING] = True
-        self.test_name = testname
-        self.options = options
-        self.env_name = envname
-        self.env_options = options["project_config"].items(env=envname, as_dict=True)
-        self._run_failed = False
-        self._output_file_generated = False
-
-    def get_transport(self):
-        transport = None
-        if self.env_options.get("platform") == "native":
-            transport = "native"
-        elif "framework" in self.env_options:
-            transport = self.env_options.get("framework")[0]
-        if "test_transport" in self.env_options:
-            transport = self.env_options["test_transport"]
-        if transport not in TRANSPORT_OPTIONS:
-            raise exception.PlatformioException(
-                "Unknown Unit Test transport `%s`. Please check a documentation how "
-                "to create an own 'Test Transport':\n"
-                "- https://docs.platformio.org/page/plus/unit-testing.html" % transport
-            )
-        return transport.lower()
-
-    def get_baudrate(self):
-        return int(self.env_options.get("test_speed", self.DEFAULT_BAUDRATE))
-
-    def print_progress(self, text):
-        click.secho(text, bold=self.options.get("verbose"))
-
-    def build_or_upload(self, target):
-        if not self._output_file_generated:
-            self.generate_output_file(
-                self.options["project_config"].get("platformio", "test_dir")
-            )
-            self._output_file_generated = True
-
-        if self.test_name != "*":
-            self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name
-
-        try:
-            # pylint: disable=import-outside-toplevel
-            from platformio.commands.run.command import cli as cmd_run
-
-            return self.cmd_ctx.invoke(
-                cmd_run,
-                project_dir=self.options["project_dir"],
-                project_conf=self.options["project_config"].path,
-                upload_port=self.options.get("upload_port"),
-                verbose=self.options["verbose"],
-                silent=self.options.get("silent"),
-                environment=[self.env_name],
-                disable_auto_clean="nobuild" in target,
-                target=target,
-            )
-        except exception.ReturnErrorCode:
-            return False
-
-    def process(self):
-        raise NotImplementedError
-
-    def run(self):
-        raise NotImplementedError
-
-    def on_run_out(self, line):
-        line = line.strip()
-        if line.endswith(":PASS"):
-            click.echo("%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green")))
-        elif ":FAIL" in line:
-            self._run_failed = True
-            click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red")))
-        else:
-            click.echo(line)
-
-    def generate_output_file(self, test_dir):
-        assert isdir(test_dir)
-
-        file_tpl = "\n".join(
-            [
-                "$include",
-                "#include <output_export.h>",
-                "",
-                "$object",
-                "",
-                "#ifdef __GNUC__",
-                "void output_start(unsigned int baudrate __attribute__((unused)))",
-                "#else",
-                "void output_start(unsigned int baudrate)",
-                "#endif",
-                "{",
-                "    $begin",
-                "}",
-                "",
-                "void output_char(int c)",
-                "{",
-                "    $putchar",
-                "}",
-                "",
-                "void output_flush(void)",
-                "{",
-                "    $flush",
-                "}",
-                "",
-                "void output_complete(void)",
-                "{",
-                "   $end",
-                "}",
-            ]
-        )
-
-        tmp_file_prefix = "tmp_pio_test_transport"
-
-        def delete_tmptest_files(test_dir):
-            for item in listdir(test_dir):
-                if item.startswith(tmp_file_prefix) and isfile(join(test_dir, item)):
-                    try:
-                        remove(join(test_dir, item))
-                    except:  # pylint: disable=bare-except
-                        click.secho(
-                            "Warning: Could not remove temporary file '%s'. "
-                            "Please remove it manually." % join(test_dir, item),
-                            fg="yellow",
-                        )
-
-        transport_options = TRANSPORT_OPTIONS[self.get_transport()]
-        tpl = Template(file_tpl).substitute(transport_options)
-        data = Template(tpl).substitute(baudrate=self.get_baudrate())
-
-        delete_tmptest_files(test_dir)
-        tmp_file = join(
-            test_dir,
-            "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")),
-        )
-        with open(tmp_file, mode="w", encoding="utf8") as fp:
-            fp.write(data)
-
-        atexit.register(delete_tmptest_files, test_dir)

platformio/commands/update.py

@@ -14,16 +14,11 @@
 
 import click
 
-from platformio.cache import cleanup_content_cache
-from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
-from platformio.commands.lib.command import lib_update as cmd_lib_update
-from platformio.commands.platform import platform_update as cmd_platform_update
-from platformio.package.manager.core import update_core_packages
-from platformio.package.manager.library import LibraryPackageManager
-
 
 @click.command(
-    "update", short_help="Update installed platforms, packages and libraries"
+    "update",
+    short_help="Update installed platforms, packages and libraries",
+    hidden=True,
 )
 @click.option("--core-packages", is_flag=True, help="Update only the core packages")
 @click.option(
@@ -35,25 +30,9 @@ from platformio.package.manager.library import LibraryPackageManager
 @click.option(
     "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
 )
-@click.pass_context
-def cli(ctx, core_packages, only_check, dry_run):
-    # cleanup lib search results, cached board and platform lists
-    cleanup_content_cache("http")
-
-    only_check = dry_run or only_check
-
-    update_core_packages(only_check)
-
-    if core_packages:
-        return
-
-    click.echo()
-    click.echo("Platform Manager")
-    click.echo("================")
-    ctx.invoke(cmd_platform_update, only_check=only_check)
-
-    click.echo()
-    click.echo("Library Manager")
-    click.echo("===============")
-    ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryPackageManager().package_dir]
-    ctx.invoke(cmd_lib_update, only_check=only_check)
+def cli(*_, **__):
+    click.secho(
+        "This command is deprecated and will be removed in the next releases. \n"
+        "Please use `pio pkg update` instead.",
+        fg="yellow",
+    )

platformio/commands/upgrade.py

@@ -20,15 +20,17 @@ from zipfile import ZipFile
 import click
 
 from platformio import VERSION, __version__, app, exception
-from platformio.clients.http import fetch_remote_content
 from platformio.compat import IS_WINDOWS
+from platformio.http import fetch_remote_content
+from platformio.package.manager.core import update_core_packages
 from platformio.proc import exec_command, get_pythonexe_path
 from platformio.project.helpers import get_project_cache_dir
 
 
-@click.command("upgrade", short_help="Upgrade PlatformIO to the latest version")
+@click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version")
 @click.option("--dev", is_flag=True, help="Use development branch")
 def cli(dev):
+    update_core_packages()
     if not dev and __version__ == get_latest_version():
         return click.secho(
             "You're up-to-date!\nPlatformIO %s is currently the "

platformio/compat.py

@@ -14,25 +14,26 @@
 
 # pylint: disable=unused-import,no-name-in-module
 
+import importlib.util
 import inspect
 import locale
 import sys
 
 from platformio.exception import UserSideException
 
-if sys.version_info >= (3,):
-    if sys.version_info >= (3, 7):
-        from asyncio import create_task as aio_create_task
-        from asyncio import get_running_loop as aio_get_running_loop
-    else:
-        from asyncio import ensure_future as aio_create_task
-        from asyncio import get_event_loop as aio_get_running_loop
+if sys.version_info >= (3, 7):
+    from asyncio import create_task as aio_create_task
+    from asyncio import get_running_loop as aio_get_running_loop
+else:
+    from asyncio import ensure_future as aio_create_task
+    from asyncio import get_event_loop as aio_get_running_loop
 
 
-PY2 = sys.version_info[0] == 2
+PY2 = sys.version_info[0] == 2  # DO NOT REMOVE IT. ESP8266/ESP32 depend on it
 IS_CYGWIN = sys.platform.startswith("cygwin")
 IS_WINDOWS = WINDOWS = sys.platform.startswith("win")
 IS_MACOS = sys.platform.startswith("darwin")
+MISSING = object()
 string_types = (str,)
 
 
@@ -57,8 +58,6 @@ def hashlib_encode_data(data):
 
 
 def load_python_module(name, pathname):
-    import importlib.util  # pylint: disable=import-outside-toplevel
-
     spec = importlib.util.spec_from_file_location(name, pathname)
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)

platformio/debug/cli.py

@@ -23,12 +23,12 @@ import subprocess
 import click
 
 from platformio import app, exception, fs, proc
-from platformio.commands.platform import init_platform
 from platformio.compat import IS_WINDOWS
 from platformio.debug import helpers
 from platformio.debug.config.factory import DebugConfigFactory
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.debug.process.gdb import GDBClientProcess
+from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.exception import ProjectEnvsNotAvailableError
 from platformio.project.helpers import is_platformio_project
@@ -38,7 +38,7 @@ from platformio.project.options import ProjectOptions
 @click.command(
     "debug",
     context_settings=dict(ignore_unknown_options=True),
-    short_help="Unified debugger",
+    short_help="Unified Debugger",
 )
 @click.option(
     "-d",
@@ -81,7 +81,29 @@ def cli(
             project_dir = os.getenv(name)
 
     with fs.cd(project_dir):
-        project_config = ProjectConfig.get_instance(project_conf)
+        return _debug_in_project_dir(
+            ctx,
+            project_dir,
+            project_conf,
+            environment,
+            load_mode,
+            verbose,
+            interface,
+            __unprocessed,
+        )
+
+
+def _debug_in_project_dir(
+    ctx,
+    project_dir,
+    project_conf,
+    environment,
+    load_mode,
+    verbose,
+    interface,
+    __unprocessed,
+):
+    project_config = ProjectConfig.get_instance(project_conf)
     project_config.validate(envs=[environment] if environment else None)
     env_name = environment or helpers.get_default_debug_env(project_config)
 
@@ -94,10 +116,11 @@ def cli(
     if "platform" not in env_options:
         raise ProjectEnvsNotAvailableError()
 
-    with fs.cd(project_dir):
-        debug_config = DebugConfigFactory.new(
-            init_platform(env_options["platform"]), project_config, env_name
-        )
+    debug_config = DebugConfigFactory.new(
+        PlatformFactory.new(env_options["platform"], autoinstall=True),
+        project_config,
+        env_name,
+    )
 
     if "--version" in __unprocessed:
         return subprocess.run(
@@ -163,18 +186,18 @@ def cli(
     loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop()
     asyncio.set_event_loop(loop)
 
-    with fs.cd(project_dir):
-        client = GDBClientProcess(project_dir, debug_config)
-        coro = client.run(__unprocessed)
-        try:
-            signal.signal(signal.SIGINT, signal.SIG_IGN)
-            loop.run_until_complete(coro)
-            if IS_WINDOWS:
-                # an issue with `asyncio` executor and STIDIN,
-                # it cannot be closed gracefully
-                proc.force_exit()
-        finally:
-            del client
-            loop.close()
+    client = GDBClientProcess(project_dir, debug_config)
+    coro = client.run(__unprocessed)
+    try:
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+        loop.run_until_complete(coro)
+        if IS_WINDOWS:
+            client.close()
+            # an issue with `asyncio` executor and STIDIN,
+            # it cannot be closed gracefully
+            proc.force_exit()
+    finally:
+        client.close()
+        loop.close()
 
     return True

platformio/debug/config/base.py

@@ -20,7 +20,7 @@ from platformio.compat import string_types
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.debug.helpers import reveal_debug_port
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import load_project_ide_data
+from platformio.project.helpers import load_build_metadata
 from platformio.project.options import ProjectOptions
 
 
@@ -147,7 +147,7 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
         )
 
     def _load_build_data(self):
-        data = load_project_ide_data(os.getcwd(), self.env_name, cache=True)
+        data = load_build_metadata(os.getcwd(), self.env_name, cache=True)
         if data:
             return data
         raise DebugInvalidOptionsError("Could not load a build configuration")
@@ -186,11 +186,7 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
                 else None
             )
             if server_package and not server_package_dir:
-                self.platform.install_packages(
-                    with_packages=[server_package],
-                    skip_default_package=True,
-                    silent=True,
-                )
+                self.platform.install_package(server_package)
                 server_package_dir = self.platform.get_package_dir(server_package)
             result.update(
                 dict(

platformio/debug/config/factory.py

@@ -23,7 +23,7 @@ class DebugConfigFactory(object):
     @staticmethod
     def get_clsname(name):
         name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I)
-        return "%s%sDebugConfig" % (name.upper()[0], name.lower()[1:])
+        return "%sDebugConfig" % name.lower().capitalize()
 
     @classmethod
     def new(cls, platform, project_config, env_name):

platformio/debug/config/generic.py

@@ -34,5 +34,5 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(GenericDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":3333"

platformio/debug/config/jlink.py

@@ -38,11 +38,9 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(JlinkDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":2331"
 
     @property
     def server_ready_pattern(self):
-        return super(JlinkDebugConfig, self).server_ready_pattern or (
-            "Waiting for GDB connection"
-        )
+        return super().server_ready_pattern or ("Waiting for GDB connection")

platformio/debug/config/mspdebug.py

@@ -32,5 +32,5 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(MspdebugDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":2000"

platformio/debug/config/qemu.py

@@ -33,5 +33,5 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(QemuDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":1234"

platformio/debug/config/renode.py

@@ -35,11 +35,11 @@ monitor start
 """
 
     def __init__(self, *args, **kwargs):
-        super(RenodeDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":3333"
 
     @property
     def server_ready_pattern(self):
-        return super(RenodeDebugConfig, self).server_ready_pattern or (
+        return super().server_ready_pattern or (
             "GDB server with all CPUs started on port"
         )

platformio/debug/helpers.py

@@ -12,22 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import re
 import sys
 import time
 from fnmatch import fnmatch
 from hashlib import sha1
 from io import BytesIO
-from os.path import isfile
 
-from platformio import util
-from platformio.commands import PlatformioCLI
-from platformio.commands.run.command import cli as cmd_run
-from platformio.commands.run.command import print_processing_header
-from platformio.commands.test.helpers import get_test_names
-from platformio.commands.test.processor import TestProcessorBase
+from platformio.cli import PlatformioCLI
 from platformio.compat import IS_WINDOWS, is_bytes
 from platformio.debug.exception import DebugInvalidOptionsError
+from platformio.device.list import list_serial_ports
+from platformio.run.cli import cli as cmd_run
+from platformio.run.cli import print_processing_header
+from platformio.test.helpers import list_test_names
+from platformio.test.result import TestSuite
+from platformio.test.runners.base import TestRunnerOptions
+from platformio.test.runners.factory import TestRunnerFactory
 
 
 class GDBMIConsoleStream(BytesIO):  # pylint: disable=too-few-public-methods
@@ -80,27 +82,25 @@ def predebug_project(
 ):  # pylint: disable=too-many-arguments
     debug_testname = project_config.get("env:" + env_name, "debug_test")
     if debug_testname:
-        test_names = get_test_names(project_config)
+        test_names = list_test_names(project_config)
         if debug_testname not in test_names:
             raise DebugInvalidOptionsError(
                 "Unknown test name `%s`. Valid names are `%s`"
                 % (debug_testname, ", ".join(test_names))
             )
         print_processing_header(env_name, project_config, verbose)
-        tp = TestProcessorBase(
-            ctx,
-            debug_testname,
-            env_name,
-            dict(
-                project_config=project_config,
-                project_dir=project_dir,
+        test_runner = TestRunnerFactory.new(
+            TestSuite(env_name, debug_testname),
+            project_config,
+            TestRunnerOptions(
+                verbose=verbose,
                 without_building=False,
-                without_uploading=True,
+                without_debugging=False,
+                without_uploading=not preload,
                 without_testing=True,
-                verbose=False,
             ),
         )
-        tp.build_or_upload(["__debug", "__test"] + (["upload"] if preload else []))
+        test_runner.start(ctx)
     else:
         ctx.invoke(
             cmd_run,
@@ -116,7 +116,7 @@ def predebug_project(
 
 
 def has_debug_symbols(prog_path):
-    if not isfile(prog_path):
+    if not os.path.isfile(prog_path):
         return False
     matched = {
         b".debug_info": False,
@@ -142,7 +142,7 @@ def has_debug_symbols(prog_path):
 
 def is_prog_obsolete(prog_path):
     prog_hash_path = prog_path + ".sha1"
-    if not isfile(prog_path):
+    if not os.path.isfile(prog_path):
         return True
     shasum = sha1()
     with open(prog_path, "rb") as fp:
@@ -153,7 +153,7 @@ def is_prog_obsolete(prog_path):
             shasum.update(data)
     new_digest = shasum.hexdigest()
     old_digest = None
-    if isfile(prog_hash_path):
+    if os.path.isfile(prog_hash_path):
         with open(prog_hash_path, encoding="utf8") as fp:
             old_digest = fp.read()
     if new_digest == old_digest:
@@ -178,7 +178,7 @@ def reveal_debug_port(env_debug_port, tool_name, tool_settings):
         return fnmatch(port, pattern)
 
     def _look_for_serial_port(hwids):
-        for item in util.get_serialports(filter_hwid=True):
+        for item in list_serial_ports(filter_hwid=True):
             if not _is_match_pattern(item["port"]):
                 continue
             port = item["port"]

platformio/debug/process/client.py

@@ -27,7 +27,7 @@ from platformio.project.helpers import get_project_cache_dir
 
 class DebugClientProcess(DebugBaseProcess):
     def __init__(self, project_dir, debug_config):
-        super(DebugClientProcess, self).__init__()
+        super().__init__()
         self.project_dir = project_dir
         self.debug_config = debug_config
 
@@ -55,7 +55,7 @@ class DebugClientProcess(DebugBaseProcess):
             self.debug_config.port = await self._server_process.run()
 
     def connection_made(self, transport):
-        super(DebugClientProcess, self).connection_made(transport)
+        super().connection_made(transport)
         self._lock_session(transport.get_pid())
         # Disable SIGINT and allow GDB's Ctrl+C interrupt
         signal.signal(signal.SIGINT, lambda *args, **kwargs: None)
@@ -64,7 +64,15 @@ class DebugClientProcess(DebugBaseProcess):
     def process_exited(self):
         if self._server_process:
             self._server_process.terminate()
-        super(DebugClientProcess, self).process_exited()
+        super().process_exited()
+
+    def close(self):
+        self._unlock_session()
+        if self.working_dir and os.path.isdir(self.working_dir):
+            fs.rmtree(self.working_dir)
+
+    def __del__(self):
+        self.close()
 
     def _kill_previous_session(self):
         assert self._session_id
@@ -94,8 +102,3 @@ class DebugClientProcess(DebugBaseProcess):
             return
         with ContentCache() as cc:
             cc.delete(self._session_id)
-
-    def __del__(self):
-        self._unlock_session()
-        if self.working_dir and os.path.isdir(self.working_dir):
-            fs.rmtree(self.working_dir)

platformio/debug/process/gdb.py

@@ -29,12 +29,12 @@ class GDBClientProcess(DebugClientProcess):
     INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed"
 
     def __init__(self, *args, **kwargs):
-        super(GDBClientProcess, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._target_is_running = False
         self._errors_buffer = b""
 
     async def run(self, extra_args):  # pylint: disable=arguments-differ
-        await super(GDBClientProcess, self).run()
+        await super().run()
 
         self.generate_init_script(os.path.join(self.working_dir, self.PIO_SRC_NAME))
         gdb_path = self.debug_config.client_executable_path or "gdb"
@@ -109,7 +109,7 @@ class GDBClientProcess(DebugClientProcess):
             fp.write("\n".join(self.debug_config.reveal_patterns(commands)))
 
     def stdin_data_received(self, data):
-        super(GDBClientProcess, self).stdin_data_received(data)
+        super().stdin_data_received(data)
         if b"-exec-run" in data:
             if self._target_is_running:
                 token, _ = data.split(b"-", 1)
@@ -127,7 +127,7 @@ class GDBClientProcess(DebugClientProcess):
         self.transport.get_pipe_transport(0).write(data)
 
     def stdout_data_received(self, data):
-        super(GDBClientProcess, self).stdout_data_received(data)
+        super().stdout_data_received(data)
         self._handle_error(data)
         # go to init break automatically
         if self.INIT_COMPLETED_BANNER.encode() in data:
@@ -170,7 +170,7 @@ class GDBClientProcess(DebugClientProcess):
         self._target_is_running = True
 
     def stderr_data_received(self, data):
-        super(GDBClientProcess, self).stderr_data_received(data)
+        super().stderr_data_received(data)
         self._handle_error(data)
 
     def _handle_error(self, data):

platformio/debug/process/server.py

@@ -30,7 +30,7 @@ class DebugServerProcess(DebugBaseProcess):
     STD_BUFFER_SIZE = 1024
 
     def __init__(self, debug_config):
-        super(DebugServerProcess, self).__init__()
+        super().__init__()
         self.debug_config = debug_config
         self._ready = False
         self._std_buffer = {"out": b"", "err": b""}
@@ -134,7 +134,7 @@ class DebugServerProcess(DebugBaseProcess):
         return self._ready
 
     def stdout_data_received(self, data):
-        super(DebugServerProcess, self).stdout_data_received(
+        super().stdout_data_received(
             escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data
         )
         self._std_buffer["out"] += data
@@ -142,7 +142,7 @@ class DebugServerProcess(DebugBaseProcess):
         self._std_buffer["out"] = self._std_buffer["out"][-1 * self.STD_BUFFER_SIZE :]
 
     def stderr_data_received(self, data):
-        super(DebugServerProcess, self).stderr_data_received(data)
+        super().stderr_data_received(data)
         self._std_buffer["err"] += data
         self._check_ready_by_pattern(self._std_buffer["err"])
         self._std_buffer["err"] = self._std_buffer["err"][-1 * self.STD_BUFFER_SIZE :]

platformio/device/cli.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.device.commands.list import device_list_cmd
+from platformio.device.commands.monitor import device_monitor_cmd
+
+
+@click.group(
+    "device",
+    commands=[
+        device_list_cmd,
+        device_monitor_cmd,
+    ],
+    short_help="Device manager & Serial/Socket monitor",
+)
+def cli():
+    pass