Merge branch 'release/v6.0.2'

* Fix filter argument used in remote agent

* Linting

* Renamed reserved variable name to filter_

.github/workflows/core.yml

@@ -8,16 +8,25 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        exclude:
+          - os: macos-latest
+            python-version: "3.6"
+          - os: windows-latest
+            python-version: "3.10"
+
     runs-on: ${{ matrix.os }}
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: "recursive"
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -26,11 +35,8 @@ jobs:
       - name: Python Lint
         run: |
           tox -e lint
+
       - name: Integration Tests
-        env:
-          TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }}
-          TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }}
-          TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }}
         run: |
           tox -e testcore
 

.github/workflows/deployment.yml

@@ -0,0 +1,42 @@
+name: Deployment
+
+on:
+  push:
+    branches:
+      - master
+      - "release/**"
+
+jobs:
+  deployment:
+    runs-on: ubuntu-latest
+    environment: production
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+
+      - name: Deployment Tests
+        env:
+          TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }}
+          TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }}
+          TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }}
+        run: |
+          tox -e testcore
+
+      - name: Publish package to PyPI
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/docs.yml

@@ -103,7 +103,7 @@ jobs:
       - name: Deploy to Github Pages
         uses: peaceiris/actions-gh-pages@v3
         with:
-          personal_token: ${{ secrets.PERSONAL_TOKEN }}
+          personal_token: ${{ secrets.DEPLOY_GH_DOCS_TOKEN }}
           external_repository: ${{ env.DOCS_REPO }}
           publish_dir: ./${{ env.DOCS_DIR }}
           commit_message: Sync Docs

.github/workflows/examples.yml

@@ -2,22 +2,28 @@ name: Examples
 
 on: [push, pull_request]
 
+
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-16.04, windows-latest, macos-latest]
-        python-version: [3.7]
+        os: [ubuntu-latest, windows-latest, macos-latest]
     runs-on: ${{ matrix.os }}
+    env:
+      PIO_INSTALL_DEVPLATFORM_OWNERNAMES: "platformio"
+      PIO_INSTALL_DEVPLATFORM_NAMES: "aceinna_imu,atmelavr,atmelmegaavr,atmelsam,espressif32,espressif8266,nordicnrf52,raspberrypi,ststm32,teensy"
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: "recursive"
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.9"
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -25,24 +31,15 @@ jobs:
 
       - name: Run on Linux
         if: startsWith(matrix.os, 'ubuntu')
-        env:
-          PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio"
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,intel_mcs51"
         run: |
-          # ChipKIT issue: install 32-bit support for GCC PIC32
-          sudo apt-get install libc6-i386
           # Free space
           sudo apt clean
           docker rmi $(docker image ls -aq)
           df -h
-          # Run
           tox -e testexamples
 
       - name: Run on macOS
         if: startsWith(matrix.os, 'macos')
-        env:
-          PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio"
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,microchippic32,lattice_ice40,gd32v"
         run: |
           df -h
           tox -e testexamples
@@ -52,8 +49,6 @@ jobs:
         env:
           PLATFORMIO_CORE_DIR: C:/pio
           PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH
-          PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio"
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap"
         run: |
           tox -e testexamples
 

.github/workflows/projects.yml

@@ -0,0 +1,69 @@
+name: Projects
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        project:
+          - marlin:
+            repository: "MarlinFirmware/Marlin"
+            folder: "Marlin"
+            config_dir: "Marlin"
+            env_name: "mega2560"
+          - esphome:
+            repository: "esphome/esphome"
+            folder: "esphome"
+            config_dir: "esphome"
+            env_name: "esp32-arduino"
+          - smartknob:
+            repository: "scottbez1/smartknob"
+            folder: "smartknob"
+            config_dir: "smartknob/firmware"
+            env_name: "view"
+          - espurna:
+            repository: "xoseperez/espurna"
+            folder: "espurna"
+            config_dir: "espurna/code"
+            env_name: "nodemcu-lolin"
+          - OpenMQTTGateway:
+            repository: "1technophile/OpenMQTTGateway"
+            folder: "OpenMQTTGateway"
+            config_dir: "OpenMQTTGateway"
+            env_name: "esp32-m5atom"
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        exclude:
+        - os: windows-latest
+          project: {"esphome": "", "repository": "esphome/esphome", "folder": "esphome", "config_dir": "esphome", "env_name": "esp32-arduino"}
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.9
+
+      - name: Install PlatformIO
+        run: pip install -U .
+
+      - name: Check out ${{ matrix.project.repository }}
+        uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
+          repository: ${{ matrix.project.repository }}
+          path: ${{ matrix.project.folder }}
+
+      - name: Install ESPHome dependencies
+        # Requires esptool package as it's used in a custom prescript
+        if: ${{ contains(matrix.project.repository, 'esphome') }}
+        run: pip install esptool==3.*
+
+      - name: Compile ${{ matrix.project.repository }}
+        run: pio run -d ${{ matrix.project.config_dir }} -e ${{ matrix.project.env_name }}
+

.gitignore

@@ -1,6 +1,6 @@
 *.egg-info
 *.pyc
-.pioenvs
+__pycache__
 .tox
 docs/_build
 dist

.pylintrc

@@ -16,6 +16,7 @@ disable=
     useless-import-alias,
     bad-option-value,
     consider-using-dict-items,
+    consider-using-f-string,
 
     ; PY2 Compat
     super-with-arguments,

.readthedocs.yml

@@ -1,12 +0,0 @@
-# See https://docs.readthedocs.io/en/stable/config-file/index.html
-
-version: 2
-
-sphinx:
-  configuration: docs/conf.py
-
-formats:
-  - pdf
-
-submodules:
-  include: all

CONTRIBUTING.md

@@ -3,7 +3,7 @@ Contributing
 
 To get started, <a href="https://cla-assistant.io/platformio/platformio-core">sign the Contributor License Agreement</a>.
 
-1. Fork the repository on GitHub.
+1. Fork the repository on GitHub
 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git`
 3. Run `pip install tox`
 4. Go to the root of project where is located `tox.ini` and run `tox -e py37`
@@ -18,4 +18,4 @@ To get started, <a href="https://cla-assistant.io/platformio/platformio-core">si
 8. Run the tests `make test`
 9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html)
 10. Commit changes to your forked repository
-11. Submit a Pull Request on GitHub.
+11. Submit a Pull Request on GitHub

HISTORY.rst

@@ -1,242 +1,133 @@
 Release Notes
 =============
 
+.. |PIOCONF| replace:: `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ configuration file
+.. |LDF| replace:: `LDF <https://docs.platformio.org/en/latest/librarymanager/ldf.html>`__
+.. |INTERPOLATION| replace:: `Interpolation of Values <https://docs.platformio.org/en/latest/projectconf/interpolation.html>`__
+.. |UNITTESTING| replace:: `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`__
+
+.. _release_notes_6:
+
+PlatformIO Core 6
+-----------------
+
+**A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.**
+
+6.0.2 (2022-06-01)
+~~~~~~~~~~~~~~~~~~
+
+* Control |UNITTESTING| verbosity with a new multilevel `pio test -v <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-v>`__ command option (`issue #4276 <https://github.com/platformio/platformio-core/issues/4276>`_)
+* Follow symbolic links during searching for the unit test suites (`issue #4288 <https://github.com/platformio/platformio-core/issues/4288>`_)
+* Show a warning when testing an empty project without a test suite (`issue #4278 <https://github.com/platformio/platformio-core/issues/4278>`_)
+* Improved support for `Asking for input (prompts) <https://docs.platformio.org/en/latest/scripting/examples/asking_for_input.html>`_
+* Fixed an issue when the `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__ option was applied outside the project scope (`issue #4277 <https://github.com/platformio/platformio-core/issues/4277>`_)
+* Fixed an issue with debugging assembly files without preprocessor (".s")
+
+6.0.1 (2022-05-17)
+~~~~~~~~~~~~~~~~~~
+
+* Improved support for the renamed configuration options (`issue #4270 <https://github.com/platformio/platformio-core/issues/4270>`_)
+* Fixed an issue when calling the built-in `pio device monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#filters>`__ filters
+* Fixed an issue when using |INTERPOLATION| and merging str+int options (`issue #4271 <https://github.com/platformio/platformio-core/issues/4271>`_)
+
+6.0.0 (2022-05-16)
+~~~~~~~~~~~~~~~~~~
+
+Please check the `Migration guide from 5.x to 6.0 <https://docs.platformio.org/en/latest/core/migration.html>`__.
+
+* **Package Management**
+
+  - New unified Package Management CLI (``pio pkg``):
+
+    * `pio pkg exec <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_exec.html>`_ - run command from package tool (`issue #4163 <https://github.com/platformio/platformio-core/issues/4163>`_)
+    * `pio pkg install <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html>`_ - install the project dependencies or custom packages
+    * `pio pkg list <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_list.html>`__ - list installed packages
+    * `pio pkg outdated <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_outdated.html>`__ - check for project outdated packages
+    * `pio pkg search <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_search.html>`__ - search for packages
+    * `pio pkg show <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_show.html>`__ - show package information
+    * `pio pkg uninstall <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_uninstall.html>`_ - uninstall the project dependencies or custom packages
+    * `pio pkg update <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_update.html>`__ - update the project dependencies or custom packages
+
+  - Package Manifest
+
+    * Added support for `"scripts" <https://docs.platformio.org/en/latest/librarymanager/config.html#scripts>`__ (`issue #485 <https://github.com/platformio/platformio-core/issues/485>`_)
+    * Added support for `multi-licensed <https://docs.platformio.org/en/latest/librarymanager/config.html#license>`__ packages using SPDX Expressions (`issue #4037 <https://github.com/platformio/platformio-core/issues/4037>`_)
+    * Added support for `"dependencies" <https://docs.platformio.org/en/latest/librarymanager/config.html#dependencies>`__ declared in a "tool" package manifest
+
+  - Added support for `symbolic links <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html#local-folder>`__ allowing pointing the local source folder to the Package Manager (`issue #3348 <https://github.com/platformio/platformio-core/issues/3348>`_)
+  - Automatically install dependencies of the local (private) project libraries (`issue #2910 <https://github.com/platformio/platformio-core/issues/2910>`_)
+  - Improved detection of a package type from the tarball archive (`issue #3828 <https://github.com/platformio/platformio-core/issues/3828>`_)
+  - Ignore files according to the patterns declared in ".gitignore" when using the `pio package pack <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_pack.html>`__ command (`issue #4188 <https://github.com/platformio/platformio-core/issues/4188>`_)
+  - Dropped automatic updates of global libraries and development platforms (`issue #4179 <https://github.com/platformio/platformio-core/issues/4179>`_)
+  - Dropped support for the "pythonPackages" field in "platform.json" manifest in favor of `Extra Python Dependencies <https://docs.platformio.org/en/latest/scripting/examples/extra_python_packages.html>`__
+  - Fixed an issue when manually removed dependencies from the |PIOCONF| were not uninstalled from the storage (`issue #3076 <https://github.com/platformio/platformio-core/issues/3076>`_)
+
+* **Unit Testing**
+
+  - Refactored from scratch |UNITTESTING| solution and its documentation
+  - New: `Test Hierarchy <https://docs.platformio.org/en/latest/advanced/unit-testing/structure.html>`_ (`issue #4135 <https://github.com/platformio/platformio-core/issues/4135>`_)
+  - New: `Doctest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/doctest.html>`__ testing framework (`issue #4240 <https://github.com/platformio/platformio-core/issues/4240>`_)
+  - New: `GoogleTest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/googletest.html>`__ testing and mocking framework (`issue #3572 <https://github.com/platformio/platformio-core/issues/3572>`_)
+  - New: `Semihosting <https://docs.platformio.org/en/latest/advanced/unit-testing/semihosting.html>`__ (`issue #3516 <https://github.com/platformio/platformio-core/issues/3516>`_)
+  - New: Hardware `Simulators <https://docs.platformio.org/en/latest/advanced/unit-testing/simulators/index.html>`__ for Unit Testing (QEMU, Renode, SimAVR, and custom solutions)
+  - New: ``test`` `build configuration <https://docs.platformio.org/en/latest/projectconf/build_configurations.html>`__
+  - Added support for a `custom testing framework <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/index.html>`_
+  - Added support for a custom `testing command <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-testing-command>`__
+  - Added support for a `custom Unity library <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/custom/examples/custom_unity_library.html>`__ (`issue #3980 <https://github.com/platformio/platformio-core/issues/3980>`_)
+  - Added support for the ``socket://`` and ``rfc2217://`` protocols using `test_port <https://docs.platformio.org/en/latest/projectconf/section_env_test.html#test-port>`__ option (`issue #4229 <https://github.com/platformio/platformio-core/issues/4229>`_)
+  - List available project tests with a new `pio test --list-tests <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-list-tests>`__ option
+  - Pass extra arguments to the testing program with a new `pio test --program-arg <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-a>`__ option (`issue #3132 <https://github.com/platformio/platformio-core/issues/3132>`_)
+  - Generate reports in JUnit and JSON formats using the `pio test <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html>`__ command (`issue #2891 <https://github.com/platformio/platformio-core/issues/2891>`_)
+  - Provide more information when the native program crashed on a host (errored with a non-zero return code) (`issue #3429 <https://github.com/platformio/platformio-core/issues/3429>`_)
+  - Improved automatic detection of a testing serial port (`issue #4076 <https://github.com/platformio/platformio-core/issues/4076>`_)
+  - Fixed an issue when command line parameters (``--ignore``, ``--filter``) do not override values defined in the |PIOCONF| (`issue #3845 <https://github.com/platformio/platformio-core/issues/3845>`_)
+  - Renamed the "test_build_project_src" project configuration option to the `test_build_src <https://docs.platformio.org/en/latest//projectconf/section_env_test.html#test-build-src>`__
+  - Removed the "test_transport" option in favor of the `Custom "unity_config.h" <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/unity.html>`_
+
+* **Static Code Analysis**
+
+  - Updated analysis tools:
+
+    * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.7 with various checker improvements and fixed false positives
+    * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.18 with improved and updated semantic analysis system
+
+  - Added support for the custom `Clang-Tidy <https://docs.platformio.org/en/latest/plus/check-tools/clang-tidy.html>`__ configuration file (`issue #4186 <https://github.com/platformio/platformio-core/issues/4186>`_)
+  - Added ability to override a tool version using the `platform_packages <https://docs.platformio.org/en/latest/projectconf/section_env_platform.html#platform-packages>`__ option (`issue #3798 <https://github.com/platformio/platformio-core/issues/3798>`_)
+  - Fixed an issue with improper handling of defects that don't specify a source file (`issue #4237 <https://github.com/platformio/platformio-core/issues/4237>`_)
+
+* **Build System**
+
+  - Show project dependency licenses when building in the verbose mode
+  - Fixed an issue when |LDF| ignores the project `lib_deps <https://docs.platformio.org/en/latest/projectconf/section_env_library.html#lib-deps>`__ while resolving library dependencies (`issue #3598 <https://github.com/platformio/platformio-core/issues/3598>`_)
+  - Fixed an issue with calling an extra script located outside a project (`issue #4220 <https://github.com/platformio/platformio-core/issues/4220>`_)
+  - Fixed an issue when GCC preprocessor was applied to the ".s" assembly files on case-sensitive OS such as Window OS (`issue #3917 <https://github.com/platformio/platformio-core/issues/3917>`_)
+  - Fixed an issue when |LDF| ignores `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__ in the "deep+" mode (`issue #4253 <https://github.com/platformio/platformio-core/issues/4253>`_)
+
+* **Integration**
+
+  - Added a new build variable (``COMPILATIONDB_INCLUDE_TOOLCHAIN``) to include toolchain paths in the compilation database (`issue #3735 <https://github.com/platformio/platformio-core/issues/3735>`_)
+  - Changed a default path for compilation database `compile_commands.json <https://docs.platformio.org/en/latest/integration/compile_commands.html>`__ to the project root
+  - Enhanced integration for Qt Creator (`issue #3046 <https://github.com/platformio/platformio-core/issues/3046>`_)
+
+* **Project Configuration**
+
+  - Extended |INTERPOLATION| with ``${this}`` pattern (`issue #3953 <https://github.com/platformio/platformio-core/issues/3953>`_)
+  - Embed environment name of the current section in the |PIOCONF| using ``${this.__env__}`` pattern
+  - Renamed the "src_build_flags" project configuration option to the `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__
+  - Renamed the "src_filter" project configuration option to the `build_src_filter <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-filter>`__
+
+* **Miscellaneous**
+
+  - Pass extra arguments to the `native <https://docs.platformio.org/en/latest/platforms/native.html>`__ program with a new `pio run --program-arg <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html#cmdoption-pio-run-a>`__ option (`issue #4246 <https://github.com/platformio/platformio-core/issues/4246>`_)
+  - Improved PIO Remote setup on credit-card sized computers (Raspberry Pi, BeagleBon, etc) (`issue #3865 <https://github.com/platformio/platformio-core/issues/3865>`_)
+  - Finally removed all tracks to the Python 2.7, the Python 3.6 is the minimum supported version.
+
 .. _release_notes_5:
 
 PlatformIO Core 5
 -----------------
 
-**A professional collaborative platform for embedded development**
-
-5.2.0 (2021-09-13)
-~~~~~~~~~~~~~~~~~~
-
-* **PlatformIO Debugging**
-
-  - Boosted `PlatformIO Debugging <https://docs.platformio.org/page/plus/debugging.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
-  - `Debug unit tests <https://docs.platformio.org/page/plus/debugging.html#debug-unit-tests>`__ created with `PlatformIO Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html>`__ solution  (`issue #948 <https://github.com/platformio/platformio-core/issues/948>`_)
-  - Debug native (desktop) applications on a host machine (`issue #980 <https://github.com/platformio/platformio-core/issues/980>`_)
-  - Support debugging on Windows using Windows CMD/CLI (`pio debug <https://docs.platformio.org/page/core/userguide/cmd_debug.html>`__) (`issue #3793 <https://github.com/platformio/platformio-core/issues/3793>`_)
-  - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern <https://docs.platformio.org/page/projectconf/section_env_debug.html#debug-server-ready-pattern>`__ option
-  - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 <https://github.com/platformio/platformio-core/issues/3756>`_)
-
-* **Package Management**
-
-  - Improved a package publishing process:
-
-    * Show package details
-    * Check for conflicting names in the PlatformIO Trusted Registry
-    * Check for duplicates and used version
-    * Validate package manifest
-
-  - Added a new option ``--non-interactive`` to `pio package publish <https://docs.platformio.org/page/core/userguide/package/cmd_publish.html>`__ command
-
-* **Build System**
-
-  - Process "precompiled" and "ldflags" properties of the "library.properties" manifest (`issue #3994 <https://github.com/platformio/platformio-core/issues/3994>`_)
-  - Upgraded build engine to the SCons 4.2 (`release notes <https://github.com/SCons/scons/blob/rel_4.2.0/CHANGES.txt>`__)
-  - Fixed an issue with broken binary file extension when a custom ``PROGNAME`` contains dot symbols (`issue #3906 <https://github.com/platformio/platformio-core/issues/3906>`_)
-  - Fixed an issue when PlatformIO archives a library that does not contain C/C++ source files (`issue #4019 <https://github.com/platformio/platformio-core/issues/4019>`_)
-
-* **Static Code Analysis**
-
-  - Updated analysis tools:
-
-    * `Clang-Tidy <https://docs.platformio.org/page/plus/check-tools/clang-tidy.html>`__ v12.0.1 with new modules and extended checks list
-    * `Cppcheck <https://docs.platformio.org/page/plus/check-tools/cppcheck.html>`__ v2.5.0 with improved code analysis and MISRA improvements
-    * `PVS-Studio <https://docs.platformio.org/page/plus/check-tools/pvs-studio.html>`__ v7.14 with support for intermodular analysis, improved MISRA support and new diagnostics
-
-* **Miscellaneous**
-
-  - Ensure that a serial port is ready before running unit tests on a remote target (`issue #3742 <https://github.com/platformio/platformio-core/issues/3742>`_)
-  - Fixed an error "Unknown development platform" when running unit tests on a clean machine (`issue #3901 <https://github.com/platformio/platformio-core/issues/3901>`_)
-  - Fixed an issue when "main.cpp" was generated for a new project for 8-bit development platforms (`issue #3872 <https://github.com/platformio/platformio-core/issues/3872>`_)
-
-5.1.1 (2021-03-17)
-~~~~~~~~~~~~~~~~~~
-
-* Fixed a "The command line is too long" issue with a linking process on Windows (`issue #3827 <https://github.com/platformio/platformio-core/issues/3827>`_)
-* Fixed an issue with `device monitor <https://docs.platformio.org/page/core/userguide/device/cmd_monitor.html>`__ when the "send_on_enter" filter didn't send EOL chars (`issue #3787 <https://github.com/platformio/platformio-core/issues/3787>`_)
-* Fixed an issue with silent mode when unwanted data is printed to stdout (`issue #3837 <https://github.com/platformio/platformio-core/issues/3837>`_)
-* Fixed an issue when code inspection fails with "Bad JSON" (`issue #3790 <https://github.com/platformio/platformio-core/issues/3790>`_)
-* Fixed an issue with overriding user-specified debugging configuration information in VSCode (`issue #3824 <https://github.com/platformio/platformio-core/issues/3824>`_)
-
-5.1.0 (2021-01-28)
-~~~~~~~~~~~~~~~~~~
-
-* **PlatformIO Home**
-
-  - Boosted `PlatformIO Home <https://docs.platformio.org/page/home/index.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
-  - Added a new ``--session-id`` option to `pio home <https://docs.platformio.org/page/core/userguide/cmd_home.html>`__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 <https://github.com/platformio/platformio-core/issues/3397>`_)
-
-* **Build System**
-
-  - Upgraded build engine to the SCons 4.1 (`release notes <https://scons.org/scons-410-is-available.html>`_)
-  - Refactored a workaround for a maximum command line character limitation (`issue #3792 <https://github.com/platformio/platformio-core/issues/3792>`_)
-  - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
-
-* **Package Management**
-
-  - New options for `pio system prune <https://docs.platformio.org/page/core/userguide/system/cmd_prune.html>`__ command:
-
-    + ``--dry-run`` option to show data that will be removed
-    + ``--core-packages`` option to remove unnecessary core packages
-    + ``--platform-packages`` option to remove unnecessary development platform packages (`issue #923 <https://github.com/platformio/platformio-core/issues/923>`_)
-
-  - Added new `check_prune_system_threshold <https://docs.platformio.org/page/core/userguide/cmd_settings.html#check-prune-system-threshold>`__ setting
-  - Disabled automatic removal of unnecessary development platform packages (`issue #3708 <https://github.com/platformio/platformio-core/issues/3708>`_, `issue #3770 <https://github.com/platformio/platformio-core/issues/3770>`_)
-  - Fixed an issue when unnecessary packages were removed in  ``update --dry-run`` mode (`issue #3809 <https://github.com/platformio/platformio-core/issues/3809>`_)
-  - Fixed a "ValueError: Invalid simple block" when uninstalling a package with a custom name and external source (`issue #3816 <https://github.com/platformio/platformio-core/issues/3816>`_)
-
-* **Debugging**
-
-  - Configure a custom debug adapter speed using a new `debug_speed <https://docs.platformio.org/page/projectconf/section_env_debug.html#debug-speed>`__ option (`issue #3799 <https://github.com/platformio/platformio-core/issues/3799>`_)
-  - Handle debugging server's "ready_pattern" in "stderr" output
-
-* **Miscellaneous**
-
-  - Improved listing of `multicast DNS services <https://docs.platformio.org/page/core/userguide/device/cmd_list.html>`_
-  - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 <https://github.com/platformio/platformio-core/issues/3804>`_)
-  - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 <https://github.com/platformio/platformio-core/issues/3808>`_)
-  - Fixed an issue when `pio device monitor --eol <https://docs.platformio.org/page/core/userguide/device/cmd_monitor.html#cmdoption-pio-device-monitor-eol>`__ and "send_on_enter" filter do not work properly (`issue #3787 <https://github.com/platformio/platformio-core/issues/3787>`_)
-
-5.0.4 (2020-12-30)
-~~~~~~~~~~~~~~~~~~
-
-- Added "Core" suffix when showing PlatformIO Core version using ``pio --version`` command
-- Improved ".ccls" configuration file for Emacs, Vim, and Sublime Text integrations
-- Updated analysis tools:
-
-  * `Cppcheck <https://docs.platformio.org/page/plus/check-tools/cppcheck.html>`__ v2.3 with improved C++ parser and several new MISRA rules
-  * `PVS-Studio <https://docs.platformio.org/page/plus/check-tools/pvs-studio.html>`__ v7.11 with new diagnostics and updated mass suppression mechanism
-
-- Show a warning message about deprecated support for Python 2 and Python 3.5
-- Do not provide "intelliSenseMode" option when generating configuration for VSCode C/C++ extension
-- Fixed a "git-sh-setup: file not found" error when installing project dependencies from Git VCS (`issue #3740 <https://github.com/platformio/platformio-core/issues/3740>`_)
-- Fixed an issue with package publishing on Windows when Unix permissions are not preserved (`issue #3776 <https://github.com/platformio/platformio-core/issues/3776>`_)
-
-5.0.3 (2020-11-12)
-~~~~~~~~~~~~~~~~~~
-
-- Added an error selector for `Sublime Text <https://docs.platformio.org/page/integration/ide/sublimetext.html>`__ build runner (`issue #3733 <https://github.com/platformio/platformio-core/issues/3733>`_)
-- Generate a working "projectEnvName" for PlatformIO IDE's debugger for VSCode
-- Force VSCode's intelliSenseMode to "gcc-x64" when GCC toolchain is used
-- Print ignored test suites and environments in the test summary report only in verbose mode (`issue #3726 <https://github.com/platformio/platformio-core/issues/3726>`_)
-- Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 <https://github.com/platformio/platformio-core/issues/3662>`_)
-- Fixed an issue when `pio package pack <https://docs.platformio.org/page/core/userguide/package/cmd_pack.html>`__ ignores some folders (`issue #3730 <https://github.com/platformio/platformio-core/issues/3730>`_)
-
-5.0.2 (2020-10-30)
-~~~~~~~~~~~~~~~~~~
-
-- Initialize a new project or update the existing passing working environment name and its options (`issue #3686 <https://github.com/platformio/platformio-core/issues/3686>`_)
-- Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry (`issue #3700 <https://github.com/platformio/platformio-core/issues/3700>`_)
-- Improved "core.call" RPC for PlatformIO Home (`issue #3671 <https://github.com/platformio/platformio-core/issues/3671>`_)
-- Fixed a "PermissionError: [WinError 5]" on Windows when an external repository is used with `lib_deps <https://docs.platformio.org/page/projectconf/section_env_library.html#lib-deps>`__ option (`issue #3664 <https://github.com/platformio/platformio-core/issues/3664>`_)
-- Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 <https://github.com/platformio/platformio-core/issues/3666>`_)
-- Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 <https://github.com/platformio/platformio-core/issues/3669>`_)
-- Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses (`issue #3677 <https://github.com/platformio/platformio-core/issues/3677>`_)
-- Fixed an issue when `pio package publish <https://docs.platformio.org/page/core/userguide/package/cmd_publish.html>`__ command removes original archive after submitting to the registry (`issue #3716 <https://github.com/platformio/platformio-core/issues/3716>`_)
-- Fixed an issue when multiple `pio lib install <https://docs.platformio.org/page/core/userguide/lib/cmd_install.html>`__ command with the same local library results in duplicates in ``lib_deps`` (`issue #3715 <https://github.com/platformio/platformio-core/issues/3715>`_)
-- Fixed an issue with a "wrong" timestamp in device monitor output using `"time" filter <https://docs.platformio.org/page/core/userguide/device/cmd_monitor.html#filters>`__ (`issue #3712 <https://github.com/platformio/platformio-core/issues/3712>`_)
-
-5.0.1 (2020-09-10)
-~~~~~~~~~~~~~~~~~~
-
-- Added support for "owner" requirement when declaring ``dependencies`` using `library.json <https://docs.platformio.org/page/librarymanager/config.html#dependencies>`__
-- Fixed an issue when using a custom git/ssh package with `platform_packages <https://docs.platformio.org/page/projectconf/section_env_platform.html#platform-packages>`__ option (`issue #3624 <https://github.com/platformio/platformio-core/issues/3624>`_)
-- Fixed an issue with "ImportError: cannot import name '_get_backend' from 'cryptography.hazmat.backends'" when using `Remote Development <https://docs.platformio.org/page/plus/pio-remote.html>`__ on RaspberryPi device (`issue #3652 <https://github.com/platformio/platformio-core/issues/3652>`_)
-- Fixed an issue when `pio package unpublish <https://docs.platformio.org/page/core/userguide/package/cmd_unpublish.html>`__ command crashes (`issue #3660 <https://github.com/platformio/platformio-core/issues/3660>`_)
-- Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 <https://github.com/platformio/platformio-core/issues/3662>`_)
-- Fixed an issue with incorrect value for C++ language standard in IDE projects when an in-progress language standard is used (`issue #3653 <https://github.com/platformio/platformio-core/issues/3653>`_)
-- Fixed an issue with "Invalid simple block (semantic_version)" from library dependency that refs to an external source (repository, ZIP/Tar archives) (`issue #3658 <https://github.com/platformio/platformio-core/issues/3658>`_)
-- Fixed an issue when can not remove update or remove external dev-platform using PlatformIO Home (`issue #3663 <https://github.com/platformio/platformio-core/issues/3663>`_)
-
-5.0.0 (2020-09-03)
-~~~~~~~~~~~~~~~~~~
-
-Please check `Migration guide from 4.x to 5.0 <https://docs.platformio.org/page/core/migration.html>`__.
-
-* Integration with the new **PlatformIO Trusted Registry**
-
-  - Enterprise-grade package storage with high availability (multi replicas)
-  - Secure, fast, and reliable global content delivery network (CDN)
-  - Universal support for all packages:
-
-    * Libraries
-    * Development platforms
-    * Toolchains
-
-  - Built-in fine-grained access control (role-based, teams, organizations)
-  - New CLI commands:
-
-    * `pio package <https://docs.platformio.org/page/core/userguide/package/index.html>`__ – manage packages in the registry
-    * `pio access <https://docs.platformio.org/page/core/userguide/access/index.html>`__ – manage package access for users, teams, and maintainers
-
-* Integration with the new **Account Management System**
-
-  - `Manage organizations <https://docs.platformio.org/page/core/userguide/org/index.html>`__
-  - `Manage teams and team memberships <https://docs.platformio.org/page/core/userguide/team/index.html>`__
-
-* New **Package Management System**
-
-  - Integrated PlatformIO Core with the new PlatformIO Registry
-  - Support for owner-based dependency declaration (resolves name conflicts) (`issue #1824 <https://github.com/platformio/platformio-core/issues/1824>`_)
-  - Automatically save dependencies to `"platformio.ini" <https://docs.platformio.org/page/projectconf.html>`__ when installing using PlatformIO CLI (`issue #2964 <https://github.com/platformio/platformio-core/issues/2964>`_)
-  - Follow SemVer complaint version constraints when checking library updates `issue #1281 <https://github.com/platformio/platformio-core/issues/1281>`_)
-  - Dropped support for "packageRepositories" section in "platform.json" manifest (please publish packages directly to the registry)
-
-* **Build System**
-
-  - Upgraded build engine to the `SCons 4.0 - a next-generation software construction tool <https://scons.org/>`__
-
-    * `Configuration files are Python scripts <https://docs.platformio.org/page/projectconf/advanced_scripting.html>`__ – use the power of a real programming language to solve build problems
-    * Built-in reliable and automatic dependency analysis
-    * Improved support for parallel builds
-    * Ability to `share built files in a cache <https://docs.platformio.org/page/projectconf/section_platformio.html#projectconf-pio-build-cache-dir>`__ to speed up multiple builds
-
-  - New `Custom Targets <https://docs.platformio.org/page/projectconf/advanced_scripting.html#custom-targets>`__
-
-    * Pre/Post processing based on dependent sources (another target, source file, etc.)
-    * Command launcher with own arguments
-    * Launch command with custom options declared in `"platformio.ini" <https://docs.platformio.org/page/projectconf.html>`__
-    * Python callback as a target (use the power of Python interpreter and PlatformIO Build API)
-    * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets <https://docs.platformio.org/page/core/userguide/cmd_run.html#cmdoption-platformio-run-list-targets>`__ command (`issue #3544 <https://github.com/platformio/platformio-core/issues/3544>`_)
-
-  - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 <https://github.com/platformio/platformio-core/issues/3570>`_)
-  - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) <https://docs.platformio.org/page/librarymanager/ldf.html#dependency-finder-mode>`__ for Arduino library when "library.property" has "depends" field (`issue #3607 <https://github.com/platformio/platformio-core/issues/3607>`_)
-  - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 <https://github.com/platformio/platformio-core/issues/3531>`_)
-  - Fixed an issue with the ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 <https://github.com/platformio/platformio-core/issues/3542>`_)
-
-* **Project Management**
-
-  - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci <https://docs.platformio.org/page/core/userguide/cmd_ci.html>`__, `src_filter <https://docs.platformio.org/page/projectconf/section_env_build.html#src-filter>`__, `check_patterns <https://docs.platformio.org/page/projectconf/section_env_check.html#check-patterns>`__, `library.json > srcFilter <https://docs.platformio.org/page/librarymanager/config.html#srcfilter>`__). Python 3.5+ is required
-  - Added a new ``-e, --environment`` option to `pio project init <https://docs.platformio.org/page/core/userguide/project/cmd_init.html#cmdoption-platformio-project-init-e>`__ command that helps to update a PlatformIO project using the existing environment
-  - Dump build system data intended for IDE extensions/plugins using a new `pio project data <https://docs.platformio.org/page/core/userguide/project/cmd_data.html>`__ command
-  - Do not generate ".travis.yml" for a new project, let the user have a choice
-
-* **Unit Testing**
-
-  - Updated PIO Unit Testing support for Mbed framework and added compatibility with Mbed OS 6
-  - Fixed an issue when running multiple test environments (`issue #3523 <https://github.com/platformio/platformio-core/issues/3523>`_)
-  - Fixed an issue when Unit Testing engine fails with a custom project configuration file (`issue #3583 <https://github.com/platformio/platformio-core/issues/3583>`_)
-
-* **Static Code Analysis**
-
-  - Updated analysis tools:
-
-    * `Cppcheck <https://docs.platformio.org/page/plus/check-tools/cppcheck.html>`__ v2.1 with a new "soundy" analysis option and improved code parser
-    * `PVS-Studio <https://docs.platformio.org/page/plus/check-tools/pvs-studio.html>`__ v7.09 with a new file list analysis mode and an extended list of analysis diagnostics
-
-  - Added Cppcheck package for ARM-based single-board computers (`issue #3559 <https://github.com/platformio/platformio-core/issues/3559>`_)
-  - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 <https://github.com/platformio/platformio-core/issues/3631>`_)
-
-* **Miscellaneous**
-
-  - Display system-wide information using a new `pio system info <https://docs.platformio.org/page/core/userguide/system/cmd_info.html>`__ command (`issue #3521 <https://github.com/platformio/platformio-core/issues/3521>`_)
-  - Remove unused data using a new `pio system prune <https://docs.platformio.org/page/core/userguide/system/cmd_prune.html>`__ command (`issue #3522 <https://github.com/platformio/platformio-core/issues/3522>`_)
-  - Show ignored project environments only in the verbose mode (`issue #3641 <https://github.com/platformio/platformio-core/issues/3641>`_)
-  - Do not escape compiler arguments in VSCode template on Windows
-  - Drop support for Python 2 and 3.5.
+See `PlatformIO Core 5.0 history <https://github.com/platformio/platformio-core/blob/v5.2.5/HISTORY.rst>`__.
 
 .. _release_notes_4:
 

Makefile

@@ -11,7 +11,7 @@ format:
 	black ./tests
 
 test:
-	py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
+	py.test --verbose --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
 
 before-commit: isort format lint
 

README.rst

@@ -1,27 +1,32 @@
-PlatformIO
-==========
+.. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg
+    :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md
+    :alt:  SWUbanner
+
+PlatformIO Core
+===============
 
 .. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg
-    :target: https://docs.platformio.org/page/core/index.html
+    :target: https://docs.platformio.org/en/latest/core/index.html
     :alt:  CI Build for PlatformIO Core
-.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
-    :target: https://github.com/platformio/platformio-examples
-    :alt:  CI Build for dev-platform examples
 .. image:: https://github.com/platformio/platformio-core/workflows/Docs/badge.svg
     :target: https://docs.platformio.org?utm_source=github&utm_medium=core
     :alt:  CI Build for Docs
+.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
+    :target: https://github.com/platformio/platformio-examples
+    :alt:  CI Build for dev-platform examples
+.. image:: https://github.com/platformio/platformio-core/workflows/Projects/badge.svg
+    :target: https://docs.platformio.org/en/latest/tutorials/index.html#projects
+    :alt:  CI Build for the Community Projects
 .. image:: https://img.shields.io/pypi/v/platformio.svg
     :target: https://pypi.python.org/pypi/platformio/
     :alt: Latest Version
-.. image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
-    :target: https://pypi.python.org/pypi/platformio/
-    :alt:  License
 .. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg
-   :alt: Community Labs
+   :alt: PlatformIO Labs
    :target: https://piolabs.com/?utm_source=github&utm_medium=core
 
-**Quick Links:** `Web <https://platformio.org?utm_source=github&utm_medium=core>`_ |
+**Quick Links:** `Homepage <https://platformio.org?utm_source=github&utm_medium=core>`_ |
 `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_ |
+`Registry <https://registry.platformio.org?utm_source=github&utm_medium=core>`_ |
 `Project Examples <https://github.com/platformio/platformio-examples/>`__ |
 `Docs <https://docs.platformio.org?utm_source=github&utm_medium=core>`_ |
 `Donate <https://platformio.org/donate?utm_source=github&utm_medium=core>`_ |
@@ -35,7 +40,7 @@ PlatformIO
 .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png
     :target: https://platformio.org?utm_source=github&utm_medium=core
 
-`PlatformIO <https://platformio.org?utm_source=github&utm_medium=core>`_ is a professional collaborative platform for embedded development
+`PlatformIO <https://platformio.org>`_ is a professional collaborative platform for embedded development.
 
 **A place where Developers and Teams have true Freedom! No more vendor lock-in!**
 
@@ -43,93 +48,36 @@ PlatformIO
 * Cross-platform IDE and Unified Debugger
 * Static Code Analyzer and Remote Unit Testing
 * Multi-platform and Multi-architecture Build System
-* Firmware File Explorer and Memory Inspection.
+* Firmware File Explorer and Memory Inspection
 
 Get Started
 -----------
 
-* `What is PlatformIO? <https://docs.platformio.org/page/what-is-platformio.html?utm_source=github&utm_medium=core>`_
+* `What is PlatformIO? <https://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
 * `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_
-* `PlatformIO Core (CLI) <https://docs.platformio.org/page/core.html?utm_source=github&utm_medium=core>`_
+* `PlatformIO Core (CLI) <https://docs.platformio.org/en/latest/core.html?utm_source=github&utm_medium=core>`_
 * `Project Examples <https://github.com/platformio/platformio-examples?utm_source=github&utm_medium=core>`__
 
 Solutions
 ---------
 
-* `Library Management <https://docs.platformio.org/page/librarymanager/index.html?utm_source=github&utm_medium=core>`_
-* `Desktop IDEs Integration <https://docs.platformio.org/page/ide.html?utm_source=github&utm_medium=core>`_
-* `Continuous Integration <https://docs.platformio.org/page/ci/index.html?utm_source=github&utm_medium=core>`_
+* `Library Management <https://docs.platformio.org/en/latest/librarymanager/index.html?utm_source=github&utm_medium=core>`_
+* `Desktop IDEs Integration <https://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core>`_
+* `Continuous Integration <https://docs.platformio.org/en/latest/ci/index.html?utm_source=github&utm_medium=core>`_
 
 **Advanced**
 
-* `Debugging <https://docs.platformio.org/page/plus/debugging.html?utm_source=github&utm_medium=core>`_
-* `Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
-* `Static Code Analysis <https://docs.platformio.org/page/plus/pio-check.html?utm_source=github&utm_medium=core>`_
-* `Remote Development <https://docs.platformio.org/page/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
+* `Debugging <https://docs.platformio.org/en/latest/plus/debugging.html?utm_source=github&utm_medium=core>`_
+* `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html?utm_source=github&utm_medium=core>`_
+* `Static Code Analysis <https://docs.platformio.org/en/latest/plus/pio-check.html?utm_source=github&utm_medium=core>`_
+* `Remote Development <https://docs.platformio.org/en/latest/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
 
 Registry
 --------
 
-* `Libraries <https://platformio.org/lib?utm_source=github&utm_medium=core>`_
-* `Development Platforms <https://platformio.org/platforms?utm_source=github&utm_medium=core>`_
-* `Frameworks <https://platformio.org/frameworks?utm_source=github&utm_medium=core>`_
-* `Embedded Boards <https://platformio.org/boards?utm_source=github&utm_medium=core>`_
-
-Development Platforms
----------------------
-
-* `Aceinna IMU <https://platformio.org/platforms/aceinna_imu?utm_source=github&utm_medium=core>`_
-* `ASR Microelectronics ASR605x <https://platformio.org/platforms/asrmicro650x?utm_source=github&utm_medium=core>`_
-* `Atmel AVR <https://platformio.org/platforms/atmelavr?utm_source=github&utm_medium=core>`_
-* `Atmel SAM <https://platformio.org/platforms/atmelsam?utm_source=github&utm_medium=core>`_
-* `Espressif 32 <https://platformio.org/platforms/espressif32?utm_source=github&utm_medium=core>`_
-* `Espressif 8266 <https://platformio.org/platforms/espressif8266?utm_source=github&utm_medium=core>`_
-* `Freescale Kinetis <https://platformio.org/platforms/freescalekinetis?utm_source=github&utm_medium=core>`_
-* `Infineon XMC <https://platformio.org/platforms/infineonxmc?utm_source=github&utm_medium=core>`_
-* `Intel ARC32 <https://platformio.org/platforms/intel_arc32?utm_source=github&utm_medium=core>`_
-* `Intel MCS-51 (8051) <https://platformio.org/platforms/intel_mcs51?utm_source=github&utm_medium=core>`_
-* `Kendryte K210 <https://platformio.org/platforms/kendryte210?utm_source=github&utm_medium=core>`_
-* `Lattice iCE40 <https://platformio.org/platforms/lattice_ice40?utm_source=github&utm_medium=core>`_
-* `Maxim 32 <https://platformio.org/platforms/maxim32?utm_source=github&utm_medium=core>`_
-* `Microchip PIC32 <https://platformio.org/platforms/microchippic32?utm_source=github&utm_medium=core>`_
-* `Nordic nRF51 <https://platformio.org/platforms/nordicnrf51?utm_source=github&utm_medium=core>`_
-* `Nordic nRF52 <https://platformio.org/platforms/nordicnrf52?utm_source=github&utm_medium=core>`_
-* `Nuclei <https://platformio.org/platforms/nuclei?utm_source=github&utm_medium=core>`_
-* `NXP LPC <https://platformio.org/platforms/nxplpc?utm_source=github&utm_medium=core>`_
-* `RISC-V <https://platformio.org/platforms/riscv?utm_source=github&utm_medium=core>`_
-* `RISC-V GAP <https://platformio.org/platforms/riscv_gap?utm_source=github&utm_medium=core>`_
-* `Shakti <https://platformio.org/platforms/shakti?utm_source=github&utm_medium=core>`_
-* `Silicon Labs EFM32 <https://platformio.org/platforms/siliconlabsefm32?utm_source=github&utm_medium=core>`_
-* `ST STM32 <https://platformio.org/platforms/ststm32?utm_source=github&utm_medium=core>`_
-* `ST STM8 <https://platformio.org/platforms/ststm8?utm_source=github&utm_medium=core>`_
-* `Teensy <https://platformio.org/platforms/teensy?utm_source=github&utm_medium=core>`_
-* `TI MSP430 <https://platformio.org/platforms/timsp430?utm_source=github&utm_medium=core>`_
-* `TI Tiva <https://platformio.org/platforms/titiva?utm_source=github&utm_medium=core>`_
-* `WIZNet W7500 <https://platformio.org/platforms/wiznet7500?utm_source=github&utm_medium=core>`_
-
-Frameworks
-----------
-
-* `Arduino <https://platformio.org/frameworks/arduino?utm_source=github&utm_medium=core>`_
-* `CMSIS <https://platformio.org/frameworks/cmsis?utm_source=github&utm_medium=core>`_
-* `ESP-IDF <https://platformio.org/frameworks/espidf?utm_source=github&utm_medium=core>`_
-* `ESP8266 Non-OS SDK <https://platformio.org/frameworks/esp8266-nonos-sdk?utm_source=github&utm_medium=core>`_
-* `ESP8266 RTOS SDK <https://platformio.org/frameworks/esp8266-rtos-sdk?utm_source=github&utm_medium=core>`_
-* `Freedom E SDK <https://platformio.org/frameworks/freedom-e-sdk?utm_source=github&utm_medium=core>`_
-* `GigaDevice GD32V SDK <https://platformio.org/frameworks/gd32vf103-sdk?utm_source=github&utm_medium=core>`_
-* `Kendryte Standalone SDK <https://platformio.org/frameworks/kendryte-standalone-sdk?utm_source=github&utm_medium=core>`_
-* `Kendryte FreeRTOS SDK <https://platformio.org/frameworks/kendryte-freertos-sdk?utm_source=github&utm_medium=core>`_
-* `libOpenCM3 <https://platformio.org/frameworks/libopencm3?utm_source=github&utm_medium=core>`_
-* `Mbed <https://platformio.org/frameworks/mbed?utm_source=github&utm_medium=core>`_
-* `Nuclei SDK <https://platformio.org/frameworks/nuclei-sdk?utm_source=github&utm_medium=core>`_
-* `PULP OS <https://platformio.org/frameworks/pulp-os?utm_source=github&utm_medium=core>`_
-* `Pumbaa <https://platformio.org/frameworks/pumbaa?utm_source=github&utm_medium=core>`_
-* `Shakti SDK <https://platformio.org/frameworks/shakti-sdk?utm_source=github&utm_medium=core>`_
-* `Simba <https://platformio.org/frameworks/simba?utm_source=github&utm_medium=core>`_
-* `SPL <https://platformio.org/frameworks/spl?utm_source=github&utm_medium=core>`_
-* `STM32Cube <https://platformio.org/frameworks/stm32cube?utm_source=github&utm_medium=core>`_
-* `WiringPi <https://platformio.org/frameworks/wiringpi?utm_source=github&utm_medium=core>`_
-* `Zephyr <https://platformio.org/frameworks/zephyr?utm_source=github&utm_medium=core>`_
+* `Libraries <https://registry.platformio.org/search?t=library&utm_source=github&utm_medium=core>`_
+* `Development Platforms <https://registry.platformio.org/search?t=platform&utm_source=github&utm_medium=core>`_
+* `Development Tools <https://registry.platformio.org/search?t=tool&utm_source=github&utm_medium=core>`_
 
 Contributing
 ------------
@@ -142,7 +90,7 @@ Telemetry / Privacy Policy
 Share minimal diagnostics and usage information to help us make PlatformIO better.
 It is enabled by default. For more information see:
 
-* `Telemetry Setting <https://docs.platformio.org/page/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
+* `Telemetry Setting <https://docs.platformio.org/en/latest/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
 
 License
 -------

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (5, 2, 0)
+VERSION = (6, 0, 2)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -38,27 +38,25 @@ __license__ = "Apache Software License"
 __copyright__ = "Copyright 2014-present PlatformIO Labs"
 
 __accounts_api__ = "https://api.accounts.platformio.org"
-__registry_api__ = [
-    "https://api.registry.platformio.org",
-    "https://api.registry.ns1.platformio.org",
+__registry_mirror_hosts__ = [
+    "registry.platformio.org",
+    "registry.nm1.platformio.org",
 ]
 __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
 
 __default_requests_timeout__ = (10, None)  # (connect, read)
 
 __core_packages__ = {
-    "contrib-piohome": "~3.3.4",
+    "contrib-piohome": "~3.4.1",
     "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
-    "tool-unity": "~1.20500.0",
-    "tool-scons": "~4.40200.0",
-    "tool-cppcheck": "~1.250.0",
+    "tool-scons": "~4.40300.0",
+    "tool-cppcheck": "~1.270.0",
     "tool-clangtidy": "~1.120001.0",
-    "tool-pvs-studio": "~7.14.0",
+    "tool-pvs-studio": "~7.18.0",
 }
 
 __check_internet_hosts__ = [
     "185.199.110.153",  # Github.com
     "88.198.170.159",  # platformio.org
     "github.com",
-    "platformio.org",
-]
+] + __registry_mirror_hosts__

platformio/__main__.py

@@ -12,31 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=import-outside-toplevel
-
 import os
 import sys
 from traceback import format_exc
 
 import click
 
-from platformio import __version__, exception
-from platformio.commands import PlatformioCLI
+from platformio import __version__, exception, maintenance
+from platformio.cli import PlatformioCLI
 from platformio.compat import IS_CYGWIN, ensure_python3
 
-try:
-    import click_completion  # pylint: disable=import-error
-
-    click_completion.init()
-except:  # pylint: disable=bare-except
-    pass
-
 
 @click.command(
     cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])
 )
 @click.version_option(__version__, prog_name="PlatformIO Core")
-@click.option("--force", "-f", is_flag=True, help="DEPRECATE")
+@click.option("--force", "-f", is_flag=True, help="DEPRECATED", hidden=True)
 @click.option("--caller", "-c", help="Caller ID (service)")
 @click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters")
 @click.pass_context
@@ -62,16 +53,12 @@ def cli(ctx, force, caller, no_ansi):
     except:  # pylint: disable=bare-except
         pass
 
-    from platformio import maintenance
-
     maintenance.on_platformio_start(ctx, force, caller)
 
 
-@cli.resultcallback()
+@cli.result_callback()
 @click.pass_context
 def process_result(ctx, result, *_, **__):
-    from platformio import maintenance
-
     maintenance.on_platformio_end(ctx, result)
 
 
@@ -118,10 +105,7 @@ def main(argv=None):
             exit_code = int(e.code)
     except Exception as e:  # pylint: disable=broad-except
         if not isinstance(e, exception.ReturnErrorCode):
-            if sys.version_info.major != 2:
-                from platformio import maintenance
-
-                maintenance.on_platformio_exception(e)
+            maintenance.on_platformio_exception(e)
             error_str = "Error: "
             if isinstance(e, exception.PlatformioException):
                 error_str += str(e)
@@ -136,7 +120,7 @@ An unexpected error occurred. Further steps:
   `pip install -U platformio` command
 
 * Try to find answer in FAQ Troubleshooting section
-  https://docs.platformio.org/page/faq.html
+  https://docs.platformio.org/page/faq/index.html
 
 * Report this problem to the developers
   https://github.com/platformio/platformio-core/issues

platformio/account/cli.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.commands.destroy import account_destroy_cmd
+from platformio.account.commands.forgot import account_forgot_cmd
+from platformio.account.commands.login import account_login_cmd
+from platformio.account.commands.logout import account_logout_cmd
+from platformio.account.commands.password import account_password_cmd
+from platformio.account.commands.register import account_register_cmd
+from platformio.account.commands.show import account_show_cmd
+from platformio.account.commands.token import account_token_cmd
+from platformio.account.commands.update import account_update_cmd
+
+
+@click.group(
+    "account",
+    commands=[
+        account_destroy_cmd,
+        account_forgot_cmd,
+        account_login_cmd,
+        account_logout_cmd,
+        account_password_cmd,
+        account_register_cmd,
+        account_show_cmd,
+        account_token_cmd,
+        account_update_cmd,
+    ],
+    short_help="Manage PlatformIO account",
+)
+def cli():
+    pass

platformio/account/client.py

@@ -16,8 +16,8 @@ import os
 import time
 
 from platformio import __accounts_api__, app
-from platformio.clients.http import HTTPClient
 from platformio.exception import PlatformioException
+from platformio.http import HTTPClient, HTTPClientError
 
 
 class AccountError(PlatformioException):
@@ -40,7 +40,7 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
     SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
 
     def __init__(self):
-        super(AccountClient, self).__init__(__accounts_api__)
+        super().__init__(__accounts_api__)
 
     @staticmethod
     def get_refresh_token():
@@ -61,13 +61,33 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         del account[key]
         app.set_state_item("account", account)
 
-    def send_auth_request(self, *args, **kwargs):
-        headers = kwargs.get("headers", {})
-        if "Authorization" not in headers:
-            token = self.fetch_authentication_token()
-            headers["Authorization"] = "Bearer %s" % token
-        kwargs["headers"] = headers
-        return self.fetch_json_data(*args, **kwargs)
+    def fetch_json_data(self, *args, **kwargs):
+        try:
+            return super().fetch_json_data(*args, **kwargs)
+        except HTTPClientError as exc:
+            raise AccountError(exc) from exc
+
+    def fetch_authentication_token(self):
+        if os.environ.get("PLATFORMIO_AUTH_TOKEN"):
+            return os.environ.get("PLATFORMIO_AUTH_TOKEN")
+        auth = app.get_state_item("account", {}).get("auth", {})
+        if auth.get("access_token") and auth.get("access_token_expire"):
+            if auth.get("access_token_expire") > time.time():
+                return auth.get("access_token")
+            if auth.get("refresh_token"):
+                try:
+                    data = self.fetch_json_data(
+                        "post",
+                        "/v1/login",
+                        headers={
+                            "Authorization": "Bearer %s" % auth.get("refresh_token")
+                        },
+                    )
+                    app.set_state_item("account", data)
+                    return data.get("auth").get("access_token")
+                except AccountError:
+                    self.delete_local_session()
+        raise AccountNotAuthorized()
 
     def login(self, username, password):
         try:
@@ -119,10 +139,11 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         return True
 
     def change_password(self, old_password, new_password):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/password",
             data={"old_password": old_password, "new_password": new_password},
+            x_with_authorization=True,
         )
 
     def registration(
@@ -150,10 +171,11 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         )
 
     def auth_token(self, password, regenerate):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/token",
             data={"password": password, "regenerate": 1 if regenerate else 0},
+            x_with_authorization=True,
         ).get("auth_token")
 
     def forgot_password(self, username):
@@ -164,18 +186,20 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         )
 
     def get_profile(self):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "get",
             "/v1/profile",
+            x_with_authorization=True,
         )
 
     def update_profile(self, profile, current_password):
         profile["current_password"] = current_password
         self.delete_local_state("summary")
-        response = self.send_auth_request(
+        response = self.fetch_json_data(
             "put",
             "/v1/profile",
             data=profile,
+            x_with_authorization=True,
         )
         return response
 
@@ -193,9 +217,10 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
                     "username": account.get("username"),
                 }
             }
-        result = self.send_auth_request(
+        result = self.fetch_json_data(
             "get",
             "/v1/summary",
+            x_with_authorization=True,
         )
         account["summary"] = dict(
             profile=result.get("profile"),
@@ -211,119 +236,121 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         return self.get_account_info(offline=True).get("profile").get("username")
 
     def destroy_account(self):
-        return self.send_auth_request("delete", "/v1/account")
+        return self.fetch_json_data(
+            "delete",
+            "/v1/account",
+            x_with_authorization=True,
+        )
 
     def create_org(self, orgname, email, displayname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/orgs",
             data={"orgname": orgname, "email": email, "displayname": displayname},
+            x_with_authorization=True,
         )
 
     def get_org(self, orgname):
-        return self.send_auth_request("get", "/v1/orgs/%s" % orgname)
+        return self.fetch_json_data(
+            "get",
+            "/v1/orgs/%s" % orgname,
+            x_with_authorization=True,
+        )
 
     def list_orgs(self):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "get",
             "/v1/orgs",
+            x_with_authorization=True,
         )
 
     def update_org(self, orgname, data):
-        return self.send_auth_request(
-            "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v}
+        return self.fetch_json_data(
+            "put",
+            "/v1/orgs/%s" % orgname,
+            data={k: v for k, v in data.items() if v},
+            x_with_authorization=True,
         )
 
     def destroy_org(self, orgname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s" % orgname,
+            x_with_authorization=True,
         )
 
     def add_org_owner(self, orgname, username):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/orgs/%s/owners" % orgname,
             data={"username": username},
+            x_with_authorization=True,
         )
 
     def list_org_owners(self, orgname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "get",
             "/v1/orgs/%s/owners" % orgname,
+            x_with_authorization=True,
         )
 
     def remove_org_owner(self, orgname, username):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s/owners" % orgname,
             data={"username": username},
+            x_with_authorization=True,
         )
 
     def create_team(self, orgname, teamname, description):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/orgs/%s/teams" % orgname,
             data={"name": teamname, "description": description},
+            x_with_authorization=True,
         )
 
     def destroy_team(self, orgname, teamname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s/teams/%s" % (orgname, teamname),
+            x_with_authorization=True,
         )
 
     def get_team(self, orgname, teamname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "get",
             "/v1/orgs/%s/teams/%s" % (orgname, teamname),
+            x_with_authorization=True,
         )
 
     def list_teams(self, orgname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "get",
             "/v1/orgs/%s/teams" % orgname,
+            x_with_authorization=True,
         )
 
     def update_team(self, orgname, teamname, data):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "put",
             "/v1/orgs/%s/teams/%s" % (orgname, teamname),
             data={k: v for k, v in data.items() if v},
+            x_with_authorization=True,
         )
 
     def add_team_member(self, orgname, teamname, username):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/orgs/%s/teams/%s/members" % (orgname, teamname),
             data={"username": username},
+            x_with_authorization=True,
         )
 
     def remove_team_member(self, orgname, teamname, username):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s/teams/%s/members" % (orgname, teamname),
             data={"username": username},
+            x_with_authorization=True,
         )
-
-    def fetch_authentication_token(self):
-        if os.environ.get("PLATFORMIO_AUTH_TOKEN"):
-            return os.environ.get("PLATFORMIO_AUTH_TOKEN")
-        auth = app.get_state_item("account", {}).get("auth", {})
-        if auth.get("access_token") and auth.get("access_token_expire"):
-            if auth.get("access_token_expire") > time.time():
-                return auth.get("access_token")
-            if auth.get("refresh_token"):
-                try:
-                    data = self.fetch_json_data(
-                        "post",
-                        "/v1/login",
-                        headers={
-                            "Authorization": "Bearer %s" % auth.get("refresh_token")
-                        },
-                    )
-                    app.set_state_item("account", data)
-                    return data.get("auth").get("access_token")
-                except AccountError:
-                    self.delete_local_session()
-        raise AccountNotAuthorized()

platformio/account/commands/destroy.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient, AccountNotAuthorized
+
+
+@click.command("destroy", short_help="Destroy account")
+def account_destroy_cmd():
+    client = AccountClient()
+    click.confirm(
+        "Are you sure you want to delete the %s user account?\n"
+        "Warning! All linked data will be permanently removed and can not be restored."
+        % client.get_logged_username(),
+        abort=True,
+    )
+    client.destroy_account()
+    try:
+        client.logout()
+    except AccountNotAuthorized:
+        pass
+    click.secho(
+        "User account has been destroyed.",
+        fg="green",
+    )

platformio/account/commands/forgot.py

@@ -12,16 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=unused-argument
+import click
 
-from platformio.commands.update import cli as cmd_update
+from platformio.account.client import AccountClient
 
 
-def test_update(clirunner, validate_cliresult, isolated_pio_core):
-    matches = ("Platform Manager", "Library Manager")
-    result = clirunner.invoke(cmd_update, ["--only-check"])
-    validate_cliresult(result)
-    assert all(m in result.output for m in matches)
-    result = clirunner.invoke(cmd_update)
-    validate_cliresult(result)
-    assert all(m in result.output for m in matches)
+@click.command("forgot", short_help="Forgot password")
+@click.option("--username", prompt="Username or email")
+def account_forgot_cmd(username):
+    client = AccountClient()
+    client.forgot_password(username)
+    click.secho(
+        "If this account is registered, we will send the "
+        "further instructions to your email.",
+        fg="green",
+    )

platformio/account/commands/login.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("login", short_help="Log in to PlatformIO Account")
+@click.option("-u", "--username", prompt="Username or email")
+@click.option("-p", "--password", prompt=True, hide_input=True)
+def account_login_cmd(username, password):
+    client = AccountClient()
+    client.login(username, password)
+    click.secho("Successfully logged in!", fg="green")

platformio/account/commands/logout.py

@@ -12,19 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
+import click
 
-from platformio import exception
+from platformio.account.client import AccountClient
 
 
-def get_test_names(config):
-    test_dir = config.get_optional_dir("test")
-    if not os.path.isdir(test_dir):
-        raise exception.TestDirNotExists(test_dir)
-    names = []
-    for item in sorted(os.listdir(test_dir)):
-        if os.path.isdir(os.path.join(test_dir, item)):
-            names.append(item)
-    if not names:
-        names = ["*"]
-    return names
+@click.command("logout", short_help="Log out of PlatformIO Account")
+def account_logout_cmd():
+    client = AccountClient()
+    client.logout()
+    click.secho("Successfully logged out!", fg="green")

platformio/account/commands/password.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("password", short_help="Change password")
+@click.option("--old-password", prompt=True, hide_input=True)
+@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
+def account_password_cmd(old_password, new_password):
+    client = AccountClient()
+    client.change_password(old_password, new_password)
+    click.secho("Password successfully changed!", fg="green")

platformio/account/commands/register.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import (
+    validate_email,
+    validate_password,
+    validate_username,
+)
+
+
+@click.command("register", short_help="Create new PlatformIO Account")
+@click.option(
+    "-u",
+    "--username",
+    prompt=True,
+    callback=lambda _, __, value: validate_username(value),
+)
+@click.option(
+    "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value)
+)
+@click.option(
+    "-p",
+    "--password",
+    prompt=True,
+    hide_input=True,
+    confirmation_prompt=True,
+    callback=lambda _, __, value: validate_password(value),
+)
+@click.option("--firstname", prompt=True)
+@click.option("--lastname", prompt=True)
+def account_register_cmd(username, email, password, firstname, lastname):
+    client = AccountClient()
+    client.registration(username, email, password, firstname, lastname)
+    click.secho(
+        "An account has been successfully created. "
+        "Please check your mail to activate your account and verify your email address.",
+        fg="green",
+    )

platformio/account/commands/show.py

@@ -0,0 +1,116 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio import util
+from platformio.account.client import AccountClient
+
+
+@click.command("show", short_help="PlatformIO Account information")
+@click.option("--offline", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_show_cmd(offline, json_output):
+    client = AccountClient()
+    info = client.get_account_info(offline)
+    if json_output:
+        click.echo(json.dumps(info))
+        return
+    click.echo()
+    if info.get("profile"):
+        print_profile(info["profile"])
+    if info.get("packages"):
+        print_packages(info["packages"])
+    if info.get("subscriptions"):
+        print_subscriptions(info["subscriptions"])
+    click.echo()
+
+
+def print_profile(profile):
+    click.secho("Profile", fg="cyan", bold=True)
+    click.echo("=" * len("Profile"))
+    data = []
+    if profile.get("username"):
+        data.append(("Username:", profile["username"]))
+    if profile.get("email"):
+        data.append(("Email:", profile["email"]))
+    if profile.get("firstname"):
+        data.append(("First name:", profile["firstname"]))
+    if profile.get("lastname"):
+        data.append(("Last name:", profile["lastname"]))
+    click.echo(tabulate(data, tablefmt="plain"))
+
+
+def print_packages(packages):
+    click.echo()
+    click.secho("Packages", fg="cyan")
+    click.echo("=" * len("Packages"))
+    for package in packages:
+        click.echo()
+        click.secho(package.get("name"), bold=True)
+        click.echo("-" * len(package.get("name")))
+        if package.get("description"):
+            click.echo(package.get("description"))
+        data = []
+        expire = "-"
+        if "subscription" in package:
+            expire = util.parse_datetime(
+                package["subscription"].get("end_at")
+                or package["subscription"].get("next_bill_at")
+            ).strftime("%Y-%m-%d")
+        data.append(("Expire:", expire))
+        services = []
+        for key in package:
+            if not key.startswith("service."):
+                continue
+            if isinstance(package[key], dict):
+                services.append(package[key].get("title"))
+            else:
+                services.append(package[key])
+        if services:
+            data.append(("Services:", ", ".join(services)))
+        click.echo(tabulate(data, tablefmt="plain"))
+
+
+def print_subscriptions(subscriptions):
+    click.echo()
+    click.secho("Subscriptions", fg="cyan")
+    click.echo("=" * len("Subscriptions"))
+    for subscription in subscriptions:
+        click.echo()
+        click.secho(subscription.get("product_name"), bold=True)
+        click.echo("-" * len(subscription.get("product_name")))
+        data = [("State:", subscription.get("status"))]
+        begin_at = util.parse_datetime(subscription.get("begin_at")).strftime("%c")
+        data.append(("Start date:", begin_at or "-"))
+        end_at = subscription.get("end_at")
+        if end_at:
+            end_at = util.parse_datetime(subscription.get("end_at")).strftime("%c")
+        data.append(("End date:", end_at or "-"))
+        next_bill_at = subscription.get("next_bill_at")
+        if next_bill_at:
+            next_bill_at = util.parse_datetime(
+                subscription.get("next_bill_at")
+            ).strftime("%c")
+        data.append(("Next payment:", next_bill_at or "-"))
+        data.append(
+            ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")
+        )
+        data.append(
+            ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-")
+        )
+        click.echo(tabulate(data, tablefmt="plain"))

platformio/account/commands/token.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("token", short_help="Get or regenerate Authentication Token")
+@click.option("-p", "--password", prompt=True, hide_input=True)
+@click.option("--regenerate", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_token_cmd(password, regenerate, json_output):
+    client = AccountClient()
+    auth_token = client.auth_token(password, regenerate)
+    if json_output:
+        click.echo(json.dumps({"status": "success", "result": auth_token}))
+        return
+    click.secho("Personal Authentication Token: %s" % auth_token, fg="green")

platformio/account/commands/update.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient, AccountNotAuthorized
+from platformio.account.validate import validate_email, validate_username
+
+
+@click.command("update", short_help="Update profile information")
+@click.option("--current-password", prompt=True, hide_input=True)
+@click.option("--username")
+@click.option("--email")
+@click.option("--firstname")
+@click.option("--lastname")
+def account_update_cmd(current_password, **kwargs):
+    client = AccountClient()
+    profile = client.get_profile()
+    new_profile = profile.copy()
+    if not any(kwargs.values()):
+        for field in profile:
+            new_profile[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=profile[field]
+            )
+            if field == "email":
+                validate_email(new_profile[field])
+            if field == "username":
+                validate_username(new_profile[field])
+    else:
+        new_profile.update({key: value for key, value in kwargs.items() if value})
+    client.update_profile(new_profile, current_password)
+    click.secho("Profile successfully updated!", fg="green")
+    username_changed = new_profile["username"] != profile["username"]
+    email_changed = new_profile["email"] != profile["email"]
+    if not username_changed and not email_changed:
+        return None
+    try:
+        client.logout()
+    except AccountNotAuthorized:
+        pass
+    if email_changed:
+        click.secho(
+            "Please check your mail to verify your new email address and re-login. ",
+            fg="yellow",
+        )
+        return None
+    click.secho("Please re-login.", fg="yellow")
+    return None

platformio/account/org/cli.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.org.commands.add import org_add_cmd
+from platformio.account.org.commands.create import org_create_cmd
+from platformio.account.org.commands.destroy import org_destroy_cmd
+from platformio.account.org.commands.list import org_list_cmd
+from platformio.account.org.commands.remove import org_remove_cmd
+from platformio.account.org.commands.update import org_update_cmd
+
+
+@click.group(
+    "account",
+    commands=[
+        org_add_cmd,
+        org_create_cmd,
+        org_destroy_cmd,
+        org_list_cmd,
+        org_remove_cmd,
+        org_update_cmd,
+    ],
+    short_help="Manage organizations",
+)
+def cli():
+    pass

platformio/account/org/commands/add.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("add", short_help="Add a new owner to organization")
+@click.argument(
+    "orgname",
+)
+@click.argument(
+    "username",
+)
+def org_add_cmd(orgname, username):
+    client = AccountClient()
+    client.add_org_owner(orgname, username)
+    return click.secho(
+        "The new owner `%s` has been successfully added to the `%s` organization."
+        % (username, orgname),
+        fg="green",
+    )

platformio/account/org/commands/create.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_email, validate_orgname
+
+
+@click.command("create", short_help="Create a new organization")
+@click.argument(
+    "orgname",
+    callback=lambda _, __, value: validate_orgname(value),
+)
+@click.option(
+    "--email", callback=lambda _, __, value: validate_email(value) if value else value
+)
+@click.option(
+    "--displayname",
+)
+def org_create_cmd(orgname, email, displayname):
+    client = AccountClient()
+    client.create_org(orgname, email, displayname)
+    return click.secho(
+        "The organization `%s` has been successfully created." % orgname,
+        fg="green",
+    )

platformio/account/org/commands/destroy.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("destroy", short_help="Destroy organization")
+@click.argument("orgname")
+def org_destroy_cmd(orgname):
+    client = AccountClient()
+    click.confirm(
+        "Are you sure you want to delete the `%s` organization account?\n"
+        "Warning! All linked data will be permanently removed and can not be restored."
+        % orgname,
+        abort=True,
+    )
+    client.destroy_org(orgname)
+    return click.secho(
+        "Organization `%s` has been destroyed." % orgname,
+        fg="green",
+    )

platformio/account/org/commands/list.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio.account.client import AccountClient
+
+
+@click.command("list", short_help="List organizations and their members")
+@click.option("--json-output", is_flag=True)
+def org_list_cmd(json_output):
+    client = AccountClient()
+    orgs = client.list_orgs()
+    if json_output:
+        return click.echo(json.dumps(orgs))
+    if not orgs:
+        return click.echo("You do not have any organization")
+    for org in orgs:
+        click.echo()
+        click.secho(org.get("orgname"), fg="cyan")
+        click.echo("-" * len(org.get("orgname")))
+        data = []
+        if org.get("displayname"):
+            data.append(("Display Name:", org.get("displayname")))
+        if org.get("email"):
+            data.append(("Email:", org.get("email")))
+        data.append(
+            (
+                "Owners:",
+                ", ".join((owner.get("username") for owner in org.get("owners"))),
+            )
+        )
+        click.echo(tabulate(data, tablefmt="plain"))
+    return click.echo()

platformio/account/org/commands/remove.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("remove", short_help="Remove an owner from organization")
+@click.argument(
+    "orgname",
+)
+@click.argument(
+    "username",
+)
+def org_remove_cmd(orgname, username):
+    client = AccountClient()
+    client.remove_org_owner(orgname, username)
+    return click.secho(
+        "The `%s` owner has been successfully removed from the `%s` organization."
+        % (username, orgname),
+        fg="green",
+    )

platformio/account/org/commands/update.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_email, validate_orgname
+
+
+@click.command("update", short_help="Update organization")
+@click.argument("cur_orgname")
+@click.option(
+    "--orgname",
+    callback=lambda _, __, value: validate_orgname(value),
+    help="A new orgname",
+)
+@click.option("--email")
+@click.option("--displayname")
+def org_update_cmd(cur_orgname, **kwargs):
+    client = AccountClient()
+    org = client.get_org(cur_orgname)
+    del org["owners"]
+    new_org = org.copy()
+    if not any(kwargs.values()):
+        for field in org:
+            new_org[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=org[field]
+            )
+            if field == "email":
+                validate_email(new_org[field])
+            if field == "orgname":
+                validate_orgname(new_org[field])
+    else:
+        new_org.update(
+            {key.replace("new_", ""): value for key, value in kwargs.items() if value}
+        )
+    client.update_org(cur_orgname, new_org)
+    return click.secho(
+        "The organization `%s` has been successfully updated." % cur_orgname,
+        fg="green",
+    )

platformio/account/team/cli.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.team.commands.add import team_add_cmd
+from platformio.account.team.commands.create import team_create_cmd
+from platformio.account.team.commands.destroy import team_destroy_cmd
+from platformio.account.team.commands.list import team_list_cmd
+from platformio.account.team.commands.remove import team_remove_cmd
+from platformio.account.team.commands.update import team_update_cmd
+
+
+@click.group(
+    "team",
+    commands=[
+        team_add_cmd,
+        team_create_cmd,
+        team_destroy_cmd,
+        team_list_cmd,
+        team_remove_cmd,
+        team_update_cmd,
+    ],
+    short_help="Manage organization teams",
+)
+def cli():
+    pass

platformio/account/team/commands/add.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("add", short_help="Add a new member to team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.argument(
+    "username",
+)
+def team_add_cmd(orgname_teamname, username):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.add_team_member(orgname, teamname, username)
+    return click.secho(
+        "The new member %s has been successfully added to the %s team."
+        % (username, teamname),
+        fg="green",
+    )

platformio/account/team/commands/create.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("create", short_help="Create a new team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(
+        value, teamname_validate=True
+    ),
+)
+@click.option(
+    "--description",
+)
+def team_create_cmd(orgname_teamname, description):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.create_team(orgname, teamname, description)
+    return click.secho(
+        "The team %s has been successfully created." % teamname,
+        fg="green",
+    )

platformio/account/team/commands/destroy.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("destroy", short_help="Destroy a team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+def team_destroy_cmd(orgname_teamname):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    click.confirm(
+        click.style(
+            "Are you sure you want to destroy the %s team?" % teamname, fg="yellow"
+        ),
+        abort=True,
+    )
+    client = AccountClient()
+    client.destroy_team(orgname, teamname)
+    return click.secho(
+        "The team %s has been successfully destroyed." % teamname,
+        fg="green",
+    )

platformio/account/team/commands/list.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio.account.client import AccountClient
+
+
+@click.command("list", short_help="List teams")
+@click.argument("orgname", required=False)
+@click.option("--json-output", is_flag=True)
+def team_list_cmd(orgname, json_output):
+    client = AccountClient()
+    data = {}
+    if not orgname:
+        for item in client.list_orgs():
+            teams = client.list_teams(item.get("orgname"))
+            data[item.get("orgname")] = teams
+    else:
+        teams = client.list_teams(orgname)
+        data[orgname] = teams
+    if json_output:
+        return click.echo(json.dumps(data[orgname] if orgname else data))
+    if not any(data.values()):
+        return click.secho("You do not have any teams.", fg="yellow")
+    for org_name in data:
+        for team in data[org_name]:
+            click.echo()
+            click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan")
+            click.echo("-" * len("%s:%s" % (org_name, team.get("name"))))
+            table_data = []
+            if team.get("description"):
+                table_data.append(("Description:", team.get("description")))
+            table_data.append(
+                (
+                    "Members:",
+                    ", ".join(
+                        (member.get("username") for member in team.get("members"))
+                    )
+                    if team.get("members")
+                    else "-",
+                )
+            )
+            click.echo(tabulate(table_data, tablefmt="plain"))
+    return click.echo()

platformio/account/team/commands/remove.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("remove", short_help="Remove a member from team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.argument("username")
+def team_remove_cmd(orgname_teamname, username):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.remove_team_member(orgname, teamname, username)
+    return click.secho(
+        "The %s member has been successfully removed from the %s team."
+        % (username, teamname),
+        fg="green",
+    )

platformio/account/team/commands/update.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname, validate_teamname
+
+
+@click.command("update", short_help="Update team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.option(
+    "--name",
+    callback=lambda _, __, value: validate_teamname(value),
+    help="A new team name",
+)
+@click.option(
+    "--description",
+)
+def team_update_cmd(orgname_teamname, **kwargs):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    team = client.get_team(orgname, teamname)
+    del team["id"]
+    del team["members"]
+    new_team = team.copy()
+    if not any(kwargs.values()):
+        for field in team:
+            new_team[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=team[field]
+            )
+            if field == "name":
+                validate_teamname(new_team[field])
+    else:
+        new_team.update({key: value for key, value in kwargs.items() if value})
+    client.update_team(orgname, teamname, new_team)
+    return click.secho(
+        "The team %s has been successfully updated." % teamname,
+        fg="green",
+    )

platformio/account/validate.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import click
+
+
+def validate_username(value, field="username"):
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I):
+        raise click.BadParameter(
+            "Invalid %s format. "
+            "%s must contain only alphanumeric characters "
+            "or single hyphens, cannot begin or end with a hyphen, "
+            "and must not be longer than 38 characters."
+            % (field.lower(), field.capitalize())
+        )
+    return value
+
+
+def validate_email(value):
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d_.+-]+@[a-z\d\-]+\.[a-z\d\-.]+$", value, flags=re.I):
+        raise click.BadParameter("Invalid email address")
+    return value
+
+
+def validate_password(value):
+    value = str(value).strip()
+    if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
+        raise click.BadParameter(
+            "Invalid password format. "
+            "Password must contain at least 8 characters"
+            " including a number and a lowercase letter"
+        )
+    return value
+
+
+def validate_orgname(value):
+    return validate_username(value, "Organization name")
+
+
+def validate_orgname_teamname(value, teamname_validate=False):
+    if ":" not in value:
+        raise click.BadParameter(
+            "Please specify organization and team name in the next"
+            " format - orgname:teamname. For example, mycompany:DreamTeam"
+        )
+    teamname = str(value.strip().split(":", 1)[1])
+    if teamname_validate:
+        validate_teamname(teamname)
+    return value
+
+
+def validate_teamname(value):
+    if not value:
+        return value
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I):
+        raise click.BadParameter(
+            "Invalid team name format. "
+            "Team name must only contain alphanumeric characters, "
+            "single hyphens, underscores, spaces. It can not "
+            "begin or end with a hyphen or a underscore and must"
+            " not be longer than 20 characters."
+        )
+    return value

platformio/app.py

@@ -21,38 +21,22 @@ import os
 import platform
 import socket
 import uuid
-from os.path import dirname, isdir, isfile, join, realpath
 
 from platformio import __version__, exception, fs, proc
 from platformio.compat import IS_WINDOWS, hashlib_encode_data
 from platformio.package.lockfile import LockFile
-from platformio.project.helpers import get_default_projects_dir, get_project_core_dir
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import get_default_projects_dir
 
 
 def projects_dir_validate(projects_dir):
-    assert isdir(projects_dir)
-    return realpath(projects_dir)
+    assert os.path.isdir(projects_dir)
+    return os.path.abspath(projects_dir)
 
 
 DEFAULT_SETTINGS = {
-    "auto_update_libraries": {
-        "description": "Automatically update libraries (Yes/No)",
-        "value": False,
-    },
-    "auto_update_platforms": {
-        "description": "Automatically update platforms (Yes/No)",
-        "value": False,
-    },
-    "check_libraries_interval": {
-        "description": "Check for the library updates interval (days)",
-        "value": 7,
-    },
     "check_platformio_interval": {
-        "description": "Check for the new PlatformIO interval (days)",
-        "value": 3,
-    },
-    "check_platforms_interval": {
-        "description": "Check for the platform updates interval (days)",
+        "description": "Check for the new PlatformIO Core interval (days)",
         "value": 7,
     },
     "check_prune_system_threshold": {
@@ -64,7 +48,7 @@ DEFAULT_SETTINGS = {
         "value": True,
     },
     "enable_telemetry": {
-        "description": ("Telemetry service <http://bit.ly/pio-telemetry> (Yes/No)"),
+        "description": ("Telemetry service <https://bit.ly/pio-telemetry> (Yes/No)"),
         "value": True,
     },
     "force_verbose": {
@@ -91,7 +75,10 @@ class State(object):
         self.path = path
         self.lock = lock
         if not self.path:
-            self.path = join(get_project_core_dir(), "appstate.json")
+            core_dir = ProjectConfig.get_instance().get("platformio", "core_dir")
+            if not os.path.isdir(core_dir):
+                os.makedirs(core_dir)
+            self.path = os.path.join(core_dir, "appstate.json")
         self._storage = {}
         self._lockfile = None
         self.modified = False
@@ -99,7 +86,7 @@ class State(object):
     def __enter__(self):
         try:
             self._lock_state_file()
-            if isfile(self.path):
+            if os.path.isfile(self.path):
                 self._storage = fs.load_json(self.path)
             assert isinstance(self._storage, dict)
         except (
@@ -117,7 +104,7 @@ class State(object):
                 with open(self.path, mode="w", encoding="utf8") as fp:
                     fp.write(json.dumps(self._storage))
             except IOError:
-                raise exception.HomeDirPermissionsError(get_project_core_dir())
+                raise exception.HomeDirPermissionsError(os.path.dirname(self.path))
         self._unlock_state_file()
 
     def _lock_state_file(self):
@@ -127,7 +114,7 @@ class State(object):
         try:
             self._lockfile.acquire()
         except IOError:
-            raise exception.HomeDirPermissionsError(dirname(self.path))
+            raise exception.HomeDirPermissionsError(os.path.dirname(self.path))
 
     def _unlock_state_file(self):
         if hasattr(self, "_lockfile") and self._lockfile:
@@ -250,29 +237,14 @@ def is_disabled_progressbar():
 
 
 def get_cid():
-    # pylint: disable=import-outside-toplevel
-    from platformio.clients.http import fetch_remote_content
-
     cid = get_state_item("cid")
     if cid:
         return cid
     uid = None
-    if os.getenv("C9_UID"):
-        uid = os.getenv("C9_UID")
+    if os.getenv("GITHUB_USER"):
+        uid = os.getenv("GITHUB_USER")
     elif os.getenv("GITPOD_GIT_USER_NAME"):
         uid = os.getenv("GITPOD_GIT_USER_NAME")
-    elif os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")):
-        try:
-            uid = json.loads(
-                fetch_remote_content(
-                    "{api}/user?token={token}".format(
-                        api=os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")),
-                        token=os.getenv("USER_TOKEN"),
-                    )
-                )
-            ).get("id")
-        except:  # pylint: disable=bare-except
-            pass
     if not uid:
         uid = uuid.getnode()
     cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest())

platformio/builder/main.py

@@ -44,25 +44,28 @@ clivars.AddVariables(
     ("PIOENV",),
     ("PIOTEST_RUNNING_NAME",),
     ("UPLOAD_PORT",),
+    ("PROGRAM_ARGS",),
 )
 
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
         "ar",
-        "as",
         "cc",
         "c++",
         "link",
+        "pioasm",
         "platformio",
-        "piotarget",
-        "pioplatform",
         "pioproject",
+        "pioplatform",
+        "piotest",
+        "piotarget",
         "piomaxlen",
         "piolib",
         "pioupload",
-        "piomisc",
-        "pioide",
         "piosize",
+        "pioino",
+        "piomisc",
+        "piointegration",
     ],
     toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")],
     variables=clivars,
@@ -72,7 +75,7 @@ DEFAULT_ENV_OPTIONS = dict(
     BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
     BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
     BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
-    COMPILATIONDB_PATH=os.path.join("$BUILD_DIR", "compile_commands.json"),
+    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
     LIBPATH=["$BUILD_DIR"],
     PROGNAME="program",
     PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
@@ -109,34 +112,39 @@ env.Replace(
 config = env.GetProjectConfig()
 env.Replace(
     PROJECT_DIR=get_project_dir(),
-    PROJECT_CORE_DIR=config.get_optional_dir("core"),
-    PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"),
-    PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"),
-    PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"),
-    PROJECT_INCLUDE_DIR=config.get_optional_dir("include"),
-    PROJECT_SRC_DIR=config.get_optional_dir("src"),
-    PROJECTSRC_DIR=config.get_optional_dir("src"),  # legacy for dev/platform
-    PROJECT_TEST_DIR=config.get_optional_dir("test"),
-    PROJECT_DATA_DIR=config.get_optional_dir("data"),
-    PROJECTDATA_DIR=config.get_optional_dir("data"),  # legacy for dev/platform
-    PROJECT_BUILD_DIR=config.get_optional_dir("build"),
-    BUILD_CACHE_DIR=config.get_optional_dir("build_cache"),
+    PROJECT_CORE_DIR=config.get("platformio", "core_dir"),
+    PROJECT_PACKAGES_DIR=config.get("platformio", "packages_dir"),
+    PROJECT_WORKSPACE_DIR=config.get("platformio", "workspace_dir"),
+    PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"),
+    PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"),
+    PROJECT_SRC_DIR=config.get("platformio", "src_dir"),
+    PROJECTSRC_DIR="$PROJECT_SRC_DIR",  # legacy for dev/platform
+    PROJECT_TEST_DIR=config.get("platformio", "test_dir"),
+    PROJECT_DATA_DIR=config.get("platformio", "data_dir"),
+    PROJECTDATA_DIR="$PROJECT_DATA_DIR",  # legacy for dev/platform
+    PROJECT_BUILD_DIR=config.get("platformio", "build_dir"),
+    BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"),
     LIBSOURCE_DIRS=[
-        config.get_optional_dir("lib"),
+        config.get("platformio", "lib_dir"),
         os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
-        config.get_optional_dir("globallib"),
+        config.get("platformio", "globallib_dir"),
     ],
 )
 
-if (
-    compat.IS_WINDOWS
-    and sys.version_info >= (3, 8)
-    and env["PROJECT_DIR"].startswith("\\\\")
-):
+if int(ARGUMENTS.get("ISATTY", 0)):
+    # pylint: disable=protected-access
+    click._compat.isatty = lambda stream: True
+
+if compat.IS_WINDOWS and sys.version_info >= (3, 8) and os.getcwd().startswith("\\\\"):
+    click.secho("!!! WARNING !!!\t\t" * 3, fg="red")
     click.secho(
-        "There is a known issue with Python 3.8+ and mapped network drives on "
-        "Windows.\nSee a solution at:\n"
-        "https://github.com/platformio/platformio-core/issues/3417",
+        "Your project is located on a mapped network drive but the "
+        "current command-line shell does not support the UNC paths.",
+        fg="yellow",
+    )
+    click.secho(
+        "Please move your project to a physical drive or check this workaround: "
+        "https://bit.ly/3kuU5mP\n",
         fg="yellow",
     )
 
@@ -145,14 +153,12 @@ if env.subst("$BUILD_CACHE_DIR"):
         os.makedirs(env.subst("$BUILD_CACHE_DIR"))
     env.CacheDir("$BUILD_CACHE_DIR")
 
-if int(ARGUMENTS.get("ISATTY", 0)):
-    # pylint: disable=protected-access
-    click._compat.isatty = lambda stream: True
-
-if env.GetOption("clean"):
-    env.PioClean(env.subst("$BUILD_DIR"))
+is_clean_all = "cleanall" in COMMAND_LINE_TARGETS
+if env.GetOption("clean") or is_clean_all:
+    env.PioClean(is_clean_all)
     env.Exit(0)
-elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
+
+if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     click.echo("Verbose mode can be enabled via `-v, --verbose` option")
 
 # Dynamically load dependent tools
@@ -219,13 +225,14 @@ if "envdump" in COMMAND_LINE_TARGETS:
     click.echo(env.Dump())
     env.Exit(0)
 
-if set(["_idedata", "idedata"]) & set(COMMAND_LINE_TARGETS):
+if env.IsIntegrationDump():
+    projenv = None
     try:
         Import("projenv")
     except:  # pylint: disable=bare-except
         projenv = env
-    data = projenv.DumpIDEData(env)
-    # dump to file for the further reading by project.helpers.load_project_ide_data
+    data = projenv.DumpIntegrationData(env)
+    # dump to file for the further reading by project.helpers.load_build_metadata
     with open(
         projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")),
         mode="w",

platformio/builder/tools/pioasm.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import SCons.Tool.asm  # pylint: disable=import-error
+
+#
+# Resolve https://github.com/platformio/platformio-core/issues/3917
+# Avoid forcing .S to bare assembly on Windows OS
+#
+
+if ".S" in SCons.Tool.asm.ASSuffixes:
+    SCons.Tool.asm.ASSuffixes.remove(".S")
+if ".S" not in SCons.Tool.asm.ASPPSuffixes:
+    SCons.Tool.asm.ASPPSuffixes.append(".S")
+
+
+generate = SCons.Tool.asm.generate
+exists = SCons.Tool.asm.exists

platformio/builder/tools/pioino.py

@@ -0,0 +1,259 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import atexit
+import glob
+import io
+import os
+import re
+import tempfile
+
+import click
+
+from platformio.compat import get_filesystem_encoding, get_locale_encoding
+
+
+class InoToCPPConverter(object):
+
+    PROTOTYPE_RE = re.compile(
+        r"""^(
+        (?:template\<.*\>\s*)?      # template
+        ([a-z_\d\&]+\*?\s+){1,2}    # return type
+        ([a-z_\d]+\s*)              # name of prototype
+        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
+        )\s*(\{|;)                  # must end with `{` or `;`
+        """,
+        re.X | re.M | re.I,
+    )
+    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
+    PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
+
+    def __init__(self, env):
+        self.env = env
+        self._main_ino = None
+        self._safe_encoding = None
+
+    def read_safe_contents(self, path):
+        error_reported = False
+        for encoding in (
+            "utf-8",
+            None,
+            get_filesystem_encoding(),
+            get_locale_encoding(),
+            "latin-1",
+        ):
+            try:
+                with io.open(path, encoding=encoding) as fp:
+                    contents = fp.read()
+                    self._safe_encoding = encoding
+                    return contents
+            except UnicodeDecodeError:
+                if not error_reported:
+                    error_reported = True
+                    click.secho(
+                        "Unicode decode error has occurred, please remove invalid "
+                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
+                        % path,
+                        fg="yellow",
+                        err=True,
+                    )
+        return ""
+
+    def write_safe_contents(self, path, contents):
+        with io.open(
+            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
+        ) as fp:
+            return fp.write(contents)
+
+    def is_main_node(self, contents):
+        return self.DETECTMAIN_RE.search(contents)
+
+    def convert(self, nodes):
+        contents = self.merge(nodes)
+        if not contents:
+            return None
+        return self.process(contents)
+
+    def merge(self, nodes):
+        assert nodes
+        lines = []
+        for node in nodes:
+            contents = self.read_safe_contents(node.get_path())
+            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
+            if self.is_main_node(contents):
+                lines = _lines + lines
+                self._main_ino = node.get_path()
+            else:
+                lines.extend(_lines)
+
+        if not self._main_ino:
+            self._main_ino = nodes[0].get_path()
+
+        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
+
+    def process(self, contents):
+        out_file = self._main_ino + ".cpp"
+        assert self._gcc_preprocess(contents, out_file)
+        contents = self.read_safe_contents(out_file)
+        contents = self._join_multiline_strings(contents)
+        self.write_safe_contents(out_file, self.append_prototypes(contents))
+        return out_file
+
+    def _gcc_preprocess(self, contents, out_file):
+        tmp_path = tempfile.mkstemp()[1]
+        self.write_safe_contents(tmp_path, contents)
+        self.env.Execute(
+            self.env.VerboseAction(
+                '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
+                    out_file, tmp_path
+                ),
+                "Converting " + os.path.basename(out_file[:-4]),
+            )
+        )
+        atexit.register(_delete_file, tmp_path)
+        return os.path.isfile(out_file)
+
+    def _join_multiline_strings(self, contents):
+        if "\\\n" not in contents:
+            return contents
+        newlines = []
+        linenum = 0
+        stropen = False
+        for line in contents.split("\n"):
+            _linenum = self._parse_preproc_line_num(line)
+            if _linenum is not None:
+                linenum = _linenum
+            else:
+                linenum += 1
+
+            if line.endswith("\\"):
+                if line.startswith('"'):
+                    stropen = True
+                    newlines.append(line[:-1])
+                    continue
+                if stropen:
+                    newlines[len(newlines) - 1] += line[:-1]
+                    continue
+            elif stropen and line.endswith(('",', '";')):
+                newlines[len(newlines) - 1] += line
+                stropen = False
+                newlines.append(
+                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
+                )
+                continue
+
+            newlines.append(line)
+
+        return "\n".join(newlines)
+
+    @staticmethod
+    def _parse_preproc_line_num(line):
+        if not line.startswith("#"):
+            return None
+        tokens = line.split(" ", 3)
+        if len(tokens) > 2 and tokens[1].isdigit():
+            return int(tokens[1])
+        return None
+
+    def _parse_prototypes(self, contents):
+        prototypes = []
+        reserved_keywords = set(["if", "else", "while"])
+        for match in self.PROTOTYPE_RE.finditer(contents):
+            if (
+                set([match.group(2).strip(), match.group(3).strip()])
+                & reserved_keywords
+            ):
+                continue
+            prototypes.append(match)
+        return prototypes
+
+    def _get_total_lines(self, contents):
+        total = 0
+        if contents.endswith("\n"):
+            contents = contents[:-1]
+        for line in contents.split("\n")[::-1]:
+            linenum = self._parse_preproc_line_num(line)
+            if linenum is not None:
+                return total + linenum
+            total += 1
+        return total
+
+    def append_prototypes(self, contents):
+        prototypes = self._parse_prototypes(contents) or []
+
+        # skip already declared prototypes
+        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
+        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
+
+        if not prototypes:
+            return contents
+
+        prototype_names = set(m.group(3).strip() for m in prototypes)
+        split_pos = prototypes[0].start()
+        match_ptrs = re.search(
+            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
+            contents[:split_pos],
+            re.M,
+        )
+        if match_ptrs:
+            split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
+
+        result = []
+        result.append(contents[:split_pos].strip())
+        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
+        result.append(
+            '#line %d "%s"'
+            % (
+                self._get_total_lines(contents[:split_pos]),
+                self._main_ino.replace("\\", "/"),
+            )
+        )
+        result.append(contents[split_pos:].strip())
+        return "\n".join(result)
+
+
+def FindInoNodes(env):
+    src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
+    return env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
+        os.path.join(src_dir, "*.pde")
+    )
+
+
+def ConvertInoToCpp(env):
+    ino_nodes = env.FindInoNodes()
+    if not ino_nodes:
+        return
+    c = InoToCPPConverter(env)
+    out_file = c.convert(ino_nodes)
+
+    atexit.register(_delete_file, out_file)
+
+
+def _delete_file(path):
+    try:
+        if os.path.isfile(path):
+            os.remove(path)
+    except:  # pylint: disable=bare-except
+        pass
+
+
+def generate(env):
+    env.AddMethod(FindInoNodes)
+    env.AddMethod(ConvertInoToCpp)
+
+
+def exists(_):
+    return True

platformio/builder/tools/piointegration.py

@@ -19,33 +19,37 @@ import os
 
 import SCons.Defaults  # pylint: disable=import-error
 import SCons.Subst  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 
-from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import exec_command, where_is_program
 
 
-def _dump_includes(env):
-    includes = {}
+def IsIntegrationDump(_):
+    return set(["_idedata", "idedata"]) & set(COMMAND_LINE_TARGETS)
 
-    includes["build"] = [
-        env.subst("$PROJECT_INCLUDE_DIR"),
-        env.subst("$PROJECT_SRC_DIR"),
-    ]
-    includes["build"].extend(
-        [os.path.realpath(env.subst(item)) for item in env.get("CPPPATH", [])]
+
+def DumpIntegrationIncludes(env):
+    result = dict(build=[], compatlib=[], toolchain=[])
+
+    result["build"].extend(
+        [
+            env.subst("$PROJECT_INCLUDE_DIR"),
+            env.subst("$PROJECT_SRC_DIR"),
+        ]
+    )
+    result["build"].extend(
+        [os.path.abspath(env.subst(item)) for item in env.get("CPPPATH", [])]
     )
 
     # installed libs
-    includes["compatlib"] = []
     for lb in env.GetLibBuilders():
-        includes["compatlib"].extend(
-            [os.path.realpath(inc) for inc in lb.get_include_dirs()]
+        result["compatlib"].extend(
+            [os.path.abspath(inc) for inc in lb.get_include_dirs()]
         )
 
     # includes from toolchains
     p = env.PioPlatform()
-    includes["toolchain"] = []
-    for pkg in p.get_installed_packages():
+    for pkg in p.get_installed_packages(with_optional=False):
         if p.get_package_type(pkg.metadata.name) != "toolchain":
             continue
         toolchain_dir = glob.escape(pkg.path)
@@ -56,27 +60,12 @@ def _dump_includes(env):
             os.path.join(toolchain_dir, "*", "include*"),
         ]
         for g in toolchain_incglobs:
-            includes["toolchain"].extend(
-                [os.path.realpath(inc) for inc in glob.glob(g)]
-            )
+            result["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)])
 
-    # include Unity framework if there are tests in project
-    includes["unity"] = []
-    auto_install_unity = False
-    test_dir = env.GetProjectConfig().get_optional_dir("test")
-    if os.path.isdir(test_dir) and os.listdir(test_dir) != ["README"]:
-        auto_install_unity = True
-    unity_dir = get_core_package_dir(
-        "tool-unity",
-        auto_install=auto_install_unity,
-    )
-    if unity_dir:
-        includes["unity"].append(unity_dir)
-
-    return includes
+    return result
 
 
-def _get_gcc_defines(env):
+def get_gcc_defines(env):
     items = []
     try:
         sysenv = os.environ.copy()
@@ -99,7 +88,7 @@ def _get_gcc_defines(env):
     return items
 
 
-def _dump_defines(env):
+def dump_defines(env):
     defines = []
     # global symbols
     for item in SCons.Defaults.processDefines(env.get("CPPDEFINES", [])):
@@ -124,15 +113,15 @@ def _dump_defines(env):
 
     # built-in GCC marcos
     # if env.GetCompilerType() == "gcc":
-    #     defines.extend(_get_gcc_defines(env))
+    #     defines.extend(get_gcc_defines(env))
 
     return defines
 
 
-def _get_svd_path(env):
+def dump_svd_path(env):
     svd_path = env.GetProjectOption("debug_svd_path")
     if svd_path:
-        return os.path.realpath(svd_path)
+        return os.path.abspath(svd_path)
 
     if "BOARD" not in env:
         return None
@@ -147,7 +136,7 @@ def _get_svd_path(env):
     # default file from ./platform/misc/svd folder
     p = env.PioPlatform()
     if os.path.isfile(os.path.join(p.get_dir(), "misc", "svd", svd_path)):
-        return os.path.realpath(os.path.join(p.get_dir(), "misc", "svd", svd_path))
+        return os.path.abspath(os.path.join(p.get_dir(), "misc", "svd", svd_path))
     return None
 
 
@@ -156,19 +145,19 @@ def _subst_cmd(env, cmd):
     return " ".join([SCons.Subst.quote_spaces(arg) for arg in args])
 
 
-def DumpIDEData(env, globalenv):
+def DumpIntegrationData(env, globalenv):
     """env here is `projenv`"""
 
     data = {
         "env_name": env["PIOENV"],
         "libsource_dirs": [env.subst(item) for item in env.GetLibSourceDirs()],
-        "defines": _dump_defines(env),
-        "includes": _dump_includes(env),
+        "defines": dump_defines(env),
+        "includes": env.DumpIntegrationIncludes(),
         "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
         "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
         "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
         "prog_path": env.subst("$PROG_PATH"),
-        "svd_path": _get_svd_path(env),
+        "svd_path": dump_svd_path(env),
         "compiler_type": env.GetCompilerType(),
         "targets": globalenv.DumpTargets(),
         "extra": dict(
@@ -178,7 +167,9 @@ def DumpIDEData(env, globalenv):
             ]
         ),
     }
-    data["extra"].update(env.get("IDE_EXTRA_DATA", {}))
+    data["extra"].update(
+        env.get("INTEGRATION_EXTRA_DATA", env.get("IDE_EXTRA_DATA", {}))
+    )
 
     env_ = env.Clone()
     # https://github.com/platformio/platformio-atom-ide/issues/34
@@ -207,5 +198,7 @@ def exists(_):
 
 
 def generate(env):
-    env.AddMethod(DumpIDEData)
+    env.AddMethod(IsIntegrationDump)
+    env.AddMethod(DumpIntegrationIncludes)
+    env.AddMethod(DumpIntegrationData)
     return env

platformio/builder/tools/piolib.py

@@ -27,14 +27,16 @@ import sys
 import click
 import SCons.Scanner  # pylint: disable=import-error
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
-from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import exception, fs, util
 from platformio.builder.tools import platformio as piotool
-from platformio.clients.http import InternetIsOffline
 from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
-from platformio.package.exception import UnknownPackageError
+from platformio.http import HTTPClientError, InternetIsOffline
+from platformio.package.exception import (
+    MissingPackageManifestError,
+    UnknownPackageError,
+)
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manifest.parser import (
     ManifestParserError,
@@ -54,11 +56,21 @@ class LibBuilderFactory(object):
             used_frameworks = LibBuilderFactory.get_used_frameworks(env, path)
             common_frameworks = set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks)
             if common_frameworks:
-                clsname = "%sLibBuilder" % list(common_frameworks)[0].title()
+                clsname = "%sLibBuilder" % list(common_frameworks)[0].capitalize()
             elif used_frameworks:
-                clsname = "%sLibBuilder" % used_frameworks[0].title()
+                clsname = "%sLibBuilder" % used_frameworks[0].capitalize()
+
+        obj = globals()[clsname](env, path, verbose=verbose)
+
+        # Handle PlatformIOLibBuilder.manifest.build.builder
+        # pylint: disable=protected-access
+        if isinstance(obj, PlatformIOLibBuilder) and obj._manifest.get("build", {}).get(
+            "builder"
+        ):
+            obj = globals()[obj._manifest.get("build", {}).get("builder")](
+                env, path, verbose=verbose
+            )
 
-        obj = getattr(sys.modules[__name__], clsname)(env, path, verbose=verbose)
         assert isinstance(obj, LibBuilderBase)
         return obj
 
@@ -99,7 +111,7 @@ class LibBuilderFactory(object):
         return []
 
 
-class LibBuilderBase(object):
+class LibBuilderBase:
 
     CLASSIC_SCANNER = SCons.Scanner.C.CScanner()
     CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner()
@@ -115,7 +127,7 @@ class LibBuilderBase(object):
     def __init__(self, env, path, manifest=None, verbose=False):
         self.env = env.Clone()
         self.envorigin = env.Clone()
-        self.path = os.path.realpath(env.subst(path))
+        self.path = os.path.abspath(env.subst(path))
         self.verbose = verbose
 
         try:
@@ -126,9 +138,11 @@ class LibBuilderBase(object):
             )
             self._manifest = {}
 
-        self._is_dependent = False
-        self._is_built = False
-        self._depbuilders = []
+        self.is_dependent = False
+        self.is_built = False
+        self.depbuilders = []
+
+        self._deps_are_processed = False
         self._circular_deps = []
         self._processed_files = []
 
@@ -149,7 +163,12 @@ class LibBuilderBase(object):
             p2 = p2.lower()
         if p1 == p2:
             return True
-        return os.path.commonprefix((p1 + os.path.sep, p2)) == p1 + os.path.sep
+        if os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep:
+            return True
+        # try to resolve paths
+        p1 = os.path.os.path.realpath(p1)
+        p2 = os.path.os.path.realpath(p2)
+        return os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep
 
     @property
     def name(self):
@@ -159,6 +178,11 @@ class LibBuilderBase(object):
     def version(self):
         return self._manifest.get("version")
 
+    @property
+    def dependent(self):
+        """Backward compatibility with ESP-IDF"""
+        return self.is_dependent
+
     @property
     def dependencies(self):
         return self._manifest.get("dependencies")
@@ -174,19 +198,19 @@ class LibBuilderBase(object):
 
     @property
     def include_dir(self):
-        if not all(
-            os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src")
-        ):
-            return None
-        return os.path.join(self.path, "include")
+        for name in ("include", "Include"):
+            d = os.path.join(self.path, name)
+            if os.path.isdir(d):
+                return d
+        return None
 
     @property
     def src_dir(self):
-        return (
-            os.path.join(self.path, "src")
-            if os.path.isdir(os.path.join(self.path, "src"))
-            else self.path
-        )
+        for name in ("src", "Src"):
+            d = os.path.join(self.path, name)
+            if os.path.isdir(d):
+                return d
+        return self.path
 
     def get_include_dirs(self):
         items = []
@@ -215,18 +239,6 @@ class LibBuilderBase(object):
     def extra_script(self):
         return None
 
-    @property
-    def depbuilders(self):
-        return self._depbuilders
-
-    @property
-    def dependent(self):
-        return self._is_dependent
-
-    @property
-    def is_built(self):
-        return self._is_built
-
     @property
     def lib_archive(self):
         return self.env.GetProjectOption("lib_archive")
@@ -280,14 +292,15 @@ class LibBuilderBase(object):
             if self.extra_script:
                 self.env.SConscriptChdir(1)
                 self.env.SConscript(
-                    os.path.realpath(self.extra_script),
+                    os.path.abspath(self.extra_script),
                     exports={"env": self.env, "pio_lib_builder": self},
                 )
             self.env.ProcessUnFlags(self.build_unflags)
 
     def process_dependencies(self):
-        if not self.dependencies:
+        if not self.dependencies or self._deps_are_processed:
             return
+        self._deps_are_processed = True
         for item in self.dependencies:
             found = False
             for lb in self.env.GetLibBuilders():
@@ -295,7 +308,7 @@ class LibBuilderBase(object):
                     continue
                 found = True
                 if lb not in self.depbuilders:
-                    self.depend_recursive(lb)
+                    self.depend_on(lb)
                 break
 
             if not found and self.verbose:
@@ -390,7 +403,29 @@ class LibBuilderBase(object):
 
         return result
 
-    def depend_recursive(self, lb, search_files=None):
+    def search_deps_recursive(self, search_files=None):
+        self.process_dependencies()
+
+        # when LDF is disabled
+        if self.lib_ldf_mode == "off":
+            return
+
+        if self.lib_ldf_mode.startswith("deep"):
+            search_files = self.get_search_files()
+
+        lib_inc_map = {}
+        for inc in self._get_found_includes(search_files):
+            for lb in self.env.GetLibBuilders():
+                if inc.get_abspath() in lb:
+                    if lb not in lib_inc_map:
+                        lib_inc_map[lb] = []
+                    lib_inc_map[lb].append(inc.get_abspath())
+                    break
+
+        for lb, lb_search_files in lib_inc_map.items():
+            self.depend_on(lb, search_files=lb_search_files)
+
+    def depend_on(self, lb, search_files=None, recursive=True):
         def _already_depends(_lb):
             if self in _lb.depbuilders:
                 return True
@@ -408,49 +443,34 @@ class LibBuilderBase(object):
                         "between `%s` and `%s`\n" % (self.path, lb.path)
                     )
                 self._circular_deps.append(lb)
-            elif lb not in self._depbuilders:
-                self._depbuilders.append(lb)
+            elif lb not in self.depbuilders:
+                self.depbuilders.append(lb)
+                lb.is_dependent = True
                 LibBuilderBase._INCLUDE_DIRS_CACHE = None
-        lb.search_deps_recursive(search_files)
 
-    def search_deps_recursive(self, search_files=None):
-        if not self._is_dependent:
-            self._is_dependent = True
-            self.process_dependencies()
-
-            if self.lib_ldf_mode.startswith("deep"):
-                search_files = self.get_search_files()
-
-        # when LDF is disabled
-        if self.lib_ldf_mode == "off":
-            return
-
-        lib_inc_map = {}
-        for inc in self._get_found_includes(search_files):
-            for lb in self.env.GetLibBuilders():
-                if inc.get_abspath() in lb:
-                    if lb not in lib_inc_map:
-                        lib_inc_map[lb] = []
-                    lib_inc_map[lb].append(inc.get_abspath())
-                    break
-
-        for lb, lb_search_files in lib_inc_map.items():
-            self.depend_recursive(lb, lb_search_files)
+        if recursive:
+            lb.search_deps_recursive(search_files)
 
     def build(self):
         libs = []
-        for lb in self._depbuilders:
+        shared_scopes = ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS")
+        for lb in self.depbuilders:
             libs.extend(lb.build())
             # copy shared information to self env
-            for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
-                self.env.PrependUnique(**{key: lb.env.get(key)})
+            self.env.PrependUnique(
+                **{
+                    scope: lb.env.get(scope)
+                    for scope in shared_scopes
+                    if lb.env.get(scope)
+                }
+            )
 
         for lb in self._circular_deps:
             self.env.PrependUnique(CPPPATH=lb.get_include_dirs())
 
-        if self._is_built:
+        if self.is_built:
             return libs
-        self._is_built = True
+        self.is_built = True
 
         self.env.PrependUnique(CPPPATH=self.get_include_dirs())
 
@@ -458,8 +478,13 @@ class LibBuilderBase(object):
             for lb in self.env.GetLibBuilders():
                 if self == lb or not lb.is_built:
                     continue
-                for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
-                    self.env.PrependUnique(**{key: lb.env.get(key)})
+                self.env.PrependUnique(
+                    **{
+                        scope: lb.env.get(scope)
+                        for scope in shared_scopes
+                        if lb.env.get(scope)
+                    }
+                )
 
         do_not_archive = not self.lib_archive
         if not do_not_archive:
@@ -491,8 +516,16 @@ class ArduinoLibBuilder(LibBuilderBase):
             return {}
         return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
+    @property
+    def include_dir(self):
+        if not all(
+            os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src")
+        ):
+            return None
+        return os.path.join(self.path, "include")
+
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
+        include_dirs = super().get_include_dirs()
         if os.path.isdir(os.path.join(self.path, "src")):
             return include_dirs
         if os.path.isdir(os.path.join(self.path, "utility")):
@@ -566,9 +599,12 @@ class ArduinoLibBuilder(LibBuilderBase):
         if self._manifest.get("precompiled") in ("true", "full"):
             # add to LDPATH {build.mcu} folder
             board_config = self.env.BoardConfig()
-            self.env.PrependUnique(
-                LIBPATH=os.path.join(self.src_dir, board_config.get("build.cpu"))
-            )
+            for key in ("build.mcu", "build.cpu"):
+                libpath = os.path.join(self.src_dir, board_config.get(key, ""))
+                if not os.path.isdir(libpath):
+                    continue
+                self.env.PrependUnique(LIBPATH=libpath)
+                break
         ldflags = [flag for flag in ldflags if flag]  # remove empty
         return " ".join(ldflags) if ldflags else None
 
@@ -580,12 +616,6 @@ class MbedLibBuilder(LibBuilderBase):
             return {}
         return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
-    @property
-    def include_dir(self):
-        if os.path.isdir(os.path.join(self.path, "include")):
-            return os.path.join(self.path, "include")
-        return None
-
     @property
     def src_dir(self):
         if os.path.isdir(os.path.join(self.path, "source")):
@@ -593,7 +623,7 @@ class MbedLibBuilder(LibBuilderBase):
         return LibBuilderBase.src_dir.fget(self)  # pylint: disable=no-member
 
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
+        include_dirs = super().get_include_dirs()
         if self.path not in include_dirs:
             include_dirs.append(self.path)
 
@@ -617,7 +647,7 @@ class MbedLibBuilder(LibBuilderBase):
 
     def process_extra_options(self):
         self._process_mbed_lib_confs()
-        return super(MbedLibBuilder, self).process_extra_options()
+        return super().process_extra_options()
 
     def _process_mbed_lib_confs(self):
         mbed_lib_paths = [
@@ -735,14 +765,14 @@ class PlatformIOLibBuilder(LibBuilderBase):
     def include_dir(self):
         if "includeDir" in self._manifest.get("build", {}):
             with fs.cd(self.path):
-                return os.path.realpath(self._manifest.get("build").get("includeDir"))
+                return os.path.abspath(self._manifest.get("build").get("includeDir"))
         return LibBuilderBase.include_dir.fget(self)  # pylint: disable=no-member
 
     @property
     def src_dir(self):
         if "srcDir" in self._manifest.get("build", {}):
             with fs.cd(self.path):
-                return os.path.realpath(self._manifest.get("build").get("srcDir"))
+                return os.path.abspath(self._manifest.get("build").get("srcDir"))
         return LibBuilderBase.src_dir.fget(self)  # pylint: disable=no-member
 
     @property
@@ -814,7 +844,7 @@ class PlatformIOLibBuilder(LibBuilderBase):
         return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"])
 
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
+        include_dirs = super().get_include_dirs()
 
         # backwards compatibility with PlatformIO 2.0
         if (
@@ -836,7 +866,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
     def __init__(self, env, *args, **kwargs):
         # backup original value, will be reset in base.__init__
         project_src_filter = env.get("SRC_FILTER")
-        super(ProjectAsLibBuilder, self).__init__(env, *args, **kwargs)
+        super().__init__(env, *args, **kwargs)
         self.env["SRC_FILTER"] = project_src_filter
 
     @property
@@ -853,16 +883,16 @@ class ProjectAsLibBuilder(LibBuilderBase):
         project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
         if os.path.isdir(project_include_dir):
             include_dirs.append(project_include_dir)
-        for include_dir in LibBuilderBase.get_include_dirs(self):
+        for include_dir in super().get_include_dirs():
             if include_dir not in include_dirs:
                 include_dirs.append(include_dir)
         return include_dirs
 
     def get_search_files(self):
         # project files
-        items = LibBuilderBase.get_search_files(self)
+        items = super().get_search_files()
         # test files
-        if "__test" in COMMAND_LINE_TARGETS:
+        if "test" in self.env.GetBuildType():
             items.extend(
                 [
                     os.path.join("$PROJECT_TEST_DIR", item)
@@ -886,13 +916,19 @@ class ProjectAsLibBuilder(LibBuilderBase):
         # pylint: disable=no-member
         return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self)
 
+    @property
+    def build_flags(self):
+        # pylint: disable=no-member
+        return self.env.get("SRC_BUILD_FLAGS") or LibBuilderBase.build_flags.fget(self)
+
     @property
     def dependencies(self):
         return self.env.GetProjectOption("lib_deps", [])
 
     def process_extra_options(self):
-        # skip for project, options are already processed
-        pass
+        with fs.cd(self.path):
+            self.env.ProcessFlags(self.build_flags)
+            self.env.ProcessUnFlags(self.build_unflags)
 
     def install_dependencies(self):
         def _is_builtin(spec):
@@ -924,7 +960,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
             try:
                 lm.install(spec)
                 did_install = True
-            except (UnknownPackageError, InternetIsOffline) as e:
+            except (HTTPClientError, UnknownPackageError, InternetIsOffline) as e:
                 click.secho("Warning! %s" % e, fg="yellow")
 
         # reset cache
@@ -932,6 +968,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
             DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None)
 
     def process_dependencies(self):  # pylint: disable=too-many-branches
+        found_lbs = []
         for spec in self.dependencies:
             found = False
             for storage_dir in self.env.GetLibSourceDirs():
@@ -945,7 +982,8 @@ class ProjectAsLibBuilder(LibBuilderBase):
                     if pkg.path != lb.path:
                         continue
                     if lb not in self.depbuilders:
-                        self.depend_recursive(lb)
+                        self.depend_on(lb, recursive=False)
+                        found_lbs.append(lb)
                     found = True
                     break
             if found:
@@ -957,13 +995,17 @@ class ProjectAsLibBuilder(LibBuilderBase):
                 if lb.name != spec:
                     continue
                 if lb not in self.depbuilders:
-                    self.depend_recursive(lb)
+                    self.depend_on(lb)
                 found = True
                 break
 
+        # process library dependencies
+        for lb in found_lbs:
+            lb.search_deps_recursive()
+
     def build(self):
-        self._is_built = True  # do not build Project now
-        result = LibBuilderBase.build(self)
+        self.is_built = True  # do not build Project now
+        result = super().build()
         self.env.PrependUnique(CPPPATH=self.get_include_dirs())
         return result
 
@@ -996,25 +1038,30 @@ def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)))
     return True
 
 
-def GetLibBuilders(env):  # pylint: disable=too-many-branches
-    if DefaultEnvironment().get("__PIO_LIB_BUILDERS", None) is not None:
+def GetLibBuilders(_):  # pylint: disable=too-many-branches
+    env = DefaultEnvironment()
+    if env.get("__PIO_LIB_BUILDERS", None) is not None:
         return sorted(
-            DefaultEnvironment()["__PIO_LIB_BUILDERS"],
-            key=lambda lb: 0 if lb.dependent else 1,
+            env["__PIO_LIB_BUILDERS"],
+            key=lambda lb: 0 if lb.is_dependent else 1,
         )
 
-    DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=[])
+    env.Replace(__PIO_LIB_BUILDERS=[])
 
     verbose = int(ARGUMENTS.get("PIOVERBOSE", 0))
     found_incompat = False
 
     for storage_dir in env.GetLibSourceDirs():
-        storage_dir = os.path.realpath(storage_dir)
+        storage_dir = os.path.abspath(storage_dir)
         if not os.path.isdir(storage_dir):
             continue
         for item in sorted(os.listdir(storage_dir)):
             lib_dir = os.path.join(storage_dir, item)
-            if item == "__cores__" or not os.path.isdir(lib_dir):
+            if item == "__cores__":
+                continue
+            if LibraryPackageManager.is_symlink(lib_dir):
+                lib_dir, _ = LibraryPackageManager.resolve_symlink(lib_dir)
+            if not lib_dir or not os.path.isdir(lib_dir):
                 continue
             try:
                 lb = LibBuilderFactory.new(env, lib_dir)
@@ -1025,13 +1072,13 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
                     )
                 continue
             if env.IsCompatibleLibBuilder(lb):
-                DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
+                env.Append(__PIO_LIB_BUILDERS=[lb])
             else:
                 found_incompat = True
 
     for lb in env.get("EXTRA_LIB_BUILDERS", []):
         if env.IsCompatibleLibBuilder(lb):
-            DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
+            env.Append(__PIO_LIB_BUILDERS=[lb])
         else:
             found_incompat = True
 
@@ -1042,13 +1089,25 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             "ldf-compat-mode\n"
         )
 
-    return DefaultEnvironment()["__PIO_LIB_BUILDERS"]
+    return env["__PIO_LIB_BUILDERS"]
 
 
 def ConfigureProjectLibBuilder(env):
+    _pm_storage = {}
+
+    def _get_lib_license(pkg):
+        storage_dir = os.path.dirname(os.path.dirname(pkg.path))
+        if storage_dir not in _pm_storage:
+            _pm_storage[storage_dir] = LibraryPackageManager(storage_dir)
+        try:
+            return (_pm_storage[storage_dir].load_manifest(pkg) or {}).get("license")
+        except MissingPackageManifestError:
+            pass
+        return None
+
     def _correct_found_libs(lib_builders):
         # build full dependency graph
-        found_lbs = [lb for lb in lib_builders if lb.dependent]
+        found_lbs = [lb for lb in lib_builders if lb.is_dependent]
         for lb in lib_builders:
             if lb in found_lbs:
                 lb.search_deps_recursive(lb.get_search_files())
@@ -1060,18 +1119,20 @@ def ConfigureProjectLibBuilder(env):
     def _print_deps_tree(root, level=0):
         margin = "|   " * (level)
         for lb in root.depbuilders:
-            title = "<%s>" % lb.name
+            title = lb.name
             pkg = PackageItem(lb.path)
             if pkg.metadata:
-                title += " %s" % pkg.metadata.version
+                title += " @ %s" % pkg.metadata.version
             elif lb.version:
-                title += " %s" % lb.version
+                title += " @ %s" % lb.version
             click.echo("%s|-- %s" % (margin, title), nl=False)
             if int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                click.echo(
+                    " (License: %s, " % (_get_lib_license(pkg) or "Unknown"), nl=False
+                )
                 if pkg.metadata and pkg.metadata.spec.external:
-                    click.echo(" [%s]" % pkg.metadata.spec.url, nl=False)
-                click.echo(" (", nl=False)
-                click.echo(lb.path, nl=False)
+                    click.echo("URI: %s, " % pkg.metadata.spec.uri, nl=False)
+                click.echo("Path: %s" % lb.path, nl=False)
                 click.echo(")", nl=False)
             click.echo("")
             if lb.depbuilders:
@@ -1080,7 +1141,7 @@ def ConfigureProjectLibBuilder(env):
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
     ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project)  # pylint: disable=no-member
 
-    click.echo("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf")
+    click.echo("LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf")
     click.echo(
         "LDF Modes: Finder ~ %s, Compatibility ~ %s"
         % (ldf_mode, project.lib_compat_mode)

platformio/builder/tools/piomisc.py

@@ -14,244 +14,15 @@
 
 from __future__ import absolute_import
 
-import atexit
-import glob
-import io
 import os
-import re
 import sys
-import tempfile
-
-import click
 
 from platformio import fs, util
-from platformio.compat import get_filesystem_encoding, get_locale_encoding
-from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import exec_command
 
 
-class InoToCPPConverter(object):
-
-    PROTOTYPE_RE = re.compile(
-        r"""^(
-        (?:template\<.*\>\s*)?      # template
-        ([a-z_\d\&]+\*?\s+){1,2}    # return type
-        ([a-z_\d]+\s*)              # name of prototype
-        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
-        )\s*(\{|;)                  # must end with `{` or `;`
-        """,
-        re.X | re.M | re.I,
-    )
-    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
-    PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
-
-    def __init__(self, env):
-        self.env = env
-        self._main_ino = None
-        self._safe_encoding = None
-
-    def read_safe_contents(self, path):
-        error_reported = False
-        for encoding in (
-            "utf-8",
-            None,
-            get_filesystem_encoding(),
-            get_locale_encoding(),
-            "latin-1",
-        ):
-            try:
-                with io.open(path, encoding=encoding) as fp:
-                    contents = fp.read()
-                    self._safe_encoding = encoding
-                    return contents
-            except UnicodeDecodeError:
-                if not error_reported:
-                    error_reported = True
-                    click.secho(
-                        "Unicode decode error has occurred, please remove invalid "
-                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
-                        % path,
-                        fg="yellow",
-                        err=True,
-                    )
-        return ""
-
-    def write_safe_contents(self, path, contents):
-        with io.open(
-            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
-        ) as fp:
-            return fp.write(contents)
-
-    def is_main_node(self, contents):
-        return self.DETECTMAIN_RE.search(contents)
-
-    def convert(self, nodes):
-        contents = self.merge(nodes)
-        if not contents:
-            return None
-        return self.process(contents)
-
-    def merge(self, nodes):
-        assert nodes
-        lines = []
-        for node in nodes:
-            contents = self.read_safe_contents(node.get_path())
-            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
-            if self.is_main_node(contents):
-                lines = _lines + lines
-                self._main_ino = node.get_path()
-            else:
-                lines.extend(_lines)
-
-        if not self._main_ino:
-            self._main_ino = nodes[0].get_path()
-
-        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
-
-    def process(self, contents):
-        out_file = self._main_ino + ".cpp"
-        assert self._gcc_preprocess(contents, out_file)
-        contents = self.read_safe_contents(out_file)
-        contents = self._join_multiline_strings(contents)
-        self.write_safe_contents(out_file, self.append_prototypes(contents))
-        return out_file
-
-    def _gcc_preprocess(self, contents, out_file):
-        tmp_path = tempfile.mkstemp()[1]
-        self.write_safe_contents(tmp_path, contents)
-        self.env.Execute(
-            self.env.VerboseAction(
-                '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
-                    out_file, tmp_path
-                ),
-                "Converting " + os.path.basename(out_file[:-4]),
-            )
-        )
-        atexit.register(_delete_file, tmp_path)
-        return os.path.isfile(out_file)
-
-    def _join_multiline_strings(self, contents):
-        if "\\\n" not in contents:
-            return contents
-        newlines = []
-        linenum = 0
-        stropen = False
-        for line in contents.split("\n"):
-            _linenum = self._parse_preproc_line_num(line)
-            if _linenum is not None:
-                linenum = _linenum
-            else:
-                linenum += 1
-
-            if line.endswith("\\"):
-                if line.startswith('"'):
-                    stropen = True
-                    newlines.append(line[:-1])
-                    continue
-                if stropen:
-                    newlines[len(newlines) - 1] += line[:-1]
-                    continue
-            elif stropen and line.endswith(('",', '";')):
-                newlines[len(newlines) - 1] += line
-                stropen = False
-                newlines.append(
-                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
-                )
-                continue
-
-            newlines.append(line)
-
-        return "\n".join(newlines)
-
-    @staticmethod
-    def _parse_preproc_line_num(line):
-        if not line.startswith("#"):
-            return None
-        tokens = line.split(" ", 3)
-        if len(tokens) > 2 and tokens[1].isdigit():
-            return int(tokens[1])
-        return None
-
-    def _parse_prototypes(self, contents):
-        prototypes = []
-        reserved_keywords = set(["if", "else", "while"])
-        for match in self.PROTOTYPE_RE.finditer(contents):
-            if (
-                set([match.group(2).strip(), match.group(3).strip()])
-                & reserved_keywords
-            ):
-                continue
-            prototypes.append(match)
-        return prototypes
-
-    def _get_total_lines(self, contents):
-        total = 0
-        if contents.endswith("\n"):
-            contents = contents[:-1]
-        for line in contents.split("\n")[::-1]:
-            linenum = self._parse_preproc_line_num(line)
-            if linenum is not None:
-                return total + linenum
-            total += 1
-        return total
-
-    def append_prototypes(self, contents):
-        prototypes = self._parse_prototypes(contents) or []
-
-        # skip already declared prototypes
-        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
-        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
-
-        if not prototypes:
-            return contents
-
-        prototype_names = set(m.group(3).strip() for m in prototypes)
-        split_pos = prototypes[0].start()
-        match_ptrs = re.search(
-            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
-            contents[:split_pos],
-            re.M,
-        )
-        if match_ptrs:
-            split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
-
-        result = []
-        result.append(contents[:split_pos].strip())
-        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
-        result.append(
-            '#line %d "%s"'
-            % (
-                self._get_total_lines(contents[:split_pos]),
-                self._main_ino.replace("\\", "/"),
-            )
-        )
-        result.append(contents[split_pos:].strip())
-        return "\n".join(result)
-
-
-def ConvertInoToCpp(env):
-    src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
-    ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
-        os.path.join(src_dir, "*.pde")
-    )
-    if not ino_nodes:
-        return
-    c = InoToCPPConverter(env)
-    out_file = c.convert(ino_nodes)
-
-    atexit.register(_delete_file, out_file)
-
-
-def _delete_file(path):
-    try:
-        if os.path.isfile(path):
-            os.remove(path)
-    except:  # pylint: disable=bare-except
-        pass
-
-
 @util.memoized()
-def _get_compiler_type(env):
+def GetCompilerType(env):
     if env.subst("$CC").endswith("-gcc"):
         return "gcc"
     try:
@@ -270,10 +41,6 @@ def _get_compiler_type(env):
     return None
 
 
-def GetCompilerType(env):
-    return _get_compiler_type(env)
-
-
 def GetActualLDScript(env):
     def _lookup_in_ldpath(script):
         for d in env.get("LIBPATH", []):
@@ -319,7 +86,7 @@ def GetActualLDScript(env):
     env.Exit(1)
 
 
-def ConfigureDebugFlags(env):
+def ConfigureDebugTarget(env):
     def _cleanup_debug_flags(scope):
         if scope not in env:
             return
@@ -347,23 +114,15 @@ def ConfigureDebugFlags(env):
     ]
 
     if optimization_flags:
-        env.AppendUnique(ASFLAGS=optimization_flags, LINKFLAGS=optimization_flags)
-
-
-def ConfigureTestTarget(env):
-    env.Append(
-        CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
-        CPPPATH=[os.path.join("$BUILD_DIR", "UnityTestLib")],
-    )
-    unitylib = env.BuildLibrary(
-        os.path.join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity")
-    )
-    env.Prepend(LIBS=[unitylib])
-
-    src_filter = ["+<*.cpp>", "+<*.c>"]
-    if "PIOTEST_RUNNING_NAME" in env:
-        src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], os.path.sep))
-    env.Replace(PIOTEST_SRC_FILTER=src_filter)
+        env.AppendUnique(
+            ASFLAGS=[
+                # skip -O flags for assembler
+                f
+                for f in optimization_flags
+                if f.startswith("-g")
+            ],
+            LINKFLAGS=optimization_flags,
+        )
 
 
 def GetExtraScripts(env, scope):
@@ -376,18 +135,17 @@ def GetExtraScripts(env, scope):
     if not items:
         return items
     with fs.cd(env.subst("$PROJECT_DIR")):
-        return [os.path.realpath(item) for item in items]
+        return [os.path.abspath(env.subst(item)) for item in items]
+
+
+def generate(env):
+    env.AddMethod(GetCompilerType)
+    env.AddMethod(GetActualLDScript)
+    env.AddMethod(ConfigureDebugTarget)
+    env.AddMethod(GetExtraScripts)
+    # bakward-compatibility with Zephyr build script
+    env.AddMethod(ConfigureDebugTarget, "ConfigureDebugFlags")
 
 
 def exists(_):
     return True
-
-
-def generate(env):
-    env.AddMethod(ConvertInoToCpp)
-    env.AddMethod(GetCompilerType)
-    env.AddMethod(GetActualLDScript)
-    env.AddMethod(ConfigureDebugFlags)
-    env.AddMethod(ConfigureTestTarget)
-    env.AddMethod(GetExtraScripts)
-    return env

platformio/builder/tools/pioplatform.py

@@ -19,6 +19,7 @@ import sys
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
+from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import fs, util
 from platformio.compat import IS_MACOS, IS_WINDOWS
@@ -32,16 +33,17 @@ from platformio.project.config import ProjectOptions
 
 
 @util.memoized()
-def PioPlatform(env):
-    variables = env.GetProjectOptions(as_dict=True)
-    if "framework" in variables:
-        # support PIO Core 3.0 dev/platforms
-        variables["pioframework"] = variables["framework"]
+def _PioPlatform():
+    env = DefaultEnvironment()
     p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"]))
-    p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
+    p.configure_project_packages(env["PIOENV"], COMMAND_LINE_TARGETS)
     return p
 
 
+def PioPlatform(_):
+    return _PioPlatform()
+
+
 def BoardConfig(env, board=None):
     with fs.cd(env.subst("$PROJECT_DIR")):
         try:
@@ -160,7 +162,7 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
             and pkg_metadata
             and pkg_metadata.spec.external
         ):
-            data.append("(%s)" % pkg_metadata.spec.url)
+            data.append("(%s)" % pkg_metadata.spec.uri)
         if board_config:
             data.extend([">", board_config.get("name")])
         return data
@@ -213,7 +215,7 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
         data = []
         for item in platform.dump_used_packages():
             original_version = get_original_version(item["version"])
-            info = "%s %s" % (item["name"], item["version"])
+            info = "%s @ %s" % (item["name"], item["version"])
             extra = []
             if original_version:
                 extra.append(original_version)

platformio/builder/tools/pioproject.py

@@ -14,7 +14,8 @@
 
 from __future__ import absolute_import
 
-from platformio.project.config import MISSING, ProjectConfig, ProjectOptions
+from platformio.compat import MISSING
+from platformio.project.config import ProjectConfig
 
 
 def GetProjectConfig(env):
@@ -30,15 +31,17 @@ def GetProjectOption(env, option, default=MISSING):
 
 
 def LoadProjectOptions(env):
-    for option, value in env.GetProjectOptions():
-        option_meta = ProjectOptions.get("env." + option)
+    config = env.GetProjectConfig()
+    section = "env:" + env["PIOENV"]
+    for option in config.options(section):
+        option_meta = config.find_option_meta(section, option)
         if (
             not option_meta
             or not option_meta.buildenvvar
             or option_meta.buildenvvar in env
         ):
             continue
-        env[option_meta.buildenvvar] = value
+        env[option_meta.buildenvvar] = config.get(section, option)
 
 
 def exists(_):

platformio/builder/tools/piotarget.py

@@ -29,7 +29,7 @@ def VerboseAction(_, act, actstr):
     return Action(act, actstr)
 
 
-def PioClean(env, clean_dir):
+def PioClean(env, clean_all=False):
     def _relpath(path):
         if compat.IS_WINDOWS:
             prefix = os.getcwd()[:2].lower()
@@ -41,21 +41,30 @@ def PioClean(env, clean_dir):
                 return path
         return os.path.relpath(path)
 
-    if not os.path.isdir(clean_dir):
+    def _clean_dir(path):
+        clean_rel_path = _relpath(path)
+        for root, _, files in os.walk(path):
+            for f in files:
+                dst = os.path.join(root, f)
+                os.remove(dst)
+                print(
+                    "Removed %s"
+                    % (dst if not clean_rel_path.startswith(".") else _relpath(dst))
+                )
+
+    build_dir = env.subst("$BUILD_DIR")
+    libdeps_dir = env.subst("$PROJECT_LIBDEPS_DIR")
+    if os.path.isdir(build_dir):
+        _clean_dir(build_dir)
+        fs.rmtree(build_dir)
+    else:
         print("Build environment is clean")
-        env.Exit(0)
-    clean_rel_path = _relpath(clean_dir)
-    for root, _, files in os.walk(clean_dir):
-        for f in files:
-            dst = os.path.join(root, f)
-            os.remove(dst)
-            print(
-                "Removed %s"
-                % (dst if not clean_rel_path.startswith(".") else _relpath(dst))
-            )
+
+    if clean_all and os.path.isdir(libdeps_dir):
+        _clean_dir(libdeps_dir)
+        fs.rmtree(libdeps_dir)
+
     print("Done cleaning")
-    fs.rmtree(clean_dir)
-    env.Exit(0)
 
 
 def AddTarget(  # pylint: disable=too-many-arguments
@@ -65,7 +74,7 @@ def AddTarget(  # pylint: disable=too-many-arguments
     actions,
     title=None,
     description=None,
-    group="Generic",
+    group="General",
     always_build=True,
 ):
     if "__PIO_TARGETS" not in env:
@@ -101,7 +110,13 @@ def DumpTargets(env):
         description="Generate compilation database `compile_commands.json`",
         group="Advanced",
     )
-    targets["clean"] = dict(name="clean", title="Clean", group="Generic")
+    targets["clean"] = dict(name="clean", title="Clean", group="General")
+    targets["cleanall"] = dict(
+        name="cleanall",
+        title="Clean All",
+        group="General",
+        description="Clean a build environment and installed library dependencies",
+    )
     return list(targets.values())
 
 

platformio/builder/tools/piotest.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import os
+
+from platformio.builder.tools import platformio as piotool
+from platformio.test.result import TestSuite
+from platformio.test.runners.factory import TestRunnerFactory
+
+
+def ConfigureTestTarget(env):
+    env.Append(
+        CPPDEFINES=["UNIT_TEST", "PIO_UNIT_TESTING"],
+        PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piotool.SRC_BUILD_EXT],
+    )
+    env.Prepend(CPPPATH=["$PROJECT_TEST_DIR"])
+
+    if "PIOTEST_RUNNING_NAME" in env:
+        test_name = env["PIOTEST_RUNNING_NAME"]
+        while True:
+            test_name = os.path.dirname(test_name)  # parent dir
+            # skip nested tests (user's side issue?)
+            if not test_name or os.path.basename(test_name).startswith("test_"):
+                break
+            env.Prepend(
+                PIOTEST_SRC_FILTER=[
+                    f"+<{test_name}{os.path.sep}*.{ext}>"
+                    for ext in piotool.SRC_BUILD_EXT
+                ],
+                CPPPATH=[os.path.join("$PROJECT_TEST_DIR", test_name)],
+            )
+
+        env.Prepend(
+            PIOTEST_SRC_FILTER=[f"+<$PIOTEST_RUNNING_NAME{os.path.sep}>"],
+            CPPPATH=[os.path.join("$PROJECT_TEST_DIR", "$PIOTEST_RUNNING_NAME")],
+        )
+
+    test_runner = TestRunnerFactory.new(
+        TestSuite(env["PIOENV"], env.get("PIOTEST_RUNNING_NAME", "*")),
+        env.GetProjectConfig(),
+    )
+    test_runner.configure_build_env(env)
+
+
+def generate(env):
+    env.AddMethod(ConfigureTestTarget)
+
+
+def exists(_):
+    return True

platformio/builder/tools/pioupload.py

@@ -12,25 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=unused-argument
+
 from __future__ import absolute_import
 
+import os
 import re
 import sys
-from fnmatch import fnmatch
-from os import environ
-from os.path import isfile, join
 from shutil import copyfile
 from time import sleep
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from serial import Serial, SerialException
 
-from platformio import exception, fs, util
-from platformio.compat import IS_WINDOWS
+from platformio import exception, fs
+from platformio.device.finder import find_mbed_disk, find_serial_port, is_pattern_port
+from platformio.device.list import list_serial_ports
 from platformio.proc import exec_command
 
-# pylint: disable=unused-argument
-
 
 def FlushSerialBuffer(env, port):
     s = Serial(env.subst(port))
@@ -62,7 +61,7 @@ def WaitForNewSerialPort(env, before):
     elapsed = 0
     before = [p["port"] for p in before]
     while elapsed < 5 and new_port is None:
-        now = [p["port"] for p in util.get_serial_ports()]
+        now = [p["port"] for p in list_serial_ports()]
         for p in now:
             if p not in before:
                 new_port = p
@@ -97,67 +96,28 @@ def WaitForNewSerialPort(env, before):
 
 def AutodetectUploadPort(*args, **kwargs):
     env = args[0]
-
-    def _get_pattern():
-        if "UPLOAD_PORT" not in env:
-            return None
-        if set(["*", "?", "[", "]"]) & set(env["UPLOAD_PORT"]):
-            return env["UPLOAD_PORT"]
-        return None
-
-    def _is_match_pattern(port):
-        pattern = _get_pattern()
-        if not pattern:
-            return True
-        return fnmatch(port, pattern)
-
-    def _look_for_mbed_disk():
-        msdlabels = ("mbed", "nucleo", "frdm", "microbit")
-        for item in util.get_logical_devices():
-            if item["path"].startswith("/net") or not _is_match_pattern(item["path"]):
-                continue
-            mbed_pages = [join(item["path"], n) for n in ("mbed.htm", "mbed.html")]
-            if any(isfile(p) for p in mbed_pages):
-                return item["path"]
-            if item["name"] and any(l in item["name"].lower() for l in msdlabels):
-                return item["path"]
-        return None
-
-    def _look_for_serial_port():
-        port = None
-        board_hwids = []
-        upload_protocol = env.subst("$UPLOAD_PROTOCOL")
-        if "BOARD" in env and "build.hwids" in env.BoardConfig():
-            board_hwids = env.BoardConfig().get("build.hwids")
-        for item in util.get_serial_ports(filter_hwid=True):
-            if not _is_match_pattern(item["port"]):
-                continue
-            port = item["port"]
-            if upload_protocol.startswith("blackmagic"):
-                if IS_WINDOWS and port.startswith("COM") and len(port) > 4:
-                    port = "\\\\.\\%s" % port
-                if "GDB" in item["description"]:
-                    return port
-            for hwid in board_hwids:
-                hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                if hwid_str in item["hwid"]:
-                    return port
-        return port
-
-    if "UPLOAD_PORT" in env and not _get_pattern():
-        print(env.subst("Use manually specified: $UPLOAD_PORT"))
+    initial_port = env.subst("$UPLOAD_PORT")
+    upload_protocol = env.subst("$UPLOAD_PROTOCOL")
+    if initial_port and not is_pattern_port(initial_port):
+        print(env.subst("Using manually specified: $UPLOAD_PORT"))
         return
 
-    if env.subst("$UPLOAD_PROTOCOL") == "mbed" or (
-        "mbed" in env.subst("$PIOFRAMEWORK") and not env.subst("$UPLOAD_PROTOCOL")
+    if upload_protocol == "mbed" or (
+        "mbed" in env.subst("$PIOFRAMEWORK") and not upload_protocol
     ):
-        env.Replace(UPLOAD_PORT=_look_for_mbed_disk())
+        env.Replace(UPLOAD_PORT=find_mbed_disk(initial_port))
     else:
         try:
             fs.ensure_udev_rules()
         except exception.InvalidUdevRules as e:
             sys.stderr.write("\n%s\n\n" % e)
-        env.Replace(UPLOAD_PORT=_look_for_serial_port())
+        env.Replace(
+            UPLOAD_PORT=find_serial_port(
+                initial_port=initial_port,
+                board_config=env.BoardConfig() if "BOARD" in env else None,
+                upload_protocol=upload_protocol,
+            )
+        )
 
     if env.subst("$UPLOAD_PORT"):
         print(env.subst("Auto-detected: $UPLOAD_PORT"))
@@ -175,10 +135,12 @@ def UploadToDisk(_, target, source, env):
     assert "UPLOAD_PORT" in env
     progname = env.subst("$PROGNAME")
     for ext in ("bin", "hex"):
-        fpath = join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
-        if not isfile(fpath):
+        fpath = os.path.join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
+        if not os.path.isfile(fpath):
             continue
-        copyfile(fpath, join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)))
+        copyfile(
+            fpath, os.path.join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext))
+        )
     print(
         "Firmware has been successfully uploaded.\n"
         "(Some boards may require manual hard reset)"
@@ -211,7 +173,7 @@ def CheckUploadSize(_, target, source, env):
         if not isinstance(cmd, list):
             cmd = cmd.split()
         cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
-        sysenv = environ.copy()
+        sysenv = os.environ.copy()
         sysenv["PATH"] = str(env["ENV"]["PATH"])
         result = exec_command(env.subst(cmd), env=sysenv)
         if result["returncode"] != 0:

platformio/builder/tools/platformio.py

@@ -47,14 +47,16 @@ def scons_patched_match_splitext(path, suffixes=None):
 
 
 def GetBuildType(env):
-    return (
-        "debug"
-        if (
-            set(["__debug", "sizedata"]) & set(COMMAND_LINE_TARGETS)
-            or env.GetProjectOption("build_type") == "debug"
-        )
-        else "release"
-    )
+    modes = []
+    if (
+        set(["__debug", "sizedata"])  # sizedata = for memory inspection
+        & set(COMMAND_LINE_TARGETS)
+        or env.GetProjectOption("build_type") == "debug"
+    ):
+        modes.append("debug")
+    if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test":
+        modes.append("test")
+    return "+".join(modes or ["release"])
 
 
 def BuildProgram(env):
@@ -113,10 +115,6 @@ def ProcessProgramDeps(env):
 
     env.PrintConfiguration()
 
-    # fix ASM handling under non case-sensitive OS
-    if not Util.case_sensitive_suffixes(".s", ".S"):
-        env.Replace(AS="$CC", ASCOM="$ASPPCOM")
-
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
         env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
@@ -127,18 +125,25 @@ def ProcessProgramDeps(env):
     # process framework scripts
     env.BuildFrameworks(env.get("PIOFRAMEWORK"))
 
-    if env.GetBuildType() == "debug":
-        env.ConfigureDebugFlags()
+    if "debug" in env.GetBuildType():
+        env.ConfigureDebugTarget()
+    if "test" in env.GetBuildType():
+        env.ConfigureTestTarget()
 
     # remove specified flags
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
-    if "__test" in COMMAND_LINE_TARGETS:
-        env.ConfigureTestTarget()
+    if "compiledb" in COMMAND_LINE_TARGETS and env.get(
+        "COMPILATIONDB_INCLUDE_TOOLCHAIN"
+    ):
+        for scope, includes in env.DumpIntegrationIncludes().items():
+            if scope in ("toolchain",):
+                env.Append(CPPPATH=includes)
 
 
 def ProcessProjectDeps(env):
     project_lib_builder = env.ConfigureProjectLibBuilder()
+    projenv = project_lib_builder.env
 
     # prepend project libs to the beginning of list
     env.Prepend(LIBS=project_lib_builder.build())
@@ -151,19 +156,19 @@ def ProcessProjectDeps(env):
         }
     )
 
-    projenv = env.Clone()
-
-    # CPPPATH from dependencies
-    projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH"))
-    # extra build flags from `platformio.ini`
-    projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
-
-    is_test = "__test" in COMMAND_LINE_TARGETS
-    if is_test:
+    if "test" in env.GetBuildType():
+        build_files_before_nums = len(env.get("PIOBUILDFILES", []))
         projenv.BuildSources(
             "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
         )
-    if not is_test or env.GetProjectOption("test_build_project_src"):
+        if len(env.get("PIOBUILDFILES", [])) - build_files_before_nums < 1:
+            sys.stderr.write(
+                "Error: Nothing to build. Please put your test suites "
+                "to the '%s' folder\n" % env.subst("$PROJECT_TEST_DIR")
+            )
+            env.Exit(1)
+
+    if "test" not in env.GetBuildType() or env.GetProjectOption("test_build_src"):
         projenv.BuildSources(
             "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER")
         )
@@ -171,7 +176,7 @@ def ProcessProjectDeps(env):
     if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
         sys.stderr.write(
             "Error: Nothing to build. Please put your source code files "
-            "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
+            "to the '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
         )
         env.Exit(1)
 
@@ -207,12 +212,12 @@ def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
     for k in ("CPPPATH", "LIBPATH"):
         for i, p in enumerate(result.get(k, [])):
             if os.path.isdir(p):
-                result[k][i] = os.path.realpath(p)
+                result[k][i] = os.path.abspath(p)
 
     # fix relative path for "-include"
     for i, f in enumerate(result.get("CCFLAGS", [])):
         if isinstance(f, tuple) and f[0] == "-include":
-            result["CCFLAGS"][i] = (f[0], env.File(os.path.realpath(f[1].get_path())))
+            result["CCFLAGS"][i] = (f[0], env.File(os.path.abspath(f[1].get_path())))
 
     return result
 
@@ -324,25 +329,18 @@ def BuildFrameworks(env, frameworks):
         )
         env.Exit(1)
 
-    board_frameworks = env.BoardConfig().get("frameworks", [])
-    if frameworks == ["platformio"]:
-        if board_frameworks:
-            frameworks.insert(0, board_frameworks[0])
-        else:
-            sys.stderr.write("Error: Please specify `board` in `platformio.ini`\n")
-            env.Exit(1)
-
-    for f in frameworks:
-        if f == "arduino":
-            # Arduino IDE appends .o the end of filename
+    supported_frameworks = env.BoardConfig().get("frameworks", [])
+    for name in frameworks:
+        if name == "arduino":
+            # Arduino IDE appends .o to the end of filename
             Builder.match_splitext = scons_patched_match_splitext
             if "nobuild" not in COMMAND_LINE_TARGETS:
                 env.ConvertInoToCpp()
 
-        if f in board_frameworks:
-            SConscript(env.GetFrameworkScript(f), exports="env")
+        if name in supported_frameworks:
+            SConscript(env.GetFrameworkScript(name), exports="env")
         else:
-            sys.stderr.write("Error: This board doesn't support %s framework!\n" % f)
+            sys.stderr.write("Error: This board doesn't support %s framework!\n" % name)
             env.Exit(1)
 
 

platformio/check/cli.py

@@ -26,13 +26,13 @@ import click
 from tabulate import tabulate
 
 from platformio import app, exception, fs, util
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools import CheckToolFactory
+from platformio.check.defect import DefectItem
+from platformio.check.tools import CheckToolFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import find_project_dir_above, get_project_dir
 
 
-@click.command("check", short_help="Static code analysis")
+@click.command("check", short_help="Static Code Analysis")
 @click.option("-e", "--environment", multiple=True)
 @click.option(
     "-d",
@@ -106,8 +106,8 @@ def cli(
                 )
 
             default_patterns = [
-                config.get_optional_dir("src"),
-                config.get_optional_dir("include"),
+                config.get("platformio", "src_dir"),
+                config.get("platformio", "include_dir"),
             ]
             tool_options = dict(
                 verbose=verbose,
@@ -118,6 +118,7 @@ def cli(
                 if silent
                 else severity or config.get("env:" + envname, "check_severity"),
                 skip_packages=skip_packages or env_options.get("check_skip_packages"),
+                platform_packages=env_options.get("platform_packages"),
             )
 
             for tool in config.get("env:" + envname, "check_tool"):
@@ -166,7 +167,10 @@ def cli(
         if json_output:
             click.echo(json.dumps(results_to_json(results)))
         elif not silent:
-            print_check_summary(results)
+            print_check_summary(results, verbose=verbose)
+
+    # Reset custom project config
+    app.set_session_var("custom_project_conf", None)
 
     command_failed = any(r.get("succeeded") is False for r in results)
     if command_failed:
@@ -267,7 +271,7 @@ def print_defects_stats(results):
     click.echo()
 
 
-def print_check_summary(results):
+def print_check_summary(results, verbose=False):
     click.echo()
 
     tabular_data = []
@@ -284,6 +288,8 @@ def print_check_summary(results):
             status_str = click.style("FAILED", fg="red")
         elif result.get("succeeded") is None:
             status_str = "IGNORED"
+            if not verbose:
+                continue
         else:
             succeeded_nums += 1
             status_str = click.style("PASSED", fg="green")

platformio/check/defect.py

@@ -34,7 +34,7 @@ class DefectItem(object):
         severity,
         category,
         message,
-        file="unknown",
+        file=None,
         line=0,
         column=0,
         id=None,
@@ -50,7 +50,7 @@ class DefectItem(object):
         self.callstack = callstack
         self.cwe = cwe
         self.id = id
-        self.file = file
+        self.file = file or "unknown"
         if file.lower().startswith(get_project_dir().lower()):
             self.file = os.path.relpath(file, get_project_dir())
 
@@ -86,7 +86,7 @@ class DefectItem(object):
             "severity": self.SEVERITY_LABELS[self.severity],
             "category": self.category,
             "message": self.message,
-            "file": os.path.realpath(self.file),
+            "file": os.path.abspath(self.file),
             "line": self.line,
             "column": self.column,
             "callstack": self.callstack,

platformio/check/tools/__init__.py

@@ -13,9 +13,9 @@
 # limitations under the License.
 
 from platformio import exception
-from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool
-from platformio.commands.check.tools.cppcheck import CppcheckCheckTool
-from platformio.commands.check.tools.pvsstudio import PvsStudioCheckTool
+from platformio.check.tools.clangtidy import ClangtidyCheckTool
+from platformio.check.tools.cppcheck import CppcheckCheckTool
+from platformio.check.tools.pvsstudio import PvsStudioCheckTool
 
 
 class CheckToolFactory(object):

platformio/check/tools/base.py

@@ -19,8 +19,10 @@ import tempfile
 import click
 
 from platformio import fs, proc
-from platformio.commands.check.defect import DefectItem
-from platformio.project.helpers import load_project_ide_data
+from platformio.check.defect import DefectItem
+from platformio.package.manager.core import get_core_package_dir
+from platformio.package.meta import PackageSpec
+from platformio.project.helpers import load_build_metadata
 
 
 class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
@@ -55,7 +57,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         ]
 
     def _load_cpp_data(self, project_dir):
-        data = load_project_ide_data(project_dir, self.envname)
+        data = load_build_metadata(project_dir, self.envname)
         if not data:
             return
         self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", ""))
@@ -66,6 +68,13 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         self.cxx_path = data.get("cxx_path")
         self.toolchain_defines = self._get_toolchain_defines()
 
+    def get_tool_dir(self, pkg_name):
+        for spec in self.options["platform_packages"] or []:
+            spec = PackageSpec(spec)
+            if spec.name == pkg_name:
+                return get_core_package_dir(pkg_name, spec=spec)
+        return get_core_package_dir(pkg_name)
+
     def get_flags(self, tool):
         result = []
         flags = self.options.get("flags") or []
@@ -201,11 +210,11 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
 
         def _add_file(path):
             if path.endswith(header_extensions):
-                result["headers"].append(os.path.realpath(path))
+                result["headers"].append(os.path.abspath(path))
             elif path.endswith(c_extension):
-                result["c"].append(os.path.realpath(path))
+                result["c"].append(os.path.abspath(path))
             elif path.endswith(cpp_extensions):
-                result["c++"].append(os.path.realpath(path))
+                result["c++"].append(os.path.abspath(path))
 
         for pattern in patterns:
             for item in glob.glob(pattern, recursive=True):

platformio/check/tools/clangtidy.py

@@ -15,13 +15,12 @@
 import re
 from os.path import join
 
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
-from platformio.package.manager.core import get_core_package_dir
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 
 
 class ClangtidyCheckTool(CheckToolBase):
-    def tool_output_filter(self, line):
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
         if not self.options.get("verbose") and "[clang-diagnostic-error]" in line:
             return ""
 
@@ -34,7 +33,7 @@ class ClangtidyCheckTool(CheckToolBase):
 
         return ""
 
-    def parse_defect(self, raw_line):
+    def parse_defect(self, raw_line):  # pylint: disable=arguments-differ
         match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line)
         if not match:
             return raw_line
@@ -56,11 +55,13 @@ class ClangtidyCheckTool(CheckToolBase):
         return cmd_result["returncode"] < 2
 
     def configure_command(self):
-        tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy")
+        tool_path = join(self.get_tool_dir("tool-clangtidy"), "clang-tidy")
 
         cmd = [tool_path, "--quiet"]
         flags = self.get_flags("clangtidy")
-        if not self.is_flag_set("--checks", flags):
+        if not (
+            self.is_flag_set("--checks", flags) or self.is_flag_set("--config", flags)
+        ):
             cmd.append("--checks=*")
 
         project_files = self.get_project_target_files(self.options["patterns"])
@@ -77,7 +78,7 @@ class ClangtidyCheckTool(CheckToolBase):
         includes = []
         for inc in self.cpp_includes:
             if self.options.get("skip_packages") and inc.lower().startswith(
-                self.config.get_optional_dir("packages").lower()
+                self.config.get("platformio", "packages_dir").lower()
             ):
                 continue
             includes.append(inc)

platformio/check/tools/cppcheck.py

@@ -17,13 +17,13 @@ import os
 import click
 
 from platformio import proc
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
-from platformio.package.manager.core import get_core_package_dir
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 
 
 class CppcheckCheckTool(CheckToolBase):
     def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self._field_delimiter = "<&PIO&>"
         self._buffer = ""
         self.defect_fields = [
@@ -36,9 +36,8 @@ class CppcheckCheckTool(CheckToolBase):
             "cwe",
             "id",
         ]
-        super(CppcheckCheckTool, self).__init__(*args, **kwargs)
 
-    def tool_output_filter(self, line):
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
         if (
             not self.options.get("verbose")
             and "--suppress=unmatchedSuppression:" in line
@@ -50,13 +49,14 @@ class CppcheckCheckTool(CheckToolBase):
             for msg in (
                 "No C or C++ source files found",
                 "unrecognized command line option",
+                "there was an internal error",
             )
         ):
             self._bad_input = True
 
         return line
 
-    def parse_defect(self, raw_line):
+    def parse_defect(self, raw_line):  # pylint: disable=arguments-differ
         if self._field_delimiter not in raw_line:
             return None
 
@@ -84,7 +84,7 @@ class CppcheckCheckTool(CheckToolBase):
         if (
             args.get("file", "")
             .lower()
-            .startswith(self.config.get_optional_dir("packages").lower())
+            .startswith(self.config.get("platformio", "packages_dir").lower())
         ):
             if args["id"] in breaking_defect_ids:
                 if self.options.get("verbose"):
@@ -103,7 +103,7 @@ class CppcheckCheckTool(CheckToolBase):
         return DefectItem(**args)
 
     def configure_command(self, language, src_file):  # pylint: disable=arguments-differ
-        tool_path = os.path.join(get_core_package_dir("tool-cppcheck"), "cppcheck")
+        tool_path = os.path.join(self.get_tool_dir("tool-cppcheck"), "cppcheck")
 
         cmd = [
             tool_path,
@@ -141,10 +141,11 @@ class CppcheckCheckTool(CheckToolBase):
 
         build_flags = self.cxx_flags if language == "c++" else self.cc_flags
 
-        for flag in build_flags:
-            if "-std" in flag:
-                # Standards with GNU extensions are not allowed
-                cmd.append("-" + flag.replace("gnu", "c"))
+        if not self.is_flag_set("--std", flags):
+            # Try to guess the standard version from the build flags
+            for flag in build_flags:
+                if "-std" in flag:
+                    cmd.append("-" + self.convert_language_standard(flag))
 
         cmd.extend(
             ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines[language]]
@@ -200,14 +201,14 @@ class CppcheckCheckTool(CheckToolBase):
         result = []
         for inc in self.cpp_includes:
             if self.options.get("skip_packages") and inc.lower().startswith(
-                self.config.get_optional_dir("packages").lower()
+                self.config.get("platformio", "packages_dir").lower()
             ):
                 continue
             result.append(inc)
         return self._create_tmp_file("\n".join(result))
 
     def clean_up(self):
-        super(CppcheckCheckTool, self).clean_up()
+        super().clean_up()
 
         # delete temporary dump files generated by addons
         if not self.is_flag_set("--addon", self.get_flags("cppcheck")):
@@ -224,6 +225,21 @@ class CppcheckCheckTool(CheckToolBase):
         # Cppcheck is configured to return '3' if a defect is found
         return cmd_result["returncode"] in (0, 3)
 
+    @staticmethod
+    def convert_language_standard(flag):
+        cpp_standards_map = {
+            "0x": "11",
+            "1y": "14",
+            "1z": "17",
+            "2a": "20",
+        }
+
+        standard = flag[-2:]
+        # Note: GNU extensions are not supported and converted to regular standards
+        return flag.replace("gnu", "c").replace(
+            standard, cpp_standards_map.get(standard, standard)
+        )
+
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
 

platformio/check/tools/pvsstudio.py

@@ -20,30 +20,29 @@ from xml.etree.ElementTree import fromstring
 import click
 
 from platformio import proc
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 from platformio.compat import IS_WINDOWS
-from platformio.package.manager.core import get_core_package_dir
 
 
 class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-attributes
     def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self._tmp_dir = tempfile.mkdtemp(prefix="piocheck")
         self._tmp_preprocessed_file = self._generate_tmp_file_path() + ".i"
         self._tmp_output_file = self._generate_tmp_file_path() + ".pvs"
         self._tmp_cfg_file = self._generate_tmp_file_path() + ".cfg"
         self._tmp_cmd_file = self._generate_tmp_file_path() + ".cmd"
         self.tool_path = os.path.join(
-            get_core_package_dir("tool-pvs-studio"),
+            self.get_tool_dir("tool-pvs-studio"),
             "x64" if IS_WINDOWS else "bin",
             "pvs-studio",
         )
-        super(PvsStudioCheckTool, self).__init__(*args, **kwargs)
 
         with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp:
             fp.write(
                 "exclude-path = "
-                + self.config.get_optional_dir("packages").replace("\\", "/")
+                + self.config.get("platformio", "packages_dir").replace("\\", "/")
             )
 
         with open(self._tmp_cmd_file, mode="w", encoding="utf8") as fp:
@@ -53,8 +52,14 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
                 )
             )
 
-    def tool_output_filter(self, line):
-        if "license was not entered" in line.lower():
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
+        if any(
+            err_msg in line.lower()
+            for err_msg in (
+                "license was not entered",
+                "license information is incorrect",
+            )
+        ):
             self._bad_input = True
         return line
 
@@ -70,7 +75,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
 
     def _demangle_report(self, output_file):
         converter_tool = os.path.join(
-            get_core_package_dir("tool-pvs-studio"),
+            self.get_tool_dir("tool-pvs-studio"),
             "HtmlGenerator" if IS_WINDOWS else os.path.join("bin", "plog-converter"),
         )
 
@@ -194,7 +199,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
             '"%s"' % self._tmp_preprocessed_file,
         ]
         cmd.extend([f for f in flags if f])
-        cmd.extend(["-D%s" % d for d in self.cpp_defines])
+        cmd.extend(['"-D%s"' % d.replace('"', '\\"') for d in self.cpp_defines])
         cmd.append('@"%s"' % self._tmp_cmd_file)
 
         # Explicitly specify C++ as the language used in .ino files
@@ -209,7 +214,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
             self._bad_input = True
 
     def clean_up(self):
-        super(PvsStudioCheckTool, self).clean_up()
+        super().clean_up()
         if os.path.isdir(self._tmp_dir):
             shutil.rmtree(self._tmp_dir)
 

platformio/cli.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+from pathlib import Path
+
+import click
+
+
+class PlatformioCLI(click.MultiCommand):
+
+    leftover_args = []
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._pio_root_path = Path(__file__).parent
+        self._pio_cmd_aliases = dict(package="pkg")
+
+    def _find_pio_commands(self):
+        def _to_module_path(p):
+            return (
+                "platformio." + ".".join(p.relative_to(self._pio_root_path).parts)[:-3]
+            )
+
+        result = {}
+        for p in self._pio_root_path.rglob("cli.py"):
+            # skip this module
+            if p.parent == self._pio_root_path:
+                continue
+            cmd_name = p.parent.name
+            result[self._pio_cmd_aliases.get(cmd_name, cmd_name)] = _to_module_path(p)
+
+        # find legacy commands
+        for p in (self._pio_root_path / "commands").iterdir():
+            if p.name.startswith("_"):
+                continue
+            if (p / "command.py").is_file():
+                result[p.name] = _to_module_path(p / "command.py")
+            elif p.name.endswith(".py"):
+                result[p.name[:-3]] = _to_module_path(p)
+
+        return result
+
+    @staticmethod
+    def in_silence():
+        args = PlatformioCLI.leftover_args
+        return args and any(
+            [
+                args[0] == "debug" and "--interpreter" in " ".join(args),
+                args[0] == "upgrade",
+                "--json-output" in args,
+                "--version" in args,
+            ]
+        )
+
+    def invoke(self, ctx):
+        PlatformioCLI.leftover_args = ctx.args
+        if hasattr(ctx, "protected_args"):
+            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
+        return super().invoke(ctx)
+
+    def list_commands(self, ctx):
+        return sorted(list(self._find_pio_commands()))
+
+    def get_command(self, ctx, cmd_name):
+        commands = self._find_pio_commands()
+        if cmd_name not in commands:
+            return self._handle_obsolate_command(ctx, cmd_name)
+        module = importlib.import_module(commands[cmd_name])
+        return getattr(module, "cli")
+
+    @staticmethod
+    def _handle_obsolate_command(ctx, cmd_name):
+        # pylint: disable=import-outside-toplevel
+        if cmd_name == "init":
+            from platformio.project.commands.init import project_init_cmd
+
+            return project_init_cmd
+
+        if cmd_name == "package":
+            from platformio.package.cli import cli
+
+            return cli
+
+        raise click.UsageError('No such command "%s"' % cmd_name, ctx)

platformio/commands/__init__.py

@@ -11,70 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import os
-
-import click
-
-
-class PlatformioCLI(click.MultiCommand):
-
-    leftover_args = []
-
-    def __init__(self, *args, **kwargs):
-        super(PlatformioCLI, self).__init__(*args, **kwargs)
-        self._pio_cmds_dir = os.path.dirname(__file__)
-
-    @staticmethod
-    def in_silence():
-        args = PlatformioCLI.leftover_args
-        return args and any(
-            [
-                args[0] == "debug" and "--interpreter" in " ".join(args),
-                args[0] == "upgrade",
-                "--json-output" in args,
-                "--version" in args,
-            ]
-        )
-
-    def invoke(self, ctx):
-        PlatformioCLI.leftover_args = ctx.args
-        if hasattr(ctx, "protected_args"):
-            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
-        return super(PlatformioCLI, self).invoke(ctx)
-
-    def list_commands(self, ctx):
-        cmds = []
-        for cmd_name in os.listdir(self._pio_cmds_dir):
-            if cmd_name.startswith("__init__"):
-                continue
-            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
-                cmds.append(cmd_name)
-            elif cmd_name.endswith(".py"):
-                cmds.append(cmd_name[:-3])
-        cmds.sort()
-        return cmds
-
-    def get_command(self, ctx, cmd_name):
-        mod = None
-        try:
-            mod_path = "platformio.commands." + cmd_name
-            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
-                mod_path = "platformio.commands.%s.command" % cmd_name
-            mod = __import__(mod_path, None, None, ["cli"])
-        except ImportError:
-            try:
-                return self._handle_obsolate_command(cmd_name)
-            except AttributeError:
-                pass
-            raise click.UsageError('No such command "%s"' % cmd_name, ctx)
-        return mod.cli
-
-    @staticmethod
-    def _handle_obsolate_command(name):
-        # pylint: disable=import-outside-toplevel
-        if name == "init":
-            from platformio.commands.project import project_init
-
-            return project_init
-        raise AttributeError()

platformio/commands/access.py

@@ -1,146 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import json
-import re
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.registry import RegistryClient
-from platformio.commands.account import validate_username
-from platformio.commands.team import validate_orgname_teamname
-
-
-def validate_client(value):
-    if ":" in value:
-        validate_orgname_teamname(value)
-    else:
-        validate_username(value)
-    return value
-
-
-@click.group("access", short_help="Manage resource access")
-def cli():
-    pass
-
-
-def validate_urn(value):
-    value = str(value).strip()
-    if not re.match(r"^prn:reg:pkg:(\d+):(\w+)$", value, flags=re.I):
-        raise click.BadParameter("Invalid URN format.")
-    return value
-
-
-@cli.command("public", short_help="Make resource public")
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_public(urn, urn_type):
-    client = RegistryClient()
-    client.update_resource(urn=urn, private=0)
-    return click.secho(
-        "The resource %s has been successfully updated." % urn,
-        fg="green",
-    )
-
-
-@cli.command("private", short_help="Make resource private")
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_private(urn, urn_type):
-    client = RegistryClient()
-    client.update_resource(urn=urn, private=1)
-    return click.secho(
-        "The resource %s has been successfully updated." % urn,
-        fg="green",
-    )
-
-
-@cli.command("grant", short_help="Grant access")
-@click.argument("level", type=click.Choice(["admin", "maintainer", "guest"]))
-@click.argument(
-    "client",
-    metavar="[<ORGNAME:TEAMNAME>|<USERNAME>]",
-    callback=lambda _, __, value: validate_client(value),
-)
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_grant(level, client, urn, urn_type):
-    reg_client = RegistryClient()
-    reg_client.grant_access_for_resource(urn=urn, client=client, level=level)
-    return click.secho(
-        "Access for resource %s has been granted for %s" % (urn, client),
-        fg="green",
-    )
-
-
-@cli.command("revoke", short_help="Revoke access")
-@click.argument(
-    "client",
-    metavar="[ORGNAME:TEAMNAME|USERNAME]",
-    callback=lambda _, __, value: validate_client(value),
-)
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_revoke(client, urn, urn_type):
-    reg_client = RegistryClient()
-    reg_client.revoke_access_from_resource(urn=urn, client=client)
-    return click.secho(
-        "Access for resource %s has been revoked for %s" % (urn, client),
-        fg="green",
-    )
-
-
-@cli.command("list", short_help="List published resources")
-@click.argument("owner", required=False)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-@click.option("--json-output", is_flag=True)
-def access_list(owner, urn_type, json_output):
-    reg_client = RegistryClient()
-    resources = reg_client.list_resources(owner=owner)
-    if json_output:
-        return click.echo(json.dumps(resources))
-    if not resources:
-        return click.secho("You do not have any resources.", fg="yellow")
-    for resource in resources:
-        click.echo()
-        click.secho(resource.get("name"), fg="cyan")
-        click.echo("-" * len(resource.get("name")))
-        table_data = []
-        table_data.append(("URN:", resource.get("urn")))
-        table_data.append(("Owner:", resource.get("owner")))
-        table_data.append(
-            (
-                "Access level(s):",
-                ", ".join(
-                    (level.capitalize() for level in resource.get("access_levels"))
-                ),
-            )
-        )
-        click.echo(tabulate(table_data, tablefmt="plain"))
-    return click.echo()

platformio/commands/account.py

@@ -1,299 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import datetime
-import json
-import re
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.account import AccountClient, AccountNotAuthorized
-
-
-@click.group("account", short_help="Manage PlatformIO account")
-def cli():
-    pass
-
-
-def validate_username(value, field="username"):
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I):
-        raise click.BadParameter(
-            "Invalid %s format. "
-            "%s must contain only alphanumeric characters "
-            "or single hyphens, cannot begin or end with a hyphen, "
-            "and must not be longer than 38 characters."
-            % (field.lower(), field.capitalize())
-        )
-    return value
-
-
-def validate_email(value):
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d_.+-]+@[a-z\d\-]+\.[a-z\d\-.]+$", value, flags=re.I):
-        raise click.BadParameter("Invalid email address")
-    return value
-
-
-def validate_password(value):
-    value = str(value).strip()
-    if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
-        raise click.BadParameter(
-            "Invalid password format. "
-            "Password must contain at least 8 characters"
-            " including a number and a lowercase letter"
-        )
-    return value
-
-
-@cli.command("register", short_help="Create new PlatformIO Account")
-@click.option(
-    "-u",
-    "--username",
-    prompt=True,
-    callback=lambda _, __, value: validate_username(value),
-)
-@click.option(
-    "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value)
-)
-@click.option(
-    "-p",
-    "--password",
-    prompt=True,
-    hide_input=True,
-    confirmation_prompt=True,
-    callback=lambda _, __, value: validate_password(value),
-)
-@click.option("--firstname", prompt=True)
-@click.option("--lastname", prompt=True)
-def account_register(username, email, password, firstname, lastname):
-    client = AccountClient()
-    client.registration(username, email, password, firstname, lastname)
-    return click.secho(
-        "An account has been successfully created. "
-        "Please check your mail to activate your account and verify your email address.",
-        fg="green",
-    )
-
-
-@cli.command("login", short_help="Log in to PlatformIO Account")
-@click.option("-u", "--username", prompt="Username or email")
-@click.option("-p", "--password", prompt=True, hide_input=True)
-def account_login(username, password):
-    client = AccountClient()
-    client.login(username, password)
-    return click.secho("Successfully logged in!", fg="green")
-
-
-@cli.command("logout", short_help="Log out of PlatformIO Account")
-def account_logout():
-    client = AccountClient()
-    client.logout()
-    return click.secho("Successfully logged out!", fg="green")
-
-
-@cli.command("password", short_help="Change password")
-@click.option("--old-password", prompt=True, hide_input=True)
-@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
-def account_password(old_password, new_password):
-    client = AccountClient()
-    client.change_password(old_password, new_password)
-    return click.secho("Password successfully changed!", fg="green")
-
-
-@cli.command("token", short_help="Get or regenerate Authentication Token")
-@click.option("-p", "--password", prompt=True, hide_input=True)
-@click.option("--regenerate", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_token(password, regenerate, json_output):
-    client = AccountClient()
-    auth_token = client.auth_token(password, regenerate)
-    if json_output:
-        return click.echo(json.dumps({"status": "success", "result": auth_token}))
-    return click.secho("Personal Authentication Token: %s" % auth_token, fg="green")
-
-
-@cli.command("forgot", short_help="Forgot password")
-@click.option("--username", prompt="Username or email")
-def account_forgot(username):
-    client = AccountClient()
-    client.forgot_password(username)
-    return click.secho(
-        "If this account is registered, we will send the "
-        "further instructions to your email.",
-        fg="green",
-    )
-
-
-@cli.command("update", short_help="Update profile information")
-@click.option("--current-password", prompt=True, hide_input=True)
-@click.option("--username")
-@click.option("--email")
-@click.option("--firstname")
-@click.option("--lastname")
-def account_update(current_password, **kwargs):
-    client = AccountClient()
-    profile = client.get_profile()
-    new_profile = profile.copy()
-    if not any(kwargs.values()):
-        for field in profile:
-            new_profile[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=profile[field]
-            )
-            if field == "email":
-                validate_email(new_profile[field])
-            if field == "username":
-                validate_username(new_profile[field])
-    else:
-        new_profile.update({key: value for key, value in kwargs.items() if value})
-    client.update_profile(new_profile, current_password)
-    click.secho("Profile successfully updated!", fg="green")
-    username_changed = new_profile["username"] != profile["username"]
-    email_changed = new_profile["email"] != profile["email"]
-    if not username_changed and not email_changed:
-        return None
-    try:
-        client.logout()
-    except AccountNotAuthorized:
-        pass
-    if email_changed:
-        return click.secho(
-            "Please check your mail to verify your new email address and re-login. ",
-            fg="yellow",
-        )
-    return click.secho("Please re-login.", fg="yellow")
-
-
-@cli.command("destroy", short_help="Destroy account")
-def account_destroy():
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the %s user account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % client.get_logged_username(),
-        abort=True,
-    )
-    client.destroy_account()
-    try:
-        client.logout()
-    except AccountNotAuthorized:
-        pass
-    return click.secho(
-        "User account has been destroyed.",
-        fg="green",
-    )
-
-
-@cli.command("show", short_help="PlatformIO Account information")
-@click.option("--offline", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_show(offline, json_output):
-    client = AccountClient()
-    info = client.get_account_info(offline)
-    if json_output:
-        return click.echo(json.dumps(info))
-    click.echo()
-    if info.get("profile"):
-        print_profile(info["profile"])
-    if info.get("packages"):
-        print_packages(info["packages"])
-    if info.get("subscriptions"):
-        print_subscriptions(info["subscriptions"])
-    return click.echo()
-
-
-def print_profile(profile):
-    click.secho("Profile", fg="cyan", bold=True)
-    click.echo("=" * len("Profile"))
-    data = []
-    if profile.get("username"):
-        data.append(("Username:", profile["username"]))
-    if profile.get("email"):
-        data.append(("Email:", profile["email"]))
-    if profile.get("firstname"):
-        data.append(("First name:", profile["firstname"]))
-    if profile.get("lastname"):
-        data.append(("Last name:", profile["lastname"]))
-    click.echo(tabulate(data, tablefmt="plain"))
-
-
-def print_packages(packages):
-    click.echo()
-    click.secho("Packages", fg="cyan")
-    click.echo("=" * len("Packages"))
-    for package in packages:
-        click.echo()
-        click.secho(package.get("name"), bold=True)
-        click.echo("-" * len(package.get("name")))
-        if package.get("description"):
-            click.echo(package.get("description"))
-        data = []
-        expire = "-"
-        if "subscription" in package:
-            expire = datetime.datetime.strptime(
-                (
-                    package["subscription"].get("end_at")
-                    or package["subscription"].get("next_bill_at")
-                ),
-                "%Y-%m-%dT%H:%M:%SZ",
-            ).strftime("%Y-%m-%d")
-        data.append(("Expire:", expire))
-        services = []
-        for key in package:
-            if not key.startswith("service."):
-                continue
-            if isinstance(package[key], dict):
-                services.append(package[key].get("title"))
-            else:
-                services.append(package[key])
-        if services:
-            data.append(("Services:", ", ".join(services)))
-        click.echo(tabulate(data, tablefmt="plain"))
-
-
-def print_subscriptions(subscriptions):
-    click.echo()
-    click.secho("Subscriptions", fg="cyan")
-    click.echo("=" * len("Subscriptions"))
-    for subscription in subscriptions:
-        click.echo()
-        click.secho(subscription.get("product_name"), bold=True)
-        click.echo("-" * len(subscription.get("product_name")))
-        data = [("State:", subscription.get("status"))]
-        begin_at = datetime.datetime.strptime(
-            subscription.get("begin_at"), "%Y-%m-%dT%H:%M:%SZ"
-        ).strftime("%Y-%m-%d %H:%M:%S")
-        data.append(("Start date:", begin_at or "-"))
-        end_at = subscription.get("end_at")
-        if end_at:
-            end_at = datetime.datetime.strptime(
-                subscription.get("end_at"), "%Y-%m-%dT%H:%M:%SZ"
-            ).strftime("%Y-%m-%d %H:%M:%S")
-        data.append(("End date:", end_at or "-"))
-        next_bill_at = subscription.get("next_bill_at")
-        if next_bill_at:
-            next_bill_at = datetime.datetime.strptime(
-                subscription.get("next_bill_at"), "%Y-%m-%dT%H:%M:%SZ"
-            ).strftime("%Y-%m-%d %H:%M:%S")
-        data.append(("Next payment:", next_bill_at or "-"))
-        data.append(
-            ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")
-        )
-        data.append(
-            ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-")
-        )
-        click.echo(tabulate(data, tablefmt="plain"))

platformio/commands/boards.py

@@ -22,7 +22,7 @@ from platformio import fs
 from platformio.package.manager.platform import PlatformPackageManager
 
 
-@click.command("boards", short_help="Embedded board explorer")
+@click.command("boards", short_help="Board Explorer")
 @click.argument("query", required=False)
 @click.option("--installed", is_flag=True)
 @click.option("--json-output", is_flag=True)

platformio/commands/ci.py

@@ -20,11 +20,10 @@ import tempfile
 import click
 
 from platformio import app, fs
-from platformio.commands.project import project_init as cmd_project_init
-from platformio.commands.project import validate_boards
-from platformio.commands.run.command import cli as cmd_run
 from platformio.exception import CIBuildEnvsEmpty
+from platformio.project.commands.init import project_init_cmd, validate_boards
 from platformio.project.config import ProjectConfig
+from platformio.run.cli import cli as cmd_run
 
 
 def validate_path(ctx, param, value):  # pylint: disable=unused-argument
@@ -33,7 +32,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
     for i, p in enumerate(value):
         if p.startswith("~"):
             value[i] = fs.expanduser(p)
-        value[i] = os.path.realpath(value[i])
+        value[i] = os.path.abspath(value[i])
         if not glob.glob(value[i], recursive=True):
             invalid_path = p
             break
@@ -44,7 +43,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
         raise click.BadParameter("Found invalid path: %s" % invalid_path)
 
 
-@click.command("ci", short_help="Continuous integration")
+@click.command("ci", short_help="Continuous Integration")
 @click.argument("src", nargs=-1, callback=validate_path)
 @click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY")
 @click.option("--exclude", multiple=True)
@@ -109,7 +108,7 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
 
         # initialise project
         ctx.invoke(
-            cmd_project_init,
+            project_init_cmd,
             project_dir=build_dir,
             board=board,
             project_option=project_option,
@@ -122,7 +121,7 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
             fs.rmtree(build_dir)
 
 
-def _copy_contents(dst_dir, contents):
+def _copy_contents(dst_dir, contents):  # pylint: disable=too-many-branches
     items = {"dirs": set(), "files": set()}
 
     for path in contents:
@@ -134,14 +133,15 @@ def _copy_contents(dst_dir, contents):
     dst_dir_name = os.path.basename(dst_dir)
 
     if dst_dir_name == "src" and len(items["dirs"]) == 1:
-        shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
+        if not os.path.isdir(dst_dir):
+            shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
     else:
         if not os.path.isdir(dst_dir):
             os.makedirs(dst_dir)
         for d in items["dirs"]:
-            shutil.copytree(
-                d, os.path.join(dst_dir, os.path.basename(d)), symlinks=True
-            )
+            src_dst_dir = os.path.join(dst_dir, os.path.basename(d))
+            if not os.path.isdir(src_dst_dir):
+                shutil.copytree(d, src_dst_dir, symlinks=True)
 
     if not items["files"]:
         return
@@ -161,7 +161,7 @@ def _exclude_contents(dst_dir, patterns):
     for p in patterns:
         contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True)
     for path in contents:
-        path = os.path.realpath(path)
+        path = os.path.abspath(path)
         if os.path.isdir(path):
             fs.rmtree(path)
         elif os.path.isfile(path):

platformio/commands/device/__init__.py

@@ -12,4 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio.commands.device.filters.base import DeviceMonitorFilter
+# pylint: disable=unused-import
+from platformio.device.filters.base import (
+    DeviceMonitorFilterBase as DeviceMonitorFilter,
+)

platformio/commands/device/command.py

@@ -1,243 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import os
-import sys
-from fnmatch import fnmatch
-
-import click
-from serial.tools import miniterm
-
-from platformio import exception, fs, util
-from platformio.commands.device import helpers as device_helpers
-from platformio.platform.factory import PlatformFactory
-from platformio.project.exception import NotPlatformIOProjectError
-
-
-@click.group(short_help="Device manager & serial/socket monitor")
-def cli():
-    pass
-
-
-@cli.command("list", short_help="List devices")
-@click.option("--serial", is_flag=True, help="List serial ports, default")
-@click.option("--logical", is_flag=True, help="List logical devices")
-@click.option("--mdns", is_flag=True, help="List multicast DNS services")
-@click.option("--json-output", is_flag=True)
-def device_list(  # pylint: disable=too-many-branches
-    serial, logical, mdns, json_output
-):
-    if not logical and not mdns:
-        serial = True
-    data = {}
-    if serial:
-        data["serial"] = util.get_serial_ports()
-    if logical:
-        data["logical"] = util.get_logical_devices()
-    if mdns:
-        data["mdns"] = util.get_mdns_services()
-
-    single_key = list(data)[0] if len(list(data)) == 1 else None
-
-    if json_output:
-        return click.echo(json.dumps(data[single_key] if single_key else data))
-
-    titles = {
-        "serial": "Serial Ports",
-        "logical": "Logical Devices",
-        "mdns": "Multicast DNS Services",
-    }
-
-    for key, value in data.items():
-        if not single_key:
-            click.secho(titles[key], bold=True)
-            click.echo("=" * len(titles[key]))
-
-        if key == "serial":
-            for item in value:
-                click.secho(item["port"], fg="cyan")
-                click.echo("-" * len(item["port"]))
-                click.echo("Hardware ID: %s" % item["hwid"])
-                click.echo("Description: %s" % item["description"])
-                click.echo("")
-
-        if key == "logical":
-            for item in value:
-                click.secho(item["path"], fg="cyan")
-                click.echo("-" * len(item["path"]))
-                click.echo("Name: %s" % item["name"])
-                click.echo("")
-
-        if key == "mdns":
-            for item in value:
-                click.secho(item["name"], fg="cyan")
-                click.echo("-" * len(item["name"]))
-                click.echo("Type: %s" % item["type"])
-                click.echo("IP: %s" % item["ip"])
-                click.echo("Port: %s" % item["port"])
-                if item["properties"]:
-                    click.echo(
-                        "Properties: %s"
-                        % (
-                            "; ".join(
-                                [
-                                    "%s=%s" % (k, v)
-                                    for k, v in item["properties"].items()
-                                ]
-                            )
-                        )
-                    )
-                click.echo("")
-
-        if single_key:
-            click.echo("")
-
-    return True
-
-
-@cli.command("monitor", short_help="Monitor device (Serial)")
-@click.option("--port", "-p", help="Port, a number or a device name")
-@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
-@click.option(
-    "--parity",
-    default="N",
-    type=click.Choice(["N", "E", "O", "S", "M"]),
-    help="Set parity, default=N",
-)
-@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
-@click.option(
-    "--xonxoff", is_flag=True, help="Enable software flow control, default=Off"
-)
-@click.option(
-    "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state"
-)
-@click.option(
-    "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state"
-)
-@click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
-@click.option(
-    "--encoding",
-    default="UTF-8",
-    help="Set the encoding for the serial port (e.g. hexlify, "
-    "Latin1, UTF-8), default: UTF-8",
-)
-@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations")
-@click.option(
-    "--eol",
-    default="CRLF",
-    type=click.Choice(["CR", "LF", "CRLF"]),
-    help="End of line mode, default=CRLF",
-)
-@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations")
-@click.option(
-    "--exit-char",
-    type=int,
-    default=3,
-    help="ASCII code of special character that is used to exit "
-    "the application, default=3 (Ctrl+C)",
-)
-@click.option(
-    "--menu-char",
-    type=int,
-    default=20,
-    help="ASCII code of special character that is used to "
-    "control miniterm (menu), default=20 (DEC)",
-)
-@click.option(
-    "--quiet",
-    is_flag=True,
-    help="Diagnostics: suppress non-error messages, default=Off",
-)
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
-)
-@click.option(
-    "-e",
-    "--environment",
-    help="Load configuration from `platformio.ini` and specified environment",
-)
-def device_monitor(**kwargs):  # pylint: disable=too-many-branches
-    # load default monitor filters
-    filters_dir = os.path.join(fs.get_source_dir(), "commands", "device", "filters")
-    for name in os.listdir(filters_dir):
-        if not name.endswith(".py"):
-            continue
-        device_helpers.load_monitor_filter(
-            os.path.join(filters_dir, name), options=kwargs
-        )
-
-    project_options = {}
-    try:
-        with fs.cd(kwargs["project_dir"]):
-            project_options = device_helpers.get_project_options(kwargs["environment"])
-        kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options)
-    except NotPlatformIOProjectError:
-        pass
-
-    platform = None
-    if "platform" in project_options:
-        with fs.cd(kwargs["project_dir"]):
-            platform = PlatformFactory.new(project_options["platform"])
-            device_helpers.register_platform_filters(platform, options=kwargs)
-
-    if not kwargs["port"]:
-        ports = util.get_serial_ports(filter_hwid=True)
-        if len(ports) == 1:
-            kwargs["port"] = ports[0]["port"]
-        elif "platform" in project_options and "board" in project_options:
-            board_hwids = device_helpers.get_board_hwids(
-                kwargs["project_dir"],
-                platform,
-                project_options["board"],
-            )
-            for item in ports:
-                for hwid in board_hwids:
-                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                    if hwid_str in item["hwid"]:
-                        kwargs["port"] = item["port"]
-                        break
-                if kwargs["port"]:
-                    break
-    elif kwargs["port"] and (set(["*", "?", "[", "]"]) & set(kwargs["port"])):
-        for item in util.get_serial_ports():
-            if fnmatch(item["port"], kwargs["port"]):
-                kwargs["port"] = item["port"]
-                break
-
-    # override system argv with patched options
-    sys.argv = ["monitor"] + device_helpers.options_to_argv(
-        kwargs,
-        project_options,
-        ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"),
-    )
-
-    if not kwargs["quiet"]:
-        click.echo(
-            "--- Available filters and text transformations: %s"
-            % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys()))
-        )
-        click.echo("--- More details at http://bit.ly/pio-monitor-filters")
-    try:
-        miniterm.main(
-            default_port=kwargs["port"],
-            default_baudrate=kwargs["baud"] or 9600,
-            default_rts=kwargs["rts"],
-            default_dtr=kwargs["dtr"],
-        )
-    except Exception as e:
-        raise exception.MinitermException(e)

platformio/commands/device/filters/base.py

@@ -1,43 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from serial.tools import miniterm
-
-from platformio.project.config import ProjectConfig
-
-
-class DeviceMonitorFilter(miniterm.Transform):
-    def __init__(self, options=None):
-        """Called by PlatformIO to pass context"""
-        miniterm.Transform.__init__(self)
-
-        self.options = options or {}
-        self.project_dir = self.options.get("project_dir")
-        self.environment = self.options.get("environment")
-
-        self.config = ProjectConfig.get_instance()
-        if not self.environment:
-            default_envs = self.config.default_envs()
-            if default_envs:
-                self.environment = default_envs[0]
-            elif self.config.envs():
-                self.environment = self.config.envs()[0]
-
-    def __call__(self):
-        """Called by the miniterm library when the filter is actually used"""
-        return self
-
-    @property
-    def NAME(self):
-        raise NotImplementedError("Please declare NAME attribute for the filter class")

platformio/commands/device/helpers.py

@@ -1,106 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import inspect
-import os
-
-from serial.tools import miniterm
-
-from platformio import fs
-from platformio.commands.device import DeviceMonitorFilter
-from platformio.compat import get_object_members, load_python_module
-from platformio.project.config import ProjectConfig
-
-
-def apply_project_monitor_options(cli_options, project_options):
-    for k in ("port", "speed", "rts", "dtr"):
-        k2 = "monitor_%s" % k
-        if k == "speed":
-            k = "baud"
-        if cli_options[k] is None and k2 in project_options:
-            cli_options[k] = project_options[k2]
-            if k != "port":
-                cli_options[k] = int(cli_options[k])
-    return cli_options
-
-
-def options_to_argv(cli_options, project_options, ignore=None):
-    confmon_flags = project_options.get("monitor_flags", [])
-    result = confmon_flags[::]
-
-    for f in project_options.get("monitor_filters", []):
-        result.extend(["--filter", f])
-
-    for k, v in cli_options.items():
-        if v is None or (ignore and k in ignore):
-            continue
-        k = "--" + k.replace("_", "-")
-        if k in confmon_flags:
-            continue
-        if isinstance(v, bool):
-            if v:
-                result.append(k)
-        elif isinstance(v, tuple):
-            for i in v:
-                result.extend([k, i])
-        else:
-            result.extend([k, str(v)])
-    return result
-
-
-def get_project_options(environment=None):
-    config = ProjectConfig.get_instance()
-    config.validate(envs=[environment] if environment else None)
-    if not environment:
-        default_envs = config.default_envs()
-        if default_envs:
-            environment = default_envs[0]
-        else:
-            environment = config.envs()[0]
-    return config.items(env=environment, as_dict=True)
-
-
-def get_board_hwids(project_dir, platform, board):
-    with fs.cd(project_dir):
-        return platform.board_config(board).get("build.hwids", [])
-
-
-def load_monitor_filter(path, options=None):
-    name = os.path.basename(path)
-    name = name[: name.find(".")]
-    module = load_python_module("platformio.commands.device.filters.%s" % name, path)
-    for cls in get_object_members(module).values():
-        if (
-            not inspect.isclass(cls)
-            or not issubclass(cls, DeviceMonitorFilter)
-            or cls == DeviceMonitorFilter
-        ):
-            continue
-        obj = cls(options)
-        miniterm.TRANSFORMATIONS[obj.NAME] = obj
-    return True
-
-
-def register_platform_filters(platform, options=None):
-    monitor_dir = os.path.join(platform.get_dir(), "monitor")
-    if not os.path.isdir(monitor_dir):
-        return
-
-    for name in os.listdir(monitor_dir):
-        if not name.startswith("filter_") or not name.endswith(".py"):
-            continue
-        path = os.path.join(monitor_dir, name)
-        if not os.path.isfile(path):
-            continue
-        load_monitor_filter(path, options)

platformio/commands/home/rpc/handlers/ide.py

@@ -1,48 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import time
-
-from ajsonrpc.core import JSONRPC20DispatchException
-
-from platformio.compat import aio_get_running_loop
-
-
-class IDERPC:
-    def __init__(self):
-        self._queue = {}
-
-    def send_command(self, sid, command, params):
-        if not self._queue.get(sid):
-            raise JSONRPC20DispatchException(
-                code=4005, message="PIO Home IDE agent is not started"
-            )
-        while self._queue[sid]:
-            self._queue[sid].pop().set_result(
-                {"id": time.time(), "method": command, "params": params}
-            )
-
-    async def listen_commands(self, sid=0):
-        if sid not in self._queue:
-            self._queue[sid] = []
-        self._queue[sid].append(aio_get_running_loop().create_future())
-        return await self._queue[sid][-1]
-
-    def open_project(self, sid, project_dir):
-        return self.send_command(sid, "open_project", project_dir)
-
-    def open_text_document(self, sid, path, line=None, column=None):
-        return self.send_command(
-            sid, "open_text_document", dict(path=path, line=line, column=column)
-        )

platformio/commands/lib/command.py

@@ -15,14 +15,16 @@
 # pylint: disable=too-many-branches, too-many-locals
 
 import json
+import logging
 import os
 import time
+from urllib.parse import quote
 
 import click
 from tabulate import tabulate
 
 from platformio import exception, fs, util
-from platformio.commands import PlatformioCLI
+from platformio.cli import PlatformioCLI
 from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps
 from platformio.package.exception import NotGlobalLibDir, UnknownPackageError
 from platformio.package.manager.library import LibraryPackageManager
@@ -31,11 +33,6 @@ from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import get_project_dir, is_platformio_project
 
-try:
-    from urllib.parse import quote
-except ImportError:
-    from urllib import quote
-
 CTX_META_INPUT_DIRS_KEY = __name__ + ".input_dirs"
 CTX_META_PROJECT_ENVIRONMENTS_KEY = __name__ + ".project_environments"
 CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs"
@@ -43,10 +40,10 @@ CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps"
 
 
 def get_project_global_lib_dir():
-    return ProjectConfig.get_instance().get_optional_dir("globallib")
+    return ProjectConfig.get_instance().get("platformio", "globallib_dir")
 
 
-@click.group(short_help="Library manager")
+@click.group(short_help="Library manager", hidden=True)
 @click.option(
     "-d",
     "--storage-dir",
@@ -71,6 +68,7 @@ def get_project_global_lib_dir():
 )
 @click.pass_context
 def cli(ctx, **options):
+    in_silence = PlatformioCLI.in_silence()
     storage_cmds = ("install", "uninstall", "update", "list")
     # skip commands that don't need storage folder
     if ctx.invoked_subcommand not in storage_cmds or (
@@ -97,7 +95,6 @@ def cli(ctx, **options):
             get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand
         )
 
-    in_silence = PlatformioCLI.in_silence()
     ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options["environment"]
     ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs
     ctx.meta[CTX_META_STORAGE_DIRS_KEY] = []
@@ -111,7 +108,7 @@ def cli(ctx, **options):
                 os.path.join(storage_dir, "platformio.ini")
             )
             config.validate(options["environment"], silent=in_silence)
-            libdeps_dir = config.get_optional_dir("libdeps")
+            libdeps_dir = config.get("platformio", "libdeps_dir")
             for env in config.envs():
                 if options["environment"] and env not in options["environment"]:
                     continue
@@ -144,6 +141,11 @@ def cli(ctx, **options):
 def lib_install(  # pylint: disable=too-many-arguments,unused-argument
     ctx, libraries, save, silent, interactive, force
 ):
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg install` instead.\n",
+        fg="yellow",
+    )
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, [])
 
@@ -152,16 +154,16 @@ def lib_install(  # pylint: disable=too-many-arguments,unused-argument
         if not silent and (libraries or storage_dir in storage_libdeps):
             print_storage_header(storage_dirs, storage_dir)
         lm = LibraryPackageManager(storage_dir)
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
 
         if libraries:
             installed_pkgs = {
-                library: lm.install(library, silent=silent, force=force)
-                for library in libraries
+                library: lm.install(library, force=force) for library in libraries
             }
 
         elif storage_dir in storage_libdeps:
             for library in storage_libdeps[storage_dir]:
-                lm.install(library, silent=silent, force=force)
+                lm.install(library, force=force)
 
     if save and installed_pkgs:
         _save_deps(ctx, installed_pkgs)
@@ -207,14 +209,18 @@ def _save_deps(ctx, pkgs, action="add"):
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.pass_context
 def lib_uninstall(ctx, libraries, save, silent):
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg uninstall` instead.\n",
+        fg="yellow",
+    )
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     uninstalled_pkgs = {}
     for storage_dir in storage_dirs:
         print_storage_header(storage_dirs, storage_dir)
         lm = LibraryPackageManager(storage_dir)
-        uninstalled_pkgs = {
-            library: lm.uninstall(library, silent=silent) for library in libraries
-        }
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
+        uninstalled_pkgs = {library: lm.uninstall(library) for library in libraries}
 
     if save and uninstalled_pkgs:
         _save_deps(ctx, uninstalled_pkgs, action="remove")
@@ -237,14 +243,27 @@ def lib_uninstall(ctx, libraries, save, silent):
 def lib_update(  # pylint: disable=too-many-arguments
     ctx, libraries, only_check, dry_run, silent, json_output
 ):
-    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     only_check = dry_run or only_check
+    if only_check and not json_output:
+        raise exception.UserSideException(
+            "This command is deprecated, please use `pio pkg outdated` instead"
+        )
+
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg update` instead.\n",
+            fg="yellow",
+        )
+
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     json_result = {}
     for storage_dir in storage_dirs:
         if not json_output:
             print_storage_header(storage_dirs, storage_dir)
         lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, [])
         lm = LibraryPackageManager(storage_dir)
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
         _libraries = libraries or lib_deps or lm.get_installed()
 
         if only_check and json_output:
@@ -277,9 +296,7 @@ def lib_update(  # pylint: disable=too-many-arguments
                     None if isinstance(library, PackageItem) else PackageSpec(library)
                 )
                 try:
-                    lm.update(
-                        library, to_spec=to_spec, only_check=only_check, silent=silent
-                    )
+                    lm.update(library, to_spec=to_spec)
                 except UnknownPackageError as e:
                     if library not in lib_deps:
                         raise e
@@ -298,6 +315,12 @@ def lib_update(  # pylint: disable=too-many-arguments
 @click.option("--json-output", is_flag=True)
 @click.pass_context
 def lib_list(ctx, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg list` instead.\n",
+            fg="yellow",
+        )
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     json_result = {}
     for storage_dir in storage_dirs:
@@ -341,6 +364,12 @@ def lib_list(ctx, json_output):
     help="Do not prompt, automatically paginate with delay",
 )
 def lib_search(query, json_output, page, noninteractive, **filters):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg search` instead.\n",
+            fg="yellow",
+        )
     regclient = LibraryPackageManager().get_registry_client_instance()
     if not query:
         query = []
@@ -355,7 +384,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         "get",
         "/v2/lib/search",
         params=dict(query=" ".join(query), page=page),
-        cache_valid="1d",
+        x_cache_valid="1d",
     )
 
     if json_output:
@@ -408,7 +437,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
             "get",
             "/v2/lib/search",
             params=dict(query=" ".join(query), page=int(result["page"]) + 1),
-            cache_valid="1d",
+            x_cache_valid="1d",
         )
 
 
@@ -437,10 +466,19 @@ def lib_builtin(storage, json_output):
 @click.argument("library", metavar="[LIBRARY]")
 @click.option("--json-output", is_flag=True)
 def lib_show(library, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg show` instead.\n",
+            fg="yellow",
+        )
     lm = LibraryPackageManager()
-    lib_id = lm.reveal_registry_package_id(library, silent=json_output)
+    lm.set_log_level(logging.ERROR if json_output else logging.DEBUG)
+    lib_id = lm.reveal_registry_package_id(library)
     regclient = lm.get_registry_client_instance()
-    lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h")
+    lib = regclient.fetch_json_data(
+        "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h"
+    )
     if json_output:
         return click.echo(json.dumps(lib))
 
@@ -455,7 +493,7 @@ def lib_show(library, json_output):
         "Version: %s, released %s"
         % (
             lib["version"]["name"],
-            time.strftime("%c", util.parse_date(lib["version"]["released"])),
+            util.parse_datetime(lib["version"]["released"]).strftime("%c"),
         )
     )
     click.echo("Manifest: %s" % lib["confurl"])
@@ -463,9 +501,9 @@ def lib_show(library, json_output):
         if key not in lib or not lib[key]:
             continue
         if isinstance(lib[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(lib[key])))
+            click.echo("%s: %s" % (key.capitalize(), ", ".join(lib[key])))
         else:
-            click.echo("%s: %s" % (key.title(), lib[key]))
+            click.echo("%s: %s" % (key.capitalize(), lib[key]))
 
     blocks = []
 
@@ -497,7 +535,7 @@ def lib_show(library, json_output):
             "Versions",
             [
                 "%s, released %s"
-                % (v["name"], time.strftime("%c", util.parse_date(v["released"])))
+                % (v["name"], util.parse_datetime(v["released"]).strftime("%c"))
                 for v in lib["versions"]
             ],
         )
@@ -527,7 +565,7 @@ def lib_show(library, json_output):
 @click.argument("config_url")
 def lib_register(config_url):  # pylint: disable=unused-argument
     raise exception.UserSideException(
-        "This command is deprecated. Please use `pio package publish` command."
+        "This command is deprecated. Please use `pio pkg publish` command."
     )
 
 
@@ -535,7 +573,7 @@ def lib_register(config_url):  # pylint: disable=unused-argument
 @click.option("--json-output", is_flag=True)
 def lib_stats(json_output):
     regclient = LibraryPackageManager().get_registry_client_instance()
-    result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h")
+    result = regclient.fetch_json_data("get", "/v2/lib/stats", x_cache_valid="1h")
 
     if json_output:
         return click.echo(json.dumps(result))
@@ -544,7 +582,7 @@ def lib_stats(json_output):
         tabular_data = [
             (
                 click.style(item["name"], fg="cyan"),
-                time.strftime("%c", util.parse_date(item["date"])),
+                util.parse_datetime(item["date"]).strftime("%c"),
                 "https://platformio.org/lib/show/%s/%s"
                 % (item["id"], quote(item["name"])),
             )
@@ -619,9 +657,9 @@ def print_lib_item(item):
         if key not in item or not item[key]:
             continue
         if isinstance(item[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
+            click.echo("%s: %s" % (key.capitalize(), ", ".join(item[key])))
         else:
-            click.echo("%s: %s" % (key.title(), item[key]))
+            click.echo("%s: %s" % (key.capitalize(), item[key]))
 
     for key in ("frameworks", "platforms"):
         if key not in item:

platformio/commands/lib/helpers.py

@@ -14,6 +14,7 @@
 
 import os
 
+from platformio import util
 from platformio.compat import ci_strings_are_equal
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.meta import PackageSpec
@@ -22,6 +23,7 @@ from platformio.project.config import ProjectConfig
 from platformio.project.exception import InvalidProjectConfError
 
 
+@util.memoized(expire="60s")
 def get_builtin_libs(storage_names=None):
     # pylint: disable=import-outside-toplevel
     from platformio.package.manager.library import LibraryPackageManager
@@ -45,8 +47,8 @@ def get_builtin_libs(storage_names=None):
     return items
 
 
-def is_builtin_lib(name, storages=None):
-    for storage in storages or get_builtin_libs():
+def is_builtin_lib(name):
+    for storage in get_builtin_libs():
         for lib in storage["items"]:
             if lib.get("name") == name:
                 return True

platformio/commands/org.py

@@ -1,165 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import json
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.account import AccountClient
-from platformio.commands.account import validate_email, validate_username
-
-
-@click.group("org", short_help="Manage organizations")
-def cli():
-    pass
-
-
-def validate_orgname(value):
-    return validate_username(value, "Organization name")
-
-
-@cli.command("create", short_help="Create a new organization")
-@click.argument(
-    "orgname",
-    callback=lambda _, __, value: validate_orgname(value),
-)
-@click.option(
-    "--email", callback=lambda _, __, value: validate_email(value) if value else value
-)
-@click.option(
-    "--displayname",
-)
-def org_create(orgname, email, displayname):
-    client = AccountClient()
-    client.create_org(orgname, email, displayname)
-    return click.secho(
-        "The organization `%s` has been successfully created." % orgname,
-        fg="green",
-    )
-
-
-@cli.command("list", short_help="List organizations and their members")
-@click.option("--json-output", is_flag=True)
-def org_list(json_output):
-    client = AccountClient()
-    orgs = client.list_orgs()
-    if json_output:
-        return click.echo(json.dumps(orgs))
-    if not orgs:
-        return click.echo("You do not have any organization")
-    for org in orgs:
-        click.echo()
-        click.secho(org.get("orgname"), fg="cyan")
-        click.echo("-" * len(org.get("orgname")))
-        data = []
-        if org.get("displayname"):
-            data.append(("Display Name:", org.get("displayname")))
-        if org.get("email"):
-            data.append(("Email:", org.get("email")))
-        data.append(
-            (
-                "Owners:",
-                ", ".join((owner.get("username") for owner in org.get("owners"))),
-            )
-        )
-        click.echo(tabulate(data, tablefmt="plain"))
-    return click.echo()
-
-
-@cli.command("update", short_help="Update organization")
-@click.argument("cur_orgname")
-@click.option(
-    "--orgname",
-    callback=lambda _, __, value: validate_orgname(value),
-    help="A new orgname",
-)
-@click.option("--email")
-@click.option("--displayname")
-def org_update(cur_orgname, **kwargs):
-    client = AccountClient()
-    org = client.get_org(cur_orgname)
-    del org["owners"]
-    new_org = org.copy()
-    if not any(kwargs.values()):
-        for field in org:
-            new_org[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=org[field]
-            )
-            if field == "email":
-                validate_email(new_org[field])
-            if field == "orgname":
-                validate_orgname(new_org[field])
-    else:
-        new_org.update(
-            {key.replace("new_", ""): value for key, value in kwargs.items() if value}
-        )
-    client.update_org(cur_orgname, new_org)
-    return click.secho(
-        "The organization `%s` has been successfully updated." % cur_orgname,
-        fg="green",
-    )
-
-
-@cli.command("destroy", short_help="Destroy organization")
-@click.argument("orgname")
-def account_destroy(orgname):
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the `%s` organization account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % orgname,
-        abort=True,
-    )
-    client.destroy_org(orgname)
-    return click.secho(
-        "Organization `%s` has been destroyed." % orgname,
-        fg="green",
-    )
-
-
-@cli.command("add", short_help="Add a new owner to organization")
-@click.argument(
-    "orgname",
-)
-@click.argument(
-    "username",
-)
-def org_add_owner(orgname, username):
-    client = AccountClient()
-    client.add_org_owner(orgname, username)
-    return click.secho(
-        "The new owner `%s` has been successfully added to the `%s` organization."
-        % (username, orgname),
-        fg="green",
-    )
-
-
-@cli.command("remove", short_help="Remove an owner from organization")
-@click.argument(
-    "orgname",
-)
-@click.argument(
-    "username",
-)
-def org_remove_owner(orgname, username):
-    client = AccountClient()
-    client.remove_org_owner(orgname, username)
-    return click.secho(
-        "The `%s` owner has been successfully removed from the `%s` organization."
-        % (username, orgname),
-        fg="green",
-    )

platformio/commands/platform.py

@@ -13,12 +13,14 @@
 # limitations under the License.
 
 import json
+import logging
 import os
 
 import click
 
-from platformio.cache import cleanup_content_cache
 from platformio.commands.boards import print_boards
+from platformio.exception import UserSideException
+from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.meta import PackageItem, PackageSpec
 from platformio.package.version import get_original_version
@@ -26,7 +28,7 @@ from platformio.platform.exception import UnknownPlatform
 from platformio.platform.factory import PlatformFactory
 
 
-@click.group(short_help="Platform manager")
+@click.group(short_help="Platform manager", hidden=True)
 def cli():
     pass
 
@@ -35,6 +37,12 @@ def cli():
 @click.argument("query", required=False)
 @click.option("--json-output", is_flag=True)
 def platform_search(query, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg search` instead.\n",
+            fg="yellow",
+        )
     platforms = []
     for platform in _get_registry_platforms():
         if query == "all":
@@ -61,7 +69,7 @@ def platform_frameworks(query, json_output):
     regclient = PlatformPackageManager().get_registry_client_instance()
     frameworks = []
     for framework in regclient.fetch_json_data(
-        "get", "/v2/frameworks", cache_valid="1d"
+        "get", "/v2/frameworks", x_cache_valid="1d"
     ):
         if query == "all":
             query = ""
@@ -86,6 +94,12 @@ def platform_frameworks(query, json_output):
 @cli.command("list", short_help="List installed development platforms")
 @click.option("--json-output", is_flag=True)
 def platform_list(json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg list` instead.\n",
+            fg="yellow",
+        )
     platforms = []
     pm = PlatformPackageManager()
     for pkg in pm.get_installed():
@@ -104,6 +118,12 @@ def platform_list(json_output):
 @click.argument("platform")
 @click.option("--json-output", is_flag=True)
 def platform_show(platform, json_output):  # pylint: disable=too-many-branches
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg show` instead.\n",
+            fg="yellow",
+        )
     data = _get_platform_data(platform)
     if not data:
         raise UnknownPlatform(platform)
@@ -178,7 +198,7 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     is_flag=True,
     help="Reinstall/redownload dev/platform and its packages if exist",
 )
-def platform_install(  # pylint: disable=too-many-arguments
+def platform_install(  # pylint: disable=too-many-arguments,too-many-locals
     platforms,
     with_package,
     without_package,
@@ -187,37 +207,56 @@ def platform_install(  # pylint: disable=too-many-arguments
     silent,
     force,
 ):
-    return _platform_install(
-        platforms,
-        with_package,
-        without_package,
-        skip_default_package,
-        with_all_packages,
-        silent,
-        force,
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg install` instead.\n",
+        fg="yellow",
     )
 
+    def _find_pkg_names(p, candidates):
+        result = []
+        for candidate in candidates:
+            found = False
+            # lookup by package types
+            for _name, _opts in p.packages.items():
+                if _opts.get("type") == candidate:
+                    result.append(_name)
+                    found = True
+            if (
+                p.frameworks
+                and candidate.startswith("framework-")
+                and candidate[10:] in p.frameworks
+            ):
+                result.append(p.frameworks[candidate[10:]]["package"])
+                found = True
+            if not found:
+                result.append(candidate)
+        return result
 
-def _platform_install(  # pylint: disable=too-many-arguments
-    platforms,
-    with_package=None,
-    without_package=None,
-    skip_default_package=False,
-    with_all_packages=False,
-    silent=False,
-    force=False,
-):
     pm = PlatformPackageManager()
+    pm.set_log_level(logging.WARN if silent else logging.DEBUG)
     for platform in platforms:
-        pkg = pm.install(
-            spec=platform,
-            with_packages=with_package or [],
-            without_packages=without_package or [],
-            skip_default_package=skip_default_package,
-            with_all_packages=with_all_packages,
-            silent=silent,
-            force=force,
-        )
+        if with_package or without_package or with_all_packages:
+            pkg = pm.install(platform, skip_dependencies=True)
+            p = PlatformFactory.new(pkg)
+            if with_all_packages:
+                with_package = list(p.packages)
+            with_package = set(_find_pkg_names(p, with_package or []))
+            without_package = set(_find_pkg_names(p, without_package or []))
+            upkgs = with_package | without_package
+            ppkgs = set(p.packages)
+            if not upkgs.issubset(ppkgs):
+                raise UnknownPackageError(", ".join(upkgs - ppkgs))
+            for name, options in p.packages.items():
+                if name in without_package:
+                    continue
+                if name in with_package or not (
+                    skip_default_package or options.get("optional", False)
+                ):
+                    p.pm.install(p.get_package_spec(name), force=force)
+        else:
+            pkg = pm.install(platform, skip_dependencies=skip_default_package)
+
         if pkg and not silent:
             click.secho(
                 "The platform '%s' has been successfully installed!\n"
@@ -230,7 +269,13 @@ def _platform_install(  # pylint: disable=too-many-arguments
 @cli.command("uninstall", short_help="Uninstall development platform")
 @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]")
 def platform_uninstall(platforms):
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg uninstall` instead.\n",
+        fg="yellow",
+    )
     pm = PlatformPackageManager()
+    pm.set_log_level(logging.DEBUG)
     for platform in platforms:
         if pm.uninstall(platform):
             click.secho(
@@ -256,9 +301,22 @@ def platform_uninstall(platforms):
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option("--json-output", is_flag=True)
 def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
-    platforms, only_packages, only_check, dry_run, silent, json_output
+    platforms, only_check, dry_run, silent, json_output, **_
 ):
+    if only_check and not json_output:
+        raise UserSideException(
+            "This command is deprecated, please use `pio pkg outdated` instead"
+        )
+
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg update` instead.\n",
+            fg="yellow",
+        )
+
     pm = PlatformPackageManager()
+    pm.set_log_level(logging.WARN if silent else logging.DEBUG)
     platforms = platforms or pm.get_installed()
     only_check = dry_run or only_check
 
@@ -290,9 +348,6 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
             result.append(data)
         return click.echo(json.dumps(result))
 
-    # cleanup cached board and platform lists
-    cleanup_content_cache("http")
-
     for platform in platforms:
         click.echo(
             "Platform %s"
@@ -304,9 +359,7 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
             )
         )
         click.echo("--------")
-        pm.update(
-            platform, only_packages=only_packages, only_check=only_check, silent=silent
-        )
+        pm.update(platform)
         click.echo()
 
     return True
@@ -317,15 +370,6 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
 #
 
 
-def init_platform(name, skip_default_package=True, auto_install=True):
-    try:
-        return PlatformFactory.new(name)
-    except UnknownPlatform:
-        if auto_install:
-            _platform_install([name], skip_default_package=skip_default_package)
-        return PlatformFactory.new(name)
-
-
 def _print_platforms(platforms):
     for platform in platforms:
         click.echo(
@@ -354,7 +398,7 @@ def _print_platforms(platforms):
 
 def _get_registry_platforms():
     regclient = PlatformPackageManager().get_registry_client_instance()
-    return regclient.fetch_json_data("get", "/v2/platforms", cache_valid="1d")
+    return regclient.fetch_json_data("get", "/v2/platforms", x_cache_valid="1d")
 
 
 def _get_platform_data(*args, **kwargs):

platformio/commands/system/command.py

@@ -1,214 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import platform
-import subprocess
-import sys
-
-import click
-from tabulate import tabulate
-
-from platformio import __version__, compat, fs, proc, util
-from platformio.commands.system.completion import (
-    get_completion_install_path,
-    install_completion_code,
-    uninstall_completion_code,
-)
-from platformio.commands.system.prune import (
-    prune_cached_data,
-    prune_core_packages,
-    prune_platform_packages,
-)
-from platformio.package.manager.library import LibraryPackageManager
-from platformio.package.manager.platform import PlatformPackageManager
-from platformio.package.manager.tool import ToolPackageManager
-from platformio.project.config import ProjectConfig
-
-
-@click.group("system", short_help="Miscellaneous system commands")
-def cli():
-    pass
-
-
-@cli.command("info", short_help="Display system-wide information")
-@click.option("--json-output", is_flag=True)
-def system_info(json_output):
-    project_config = ProjectConfig()
-    data = {}
-    data["core_version"] = {"title": "PlatformIO Core", "value": __version__}
-    data["python_version"] = {
-        "title": "Python",
-        "value": "{0}.{1}.{2}-{3}.{4}".format(*list(sys.version_info)),
-    }
-    data["system"] = {"title": "System Type", "value": util.get_systype()}
-    data["platform"] = {"title": "Platform", "value": platform.platform(terse=True)}
-    data["filesystem_encoding"] = {
-        "title": "File System Encoding",
-        "value": compat.get_filesystem_encoding(),
-    }
-    data["locale_encoding"] = {
-        "title": "Locale Encoding",
-        "value": compat.get_locale_encoding(),
-    }
-    data["core_dir"] = {
-        "title": "PlatformIO Core Directory",
-        "value": project_config.get_optional_dir("core"),
-    }
-    data["platformio_exe"] = {
-        "title": "PlatformIO Core Executable",
-        "value": proc.where_is_program(
-            "platformio.exe" if compat.IS_WINDOWS else "platformio"
-        ),
-    }
-    data["python_exe"] = {
-        "title": "Python Executable",
-        "value": proc.get_pythonexe_path(),
-    }
-    data["global_lib_nums"] = {
-        "title": "Global Libraries",
-        "value": len(LibraryPackageManager().get_installed()),
-    }
-    data["dev_platform_nums"] = {
-        "title": "Development Platforms",
-        "value": len(PlatformPackageManager().get_installed()),
-    }
-    data["package_tool_nums"] = {
-        "title": "Tools & Toolchains",
-        "value": len(
-            ToolPackageManager(
-                project_config.get_optional_dir("packages")
-            ).get_installed()
-        ),
-    }
-
-    click.echo(
-        json.dumps(data)
-        if json_output
-        else tabulate([(item["title"], item["value"]) for item in data.values()])
-    )
-
-
-@cli.command("prune", short_help="Remove unused data")
-@click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation")
-@click.option(
-    "--dry-run", is_flag=True, help="Do not prune, only show data that will be removed"
-)
-@click.option("--cache", is_flag=True, help="Prune only cached data")
-@click.option(
-    "--core-packages", is_flag=True, help="Prune only unnecessary core packages"
-)
-@click.option(
-    "--platform-packages",
-    is_flag=True,
-    help="Prune only unnecessary development platform packages",
-)
-def system_prune(force, dry_run, cache, core_packages, platform_packages):
-    if dry_run:
-        click.secho(
-            "Dry run mode (do not prune, only show data that will be removed)",
-            fg="yellow",
-        )
-        click.echo()
-
-    reclaimed_cache = 0
-    reclaimed_core_packages = 0
-    reclaimed_platform_packages = 0
-    prune_all = not any([cache, core_packages, platform_packages])
-
-    if cache or prune_all:
-        reclaimed_cache = prune_cached_data(force, dry_run)
-        click.echo()
-
-    if core_packages or prune_all:
-        reclaimed_core_packages = prune_core_packages(force, dry_run)
-        click.echo()
-
-    if platform_packages or prune_all:
-        reclaimed_platform_packages = prune_platform_packages(force, dry_run)
-        click.echo()
-
-    click.secho(
-        "Total reclaimed space: %s"
-        % fs.humanize_file_size(
-            reclaimed_cache + reclaimed_core_packages + reclaimed_platform_packages
-        ),
-        fg="green",
-    )
-
-
-@cli.group("completion", short_help="Shell completion support")
-def completion():
-    # pylint: disable=import-error,import-outside-toplevel
-    try:
-        import click_completion  # pylint: disable=unused-import,unused-variable
-    except ImportError:
-        click.echo("Installing dependent packages...")
-        subprocess.check_call(
-            [proc.get_pythonexe_path(), "-m", "pip", "install", "click-completion"],
-        )
-
-
-@completion.command("install", short_help="Install shell completion files/code")
-@click.option(
-    "--shell",
-    default=None,
-    type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]),
-    help="The shell type, default=auto",
-)
-@click.option(
-    "--path",
-    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
-    help="Custom installation path of the code to be evaluated by the shell. "
-    "The standard installation path is used by default.",
-)
-def completion_install(shell, path):
-
-    import click_completion  # pylint: disable=import-outside-toplevel,import-error
-
-    shell = shell or click_completion.get_auto_shell()
-    path = path or get_completion_install_path(shell)
-    install_completion_code(shell, path)
-    click.echo(
-        "PlatformIO CLI completion has been installed for %s shell to %s \n"
-        "Please restart a current shell session."
-        % (click.style(shell, fg="cyan"), click.style(path, fg="blue"))
-    )
-
-
-@completion.command("uninstall", short_help="Uninstall shell completion files/code")
-@click.option(
-    "--shell",
-    default=None,
-    type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]),
-    help="The shell type, default=auto",
-)
-@click.option(
-    "--path",
-    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
-    help="Custom installation path of the code to be evaluated by the shell. "
-    "The standard installation path is used by default.",
-)
-def completion_uninstall(shell, path):
-
-    import click_completion  # pylint: disable=import-outside-toplevel,import-error
-
-    shell = shell or click_completion.get_auto_shell()
-    path = path or get_completion_install_path(shell)
-    uninstall_completion_code(shell, path)
-    click.echo(
-        "PlatformIO CLI completion has been uninstalled for %s shell from %s \n"
-        "Please restart a current shell session."
-        % (click.style(shell, fg="cyan"), click.style(path, fg="blue"))
-    )

platformio/commands/team.py

@@ -1,212 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import json
-import re
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.account import AccountClient
-
-
-def validate_orgname_teamname(value, teamname_validate=False):
-    if ":" not in value:
-        raise click.BadParameter(
-            "Please specify organization and team name in the next"
-            " format - orgname:teamname. For example, mycompany:DreamTeam"
-        )
-    teamname = str(value.strip().split(":", 1)[1])
-    if teamname_validate:
-        validate_teamname(teamname)
-    return value
-
-
-def validate_teamname(value):
-    if not value:
-        return value
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I):
-        raise click.BadParameter(
-            "Invalid team name format. "
-            "Team name must only contain alphanumeric characters, "
-            "single hyphens, underscores, spaces. It can not "
-            "begin or end with a hyphen or a underscore and must"
-            " not be longer than 20 characters."
-        )
-    return value
-
-
-@click.group("team", short_help="Manage organization teams")
-def cli():
-    pass
-
-
-@cli.command("create", short_help="Create a new team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(
-        value, teamname_validate=True
-    ),
-)
-@click.option(
-    "--description",
-)
-def team_create(orgname_teamname, description):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.create_team(orgname, teamname, description)
-    return click.secho(
-        "The team %s has been successfully created." % teamname,
-        fg="green",
-    )
-
-
-@cli.command("list", short_help="List teams")
-@click.argument("orgname", required=False)
-@click.option("--json-output", is_flag=True)
-def team_list(orgname, json_output):
-    client = AccountClient()
-    data = {}
-    if not orgname:
-        for item in client.list_orgs():
-            teams = client.list_teams(item.get("orgname"))
-            data[item.get("orgname")] = teams
-    else:
-        teams = client.list_teams(orgname)
-        data[orgname] = teams
-    if json_output:
-        return click.echo(json.dumps(data[orgname] if orgname else data))
-    if not any(data.values()):
-        return click.secho("You do not have any teams.", fg="yellow")
-    for org_name in data:
-        for team in data[org_name]:
-            click.echo()
-            click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan")
-            click.echo("-" * len("%s:%s" % (org_name, team.get("name"))))
-            table_data = []
-            if team.get("description"):
-                table_data.append(("Description:", team.get("description")))
-            table_data.append(
-                (
-                    "Members:",
-                    ", ".join(
-                        (member.get("username") for member in team.get("members"))
-                    )
-                    if team.get("members")
-                    else "-",
-                )
-            )
-            click.echo(tabulate(table_data, tablefmt="plain"))
-    return click.echo()
-
-
-@cli.command("update", short_help="Update team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.option(
-    "--name",
-    callback=lambda _, __, value: validate_teamname(value),
-    help="A new team name",
-)
-@click.option(
-    "--description",
-)
-def team_update(orgname_teamname, **kwargs):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    team = client.get_team(orgname, teamname)
-    del team["id"]
-    del team["members"]
-    new_team = team.copy()
-    if not any(kwargs.values()):
-        for field in team:
-            new_team[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=team[field]
-            )
-            if field == "name":
-                validate_teamname(new_team[field])
-    else:
-        new_team.update({key: value for key, value in kwargs.items() if value})
-    client.update_team(orgname, teamname, new_team)
-    return click.secho(
-        "The team %s has been successfully updated." % teamname,
-        fg="green",
-    )
-
-
-@cli.command("destroy", short_help="Destroy a team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-def team_destroy(orgname_teamname):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    click.confirm(
-        click.style(
-            "Are you sure you want to destroy the %s team?" % teamname, fg="yellow"
-        ),
-        abort=True,
-    )
-    client = AccountClient()
-    client.destroy_team(orgname, teamname)
-    return click.secho(
-        "The team %s has been successfully destroyed." % teamname,
-        fg="green",
-    )
-
-
-@cli.command("add", short_help="Add a new member to team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.argument(
-    "username",
-)
-def team_add_member(orgname_teamname, username):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.add_team_member(orgname, teamname, username)
-    return click.secho(
-        "The new member %s has been successfully added to the %s team."
-        % (username, teamname),
-        fg="green",
-    )
-
-
-@cli.command("remove", short_help="Remove a member from team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.argument("username")
-def team_remove_owner(orgname_teamname, username):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.remove_team_member(orgname, teamname, username)
-    return click.secho(
-        "The %s member has been successfully removed from the %s team."
-        % (username, teamname),
-        fg="green",
-    )

platformio/commands/test/command.py

@@ -1,266 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
-
-import fnmatch
-import os
-import shutil
-from time import time
-
-import click
-from tabulate import tabulate
-
-from platformio import app, exception, fs, util
-from platformio.commands.platform import init_platform
-from platformio.commands.test.embedded import EmbeddedTestProcessor
-from platformio.commands.test.helpers import get_test_names
-from platformio.commands.test.native import NativeTestProcessor
-from platformio.project.config import ProjectConfig
-
-
-@click.command("test", short_help="Unit testing")
-@click.option("--environment", "-e", multiple=True, metavar="<environment>")
-@click.option(
-    "--filter",
-    "-f",
-    multiple=True,
-    metavar="<pattern>",
-    help="Filter tests by a pattern",
-)
-@click.option(
-    "--ignore",
-    "-i",
-    multiple=True,
-    metavar="<pattern>",
-    help="Ignore tests by a pattern",
-)
-@click.option("--upload-port")
-@click.option("--test-port")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
-)
-@click.option(
-    "-c",
-    "--project-conf",
-    type=click.Path(
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
-)
-@click.option("--without-building", is_flag=True)
-@click.option("--without-uploading", is_flag=True)
-@click.option("--without-testing", is_flag=True)
-@click.option("--no-reset", is_flag=True)
-@click.option(
-    "--monitor-rts",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial RTS line state for Serial Monitor",
-)
-@click.option(
-    "--monitor-dtr",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial DTR line state for Serial Monitor",
-)
-@click.option("--verbose", "-v", is_flag=True)
-@click.pass_context
-def cli(  # pylint: disable=redefined-builtin
-    ctx,
-    environment,
-    ignore,
-    filter,
-    upload_port,
-    test_port,
-    project_dir,
-    project_conf,
-    without_building,
-    without_uploading,
-    without_testing,
-    no_reset,
-    monitor_rts,
-    monitor_dtr,
-    verbose,
-):
-    app.set_session_var("custom_project_conf", project_conf)
-
-    with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(project_conf)
-        config.validate(envs=environment)
-        test_names = get_test_names(config)
-
-        if not verbose:
-            click.echo("Verbose mode can be enabled via `-v, --verbose` option")
-        click.secho("Collected %d items" % len(test_names), bold=True)
-
-        results = []
-        default_envs = config.default_envs()
-        for testname in test_names:
-
-            for envname in config.envs():
-                section = "env:%s" % envname
-
-                # filter and ignore patterns
-                patterns = dict(filter=list(filter), ignore=list(ignore))
-                for key in patterns:
-                    patterns[key].extend(config.get(section, "test_%s" % key, []))
-
-                skip_conditions = [
-                    environment and envname not in environment,
-                    not environment and default_envs and envname not in default_envs,
-                    testname != "*"
-                    and patterns["filter"]
-                    and not any(
-                        fnmatch.fnmatch(testname, p) for p in patterns["filter"]
-                    ),
-                    testname != "*"
-                    and any(fnmatch.fnmatch(testname, p) for p in patterns["ignore"]),
-                ]
-                if any(skip_conditions):
-                    results.append({"env": envname, "test": testname})
-                    continue
-
-                click.echo()
-                print_processing_header(testname, envname)
-
-                cls = (
-                    EmbeddedTestProcessor
-                    if config.get(section, "platform")
-                    and init_platform(config.get(section, "platform")).is_embedded()
-                    else NativeTestProcessor
-                )
-                tp = cls(
-                    ctx,
-                    testname,
-                    envname,
-                    dict(
-                        project_config=config,
-                        project_dir=project_dir,
-                        upload_port=upload_port,
-                        test_port=test_port,
-                        without_building=without_building,
-                        without_uploading=without_uploading,
-                        without_testing=without_testing,
-                        no_reset=no_reset,
-                        monitor_rts=monitor_rts,
-                        monitor_dtr=monitor_dtr,
-                        verbose=verbose,
-                        silent=not verbose,
-                    ),
-                )
-                result = {
-                    "env": envname,
-                    "test": testname,
-                    "duration": time(),
-                    "succeeded": tp.process(),
-                }
-                result["duration"] = time() - result["duration"]
-                results.append(result)
-
-                print_processing_footer(result)
-
-    if without_testing:
-        return
-
-    print_testing_summary(results, verbose)
-
-    command_failed = any(r.get("succeeded") is False for r in results)
-    if command_failed:
-        raise exception.ReturnErrorCode(1)
-
-
-def print_processing_header(test, env):
-    click.echo(
-        "Processing %s in %s environment"
-        % (
-            click.style(test, fg="yellow", bold=True),
-            click.style(env, fg="cyan", bold=True),
-        )
-    )
-    terminal_width, _ = shutil.get_terminal_size()
-    click.secho("-" * terminal_width, bold=True)
-
-
-def print_processing_footer(result):
-    is_failed = not result.get("succeeded")
-    util.print_labeled_bar(
-        "[%s] Took %.2f seconds"
-        % (
-            (
-                click.style("FAILED", fg="red", bold=True)
-                if is_failed
-                else click.style("PASSED", fg="green", bold=True)
-            ),
-            result["duration"],
-        ),
-        is_error=is_failed,
-    )
-
-
-def print_testing_summary(results, verbose=False):
-    click.echo()
-
-    tabular_data = []
-    succeeded_nums = 0
-    failed_nums = 0
-    duration = 0
-
-    for result in results:
-        duration += result.get("duration", 0)
-        if result.get("succeeded") is False:
-            failed_nums += 1
-            status_str = click.style("FAILED", fg="red")
-        elif result.get("succeeded") is None:
-            if not verbose:
-                continue
-            status_str = "IGNORED"
-        else:
-            succeeded_nums += 1
-            status_str = click.style("PASSED", fg="green")
-
-        tabular_data.append(
-            (
-                result["test"],
-                click.style(result["env"], fg="cyan"),
-                status_str,
-                util.humanize_duration_time(result.get("duration")),
-            )
-        )
-
-    click.echo(
-        tabulate(
-            tabular_data,
-            headers=[
-                click.style(s, bold=True)
-                for s in ("Test", "Environment", "Status", "Duration")
-            ],
-        ),
-        err=failed_nums,
-    )
-
-    util.print_labeled_bar(
-        "%s%d succeeded in %s"
-        % (
-            "%d failed, " % failed_nums if failed_nums else "",
-            succeeded_nums,
-            util.humanize_duration_time(duration),
-        ),
-        is_error=failed_nums,
-        fg="red" if failed_nums else "green",
-    )

platformio/commands/test/embedded.py

@@ -1,150 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from time import sleep
-
-import click
-import serial
-
-from platformio import exception, util
-from platformio.commands.test.processor import TestProcessorBase
-from platformio.platform.factory import PlatformFactory
-
-
-class EmbeddedTestProcessor(TestProcessorBase):
-
-    SERIAL_TIMEOUT = 600
-
-    def process(self):
-        if not self.options["without_building"]:
-            self.print_progress("Building...")
-            target = ["__test"]
-            if self.options["without_uploading"]:
-                target.append("checkprogsize")
-            if not self.build_or_upload(target):
-                return False
-
-        if not self.options["without_uploading"]:
-            self.print_progress("Uploading...")
-            target = ["upload"]
-            if self.options["without_building"]:
-                target.append("nobuild")
-            else:
-                target.append("__test")
-            if not self.build_or_upload(target):
-                return False
-
-        if self.options["without_testing"]:
-            return True
-
-        self.print_progress("Testing...")
-        return self.run()
-
-    def run(self):
-        click.echo(
-            "If you don't see any output for the first 10 secs, "
-            "please reset board (press reset button)"
-        )
-        click.echo()
-
-        try:
-            ser = serial.Serial(
-                baudrate=self.get_baudrate(), timeout=self.SERIAL_TIMEOUT
-            )
-            ser.port = self.get_test_port()
-            ser.rts = self.options["monitor_rts"]
-            ser.dtr = self.options["monitor_dtr"]
-            ser.open()
-        except serial.SerialException as e:
-            click.secho(str(e), fg="red", err=True)
-            return False
-
-        if not self.options["no_reset"]:
-            ser.flushInput()
-            ser.setDTR(False)
-            ser.setRTS(False)
-            sleep(0.1)
-            ser.setDTR(True)
-            ser.setRTS(True)
-            sleep(0.1)
-
-        while True:
-            line = ser.readline().strip()
-
-            # fix non-ascii output from device
-            for i, c in enumerate(line[::-1]):
-                if not isinstance(c, int):
-                    c = ord(c)
-                if c > 127:
-                    line = line[-i:]
-                    break
-
-            if not line:
-                continue
-            if isinstance(line, bytes):
-                line = line.decode("utf8", "ignore")
-            self.on_run_out(line)
-            if all(l in line for l in ("Tests", "Failures", "Ignored")):
-                break
-        ser.close()
-        return not self._run_failed
-
-    def get_test_port(self):
-        # if test port is specified manually or in config
-        if self.options.get("test_port"):
-            return self.options.get("test_port")
-        if self.env_options.get("test_port"):
-            return self.env_options.get("test_port")
-
-        assert set(["platform", "board"]) & set(self.env_options.keys())
-        p = PlatformFactory.new(self.env_options["platform"])
-        board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", [])
-        port = None
-        elapsed = 0
-        while elapsed < 5 and not port:
-            for item in util.get_serialports():
-                port = item["port"]
-                for hwid in board_hwids:
-                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                    if hwid_str in item["hwid"] and self.is_serial_port_ready(port):
-                        return port
-
-            if port and not self.is_serial_port_ready(port):
-                port = None
-
-            if not port:
-                sleep(0.25)
-                elapsed += 0.25
-
-        if not port:
-            raise exception.PlatformioException(
-                "Please specify `test_port` for environment or use "
-                "global `--test-port` option."
-            )
-        return port
-
-    @staticmethod
-    def is_serial_port_ready(port, timeout=3):
-        if not port:
-            return False
-        elapsed = 0
-        while elapsed < timeout:
-            try:
-                serial.Serial(port, timeout=1).close()
-                return True
-            except:  # pylint: disable=bare-except
-                pass
-            sleep(1)
-            elapsed += 1
-        return False

platformio/commands/test/native.py

@@ -1,41 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from os.path import join
-
-from platformio import proc
-from platformio.commands.test.processor import TestProcessorBase
-from platformio.proc import LineBufferedAsyncPipe
-
-
-class NativeTestProcessor(TestProcessorBase):
-    def process(self):
-        if not self.options["without_building"]:
-            self.print_progress("Building...")
-            if not self.build_or_upload(["__test"]):
-                return False
-        if self.options["without_testing"]:
-            return None
-        self.print_progress("Testing...")
-        return self.run()
-
-    def run(self):
-        build_dir = self.options["project_config"].get_optional_dir("build")
-        result = proc.exec_command(
-            [join(build_dir, self.env_name, "program")],
-            stdout=LineBufferedAsyncPipe(self.on_run_out),
-            stderr=LineBufferedAsyncPipe(self.on_run_out),
-        )
-        assert "returncode" in result
-        return result["returncode"] == 0 and not self._run_failed

platformio/commands/test/processor.py

@@ -1,230 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import atexit
-from os import listdir, remove
-from os.path import isdir, isfile, join
-from string import Template
-
-import click
-
-from platformio import exception
-
-TRANSPORT_OPTIONS = {
-    "arduino": {
-        "include": "#include <Arduino.h>",
-        "object": "",
-        "putchar": "Serial.write(c);",
-        "flush": "Serial.flush();",
-        "begin": "Serial.begin($baudrate);",
-        "end": "Serial.end();",
-        "language": "cpp",
-    },
-    "mbed": {
-        "include": "#include <mbed.h>",
-        "object": (
-            "#if MBED_MAJOR_VERSION == 6\nUnbufferedSerial pc(USBTX, USBRX);\n"
-            "#else\nRawSerial pc(USBTX, USBRX);\n#endif"
-        ),
-        "putchar": (
-            "#if MBED_MAJOR_VERSION == 6\npc.write(&c, 1);\n"
-            "#else\npc.putc(c);\n#endif"
-        ),
-        "flush": "",
-        "begin": "pc.baud($baudrate);",
-        "end": "",
-        "language": "cpp",
-    },
-    "espidf": {
-        "include": "#include <stdio.h>",
-        "object": "",
-        "putchar": "putchar(c);",
-        "flush": "fflush(stdout);",
-        "begin": "",
-        "end": "",
-    },
-    "zephyr": {
-        "include": "#include <sys/printk.h>",
-        "object": "",
-        "putchar": 'printk("%c", c);',
-        "flush": "",
-        "begin": "",
-        "end": "",
-    },
-    "native": {
-        "include": "#include <stdio.h>",
-        "object": "",
-        "putchar": "putchar(c);",
-        "flush": "fflush(stdout);",
-        "begin": "",
-        "end": "",
-    },
-    "custom": {
-        "include": '#include "unittest_transport.h"',
-        "object": "",
-        "putchar": "unittest_uart_putchar(c);",
-        "flush": "unittest_uart_flush();",
-        "begin": "unittest_uart_begin();",
-        "end": "unittest_uart_end();",
-        "language": "cpp",
-    },
-}
-
-CTX_META_TEST_IS_RUNNING = __name__ + ".test_running"
-CTX_META_TEST_RUNNING_NAME = __name__ + ".test_running_name"
-
-
-class TestProcessorBase(object):
-
-    DEFAULT_BAUDRATE = 115200
-
-    def __init__(self, cmd_ctx, testname, envname, options):
-        self.cmd_ctx = cmd_ctx
-        self.cmd_ctx.meta[CTX_META_TEST_IS_RUNNING] = True
-        self.test_name = testname
-        self.options = options
-        self.env_name = envname
-        self.env_options = options["project_config"].items(env=envname, as_dict=True)
-        self._run_failed = False
-        self._output_file_generated = False
-
-    def get_transport(self):
-        transport = None
-        if self.env_options.get("platform") == "native":
-            transport = "native"
-        elif "framework" in self.env_options:
-            transport = self.env_options.get("framework")[0]
-        if "test_transport" in self.env_options:
-            transport = self.env_options["test_transport"]
-        if transport not in TRANSPORT_OPTIONS:
-            raise exception.PlatformioException(
-                "Unknown Unit Test transport `%s`. Please check a documentation how "
-                "to create an own 'Test Transport':\n"
-                "- https://docs.platformio.org/page/plus/unit-testing.html" % transport
-            )
-        return transport.lower()
-
-    def get_baudrate(self):
-        return int(self.env_options.get("test_speed", self.DEFAULT_BAUDRATE))
-
-    def print_progress(self, text):
-        click.secho(text, bold=self.options.get("verbose"))
-
-    def build_or_upload(self, target):
-        if not self._output_file_generated:
-            self.generate_output_file(
-                self.options["project_config"].get_optional_dir("test")
-            )
-            self._output_file_generated = True
-
-        if self.test_name != "*":
-            self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name
-
-        try:
-            # pylint: disable=import-outside-toplevel
-            from platformio.commands.run.command import cli as cmd_run
-
-            return self.cmd_ctx.invoke(
-                cmd_run,
-                project_dir=self.options["project_dir"],
-                project_conf=self.options["project_config"].path,
-                upload_port=self.options.get("upload_port"),
-                verbose=self.options["verbose"],
-                silent=self.options.get("silent"),
-                environment=[self.env_name],
-                disable_auto_clean="nobuild" in target,
-                target=target,
-            )
-        except exception.ReturnErrorCode:
-            return False
-
-    def process(self):
-        raise NotImplementedError
-
-    def run(self):
-        raise NotImplementedError
-
-    def on_run_out(self, line):
-        line = line.strip()
-        if line.endswith(":PASS"):
-            click.echo("%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green")))
-        elif ":FAIL" in line:
-            self._run_failed = True
-            click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red")))
-        else:
-            click.echo(line)
-
-    def generate_output_file(self, test_dir):
-        assert isdir(test_dir)
-
-        file_tpl = "\n".join(
-            [
-                "$include",
-                "#include <output_export.h>",
-                "",
-                "$object",
-                "",
-                "#ifdef __GNUC__",
-                "void output_start(unsigned int baudrate __attribute__((unused)))",
-                "#else",
-                "void output_start(unsigned int baudrate)",
-                "#endif",
-                "{",
-                "    $begin",
-                "}",
-                "",
-                "void output_char(int c)",
-                "{",
-                "    $putchar",
-                "}",
-                "",
-                "void output_flush(void)",
-                "{",
-                "    $flush",
-                "}",
-                "",
-                "void output_complete(void)",
-                "{",
-                "   $end",
-                "}",
-            ]
-        )
-
-        tmp_file_prefix = "tmp_pio_test_transport"
-
-        def delete_tmptest_files(test_dir):
-            for item in listdir(test_dir):
-                if item.startswith(tmp_file_prefix) and isfile(join(test_dir, item)):
-                    try:
-                        remove(join(test_dir, item))
-                    except:  # pylint: disable=bare-except
-                        click.secho(
-                            "Warning: Could not remove temporary file '%s'. "
-                            "Please remove it manually." % join(test_dir, item),
-                            fg="yellow",
-                        )
-
-        transport_options = TRANSPORT_OPTIONS[self.get_transport()]
-        tpl = Template(file_tpl).substitute(transport_options)
-        data = Template(tpl).substitute(baudrate=self.get_baudrate())
-
-        delete_tmptest_files(test_dir)
-        tmp_file = join(
-            test_dir,
-            "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c")),
-        )
-        with open(tmp_file, mode="w", encoding="utf8") as fp:
-            fp.write(data)
-
-        atexit.register(delete_tmptest_files, test_dir)

platformio/commands/update.py

@@ -14,16 +14,11 @@
 
 import click
 
-from platformio.cache import cleanup_content_cache
-from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
-from platformio.commands.lib.command import lib_update as cmd_lib_update
-from platformio.commands.platform import platform_update as cmd_platform_update
-from platformio.package.manager.core import update_core_packages
-from platformio.package.manager.library import LibraryPackageManager
-
 
 @click.command(
-    "update", short_help="Update installed platforms, packages and libraries"
+    "update",
+    short_help="Update installed platforms, packages and libraries",
+    hidden=True,
 )
 @click.option("--core-packages", is_flag=True, help="Update only the core packages")
 @click.option(
@@ -35,25 +30,9 @@ from platformio.package.manager.library import LibraryPackageManager
 @click.option(
     "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
 )
-@click.pass_context
-def cli(ctx, core_packages, only_check, dry_run):
-    # cleanup lib search results, cached board and platform lists
-    cleanup_content_cache("http")
-
-    only_check = dry_run or only_check
-
-    update_core_packages(only_check)
-
-    if core_packages:
-        return
-
-    click.echo()
-    click.echo("Platform Manager")
-    click.echo("================")
-    ctx.invoke(cmd_platform_update, only_check=only_check)
-
-    click.echo()
-    click.echo("Library Manager")
-    click.echo("===============")
-    ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryPackageManager().package_dir]
-    ctx.invoke(cmd_lib_update, only_check=only_check)
+def cli(*_, **__):
+    click.secho(
+        "This command is deprecated and will be removed in the next releases. \n"
+        "Please use `pio pkg update` instead.",
+        fg="yellow",
+    )

platformio/commands/upgrade.py

@@ -20,15 +20,17 @@ from zipfile import ZipFile
 import click
 
 from platformio import VERSION, __version__, app, exception
-from platformio.clients.http import fetch_remote_content
 from platformio.compat import IS_WINDOWS
+from platformio.http import fetch_remote_content
+from platformio.package.manager.core import update_core_packages
 from platformio.proc import exec_command, get_pythonexe_path
 from platformio.project.helpers import get_project_cache_dir
 
 
-@click.command("upgrade", short_help="Upgrade PlatformIO to the latest version")
+@click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version")
 @click.option("--dev", is_flag=True, help="Use development branch")
 def cli(dev):
+    update_core_packages()
     if not dev and __version__ == get_latest_version():
         return click.secho(
             "You're up-to-date!\nPlatformIO %s is currently the "

platformio/compat.py

@@ -14,25 +14,26 @@
 
 # pylint: disable=unused-import,no-name-in-module
 
+import importlib.util
 import inspect
 import locale
 import sys
 
 from platformio.exception import UserSideException
 
-if sys.version_info >= (3,):
-    if sys.version_info >= (3, 7):
-        from asyncio import create_task as aio_create_task
-        from asyncio import get_running_loop as aio_get_running_loop
-    else:
-        from asyncio import ensure_future as aio_create_task
-        from asyncio import get_event_loop as aio_get_running_loop
+if sys.version_info >= (3, 7):
+    from asyncio import create_task as aio_create_task
+    from asyncio import get_running_loop as aio_get_running_loop
+else:
+    from asyncio import ensure_future as aio_create_task
+    from asyncio import get_event_loop as aio_get_running_loop
 
 
-PY2 = sys.version_info[0] == 2
+PY2 = sys.version_info[0] == 2  # DO NOT REMOVE IT. ESP8266/ESP32 depend on it
 IS_CYGWIN = sys.platform.startswith("cygwin")
 IS_WINDOWS = WINDOWS = sys.platform.startswith("win")
 IS_MACOS = sys.platform.startswith("darwin")
+MISSING = object()
 string_types = (str,)
 
 
@@ -57,8 +58,6 @@ def hashlib_encode_data(data):
 
 
 def load_python_module(name, pathname):
-    import importlib.util  # pylint: disable=import-outside-toplevel
-
     spec = importlib.util.spec_from_file_location(name, pathname)
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)

platformio/debug/cli.py

@@ -17,26 +17,28 @@
 
 import asyncio
 import os
+import signal
 import subprocess
 
 import click
 
 from platformio import app, exception, fs, proc
-from platformio.commands.platform import init_platform
 from platformio.compat import IS_WINDOWS
 from platformio.debug import helpers
 from platformio.debug.config.factory import DebugConfigFactory
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.debug.process.gdb import GDBClientProcess
+from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.exception import ProjectEnvsNotAvailableError
 from platformio.project.helpers import is_platformio_project
+from platformio.project.options import ProjectOptions
 
 
 @click.command(
     "debug",
     context_settings=dict(ignore_unknown_options=True),
-    short_help="Unified debugger",
+    short_help="Unified Debugger",
 )
 @click.option(
     "-d",
@@ -54,11 +56,21 @@ from platformio.project.helpers import is_platformio_project
     ),
 )
 @click.option("--environment", "-e", metavar="<environment>")
+@click.option("--load-mode", type=ProjectOptions["env.debug_load_mode"].type)
 @click.option("--verbose", "-v", is_flag=True)
 @click.option("--interface", type=click.Choice(["gdb"]))
 @click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
 @click.pass_context
-def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unprocessed):
+def cli(
+    ctx,
+    project_dir,
+    project_conf,
+    environment,
+    load_mode,
+    verbose,
+    interface,
+    __unprocessed,
+):
     app.set_session_var("custom_project_conf", project_conf)
 
     # use env variables from Eclipse or CLion
@@ -69,7 +81,29 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro
             project_dir = os.getenv(name)
 
     with fs.cd(project_dir):
-        project_config = ProjectConfig.get_instance(project_conf)
+        return _debug_in_project_dir(
+            ctx,
+            project_dir,
+            project_conf,
+            environment,
+            load_mode,
+            verbose,
+            interface,
+            __unprocessed,
+        )
+
+
+def _debug_in_project_dir(
+    ctx,
+    project_dir,
+    project_conf,
+    environment,
+    load_mode,
+    verbose,
+    interface,
+    __unprocessed,
+):
+    project_config = ProjectConfig.get_instance(project_conf)
     project_config.validate(envs=[environment] if environment else None)
     env_name = environment or helpers.get_default_debug_env(project_config)
 
@@ -82,10 +116,11 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro
     if "platform" not in env_options:
         raise ProjectEnvsNotAvailableError()
 
-    with fs.cd(project_dir):
-        debug_config = DebugConfigFactory.new(
-            init_platform(env_options["platform"]), project_config, env_name
-        )
+    debug_config = DebugConfigFactory.new(
+        PlatformFactory.new(env_options["platform"], autoinstall=True),
+        project_config,
+        env_name,
+    )
 
     if "--version" in __unprocessed:
         return subprocess.run(
@@ -104,7 +139,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro
 
     rebuild_prog = False
     preload = debug_config.load_cmds == ["preload"]
-    load_mode = debug_config.load_mode
+    load_mode = load_mode or debug_config.load_mode
     if load_mode == "always":
         rebuild_prog = preload or not helpers.has_debug_symbols(
             debug_config.program_path
@@ -151,17 +186,18 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro
     loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop()
     asyncio.set_event_loop(loop)
 
-    with fs.cd(project_dir):
-        client = GDBClientProcess(project_dir, debug_config)
-        coro = client.run(__unprocessed)
-        try:
-            loop.run_until_complete(coro)
-            if IS_WINDOWS:
-                # an issue with `asyncio` executor and STIDIN,
-                # it cannot be closed gracefully
-                proc.force_exit()
-        finally:
-            del client
-            loop.close()
+    client = GDBClientProcess(project_dir, debug_config)
+    coro = client.run(__unprocessed)
+    try:
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+        loop.run_until_complete(coro)
+        if IS_WINDOWS:
+            client.close()
+            # an issue with `asyncio` executor and STIDIN,
+            # it cannot be closed gracefully
+            proc.force_exit()
+    finally:
+        client.close()
+        loop.close()
 
     return True

platformio/debug/config/base.py

@@ -20,7 +20,7 @@ from platformio.compat import string_types
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.debug.helpers import reveal_debug_port
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import get_project_core_dir, load_project_ide_data
+from platformio.project.helpers import load_build_metadata
 from platformio.project.options import ProjectOptions
 
 
@@ -147,13 +147,20 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
         )
 
     def _load_build_data(self):
-        data = load_project_ide_data(os.getcwd(), self.env_name, cache=True)
+        data = load_build_metadata(os.getcwd(), self.env_name, cache=True)
         if data:
             return data
         raise DebugInvalidOptionsError("Could not load a build configuration")
 
     def _configure_server(self):
+        # user disabled server in platformio.ini
+        if "debug_server" in self.env_options and not self.env_options.get(
+            "debug_server"
+        ):
+            return None
+
         result = None
+
         # specific server per a system
         if isinstance(self.tool_settings.get("server", {}), list):
             for item in self.tool_settings["server"][:]:
@@ -179,11 +186,7 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
                 else None
             )
             if server_package and not server_package_dir:
-                self.platform.install_packages(
-                    with_packages=[server_package],
-                    skip_default_package=True,
-                    silent=True,
-                )
+                self.platform.install_package(server_package)
                 server_package_dir = self.platform.get_package_dir(server_package)
             result.update(
                 dict(
@@ -208,9 +211,9 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
     def reveal_patterns(self, source, recursive=True):
         program_path = self.program_path or ""
         patterns = {
-            "PLATFORMIO_CORE_DIR": get_project_core_dir(),
+            "PLATFORMIO_CORE_DIR": self.project_config.get("platformio", "core_dir"),
             "PYTHONEXE": proc.get_pythonexe_path(),
-            "PROJECT_DIR": self.project_config.path,
+            "PROJECT_DIR": os.getcwd(),
             "PROG_PATH": program_path,
             "PROG_DIR": os.path.dirname(program_path),
             "PROG_NAME": os.path.basename(os.path.splitext(program_path)[0]),

platformio/debug/config/factory.py

@@ -23,7 +23,7 @@ class DebugConfigFactory(object):
     @staticmethod
     def get_clsname(name):
         name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I)
-        return "%s%sDebugConfig" % (name.upper()[0], name.lower()[1:])
+        return "%sDebugConfig" % name.lower().capitalize()
 
     @classmethod
     def new(cls, platform, project_config, env_name):

platformio/debug/config/generic.py

@@ -34,5 +34,5 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(GenericDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":3333"

platformio/debug/config/jlink.py

@@ -38,11 +38,9 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(JlinkDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":2331"
 
     @property
     def server_ready_pattern(self):
-        return super(JlinkDebugConfig, self).server_ready_pattern or (
-            "Waiting for GDB connection"
-        )
+        return super().server_ready_pattern or ("Waiting for GDB connection")

platformio/debug/config/mspdebug.py

@@ -32,5 +32,5 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(MspdebugDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":2000"

platformio/debug/config/qemu.py

@@ -33,5 +33,5 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
-        super(QemuDebugConfig, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.port = ":1234"