Merge branch 'release/v6.1.10'

- Added section name to the section data so that a dev-platform can apply a regex to determine section location in memory
- Performance improvement by analyzing memory location of a firmware section only once

.github/ISSUE_TEMPLATE.md

@@ -6,9 +6,8 @@ What kind of issue is this?
       use [Community Forums](https://community.platformio.org) or [Premium Support](https://platformio.org/support)
 
 - [ ] **PlatformIO IDE**.
-      All issues related to PlatformIO IDE should be reported to appropriate repository:
-      [PlatformIO IDE for Atom](https://github.com/platformio/platformio-atom-ide/issues) or
-      [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues)
+      All issues related to PlatformIO IDE should be reported to the
+      [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues) repository
 
 - [ ] **Development Platform or Board**.
       All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms

.github/workflows/core.yml

@@ -7,35 +7,37 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
-        exclude:
-          - os: macos-latest
-            python-version: "3.6"
-          - os: windows-latest
-            python-version: "3.10"
+        os: [ubuntu-20.04, windows-latest, macos-latest]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+
     runs-on: ${{ matrix.os }}
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: "recursive"
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install tox
 
+      - name: Core System Info
+        run: |
+          tox -e py
+
       - name: Python Lint
+        if: ${{ matrix.python-version != '3.6' }}
         run: |
           tox -e lint
+
       - name: Integration Tests
-        env:
-          TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }}
-          TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }}
-          TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }}
+        if: ${{ matrix.python-version == '3.9' }}
         run: |
           tox -e testcore
 

.github/workflows/deployment.yml

@@ -0,0 +1,45 @@
+name: Deployment
+
+on:
+  push:
+    branches:
+      - "master"
+      - "release/**"
+
+jobs:
+  deployment:
+    runs-on: ubuntu-latest
+    environment: production
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox wheel
+
+      - name: Deployment Tests
+        env:
+          TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }}
+          TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }}
+          TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }}
+        run: |
+          tox -e testcore
+
+      - name: Build Python source tarball
+        run:  python setup.py sdist bdist_wheel
+
+      - name: Publish package to PyPI
+        if: ${{ github.ref == 'refs/heads/master' }}
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/docs.yml

@@ -7,13 +7,13 @@ jobs:
     name: Build Docs
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: "recursive"
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -40,7 +40,7 @@ jobs:
 
       - name: Save artifact
         if: ${{ github.event_name == 'push' }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: docs
           path: ./docs.tar.gz
@@ -57,7 +57,7 @@ jobs:
     if: ${{ github.event_name == 'push' }}
     steps:
       - name: Download artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
         with:
           name: docs
       - name: Unpack artifact
@@ -65,7 +65,7 @@ jobs:
           mkdir ./${{ env.LATEST_DOCS_DIR }}
           tar -xzf ./docs.tar.gz -C ./${{ env.LATEST_DOCS_DIR }}
       - name: Delete Artifact
-        uses: geekyeggo/delete-artifact@v1
+        uses: geekyeggo/delete-artifact@v2
         with:
           name: docs
       - name: Select Docs type
@@ -78,7 +78,7 @@ jobs:
           fi
       - name: Checkout latest Docs
         continue-on-error: true
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: ${{ env.DOCS_REPO }}
           path: ${{ env.DOCS_DIR }}

.github/workflows/examples.yml

@@ -2,22 +2,28 @@ name: Examples
 
 on: [push, pull_request]
 
+
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04, windows-latest, macos-latest]
-        python-version: [3.7]
+        os: [ubuntu-latest, windows-latest, macos-latest]
     runs-on: ${{ matrix.os }}
+    env:
+      PIO_INSTALL_DEVPLATFORM_OWNERNAMES: "platformio"
+      PIO_INSTALL_DEVPLATFORM_NAMES: "aceinna_imu,atmelavr,atmelmegaavr,atmelsam,espressif32,espressif8266,nordicnrf52,raspberrypi,ststm32,teensy"
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: "recursive"
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.9"
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -25,24 +31,15 @@ jobs:
 
       - name: Run on Linux
         if: startsWith(matrix.os, 'ubuntu')
-        env:
-          PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio"
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,intel_mcs51"
         run: |
-          # ChipKIT issue: install 32-bit support for GCC PIC32
-          sudo apt-get install libc6-i386
           # Free space
           sudo apt clean
           docker rmi $(docker image ls -aq)
           df -h
-          # Run
           tox -e testexamples
 
       - name: Run on macOS
         if: startsWith(matrix.os, 'macos')
-        env:
-          PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio"
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,microchippic32,lattice_ice40,gd32v"
         run: |
           df -h
           tox -e testexamples
@@ -52,8 +49,6 @@ jobs:
         env:
           PLATFORMIO_CORE_DIR: C:/pio
           PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH
-          PIO_INSTALL_DEVPLATFORMS_OWNERNAMES: "platformio"
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap"
         run: |
           tox -e testexamples
 

.github/workflows/projects.yml

@@ -0,0 +1,69 @@
+name: Projects
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        project:
+          - marlin:
+            repository: "MarlinFirmware/Marlin"
+            folder: "Marlin"
+            config_dir: "Marlin"
+            env_name: "mega2560"
+          - esphome:
+            repository: "esphome/esphome"
+            folder: "esphome"
+            config_dir: "esphome"
+            env_name: "esp32-arduino"
+          - smartknob:
+            repository: "scottbez1/smartknob"
+            folder: "smartknob"
+            config_dir: "smartknob"
+            env_name: "view"
+          - espurna:
+            repository: "xoseperez/espurna"
+            folder: "espurna"
+            config_dir: "espurna/code"
+            env_name: "nodemcu-lolin"
+          - OpenMQTTGateway:
+            repository: "1technophile/OpenMQTTGateway"
+            folder: "OpenMQTTGateway"
+            config_dir: "OpenMQTTGateway"
+            env_name: "esp32-m5atom-lite"
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        exclude:
+        - os: windows-latest
+          project: {"esphome": "", "repository": "esphome/esphome", "folder": "esphome", "config_dir": "esphome", "env_name": "esp32-arduino"}
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install PlatformIO
+        run: pip install -U .
+
+      - name: Check out ${{ matrix.project.repository }}
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+          repository: ${{ matrix.project.repository }}
+          path: ${{ matrix.project.folder }}
+
+      - name: Install ESPHome dependencies
+        # Requires esptool package as it's used in a custom prescript
+        if: ${{ contains(matrix.project.repository, 'esphome') }}
+        run: pip install esptool==3.*
+
+      - name: Compile ${{ matrix.project.repository }}
+        run: pio run -d ${{ matrix.project.config_dir }} -e ${{ matrix.project.env_name }}
+

.pylintrc

@@ -3,21 +3,10 @@ output-format=colorized
 
 [MESSAGES CONTROL]
 disable=
-    bad-continuation,
-    bad-whitespace,
     missing-docstring,
-    ungrouped-imports,
-    invalid-name,
-    cyclic-import,
     duplicate-code,
-    superfluous-parens,
+    invalid-name,
     too-few-public-methods,
-    useless-object-inheritance,
-    useless-import-alias,
-    bad-option-value,
-    consider-using-dict-items,
     consider-using-f-string,
-
-    ; PY2 Compat
-    super-with-arguments,
-    raise-missing-from
+    cyclic-import,
+    use-dict-literal

CODE_OF_CONDUCT.md

@@ -0,0 +1,3 @@
+# Code of Conduct
+
+See https://piolabs.com/legal/code-of-conduct.html

CONTRIBUTING.md

@@ -6,12 +6,13 @@ To get started, <a href="https://cla-assistant.io/platformio/platformio-core">si
 1. Fork the repository on GitHub
 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git`
 3. Run `pip install tox`
-4. Go to the root of project where is located `tox.ini` and run `tox -e py37`
+4. Go to the root of the PlatformIO Core project where `tox.ini` is located (``cd platformio-core``) and run `tox -e py39`.
+   You can replace `py39` with your own Python version. For example, `py311` means Python 3.11.
 5. Activate current development environment:
 
-   * Windows: `.tox\py37\Scripts\activate`
-   * Bash/ZSH: `source .tox/py37/bin/activate`
-   * Fish: `source .tox/py37/bin/activate.fish`
+   * Windows: `.tox\py39\Scripts\activate`
+   * Bash/ZSH: `source .tox/py39/bin/activate`
+   * Fish: `source .tox/py39/bin/activate.fish`
 
 6. Make changes to code, documentation, etc.
 7. Lint source code `make before-commit`

HISTORY.rst

@@ -2,15 +2,187 @@ Release Notes
 =============
 
 .. |PIOCONF| replace:: `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ configuration file
+.. |LIBRARYJSON| replace:: `library.json <https://docs.platformio.org/en/latest/manifests/library-json/index.html>`__
 .. |LDF| replace:: `LDF <https://docs.platformio.org/en/latest/librarymanager/ldf.html>`__
 .. |INTERPOLATION| replace:: `Interpolation of Values <https://docs.platformio.org/en/latest/projectconf/interpolation.html>`__
+.. |UNITTESTING| replace:: `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`__
+.. |DEBUGGING| replace:: `Debugging <https://docs.platformio.org/en/latest/plus/debugging.html>`__
 
 .. _release_notes_6:
 
 PlatformIO Core 6
 -----------------
 
-**A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.**
+Unlock the true potential of embedded software development with
+PlatformIO's collaborative ecosystem, embracing declarative principles,
+test-driven methodologies, and modern toolchains for unrivaled success.
+
+6.1.10 (2023-08-11)
+~~~~~~~~~~~~~~~~~~~
+
+* Resolved an issue that caused generated projects for `PlatformIO IDE for VSCode <https://docs.platformio.org/en/latest/integration/ide/vscode.html>`__ to break when the ``-iprefix`` compiler flag was used
+* Resolved an issue encountered while utilizing the `pio pkg exec <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_exec.html>`__ command on the Windows platform to execute Python scripts from a package
+* Implemented a crucial improvement to the `pio run <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html>`__ command, guaranteeing that the ``monitor`` target is not executed if any of the preceding targets, such as ``upload``, encounter failures
+* `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.11 with new checks, CLI commands and various analysis improvements
+* Resolved a critical issue that arose on macOS ARM platforms due to the Python "requests" module, leading to a "ModuleNotFoundError: No module named 'chardet'" (`issue #4702 <https://github.com/platformio/platformio-core/issues/4702>`_)
+
+6.1.9 (2023-07-06)
+~~~~~~~~~~~~~~~~~~
+
+* Rectified a regression bug that occurred when the ``-include`` flag was passed via the `build_flags <https://docs.platformio.org/en/latest/projectconf/sections/env/options/build/build_flags.html>`__ option as a relative path and subsequently expanded (`issue #4683 <https://github.com/platformio/platformio-core/issues/4683>`_)
+* Resolved an issue that resulted in unresolved absolute toolchain paths when generating the `Compilation database "compile_commands.json" <https://docs.platformio.org/en/latest/integration/compile_commands.html>`__ (`issue #4684 <https://github.com/platformio/platformio-core/issues/4684>`_)
+
+6.1.8 (2023-07-05)
+~~~~~~~~~~~~~~~~~~
+
+* Added a new ``--lint`` option to the `pio project config <https://docs.platformio.org/en/latest/core/userguide/project/cmd_config.html>`__ command, enabling users to efficiently perform linting on the |PIOCONF|
+* Enhanced the parsing of the |PIOCONF| to provide comprehensive diagnostic information
+* Expanded the functionality of the |LIBRARYJSON| manifest by allowing the use of the underscore symbol in the `keywords <https://docs.platformio.org/en/latest/manifests/library-json/fields/keywords.html>`__ field
+* Optimized project integration templates to address the issue of long paths on Windows (`issue #4652 <https://github.com/platformio/platformio-core/issues/4652>`_)
+* Refactored |UNITTESTING| engine to resolve compiler warnings with "-Wpedantic" option (`pull #4671 <https://github.com/platformio/platformio-core/pull/4671>`_)
+* Eliminated erroneous warning regarding the use of obsolete PlatformIO Core when downgrading to the stable version (`issue #4664 <https://github.com/platformio/platformio-core/issues/4664>`_)
+* Updated the `pio project metadata <https://docs.platformio.org/en/latest/core/userguide/project/cmd_metadata.html>`__ command to return C/C++ flags as parsed Unix shell arguments when dumping project build metadata
+* Resolved a critical issue related to the usage of the ``-include`` flag within the `build_flags <https://docs.platformio.org/en/latest/projectconf/sections/env/options/build/build_flags.html>`__ option, specifically when employing dynamic variables (`issue #4682 <https://github.com/platformio/platformio-core/issues/4682>`_)
+* Removed PlatformIO IDE for Atom from the documentation as `Atom has been deprecated <https://github.blog/2022-06-08-sunsetting-atom/>`__
+
+6.1.7 (2023-05-08)
+~~~~~~~~~~~~~~~~~~
+
+* Introduced a new ``--sample-code`` option to the `pio project init <https://docs.platformio.org/en/latest/core/userguide/project/cmd_init.html>`__ command, which allows users to include sample code in the newly created project
+* Added validation for `project working environment names <https://docs.platformio.org/en/latest/projectconf/sections/env/index.html#working-env-name>`__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen)
+* Added the ability to show a detailed library dependency tree only in `verbose mode <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html#cmdoption-pio-run-v>`__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 <https://github.com/platformio/platformio-core/issues/4517>`_)
+* Added the ability to run only the `device monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html>`__ when using the `pio run -t monitor <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html>`__ command, saving you time and resources by skipping the build process
+* Implemented a new feature to store device monitor logs in the project's ``logs`` folder, making it easier to access and review device monitor logs for your projects (`issue #4596 <https://github.com/platformio/platformio-core/issues/4596>`_)
+* Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
+* Improved source file filtering functionality for the `Static Code Analysis <https://docs.platformio.org/en/latest/advanced/static-code-analysis/index.html>`__ feature, making it easier to analyze only the code you need to
+* Upgraded the build engine to the latest version of SCons (4.5.2) to improve build performance, reliability, and compatibility with other tools and systems (`release notes <https://github.com/SCons/scons/releases/tag/4.5.2>`__)
+* Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 <https://github.com/platformio/platformio-core/issues/4532>`_)
+* Restored the project generator for the `NetBeans IDE <https://docs.platformio.org/en/latest/integration/ide/netbeans.html>`__, providing you with more flexibility and options for your development workflow
+* Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 <https://github.com/platformio/platformio-core/issues/4425>`_, `issue #4493 <https://github.com/platformio/platformio-core/issues/4493>`_, `issue #4607 <https://github.com/platformio/platformio-core/issues/4607>`_)
+* Resolved an issue where the `build_cache_dir <https://docs.platformio.org/en/latest/projectconf/sections/platformio/options/directory/build_cache_dir.html>`__ setting was not being recognized consistently across multiple environments (`issue #4574 <https://github.com/platformio/platformio-core/issues/4574>`_)
+* Resolved an issue where organization details could not be updated using the `pio org update <https://docs.platformio.org/en/latest/core/userguide/org/cmd_update.html>`__ command
+* Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 <https://github.com/platformio/platformio-core/issues/4597>`_)
+* Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 <https://github.com/platformio/platformio-core/issues/4546>`_)
+* Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 <https://github.com/platformio/platformio-core/issues/4565>`_)
+* Resolved an issue where multiple targets were not executed sequentially (`issue #4604 <https://github.com/platformio/platformio-core/issues/4604>`_)
+* Resolved an issue where upgrading PlatformIO Core fails on Windows with Python 3.11 (`issue #4540 <https://github.com/platformio/platformio-core/issues/4540>`_)
+
+6.1.6 (2023-01-23)
+~~~~~~~~~~~~~~~~~~
+
+* Added support for Python 3.11
+* Added a new `name <https://docs.platformio.org/en/latest/projectconf/sections/platformio/options/generic/description.html>`__ configuration option to customize a project name (`pull #4498 <https://github.com/platformio/platformio-core/pull/4498>`_)
+* Made assets (templates, ``99-platformio-udev.rules``) part of Python's module (`issue #4458 <https://github.com/platformio/platformio-core/issues/4458>`_)
+* Updated `Clang-Tidy <https://docs.platformio.org/en/latest/plus/check-tools/clang-tidy.html>`__ check tool to v15.0.5 with new diagnostics and bugfixes
+* Removed dependency on the "zeroconf" package and install it only when a user lists mDNS devices (issue with zeroconf's LGPL license)
+* Show the real error message instead of "Can not remove temporary directory" when |PIOCONF| is broken (`issue #4480 <https://github.com/platformio/platformio-core/issues/4480>`_)
+* Fixed an issue with an incorrect test summary when a testcase name includes a colon (`issue #4508 <https://github.com/platformio/platformio-core/issues/4508>`_)
+* Fixed an issue when `extends <https://docs.platformio.org/en/latest/projectconf/sections/env/options/advanced/extends.html>`__ did not override options in the right order (`issue #4462 <https://github.com/platformio/platformio-core/issues/4462>`_)
+* Fixed an issue when `pio pkg list <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_list.html>`__ and `pio pkg uninstall <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_uninstall.html>`__ commands fail if there are circular dependencies in the |LIBRARYJSON| manifests (`issue #4475 <https://github.com/platformio/platformio-core/issues/4475>`_)
+
+6.1.5 (2022-11-01)
+~~~~~~~~~~~~~~~~~~
+
+* Added a new `enable_proxy_strict_ssl <https://docs.platformio.org/en/latest/core/userguide/cmd_settings.html>`__ setting to disable the proxy server certificate verification (`issue #4432 <https://github.com/platformio/platformio-core/issues/4432>`_)
+* Documented `PlatformIO Core Proxy Configuration <https://docs.platformio.org/en/latest/core/installation/proxy-configuration.html>`__
+* Speeded up device port finder by avoiding loading board HWIDs from development platforms
+* Improved caching of build metadata in debug mode
+* Fixed an issue when `pio pkg install --storage-dir <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_install.html>`__ command requires PlatformIO project (`issue #4410 <https://github.com/platformio/platformio-core/issues/4410>`_)
+
+6.1.4 (2022-08-12)
+~~~~~~~~~~~~~~~~~~
+
+* Added support for accepting the original FileNode environment in a "callback" function when using `Build Middlewares <https://docs.platformio.org/en/latest/scripting/middlewares.html>`__ (`pull #4380 <https://github.com/platformio/platformio-core/pull/4380>`_)
+* Improved device port finder when using dual channel UART converter (`issue #4367 <https://github.com/platformio/platformio-core/issues/4367>`_)
+* Improved project dependency resolving when using the `pio project init --ide <https://docs.platformio.org/en/latest/core/userguide/project/cmd_init.html>`__ command
+* Upgraded build engine to the SCons 4.4.0 (`release notes <https://github.com/SCons/scons/releases/tag/4.4.0>`__)
+* Keep custom "unwantedRecommendations" when generating projects for VSCode (`issue #4383 <https://github.com/platformio/platformio-core/issues/4383>`_)
+* Do not resolve project dependencies for the ``cleanall`` target (`issue #4344 <https://github.com/platformio/platformio-core/issues/4344>`_)
+* Warn about calling "env.BuildSources" in a POST-type script (`issue #4385 <https://github.com/platformio/platformio-core/issues/4385>`_)
+* Fixed an issue when escaping macros/defines for IDE integration (`issue #4360 <https://github.com/platformio/platformio-core/issues/4360>`_)
+* Fixed an issue when the "cleanall" target removes dependencies from all working environments (`issue #4386 <https://github.com/platformio/platformio-core/issues/4386>`_)
+
+6.1.3 (2022-07-18)
+~~~~~~~~~~~~~~~~~~
+
+* Fixed a regression bug when opening device monitor without any filters (`issue #4363 <https://github.com/platformio/platformio-core/issues/4363>`_)
+
+6.1.2 (2022-07-18)
+~~~~~~~~~~~~~~~~~~
+
+* Export a ``PIO_UNIT_TESTING`` macro to the project source files and dependent libraries in the |UNITTESTING| mode
+* Improved detection of Windows architecture (`issue #4353 <https://github.com/platformio/platformio-core/issues/4353>`_)
+* Warn about unknown `device monitor filters <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#filters>`__ (`issue #4362 <https://github.com/platformio/platformio-core/issues/4362>`_)
+* Fixed a regression bug when `libArchive <https://docs.platformio.org/en/latest/manifests/library-json/fields/build/libarchive.html>`__ option declared in the |LIBRARYJSON| manifest was ignored (`issue #4351 <https://github.com/platformio/platformio-core/issues/4351>`_)
+* Fixed an issue when the `pio pkg publish <https://docs.platformio.org/en/latest/core/userguide/pkg/cmd_publish.html>`__ command didn't work with Python 3.6 (`issue #4352 <https://github.com/platformio/platformio-core/issues/4352>`_)
+
+6.1.1 (2022-07-11)
+~~~~~~~~~~~~~~~~~~
+
+* Added new ``monitor_encoding`` project configuration option to configure `Device Monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html>`__ (`issue #4350 <https://github.com/platformio/platformio-core/issues/4350>`_)
+* Allowed specifying project environments for `pio ci <https://docs.platformio.org/en/latest/core/userguide/cmd_ci.html>`__ command (`issue #4347 <https://github.com/platformio/platformio-core/issues/4347>`_)
+* Show "TimeoutError" only in the verbose mode when can not find a serial port
+* Fixed an issue when a serial port was not automatically detected if the board has predefined HWIDs
+* Fixed an issue with endless scanning of project dependencies (`issue #4349 <https://github.com/platformio/platformio-core/issues/4349>`_)
+* Fixed an issue with |LDF| when incompatible libraries were used for the working project environment with the missed framework (`pull #4346 <https://github.com/platformio/platformio-core/pull/4346>`_)
+
+6.1.0 (2022-07-06)
+~~~~~~~~~~~~~~~~~~
+
+* **Device Manager**
+
+  - Automatically reconnect device monitor if a connection fails
+  - Added new `pio device monitor --no-reconnect <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#cmdoption-pio-device-monitor-no-reconnect>`__ option to disable automatic reconnection
+  - Handle device monitor disconnects more gracefully (`issue #3939 <https://github.com/platformio/platformio-core/issues/3939>`_)
+  - Improved a serial port finder for `Black Magic Probe <https://docs.platformio.org/en/latest/plus/debug-tools/blackmagic.html>`__ (`issue #4023 <https://github.com/platformio/platformio-core/issues/4023>`_)
+  - Improved a serial port finder for a board with predefined HWIDs
+  - Replaced ``monitor_flags`` with independent project configuration options: `monitor_parity <https://docs.platformio.org/en/latest/projectconf/section_env_monitor.html#monitor-parity>`__, `monitor_eol <https://docs.platformio.org/en/latest/projectconf/section_env_monitor.html#monitor-eol>`__, `monitor_raw <https://docs.platformio.org/en/latest/projectconf/section_env_monitor.html#monitor-raw>`__, `monitor_echo <https://docs.platformio.org/en/latest/projectconf/section_env_monitor.html#monitor-echo>`__
+  - Fixed an issue when the monitor filters were not applied in their order (`issue #4320 <https://github.com/platformio/platformio-core/issues/4320>`_)
+
+* **Unit Testing**
+
+  - Updated "Getting Started" documentation for `GoogleTest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/googletest.html>`__ testing and mocking framework
+  - Export |UNITTESTING| flags only to the project build environment (``projenv``, files in "src" folder)
+  - Merged the "building" stage with "uploading" for the embedded target (`issue #4307 <https://github.com/platformio/platformio-core/issues/4307>`_)
+  - Do not resolve dependencies from the project "src" folder when the `test_build_src <https://docs.platformio.org/en/latest//projectconf/section_env_test.html#test-build-src>`__ option is not enabled
+  - Do not immediately terminate a testing program when results are received
+  - Fixed an issue when a custom `pio test --project-config <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-c>`__ was not handled properly (`issue #4299 <https://github.com/platformio/platformio-core/issues/4299>`_)
+  - Fixed an issue when testing results were wrong in the verbose mode (`issue #4336 <https://github.com/platformio/platformio-core/issues/4336>`_)
+
+* **Build System**
+
+  - Significantly improved support for `Pre & Post Actions <https://docs.platformio.org/en/latest/scripting/actions.html>`__
+
+    * Allowed to declare actions in the `PRE-type scripts <https://docs.platformio.org/en/latest/scripting/launch_types.html>`__ even if the target is not ready yet
+    * Allowed library maintainers to use Pre & Post Actions in the library `extraScript <https://docs.platformio.org/en/latest/manifests/library-json/fields/build/extrascript.html>`__
+
+  - Documented `Stringification <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#stringification>`__ – converting a macro argument into a string constant (`issue #4310 <https://github.com/platformio/platformio-core/issues/4310>`_)
+  - Added new `pio run --monitor-port <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html#cmdoption-pio-run-monitor-port>`__ option to specify custom device monitor port to the ``monitor`` target (`issue #4337 <https://github.com/platformio/platformio-core/issues/4337>`_)
+  - Added ``env.StringifyMacro(value)`` helper function for the `Advanced Scripting <https://docs.platformio.org/en/latest/scripting/index.html>`__
+  - Allowed to ``Import("projenv")`` in a library extra script (`issue #4305 <https://github.com/platformio/platformio-core/issues/4305>`_)
+  - Fixed an issue when the `build_unflags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-unflags>`__ operation ignores a flag value (`issue #4309 <https://github.com/platformio/platformio-core/issues/4309>`_)
+  - Fixed an issue when the `build_unflags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-unflags>`__ option was not applied to the ``ASPPFLAGS`` scope
+  - Fixed an issue on Windows OS when flags were wrapped to the temporary file while generating the `Compilation database "compile_commands.json" <https://docs.platformio.org/en/latest/integration/compile_commands.html>`__
+  - Fixed an issue with the |LDF| when recursively scanning dependencies in the ``chain`` mode
+  - Fixed a "PermissionError" on Windows when running "clean" or "cleanall" targets (`issue #4331 <https://github.com/platformio/platformio-core/issues/4331>`_)
+
+* **Package Management**
+
+  - Fixed an issue when library dependencies were installed for the incompatible project environment (`issue #4338 <https://github.com/platformio/platformio-core/issues/4338>`_)
+
+* **Miscellaneous**
+
+  - Warn about incompatible Bash version for the `Shell Completion <https://docs.platformio.org/en/latest/core/userguide/system/completion/index.html>`__ (`issue #4326 <https://github.com/platformio/platformio-core/issues/4326>`_)
+
+6.0.2 (2022-06-01)
+~~~~~~~~~~~~~~~~~~
+
+* Control |UNITTESTING| verbosity with a new multilevel `pio test -v <https://docs.platformio.org/en/latest/core/userguide/cmd_test.html#cmdoption-pio-test-v>`__ command option (`issue #4276 <https://github.com/platformio/platformio-core/issues/4276>`_)
+* Follow symbolic links during searching for the unit test suites (`issue #4288 <https://github.com/platformio/platformio-core/issues/4288>`_)
+* Show a warning when testing an empty project without a test suite (`issue #4278 <https://github.com/platformio/platformio-core/issues/4278>`_)
+* Improved support for `Asking for input (prompts) <https://docs.platformio.org/en/latest/scripting/examples/asking_for_input.html>`_
+* Fixed an issue when the `build_src_flags <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#build-src-flags>`__ option was applied outside the project scope (`issue #4277 <https://github.com/platformio/platformio-core/issues/4277>`_)
+* Fixed an issue with debugging assembly files without preprocessor (".s")
 
 6.0.1 (2022-05-17)
 ~~~~~~~~~~~~~~~~~~
@@ -53,7 +225,7 @@ Please check the `Migration guide from 5.x to 6.0 <https://docs.platformio.org/e
 
 * **Unit Testing**
 
-  - Refactored from scratch `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html>`_ solution and its documentation
+  - Refactored from scratch |UNITTESTING| solution and its documentation
   - New: `Test Hierarchy <https://docs.platformio.org/en/latest/advanced/unit-testing/structure.html>`_ (`issue #4135 <https://github.com/platformio/platformio-core/issues/4135>`_)
   - New: `Doctest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/doctest.html>`__ testing framework (`issue #4240 <https://github.com/platformio/platformio-core/issues/4240>`_)
   - New: `GoogleTest <https://docs.platformio.org/en/latest/advanced/unit-testing/frameworks/googletest.html>`__ testing and mocking framework (`issue #3572 <https://github.com/platformio/platformio-core/issues/3572>`_)

MANIFEST.in

@@ -1 +0,0 @@
-include LICENSE

Makefile

@@ -1,6 +1,6 @@
 lint:
-	pylint -j 6 --rcfile=./.pylintrc ./tests
-	pylint -j 6 --rcfile=./.pylintrc ./platformio
+	pylint --rcfile=./.pylintrc ./tests
+	pylint --rcfile=./.pylintrc ./platformio
 
 isort:
 	isort ./platformio

README.rst

@@ -1,25 +1,21 @@
-.. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg
-    :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md
-    :alt:  SWUbanner
-
 PlatformIO Core
 ===============
 
 .. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg
     :target: https://docs.platformio.org/en/latest/core/index.html
     :alt:  CI Build for PlatformIO Core
-.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
-    :target: https://github.com/platformio/platformio-examples
-    :alt:  CI Build for dev-platform examples
 .. image:: https://github.com/platformio/platformio-core/workflows/Docs/badge.svg
     :target: https://docs.platformio.org?utm_source=github&utm_medium=core
     :alt:  CI Build for Docs
+.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
+    :target: https://github.com/platformio/platformio-examples
+    :alt:  CI Build for dev-platform examples
+.. image:: https://github.com/platformio/platformio-core/workflows/Projects/badge.svg
+    :target: https://docs.platformio.org/en/latest/tutorials/index.html#projects
+    :alt:  CI Build for the Community Projects
 .. image:: https://img.shields.io/pypi/v/platformio.svg
     :target: https://pypi.python.org/pypi/platformio/
     :alt: Latest Version
-.. image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
-    :target: https://pypi.python.org/pypi/platformio/
-    :alt:  License
 .. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg
    :alt: PlatformIO Labs
    :target: https://piolabs.com/?utm_source=github&utm_medium=core
@@ -40,9 +36,11 @@ PlatformIO Core
 .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png
     :target: https://platformio.org?utm_source=github&utm_medium=core
 
-`PlatformIO <https://platformio.org>`_ is a professional collaborative platform for embedded development.
+`PlatformIO <https://platformio.org>`_: Your Gateway to Embedded Software Development Excellence.
 
-**A place where Developers and Teams have true Freedom! No more vendor lock-in!**
+Unlock the true potential of embedded software development with
+PlatformIO's collaborative ecosystem, embracing declarative principles,
+test-driven methodologies, and modern toolchains for unrivaled success.
 
 * Open source, maximum permissive Apache 2.0 license
 * Cross-platform IDE and Unified Debugger
@@ -99,3 +97,7 @@ Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 
 The PlatformIO is licensed under the permissive Apache 2.0 license,
 so you can use it in both commercial and personal projects with confidence.
+
+.. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg
+    :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md
+    :alt:  SWUbanner

SECURITY.md

@@ -0,0 +1,34 @@
+# Security Policy
+
+## Supported Versions
+
+We are committed to ensuring the security and protection of PlatformIO Core. 
+To this end, we support only the following versions:
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 6.1.x   | :white_check_mark: |
+| < 6.1   | :x:                |
+
+Unsupported versions of the PlatformIO Core may have known vulnerabilities or security issues that could compromise the security of our organization's systems and data. 
+Therefore, it is important that all developers use only supported versions of the PlatformIO Core.
+
+## Reporting a Vulnerability
+
+We take the security of our systems and data very seriously. We encourage responsible disclosure of any vulnerabilities or security issues that you may find in our systems or applications. If you believe you have discovered a vulnerability, please report it to us immediately.
+
+To report a vulnerability, please send an email to our security team at contact@piolabs.com. Please include as much information as possible, including:
+
+- A description of the vulnerability and how it can be exploited
+- Steps to reproduce the vulnerability
+- Any additional information that can help us understand and reproduce the vulnerability
+
+Once we receive your report, our security team will acknowledge receipt within 24 hours and will work to validate the reported vulnerability. We will provide periodic updates on the progress of the vulnerability assessment, and will notify you once a fix has been deployed.
+
+If the vulnerability is accepted, we will work to remediate the issue as quickly as possible. We may also provide credit or recognition to the individual who reported the vulnerability, at our discretion.
+
+If the vulnerability is declined, we will provide a justification for our decision and may offer guidance on how to improve the report or how to test the system more effectively.
+
+Please note that we will not take any legal action against individuals who report vulnerabilities in good faith and in accordance with this policy.
+
+Thank you for helping us keep our systems and data secure.

platformio/__init__.py

@@ -12,22 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
-VERSION = (6, 0, 1)
+VERSION = (6, 1, 10)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
 __description__ = (
-    "A professional collaborative platform for embedded development. "
-    "Cross-platform IDE and Unified Debugger. "
-    "Static Code Analyzer and Remote Unit Testing. "
-    "Multi-platform and Multi-architecture Build System. "
-    "Firmware File Explorer and Memory Inspection. "
-    "IoT, Arduino, CMSIS, ESP-IDF, FreeRTOS, libOpenCM3, mbedOS, Pulp OS, SPL, "
-    "STM32Cube, Zephyr RTOS, ARM, AVR, Espressif (ESP8266/ESP32), FPGA, "
-    "MCS-51 (8051), MSP430, Nordic (nRF51/nRF52), NXP i.MX RT, PIC32, RISC-V, "
-    "STMicroelectronics (STM8/STM32), Teensy"
+    "Your Gateway to Embedded Software Development Excellence. "
+    "Unlock the true potential of embedded software development "
+    "with PlatformIO's collaborative ecosystem, embracing "
+    "declarative principles, test-driven methodologies, and "
+    "modern toolchains for unrivaled success."
 )
 __url__ = "https://platformio.org"
 
@@ -44,14 +38,12 @@ __registry_mirror_hosts__ = [
 ]
 __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
 
-__default_requests_timeout__ = (10, None)  # (connect, read)
-
 __core_packages__ = {
-    "contrib-piohome": "~3.4.1",
-    "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
-    "tool-scons": "~4.40300.0",
-    "tool-cppcheck": "~1.270.0",
-    "tool-clangtidy": "~1.120001.0",
+    "contrib-piohome": "~3.4.2",
+    "contrib-pioremote": "~1.0.0",
+    "tool-scons": "~4.40502.0",
+    "tool-cppcheck": "~1.21100.0",
+    "tool-clangtidy": "~1.150005.0",
     "tool-pvs-studio": "~7.18.0",
 }
 
@@ -60,3 +52,22 @@ __check_internet_hosts__ = [
     "88.198.170.159",  # platformio.org
     "github.com",
 ] + __registry_mirror_hosts__
+
+__install_requires__ = [
+    # Core requirements
+    "bottle == 0.12.*",
+    "click >=8.0.4, <=8.2",
+    "colorama",
+    "marshmallow == 3.*",
+    "pyelftools == 0.29",
+    "pyserial == 3.5.*",  # keep in sync "device/monitor/terminal.py"
+    "requests == 2.*",
+    "semantic_version == 2.10.*",
+    "tabulate == 0.*",
+] + [
+    # PIO Home requirements
+    "ajsonrpc == 1.2.*",
+    "starlette >=0.19, <0.32",
+    "uvicorn >=0.16, <0.24",
+    "wsproto == 1.*",
+]

platformio/__main__.py

@@ -14,12 +14,12 @@
 
 import os
 import sys
-from traceback import format_exc
+import traceback
 
 import click
 
 from platformio import __version__, exception, maintenance
-from platformio.commands import PlatformioCLI
+from platformio.cli import PlatformioCLI
 from platformio.compat import IS_CYGWIN, ensure_python3
 
 
@@ -27,11 +27,11 @@ from platformio.compat import IS_CYGWIN, ensure_python3
     cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])
 )
 @click.version_option(__version__, prog_name="PlatformIO Core")
-@click.option("--force", "-f", is_flag=True, help="DEPRECATED")
+@click.option("--force", "-f", is_flag=True, help="DEPRECATED", hidden=True)
 @click.option("--caller", "-c", help="Caller ID (service)")
 @click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters")
 @click.pass_context
-def cli(ctx, force, caller, no_ansi):
+def cli(ctx, force, caller, no_ansi):  # pylint: disable=unused-argument
     try:
         if (
             no_ansi
@@ -53,13 +53,13 @@ def cli(ctx, force, caller, no_ansi):
     except:  # pylint: disable=bare-except
         pass
 
-    maintenance.on_platformio_start(ctx, force, caller)
+    maintenance.on_cmd_start(ctx, caller)
 
 
 @cli.result_callback()
 @click.pass_context
-def process_result(ctx, result, *_, **__):
-    maintenance.on_platformio_end(ctx, result)
+def process_result(*_, **__):
+    maintenance.on_cmd_end()
 
 
 def configure():
@@ -96,31 +96,32 @@ def main(argv=None):
     if argv:
         assert isinstance(argv, list)
         sys.argv = argv
+
     try:
         ensure_python3(raise_exception=True)
         configure()
         cli()  # pylint: disable=no-value-for-parameter
-    except SystemExit as e:
-        if e.code and str(e.code).isdigit():
-            exit_code = int(e.code)
-    except Exception as e:  # pylint: disable=broad-except
-        if not isinstance(e, exception.ReturnErrorCode):
-            maintenance.on_platformio_exception(e)
-            error_str = "Error: "
-            if isinstance(e, exception.PlatformioException):
-                error_str += str(e)
+    except SystemExit as exc:
+        if exc.code and str(exc.code).isdigit():
+            exit_code = int(exc.code)
+    except Exception as exc:  # pylint: disable=broad-except
+        if not isinstance(exc, exception.ReturnErrorCode):
+            maintenance.on_platformio_exception(exc)
+            error_str = f"{exc.__class__.__name__}: "
+            if isinstance(exc, exception.PlatformioException):
+                error_str += str(exc)
             else:
-                error_str += format_exc()
+                error_str += traceback.format_exc()
                 error_str += """
 ============================================================
 
 An unexpected error occurred. Further steps:
 
 * Verify that you have the latest version of PlatformIO using
-  `pip install -U platformio` command
+  `python -m pip install -U platformio` command
 
 * Try to find answer in FAQ Troubleshooting section
-  https://docs.platformio.org/page/faq.html
+  https://docs.platformio.org/page/faq/index.html
 
 * Report this problem to the developers
   https://github.com/platformio/platformio-core/issues
@@ -128,7 +129,9 @@ An unexpected error occurred. Further steps:
 ============================================================
 """
             click.secho(error_str, fg="red", err=True)
-        exit_code = int(str(e)) if str(e).isdigit() else 1
+        exit_code = int(str(exc)) if str(exc).isdigit() else 1
+
+    maintenance.on_platformio_exit()
     sys.argv = prev_sys_argv
     return exit_code
 

platformio/account/cli.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.commands.destroy import account_destroy_cmd
+from platformio.account.commands.forgot import account_forgot_cmd
+from platformio.account.commands.login import account_login_cmd
+from platformio.account.commands.logout import account_logout_cmd
+from platformio.account.commands.password import account_password_cmd
+from platformio.account.commands.register import account_register_cmd
+from platformio.account.commands.show import account_show_cmd
+from platformio.account.commands.token import account_token_cmd
+from platformio.account.commands.update import account_update_cmd
+
+
+@click.group(
+    "account",
+    commands=[
+        account_destroy_cmd,
+        account_forgot_cmd,
+        account_login_cmd,
+        account_logout_cmd,
+        account_password_cmd,
+        account_register_cmd,
+        account_show_cmd,
+        account_token_cmd,
+        account_update_cmd,
+    ],
+    short_help="Manage PlatformIO account",
+)
+def cli():
+    pass

platformio/account/client.py

@@ -16,27 +16,23 @@ import os
 import time
 
 from platformio import __accounts_api__, app
-from platformio.clients.http import HTTPClient, HTTPClientError
-from platformio.exception import PlatformioException
+from platformio.exception import PlatformioException, UserSideException
+from platformio.http import HTTPClient, HTTPClientError
 
 
 class AccountError(PlatformioException):
-
     MESSAGE = "{0}"
 
 
-class AccountNotAuthorized(AccountError):
-
+class AccountNotAuthorized(AccountError, UserSideException):
     MESSAGE = "You are not authorized! Please log in to PlatformIO Account."
 
 
-class AccountAlreadyAuthorized(AccountError):
-
+class AccountAlreadyAuthorized(AccountError, UserSideException):
     MESSAGE = "You are already authorized with {0} account."
 
 
 class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
-
     SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
 
     def __init__(self):
@@ -46,8 +42,8 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
     def get_refresh_token():
         try:
             return app.get_state_item("account").get("auth").get("refresh_token")
-        except:  # pylint:disable=bare-except
-            raise AccountNotAuthorized()
+        except Exception as exc:
+            raise AccountNotAuthorized() from exc
 
     @staticmethod
     def delete_local_session():
@@ -298,7 +294,7 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s/owners" % orgname,
-            data={"username": username},
+            params={"username": username},
             x_with_authorization=True,
         )
 
@@ -351,6 +347,6 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s/teams/%s/members" % (orgname, teamname),
-            data={"username": username},
+            params={"username": username},
             x_with_authorization=True,
         )

platformio/account/commands/destroy.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient, AccountNotAuthorized
+
+
+@click.command("destroy", short_help="Destroy account")
+def account_destroy_cmd():
+    client = AccountClient()
+    click.confirm(
+        "Are you sure you want to delete the %s user account?\n"
+        "Warning! All linked data will be permanently removed and can not be restored."
+        % client.get_logged_username(),
+        abort=True,
+    )
+    client.destroy_account()
+    try:
+        client.logout()
+    except AccountNotAuthorized:
+        pass
+    click.secho(
+        "User account has been destroyed.",
+        fg="green",
+    )

platformio/account/commands/forgot.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("forgot", short_help="Forgot password")
+@click.option("--username", prompt="Username or email")
+def account_forgot_cmd(username):
+    client = AccountClient()
+    client.forgot_password(username)
+    click.secho(
+        "If this account is registered, we will send the "
+        "further instructions to your email.",
+        fg="green",
+    )

platformio/account/commands/login.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("login", short_help="Log in to PlatformIO Account")
+@click.option("-u", "--username", prompt="Username or email")
+@click.option("-p", "--password", prompt=True, hide_input=True)
+def account_login_cmd(username, password):
+    client = AccountClient()
+    client.login(username, password)
+    click.secho("Successfully logged in!", fg="green")

platformio/account/commands/logout.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("logout", short_help="Log out of PlatformIO Account")
+def account_logout_cmd():
+    client = AccountClient()
+    client.logout()
+    click.secho("Successfully logged out!", fg="green")

platformio/account/commands/password.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("password", short_help="Change password")
+@click.option("--old-password", prompt=True, hide_input=True)
+@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
+def account_password_cmd(old_password, new_password):
+    client = AccountClient()
+    client.change_password(old_password, new_password)
+    click.secho("Password successfully changed!", fg="green")

platformio/account/commands/register.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import (
+    validate_email,
+    validate_password,
+    validate_username,
+)
+
+
+@click.command("register", short_help="Create new PlatformIO Account")
+@click.option(
+    "-u",
+    "--username",
+    prompt=True,
+    callback=lambda _, __, value: validate_username(value),
+)
+@click.option(
+    "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value)
+)
+@click.option(
+    "-p",
+    "--password",
+    prompt=True,
+    hide_input=True,
+    confirmation_prompt=True,
+    callback=lambda _, __, value: validate_password(value),
+)
+@click.option("--firstname", prompt=True)
+@click.option("--lastname", prompt=True)
+def account_register_cmd(username, email, password, firstname, lastname):
+    client = AccountClient()
+    client.registration(username, email, password, firstname, lastname)
+    click.secho(
+        "An account has been successfully created. "
+        "Please check your mail to activate your account and verify your email address.",
+        fg="green",
+    )

platformio/account/commands/show.py

@@ -0,0 +1,116 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio import util
+from platformio.account.client import AccountClient
+
+
+@click.command("show", short_help="PlatformIO Account information")
+@click.option("--offline", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_show_cmd(offline, json_output):
+    client = AccountClient()
+    info = client.get_account_info(offline)
+    if json_output:
+        click.echo(json.dumps(info))
+        return
+    click.echo()
+    if info.get("profile"):
+        print_profile(info["profile"])
+    if info.get("packages"):
+        print_packages(info["packages"])
+    if info.get("subscriptions"):
+        print_subscriptions(info["subscriptions"])
+    click.echo()
+
+
+def print_profile(profile):
+    click.secho("Profile", fg="cyan", bold=True)
+    click.echo("=" * len("Profile"))
+    data = []
+    if profile.get("username"):
+        data.append(("Username:", profile["username"]))
+    if profile.get("email"):
+        data.append(("Email:", profile["email"]))
+    if profile.get("firstname"):
+        data.append(("First name:", profile["firstname"]))
+    if profile.get("lastname"):
+        data.append(("Last name:", profile["lastname"]))
+    click.echo(tabulate(data, tablefmt="plain"))
+
+
+def print_packages(packages):
+    click.echo()
+    click.secho("Packages", fg="cyan")
+    click.echo("=" * len("Packages"))
+    for package in packages:
+        click.echo()
+        click.secho(package.get("name"), bold=True)
+        click.echo("-" * len(package.get("name")))
+        if package.get("description"):
+            click.echo(package.get("description"))
+        data = []
+        expire = "-"
+        if "subscription" in package:
+            expire = util.parse_datetime(
+                package["subscription"].get("end_at")
+                or package["subscription"].get("next_bill_at")
+            ).strftime("%Y-%m-%d")
+        data.append(("Expire:", expire))
+        services = []
+        for key in package:
+            if not key.startswith("service."):
+                continue
+            if isinstance(package[key], dict):
+                services.append(package[key].get("title"))
+            else:
+                services.append(package[key])
+        if services:
+            data.append(("Services:", ", ".join(services)))
+        click.echo(tabulate(data, tablefmt="plain"))
+
+
+def print_subscriptions(subscriptions):
+    click.echo()
+    click.secho("Subscriptions", fg="cyan")
+    click.echo("=" * len("Subscriptions"))
+    for subscription in subscriptions:
+        click.echo()
+        click.secho(subscription.get("product_name"), bold=True)
+        click.echo("-" * len(subscription.get("product_name")))
+        data = [("State:", subscription.get("status"))]
+        begin_at = util.parse_datetime(subscription.get("begin_at")).strftime("%c")
+        data.append(("Start date:", begin_at or "-"))
+        end_at = subscription.get("end_at")
+        if end_at:
+            end_at = util.parse_datetime(subscription.get("end_at")).strftime("%c")
+        data.append(("End date:", end_at or "-"))
+        next_bill_at = subscription.get("next_bill_at")
+        if next_bill_at:
+            next_bill_at = util.parse_datetime(
+                subscription.get("next_bill_at")
+            ).strftime("%c")
+        data.append(("Next payment:", next_bill_at or "-"))
+        data.append(
+            ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")
+        )
+        data.append(
+            ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-")
+        )
+        click.echo(tabulate(data, tablefmt="plain"))

platformio/account/commands/token.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("token", short_help="Get or regenerate Authentication Token")
+@click.option("-p", "--password", prompt=True, hide_input=True)
+@click.option("--regenerate", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_token_cmd(password, regenerate, json_output):
+    client = AccountClient()
+    auth_token = client.auth_token(password, regenerate)
+    if json_output:
+        click.echo(json.dumps({"status": "success", "result": auth_token}))
+        return
+    click.secho("Personal Authentication Token: %s" % auth_token, fg="green")

platformio/account/commands/update.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient, AccountNotAuthorized
+from platformio.account.validate import validate_email, validate_username
+
+
+@click.command("update", short_help="Update profile information")
+@click.option("--current-password", prompt=True, hide_input=True)
+@click.option("--username")
+@click.option("--email")
+@click.option("--firstname")
+@click.option("--lastname")
+def account_update_cmd(current_password, **kwargs):
+    client = AccountClient()
+    profile = client.get_profile()
+    new_profile = profile.copy()
+    if not any(kwargs.values()):
+        for field in profile:
+            new_profile[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=profile[field]
+            )
+            if field == "email":
+                validate_email(new_profile[field])
+            if field == "username":
+                validate_username(new_profile[field])
+    else:
+        new_profile.update({key: value for key, value in kwargs.items() if value})
+    client.update_profile(new_profile, current_password)
+    click.secho("Profile successfully updated!", fg="green")
+    username_changed = new_profile["username"] != profile["username"]
+    email_changed = new_profile["email"] != profile["email"]
+    if not username_changed and not email_changed:
+        return None
+    try:
+        client.logout()
+    except AccountNotAuthorized:
+        pass
+    if email_changed:
+        click.secho(
+            "Please check your mail to verify your new email address and re-login. ",
+            fg="yellow",
+        )
+        return None
+    click.secho("Please re-login.", fg="yellow")
+    return None

platformio/account/org/cli.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.org.commands.add import org_add_cmd
+from platformio.account.org.commands.create import org_create_cmd
+from platformio.account.org.commands.destroy import org_destroy_cmd
+from platformio.account.org.commands.list import org_list_cmd
+from platformio.account.org.commands.remove import org_remove_cmd
+from platformio.account.org.commands.update import org_update_cmd
+
+
+@click.group(
+    "account",
+    commands=[
+        org_add_cmd,
+        org_create_cmd,
+        org_destroy_cmd,
+        org_list_cmd,
+        org_remove_cmd,
+        org_update_cmd,
+    ],
+    short_help="Manage organizations",
+)
+def cli():
+    pass

platformio/account/org/commands/add.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("add", short_help="Add a new owner to organization")
+@click.argument(
+    "orgname",
+)
+@click.argument(
+    "username",
+)
+def org_add_cmd(orgname, username):
+    client = AccountClient()
+    client.add_org_owner(orgname, username)
+    return click.secho(
+        "The new owner `%s` has been successfully added to the `%s` organization."
+        % (username, orgname),
+        fg="green",
+    )

platformio/account/org/commands/create.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_email, validate_orgname
+
+
+@click.command("create", short_help="Create a new organization")
+@click.argument(
+    "orgname",
+    callback=lambda _, __, value: validate_orgname(value),
+)
+@click.option(
+    "--email", callback=lambda _, __, value: validate_email(value) if value else value
+)
+@click.option(
+    "--displayname",
+)
+def org_create_cmd(orgname, email, displayname):
+    client = AccountClient()
+    client.create_org(orgname, email, displayname)
+    return click.secho(
+        "The organization `%s` has been successfully created." % orgname,
+        fg="green",
+    )

platformio/account/org/commands/destroy.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("destroy", short_help="Destroy organization")
+@click.argument("orgname")
+def org_destroy_cmd(orgname):
+    client = AccountClient()
+    click.confirm(
+        "Are you sure you want to delete the `%s` organization account?\n"
+        "Warning! All linked data will be permanently removed and can not be restored."
+        % orgname,
+        abort=True,
+    )
+    client.destroy_org(orgname)
+    return click.secho(
+        "Organization `%s` has been destroyed." % orgname,
+        fg="green",
+    )

platformio/account/org/commands/list.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio.account.client import AccountClient
+
+
+@click.command("list", short_help="List organizations and their members")
+@click.option("--json-output", is_flag=True)
+def org_list_cmd(json_output):
+    client = AccountClient()
+    orgs = client.list_orgs()
+    if json_output:
+        return click.echo(json.dumps(orgs))
+    if not orgs:
+        return click.echo("You do not have any organization")
+    for org in orgs:
+        click.echo()
+        click.secho(org.get("orgname"), fg="cyan")
+        click.echo("-" * len(org.get("orgname")))
+        data = []
+        if org.get("displayname"):
+            data.append(("Display Name:", org.get("displayname")))
+        if org.get("email"):
+            data.append(("Email:", org.get("email")))
+        data.append(
+            (
+                "Owners:",
+                ", ".join((owner.get("username") for owner in org.get("owners"))),
+            )
+        )
+        click.echo(tabulate(data, tablefmt="plain"))
+    return click.echo()

platformio/account/org/commands/remove.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("remove", short_help="Remove an owner from organization")
+@click.argument(
+    "orgname",
+)
+@click.argument(
+    "username",
+)
+def org_remove_cmd(orgname, username):
+    client = AccountClient()
+    client.remove_org_owner(orgname, username)
+    return click.secho(
+        "The `%s` owner has been successfully removed from the `%s` organization."
+        % (username, orgname),
+        fg="green",
+    )

platformio/account/org/commands/update.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_email, validate_orgname
+
+
+@click.command("update", short_help="Update organization")
+@click.argument("cur_orgname")
+@click.option(
+    "--orgname",
+    callback=lambda _, __, value: validate_orgname(value) if value else value,
+    help="A new orgname",
+)
+@click.option(
+    "--email",
+    callback=lambda _, __, value: validate_email(value) if value else value,
+)
+@click.option("--displayname")
+def org_update_cmd(cur_orgname, **kwargs):
+    client = AccountClient()
+    org = client.get_org(cur_orgname)
+    new_org = {
+        key: value if value is not None else org[key] for key, value in kwargs.items()
+    }
+    if not any(kwargs.values()):
+        for key in kwargs:
+            new_org[key] = click.prompt(key.capitalize(), default=org[key])
+            if key == "email":
+                validate_email(new_org[key])
+            if key == "orgname":
+                validate_orgname(new_org[key])
+    client.update_org(cur_orgname, new_org)
+    return click.secho(
+        "The organization `%s` has been successfully updated." % cur_orgname,
+        fg="green",
+    )

platformio/account/team/cli.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.team.commands.add import team_add_cmd
+from platformio.account.team.commands.create import team_create_cmd
+from platformio.account.team.commands.destroy import team_destroy_cmd
+from platformio.account.team.commands.list import team_list_cmd
+from platformio.account.team.commands.remove import team_remove_cmd
+from platformio.account.team.commands.update import team_update_cmd
+
+
+@click.group(
+    "team",
+    commands=[
+        team_add_cmd,
+        team_create_cmd,
+        team_destroy_cmd,
+        team_list_cmd,
+        team_remove_cmd,
+        team_update_cmd,
+    ],
+    short_help="Manage organization teams",
+)
+def cli():
+    pass

platformio/account/team/commands/add.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("add", short_help="Add a new member to team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.argument(
+    "username",
+)
+def team_add_cmd(orgname_teamname, username):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.add_team_member(orgname, teamname, username)
+    return click.secho(
+        "The new member %s has been successfully added to the %s team."
+        % (username, teamname),
+        fg="green",
+    )

platformio/account/team/commands/create.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("create", short_help="Create a new team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.option(
+    "--description",
+)
+def team_create_cmd(orgname_teamname, description):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.create_team(orgname, teamname, description)
+    return click.secho(
+        "The team %s has been successfully created." % teamname,
+        fg="green",
+    )

platformio/account/team/commands/destroy.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("destroy", short_help="Destroy a team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+def team_destroy_cmd(orgname_teamname):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    click.confirm(
+        click.style(
+            "Are you sure you want to destroy the %s team?" % teamname, fg="yellow"
+        ),
+        abort=True,
+    )
+    client = AccountClient()
+    client.destroy_team(orgname, teamname)
+    return click.secho(
+        "The team %s has been successfully destroyed." % teamname,
+        fg="green",
+    )

platformio/account/team/commands/list.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio.account.client import AccountClient
+
+
+@click.command("list", short_help="List teams")
+@click.argument("orgname", required=False)
+@click.option("--json-output", is_flag=True)
+def team_list_cmd(orgname, json_output):
+    client = AccountClient()
+    data = {}
+    if not orgname:
+        for item in client.list_orgs():
+            teams = client.list_teams(item.get("orgname"))
+            data[item.get("orgname")] = teams
+    else:
+        teams = client.list_teams(orgname)
+        data[orgname] = teams
+    if json_output:
+        return click.echo(json.dumps(data[orgname] if orgname else data))
+    if not any(data.values()):
+        return click.secho("You do not have any teams.", fg="yellow")
+    for org_name, teams in data.items():
+        for team in teams:
+            click.echo()
+            click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan")
+            click.echo("-" * len("%s:%s" % (org_name, team.get("name"))))
+            table_data = []
+            if team.get("description"):
+                table_data.append(("Description:", team.get("description")))
+            table_data.append(
+                (
+                    "Members:",
+                    ", ".join(
+                        (member.get("username") for member in team.get("members"))
+                    )
+                    if team.get("members")
+                    else "-",
+                )
+            )
+            click.echo(tabulate(table_data, tablefmt="plain"))
+    return click.echo()

platformio/account/team/commands/remove.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("remove", short_help="Remove a member from team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.argument("username")
+def team_remove_cmd(orgname_teamname, username):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.remove_team_member(orgname, teamname, username)
+    return click.secho(
+        "The %s member has been successfully removed from the %s team."
+        % (username, teamname),
+        fg="green",
+    )

platformio/account/team/commands/update.py

@@ -0,0 +1,51 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname, validate_teamname
+
+
+@click.command("update", short_help="Update team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.option(
+    "--name",
+    callback=lambda _, __, value: validate_teamname(value) if value else value,
+    help="A new team name",
+)
+@click.option(
+    "--description",
+)
+def team_update_cmd(orgname_teamname, **kwargs):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    team = client.get_team(orgname, teamname)
+    new_team = {
+        key: value if value is not None else team[key] for key, value in kwargs.items()
+    }
+    if not any(kwargs.values()):
+        for key in kwargs:
+            new_team[key] = click.prompt(key.capitalize(), default=team[key])
+            if key == "name":
+                validate_teamname(new_team[key])
+    client.update_team(orgname, teamname, new_team)
+    return click.secho(
+        "The team %s has been successfully updated." % teamname,
+        fg="green",
+    )

platformio/account/validate.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import click
+
+
+def validate_username(value, field="username"):
+    value = str(value).strip() if value else None
+    if not value or not re.match(
+        r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I
+    ):
+        raise click.BadParameter(
+            "Invalid %s format. "
+            "%s must contain only alphanumeric characters "
+            "or single hyphens, cannot begin or end with a hyphen, "
+            "and must not be longer than 38 characters."
+            % (field.lower(), field.capitalize())
+        )
+    return value
+
+
+def validate_orgname(value):
+    return validate_username(value, "Organization name")
+
+
+def validate_email(value):
+    value = str(value).strip() if value else None
+    if not value or not re.match(
+        r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I
+    ):
+        raise click.BadParameter("Invalid email address")
+    return value
+
+
+def validate_password(value):
+    value = str(value).strip() if value else None
+    if not value or not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
+        raise click.BadParameter(
+            "Invalid password format. "
+            "Password must contain at least 8 characters"
+            " including a number and a lowercase letter"
+        )
+    return value
+
+
+def validate_teamname(value):
+    value = str(value).strip() if value else None
+    if not value or not re.match(
+        r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I
+    ):
+        raise click.BadParameter(
+            "Invalid team name format. "
+            "Team name must only contain alphanumeric characters, "
+            "single hyphens, underscores, spaces. It can not "
+            "begin or end with a hyphen or a underscore and must"
+            " not be longer than 20 characters."
+        )
+    return value
+
+
+def validate_orgname_teamname(value):
+    value = str(value).strip() if value else None
+    if not value or ":" not in value:
+        raise click.BadParameter(
+            "Please specify organization and team name using the following"
+            " format - orgname:teamname. For example, mycompany:DreamTeam"
+        )
+    orgname, teamname = value.split(":", 1)
+    validate_orgname(orgname)
+    validate_teamname(teamname)
+    return value

platformio/app.py

@@ -12,14 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import getpass
 import hashlib
 import json
 import os
 import platform
 import socket
+import time
 import uuid
 
 from platformio import __version__, exception, fs, proc
@@ -60,25 +59,33 @@ DEFAULT_SETTINGS = {
         "value": get_default_projects_dir(),
         "validator": projects_dir_validate,
     },
+    "enable_proxy_strict_ssl": {
+        "description": "Verify the proxy server certificate against the list of supplied CAs",
+        "value": True,
+    },
 }
 
 SESSION_VARS = {
     "command_ctx": None,
-    "force_option": False,
     "caller_id": None,
     "custom_project_conf": None,
+    "pause_telemetry": False,
 }
 
 
-class State(object):
+def resolve_state_path(conf_option_dir, file_name, ensure_dir_exists=True):
+    state_dir = ProjectConfig.get_instance().get("platformio", conf_option_dir)
+    if ensure_dir_exists and not os.path.isdir(state_dir):
+        os.makedirs(state_dir)
+    return os.path.join(state_dir, file_name)
+
+
+class State:
     def __init__(self, path=None, lock=False):
         self.path = path
         self.lock = lock
         if not self.path:
-            core_dir = ProjectConfig.get_instance().get("platformio", "core_dir")
-            if not os.path.isdir(core_dir):
-                os.makedirs(core_dir)
-            self.path = os.path.join(core_dir, "appstate.json")
+            self.path = resolve_state_path("core_dir", "appstate.json")
         self._storage = {}
         self._lockfile = None
         self.modified = False
@@ -103,8 +110,10 @@ class State(object):
             try:
                 with open(self.path, mode="w", encoding="utf8") as fp:
                     fp.write(json.dumps(self._storage))
-            except IOError:
-                raise exception.HomeDirPermissionsError(os.path.dirname(self.path))
+            except IOError as exc:
+                raise exception.HomeDirPermissionsError(
+                    os.path.dirname(self.path)
+                ) from exc
         self._unlock_state_file()
 
     def _lock_state_file(self):
@@ -113,8 +122,8 @@ class State(object):
         self._lockfile = LockFile(self.path)
         try:
             self._lockfile.acquire()
-        except IOError:
-            raise exception.HomeDirPermissionsError(os.path.dirname(self.path))
+        except IOError as exc:
+            raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) from exc
 
     def _unlock_state_file(self):
         if hasattr(self, "_lockfile") and self._lockfile:
@@ -169,8 +178,8 @@ def sanitize_setting(name, value):
                 value = str(value).lower() in ("true", "yes", "y", "1")
         elif isinstance(defdata["value"], int):
             value = int(value)
-    except Exception:
-        raise exception.InvalidSettingValue(value, name)
+    except Exception as exc:
+        raise exception.InvalidSettingValue(value, name) from exc
     return value
 
 
@@ -227,13 +236,7 @@ def set_session_var(name, value):
 
 
 def is_disabled_progressbar():
-    return any(
-        [
-            get_session_var("force_option"),
-            proc.is_ci(),
-            os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true",
-        ]
-    )
+    return os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true"
 
 
 def get_cid():
@@ -251,9 +254,14 @@ def get_cid():
     cid = str(cid)
     if IS_WINDOWS or os.getuid() > 0:  # pylint: disable=no-member
         set_state_item("cid", cid)
+        set_state_item("created_at", int(time.time()))
     return cid
 
 
+def get_project_id(project_dir):
+    return hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()
+
+
 def get_user_agent():
     data = [
         "PlatformIO/%s" % __version__,
@@ -266,6 +274,8 @@ def get_user_agent():
         data.append("IDE/%s" % os.getenv("PLATFORMIO_IDE"))
     data.append("Python/%s" % platform.python_version())
     data.append("Platform/%s" % platform.platform())
+    if not get_setting("enable_telemetry"):
+        data.append("Telemetry/0")
     return " ".join(data)
 
 

platformio/assets/schema/library.json

@@ -0,0 +1,487 @@
+{
+  "$id": "https://example.com/library.json",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "library.json schema",
+  "type": "object",
+  "properties": {
+    "name": {
+      "type": "string",
+      "maxLength": 50,
+      "description": "A name of a library.\nMust be unique in the PlatformIO Registry\nShould be slug style for simplicity, consistency, and compatibility. Example: HelloWorld\nCan contain a-z, digits, and dashes (but not start/end with them)\nConsecutive dashes and [:;/,@<>] chars are not allowed.",
+      "required": true
+    },
+    "version": {
+      "type": "string",
+      "maxLength": 20,
+      "description": "A version of a current library source code. Can contain a-z, digits, dots or dash and should be Semantic Versioning compatible.",
+      "required": true
+    },
+    "description": {
+      "type": "string",
+      "maxLength": 255,
+      "description": "The field helps users to identify and search for your library with a brief description. Describe the hardware devices (sensors, boards and etc.) which are suitable with it.",
+      "required": true
+    },
+    "keywords": {
+      "anyOf": [
+        {
+          "type": "string",
+          "maxLength": 255
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "maxLength": 255
+          }
+        }
+      ],
+      "description": "Used for search by keyword. Helps to make your library easier to discover without people needing to know its name.\nThe keyword should be lowercased, can contain a-z, digits and dash (but not start/end with them). A list from the keywords can be specified with separator , or declared as Array.",
+      "required": true
+    },
+    "homepage": {
+      "type": "string",
+      "maxLength": 255,
+      "description": "Home page of a library (if is different from repository url).",
+      "required": false
+    },
+    "repository": {
+      "type": "object",
+      "properties": {
+        "type": {
+          "enum": [
+            "git",
+            "hg",
+            "svn"
+          ],
+          "description": "only “git”, “hg” or “svn” are supported"
+        },
+        "url": {
+          "type": "string"
+        },
+        "branch": {
+          "type": "string",
+          "description": "if is not specified, default branch will be used. This field will be ignored if tag/release exists with the value of version."
+        }
+      },
+      "description": "The repository in which the source code can be found.",
+      "required": false
+    },
+    "authors": {
+      "anyOf": [
+        {
+          "type": "object",
+          "properties": {
+            "name": {
+              "type": "string",
+              "required": true,
+              "description": "Full name"
+            },
+            "email": {
+              "type": "string"
+            },
+            "url": {
+              "type": "string",
+              "description": "An author’s contact page"
+            },
+            "maintainer": {
+              "type": "boolean",
+              "description": "Specify “maintainer” status"
+            }
+          }
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string",
+                "required": true,
+                "description": "Full name"
+              },
+              "email": {
+                "type": "string"
+              },
+              "url": {
+                "type": "string",
+                "description": "An author’s contact page"
+              },
+              "maintainer": {
+                "type": "boolean",
+                "description": "Specify “maintainer” status"
+              }
+            }
+          }
+        }
+      ],
+      "description": "An author contact information\nIf authors field is not defined, PlatformIO will try to fetch data from VCS provider (Github, Gitlab, etc) if repository is declared.",
+      "required": false
+    },
+    "license": {
+      "type": "string",
+      "description": "A SPDX license ID or SPDX Expression. You can check the full list of SPDX license IDs (see “Identifier” column).",
+      "required": false
+    },
+    "frameworks": {
+      "anyOf": [
+        {
+          "type": "string",
+          "description": "espidf, freertos, *, etc'"
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "description": "espidf, freertos, *, etc'"
+          }
+        }
+      ],
+      "description": "A list with compatible frameworks. The available framework names are defined in the Frameworks section.\nIf the library is compatible with the all frameworks, then do not declare this field or you use *",
+      "required": false
+    },
+    "platforms": {
+      "anyOf": [
+        {
+          "type": "string",
+          "description": "atmelavr, espressif8266, *, etc'"
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "description": "atmelavr, espressif8266, *, etc'"
+          }
+        }
+      ],
+      "description": "A list with compatible development platforms. The available platform name are defined in Development Platforms section.\nIf the library is compatible with the all platforms, then do not declare this field or use *.\nPlatformIO does not check platforms for compatibility in default mode. See Compatibility Mode for details. If you need a strict checking for compatible platforms for a library, please set libCompatMode to strict.",
+      "required": false
+    },
+    "headers": {
+      "anyOf": [
+        {
+          "type": "string",
+          "description": "MyLibrary.h"
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "description": "FooCore.h, FooFeature.h"
+          }
+        }
+      ],
+      "description": "A list of header files that can be included in a project source files using #include <...> directive.",
+      "required": false
+    },
+    "examples": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "base": {
+            "type": "string"
+          },
+          "files": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        }
+      },
+      "description": "A list of example patterns.",
+      "required": "false"
+    },
+    "dependencies": {
+      "anyOf": [
+        {
+          "type": "object",
+          "properties": {
+            "owner": {
+              "type": "string",
+              "description": "an owner name (username) from the PlatformIO Registry"
+            },
+            "name": {
+              "type": "string",
+              "description": "library name"
+            },
+            "version": {
+              "type": "string",
+              "description": "Version Requirements or Package Specifications"
+            },
+            "frameworks": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              ],
+              "description": "project compatible Frameworks"
+            },
+            "platforms": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              ],
+              "description": " project compatible Development Platforms"
+            }
+          }
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "owner": {
+                "type": "string",
+                "description": "an owner name (username) from the PlatformIO Registry"
+              },
+              "name": {
+                "type": "string",
+                "description": "library name"
+              },
+              "version": {
+                "type": "string",
+                "description": "Version Requirements or Package Specifications"
+              },
+              "frameworks": {
+                "anyOf": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  }
+                ],
+                "description": "project compatible Frameworks"
+              },
+              "platforms": {
+                "anyOf": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    }
+                  }
+                ],
+                "description": " project compatible Development Platforms"
+              }
+            }
+          }
+        }
+      ],
+      "description": "A list of dependent libraries that will be automatically installed.",
+      "required": false
+    },
+    "export": {
+      "type": "object",
+      "properties": {
+        "include": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Export only files that matched declared patterns.\n* - matches everything\n? - matches any single character\n[seq] - matches any character in seq\n[!seq] - matches any character not in seq"
+        },
+        "exclude": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Exclude the directories and files which match with exclude patterns."
+        }
+      },
+      "description": "This option is useful if you need to exclude extra data (test code, docs, images, PDFs, etc). It allows one to reduce the size of the final archive.\nTo check which files will be included in the final packages, please use pio pkg pack command.",
+      "required": false
+    },
+    "scripts": {
+      "type": "object",
+      "properties": {
+        "postinstall": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          ],
+          "description": "runs a script AFTER the package has been installed.\nRun a custom Python script located in the package “scripts” folder AFTER the package is installed. Please note that you don’t need to specify a Python interpreter for Python scripts"
+        },
+        "preuninstall": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          ],
+          "description": "runs a script BEFORE the package is removed.\nRun a custom Bash script BEFORE the package is uninstalled. The script is declared as a list of command arguments and is located at the root of a package"
+        }
+      },
+      "description": "Execute custom scripts during the special Package Management CLI life cycle events",
+      "required": false
+    },
+    "build": {
+      "type": "object",
+      "properties": {
+        "flags": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          ],
+          "description": "Extra flags to control preprocessing, compilation, assembly, and linking processes. More details build_flags.\nKeep in mind when operating with the -I flag (directories to be searched for header files). The path should be relative to the root directory where the library.json manifest is located."
+        },
+        "unflags": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          ],
+          "description": "Remove base/initial flags which were set by development platform. More details build_unflags."
+        },
+        "includeDir": {
+          "type": "string",
+          "description": "Custom directory to be searched for header files. A default value is include and means that folder is located at the root of a library.\nThe Library Dependency Finder (LDF) will pick a library automatically only when a project or other dependent libraries include any header file located in includeDir or srcDir.",
+          "required": false
+        },
+        "srcDir": {
+          "type": "string",
+          "description": "Custom location of library source code. A default value is src and means that folder is located in the root of a library.",
+          "required": "false"
+        },
+        "srcFilter": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          ],
+          "description": "Specify which source files should be included/excluded from build process. The path in filter should be relative to the srcDir option of a library.\nSee syntax for build_src_filter.\nPlease note that you can generate source filter “on-the-fly” using extraScript",
+          "required": false
+        },
+        "extraScript": {
+          "type": "string",
+          "description": "Launch extra script before a build process.",
+          "required": "false"
+        },
+        "libArchive": {
+          "type": "boolean",
+          "description": "Create an archive (*.a, static library) from the object files and link it into a firmware (program). This is default behavior of PlatformIO Build System (\"libArchive\": true).\nSetting \"libArchive\": false will instruct PlatformIO Build System to link object files directly (in-line). This could be useful if you need to override weak symbols defined in framework or other libraries.\nYou can disable library archiving globally using lib_archive option in “platformio.ini” (Project Configuration File).",
+          "required": "false"
+        },
+        "libLDFMode": {
+          "anyOf": [
+            {
+              "enum": [
+                "off"
+              ],
+              "description": "“Manual mode”, does not process source files of a project and dependencies. Builds only the libraries that are specified in manifests (library.json, module.json) or using lib_deps option."
+            },
+            {
+              "enum": [
+                "chain"
+              ],
+              "description": "[DEFAULT] Parses ALL C/C++ source files of the project and follows only by nested includes (#include ..., chain...) from the libraries. It also parses C, CC, CPP files from libraries which have the same name as included header file. Does not evaluate C/C++ Preprocessor conditional syntax."
+            },
+            {
+              "enum": [
+                "deep"
+              ],
+              "description": "Parses ALL C/C++ source files of the project and parses ALL C/C++ source files of the each found dependency (recursively). Does not evaluate C/C++ Preprocessor conditional syntax."
+            },
+            {
+              "enum": [
+                "chain+"
+              ],
+              "description": "The same behavior as for the chain but evaluates C/C++ Preprocessor conditional syntax."
+            },
+            {
+              "enum": [
+                "deep+"
+              ],
+              "description": "The same behavior as for the deep but evaluates C/C++ Preprocessor conditional syntax."
+            }
+          ],
+          "description": "Specify Library Dependency Finder Mode. See Dependency Finder Mode for details.",
+          "required": false
+        },
+        "libCompatMode": {
+          "type": "string",
+          "description": "Specify Library Compatibility Mode. See Compatibility Mode for details.",
+          "required": false
+        },
+        "builder": {
+          "anyOf": [
+            {
+              "enum": [
+                "PlatformIOLibBuilder"
+              ],
+              "description": "Default Builder"
+            },
+            {
+              "enum": [
+                "ArduinoLibBuilder"
+              ]
+            },
+            {
+              "enum": [
+                "MbedLibBuilder"
+              ]
+            }
+          ],
+          "description": "Override default PlatformIOLibBuilder with another builder.",
+          "required": false
+        }
+      },
+      "required": false
+    }
+  }
+}

platformio/assets/system/99-platformio-udev.rules

@@ -16,7 +16,7 @@
 #
 # INSTALLATION
 #
-# Please visit > https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules
+# Please visit > https://docs.platformio.org/en/latest/core/installation/udev-rules.html
 #
 #####################################################################################
 
@@ -25,7 +25,8 @@
 #
 
 # CP210X USB UART
-ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea60", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="ea[67][013]", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="10c4", ATTRS{idProduct}=="80a9", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # FT231XS USB UART
 ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6015", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
@@ -35,6 +36,8 @@ ATTRS{idVendor}=="067b", ATTRS{idProduct}=="2303", MODE:="0666", ENV{ID_MM_DEVIC
 
 # QinHeng Electronics HL-340 USB-Serial adapter
 ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7523", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+# QinHeng Electronics CH9102 USB-Serial adapter
+ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="55d4", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Arduino boards
 ATTRS{idVendor}=="2341", ATTRS{idProduct}=="[08][023]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
@@ -61,18 +64,28 @@ ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789A]?", ENV{MTP_NO_PROBE}="1"
 SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", MODE:="0666"
 KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", MODE:="0666"
 
-#TI Stellaris Launchpad
+# TI Stellaris Launchpad
 ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
-#TI MSP430 Launchpad
+# TI MSP430 Launchpad
 ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
-#GD32V DFU Bootloader
+# GD32V DFU Bootloader
 ATTRS{idVendor}=="28e9", ATTRS{idProduct}=="0189", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # FireBeetle-ESP32
 ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7522", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
+# Wio Terminal
+ATTRS{idVendor}=="2886", ATTRS{idProduct}=="[08]02d", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+
+# Raspberry Pi Pico
+ATTRS{idVendor}=="2e8a", ATTRS{idProduct}=="[01]*", MODE:="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+
+# AIR32F103
+ATTRS{idVendor}=="0d28", ATTRS{idProduct}=="0204", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+
+
 #
 # Debuggers
 #
@@ -84,44 +97,29 @@ SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port", MODE="0666", ENV{ID
 # opendous and estick
 ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
-# Original FT232/FT245 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
-
-# Original FT2232 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
-
-# Original FT4232 VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
-
-# Original FT232H VID:PID
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+# Original FT232/FT245/FT2232/FT232H/FT4232
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="60[01][104]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # DISTORTEC JTAG-lock-pick Tiny 2
 ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # TUMPA, TUMPA Lite
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a9[89]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # XDS100v2
 ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE)
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca[01]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # TI/Luminary Stellaris Evaluation Board FTDI (several)
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
-
-# TI/Luminary Stellaris In-Circuit Debug Interface FTDI (ICDI) Board
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd[9a]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # egnite Turtelizer 2
 ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Section5 ICEbear
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
-ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
+ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c14[01]", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
 
 # Amontec JTAGkey and JTAGkey-tiny
 ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
@@ -173,6 +171,3 @@ ATTRS{product}=="*CMSIS-DAP*", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID
 
 # Atmel AVR Dragon
 ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2107", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1"
-
-# Raspberry Pi Pico
-SUBSYSTEMS=="usb", ATTRS{idVendor}=="2e8a", ATTRS{idProduct}=="[01]*", MODE:="0666"

platformio/builder/main.py

@@ -28,7 +28,7 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
-from platformio import compat, fs
+from platformio import app, fs
 from platformio.platform.base import PlatformBase
 from platformio.proc import get_pythonexe_path
 from platformio.project.helpers import get_project_dir
@@ -38,7 +38,6 @@ AllowSubstExceptions(NameError)
 # append CLI arguments to build environment
 clivars = Variables(None)
 clivars.AddVariables(
-    ("PLATFORM_MANIFEST",),
     ("BUILD_SCRIPT",),
     ("PROJECT_CONFIG",),
     ("PIOENV",),
@@ -53,19 +52,20 @@ DEFAULT_ENV_OPTIONS = dict(
         "cc",
         "c++",
         "link",
+        "piohooks",
         "pioasm",
-        "platformio",
+        "piobuild",
         "pioproject",
         "pioplatform",
         "piotest",
         "piotarget",
-        "piomaxlen",
         "piolib",
         "pioupload",
         "piosize",
         "pioino",
         "piomisc",
         "piointegration",
+        "piomaxlen",
     ],
     toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")],
     variables=clivars,
@@ -78,9 +78,9 @@ DEFAULT_ENV_OPTIONS = dict(
     COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
     LIBPATH=["$BUILD_DIR"],
     PROGNAME="program",
-    PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
+    PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
+    PROG_PATH="$PROGPATH",  # deprecated
     PYTHONEXE=get_pythonexe_path(),
-    IDE_EXTRA_DATA={},
 )
 
 # Declare command verbose messages
@@ -98,6 +98,7 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
         DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value)
 
 env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
+env.SConscriptChdir(False)
 
 # Load variables from CLI
 env.Replace(
@@ -110,6 +111,8 @@ env.Replace(
 
 # Setup project optional directories
 config = env.GetProjectConfig()
+app.set_session_var("custom_project_conf", config.path)
+
 env.Replace(
     PROJECT_DIR=get_project_dir(),
     PROJECT_CORE_DIR=config.get("platformio", "core_dir"),
@@ -123,6 +126,7 @@ env.Replace(
     PROJECT_DATA_DIR=config.get("platformio", "data_dir"),
     PROJECTDATA_DIR="$PROJECT_DATA_DIR",  # legacy for dev/platform
     PROJECT_BUILD_DIR=config.get("platformio", "build_dir"),
+    BUILD_TYPE=env.GetBuildType(),
     BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"),
     LIBSOURCE_DIRS=[
         config.get("platformio", "lib_dir"),
@@ -135,29 +139,11 @@ if int(ARGUMENTS.get("ISATTY", 0)):
     # pylint: disable=protected-access
     click._compat.isatty = lambda stream: True
 
-if compat.IS_WINDOWS and sys.version_info >= (3, 8) and os.getcwd().startswith("\\\\"):
-    click.secho("!!! WARNING !!!\t\t" * 3, fg="red")
-    click.secho(
-        "Your project is located on a mapped network drive but the "
-        "current command-line shell does not support the UNC paths.",
-        fg="yellow",
-    )
-    click.secho(
-        "Please move your project to a physical drive or check this workaround: "
-        "https://bit.ly/3kuU5mP\n",
-        fg="yellow",
-    )
-
 if env.subst("$BUILD_CACHE_DIR"):
     if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")):
         os.makedirs(env.subst("$BUILD_CACHE_DIR"))
     env.CacheDir("$BUILD_CACHE_DIR")
 
-is_clean_all = "cleanall" in COMMAND_LINE_TARGETS
-if env.GetOption("clean") or is_clean_all:
-    env.PioClean(is_clean_all)
-    env.Exit(0)
-
 if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     click.echo("Verbose mode can be enabled via `-v, --verbose` option")
 
@@ -171,15 +157,18 @@ if not os.path.isdir(env.subst("$BUILD_DIR")):
 env.LoadProjectOptions()
 env.LoadPioPlatform()
 
-env.SConscriptChdir(0)
 env.SConsignFile(
     os.path.join(
-        "$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1])
+        "$BUILD_CACHE_DIR" if env.subst("$BUILD_CACHE_DIR") else "$BUILD_DIR",
+        ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]),
     )
 )
 
-for item in env.GetExtraScripts("pre"):
-    env.SConscript(item, exports="env")
+env.SConscript(env.GetExtraScripts("pre"), exports="env")
+
+if env.IsCleanTarget():
+    env.CleanProject(fullclean=int(ARGUMENTS.get("FULLCLEAN", 0)))
+    env.Exit(0)
 
 env.SConscript("$BUILD_SCRIPT")
 
@@ -188,8 +177,7 @@ if "UPLOAD_FLAGS" in env:
 if env.GetProjectOption("upload_command"):
     env.Replace(UPLOADCMD=env.GetProjectOption("upload_command"))
 
-for item in env.GetExtraScripts("post"):
-    env.SConscript(item, exports="env")
+env.SConscript(env.GetExtraScripts("post"), exports="env")
 
 ##############################################################################
 
@@ -197,7 +185,7 @@ for item in env.GetExtraScripts("post"):
 if env.get("SIZETOOL") and not (
     set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS)
 ):
-    env.Depends(["upload", "program"], "checkprogsize")
+    env.Depends("upload", "checkprogsize")
     # Replace platform's "size" target with our
     _new_targets = [t for t in DEFAULT_TARGETS if str(t) != "size"]
     Default(None)
@@ -209,7 +197,7 @@ if "compiledb" in COMMAND_LINE_TARGETS:
 
 # Print configured protocols
 env.AddPreAction(
-    ["upload", "program"],
+    "upload",
     env.VerboseAction(
         lambda source, target, env: env.PrintUploadInfo(),
         "Configuring upload protocol...",
@@ -219,13 +207,15 @@ env.AddPreAction(
 AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS))
 AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS))
 
+env.ProcessDelayedActions()
+
 ##############################################################################
 
 if "envdump" in COMMAND_LINE_TARGETS:
     click.echo(env.Dump())
     env.Exit(0)
 
-if set(["_idedata", "idedata"]) & set(COMMAND_LINE_TARGETS):
+if env.IsIntegrationDump():
     projenv = None
     try:
         Import("projenv")
@@ -252,3 +242,9 @@ if "sizedata" in COMMAND_LINE_TARGETS:
     )
 
     Default("sizedata")
+
+# issue #4604: process targets sequentially
+for index, target in enumerate(
+    [t for t in COMMAND_LINE_TARGETS if not t.startswith("__")][1:]
+):
+    env.Depends(target, COMMAND_LINE_TARGETS[index])

platformio/builder/tools/compilation_db.py

@@ -1,226 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-# Copyright 2020 MongoDB Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-# pylint: disable=unused-argument, protected-access, unused-variable, import-error
-# Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py
-
-from __future__ import absolute_import
-
-import itertools
-import json
-import os
-
-import SCons
-
-from platformio.builder.tools.platformio import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT
-from platformio.proc import where_is_program
-
-# Implements the ability for SCons to emit a compilation database for the MongoDB project. See
-# http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation
-# database is, and why you might want one. The only user visible entry point here is
-# 'env.CompilationDatabase'. This method takes an optional 'target' to name the file that
-# should hold the compilation database, otherwise, the file defaults to compile_commands.json,
-# which is the name that most clang tools search for by default.
-
-# Is there a better way to do this than this global? Right now this exists so that the
-# emitter we add can record all of the things it emits, so that the scanner for the top level
-# compilation database can access the complete list, and also so that the writer has easy
-# access to write all of the files. But it seems clunky. How can the emitter and the scanner
-# communicate more gracefully?
-__COMPILATION_DB_ENTRIES = []
-
-
-# We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even
-# integrate with the cache, but there doesn't seem to be much call for it.
-class __CompilationDbNode(SCons.Node.Python.Value):
-    def __init__(self, value):
-        SCons.Node.Python.Value.__init__(self, value)
-        self.Decider(changed_since_last_build_node)
-
-
-def changed_since_last_build_node(*args, **kwargs):
-    """Dummy decider to force always building"""
-    return True
-
-
-def makeEmitCompilationDbEntry(comstr):
-    """
-    Effectively this creates a lambda function to capture:
-    * command line
-    * source
-    * target
-    :param comstr: unevaluated command line
-    :return: an emitter which has captured the above
-    """
-    user_action = SCons.Action.Action(comstr)
-
-    def EmitCompilationDbEntry(target, source, env):
-        """
-        This emitter will be added to each c/c++ object build to capture the info needed
-        for clang tools
-        :param target: target node(s)
-        :param source: source node(s)
-        :param env: Environment for use building this node
-        :return: target(s), source(s)
-        """
-
-        # Resolve absolute path of toolchain
-        for cmd in ("CC", "CXX", "AS"):
-            if cmd not in env:
-                continue
-            if os.path.isabs(env[cmd]):
-                continue
-            env[cmd] = where_is_program(
-                env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")
-            )
-
-        dbtarget = __CompilationDbNode(source)
-
-        entry = env.__COMPILATIONDB_Entry(
-            target=dbtarget,
-            source=[],
-            __COMPILATIONDB_UTARGET=target,
-            __COMPILATIONDB_USOURCE=source,
-            __COMPILATIONDB_UACTION=user_action,
-            __COMPILATIONDB_ENV=env,
-        )
-
-        # Technically, these next two lines should not be required: it should be fine to
-        # cache the entries. However, they don't seem to update properly. Since they are quick
-        # to re-generate disable caching and sidestep this problem.
-        env.AlwaysBuild(entry)
-        env.NoCache(entry)
-
-        __COMPILATION_DB_ENTRIES.append(dbtarget)
-
-        return target, source
-
-    return EmitCompilationDbEntry
-
-
-def CompilationDbEntryAction(target, source, env, **kw):
-    """
-    Create a dictionary with evaluated command line, target, source
-    and store that info as an attribute on the target
-    (Which has been stored in __COMPILATION_DB_ENTRIES array
-    :param target: target node(s)
-    :param source: source node(s)
-    :param env: Environment for use building this node
-    :param kw:
-    :return: None
-    """
-
-    command = env["__COMPILATIONDB_UACTION"].strfunction(
-        target=env["__COMPILATIONDB_UTARGET"],
-        source=env["__COMPILATIONDB_USOURCE"],
-        env=env["__COMPILATIONDB_ENV"],
-    )
-
-    entry = {
-        "directory": env.Dir("#").abspath,
-        "command": command,
-        "file": str(env["__COMPILATIONDB_USOURCE"][0]),
-    }
-
-    target[0].write(entry)
-
-
-def WriteCompilationDb(target, source, env):
-    entries = []
-
-    for s in __COMPILATION_DB_ENTRIES:
-        item = s.read()
-        item["file"] = os.path.abspath(item["file"])
-        entries.append(item)
-
-    with open(str(target[0]), mode="w", encoding="utf8") as target_file:
-        json.dump(
-            entries, target_file, sort_keys=True, indent=4, separators=(",", ": ")
-        )
-
-
-def ScanCompilationDb(node, env, path):
-    return __COMPILATION_DB_ENTRIES
-
-
-def generate(env, **kwargs):
-    static_obj, shared_obj = SCons.Tool.createObjBuilders(env)
-
-    env["COMPILATIONDB_COMSTR"] = kwargs.get(
-        "COMPILATIONDB_COMSTR", "Building compilation database $TARGET"
-    )
-
-    components_by_suffix = itertools.chain(
-        itertools.product(
-            [".%s" % ext for ext in SRC_C_EXT],
-            [
-                (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"),
-                (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"),
-            ],
-        ),
-        itertools.product(
-            [".%s" % ext for ext in SRC_CXX_EXT],
-            [
-                (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"),
-                (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"),
-            ],
-        ),
-        itertools.product(
-            [".%s" % ext for ext in SRC_ASM_EXT],
-            [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASCOM")],
-        ),
-    )
-
-    for entry in components_by_suffix:
-        suffix = entry[0]
-        builder, base_emitter, command = entry[1]
-
-        # Assumes a dictionary emitter
-        emitter = builder.emitter[suffix]
-        builder.emitter[suffix] = SCons.Builder.ListEmitter(
-            [emitter, makeEmitCompilationDbEntry(command)]
-        )
-
-    env["BUILDERS"]["__COMPILATIONDB_Entry"] = SCons.Builder.Builder(
-        action=SCons.Action.Action(CompilationDbEntryAction, None),
-    )
-
-    env["BUILDERS"]["__COMPILATIONDB_Database"] = SCons.Builder.Builder(
-        action=SCons.Action.Action(WriteCompilationDb, "$COMPILATIONDB_COMSTR"),
-        target_scanner=SCons.Scanner.Scanner(
-            function=ScanCompilationDb, node_class=None
-        ),
-    )
-
-    def CompilationDatabase(env, target):
-        result = env.__COMPILATIONDB_Database(target=target, source=[])
-
-        env.AlwaysBuild(result)
-        env.NoCache(result)
-
-        return result
-
-    env.AddMethod(CompilationDatabase, "CompilationDatabase")
-
-
-def exists(env):
-    return True

platformio/builder/tools/pioasm.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import SCons.Tool.asm  # pylint: disable=import-error
 
 #

platformio/builder/tools/piobuild.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import fnmatch
 import os
 import sys
@@ -23,12 +21,12 @@ from SCons.Node import FS  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 from SCons.Script import AlwaysBuild  # pylint: disable=import-error
 from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
-from SCons.Script import Export  # pylint: disable=import-error
 from SCons.Script import SConscript  # pylint: disable=import-error
 
 from platformio import __version__, fs
 from platformio.compat import IS_MACOS, string_types
 from platformio.package.version import pepver_to_semver
+from platformio.proc import where_is_program
 
 SRC_HEADER_EXT = ["h", "hpp"]
 SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"]
@@ -76,10 +74,7 @@ def BuildProgram(env):
         env.Prepend(_LIBFLAGS="-Wl,--start-group ")
         env.Append(_LIBFLAGS=" -Wl,--end-group")
 
-    program = env.Program(
-        os.path.join("$BUILD_DIR", env.subst("$PROGNAME$PROGSUFFIX")),
-        env["PIOBUILDFILES"],
-    )
+    program = env.Program(env.subst("$PROGPATH"), env["PIOBUILDFILES"])
     env.Replace(PIOMAINPROG=program)
 
     AlwaysBuild(
@@ -90,7 +85,7 @@ def BuildProgram(env):
         )
     )
 
-    print("Building in %s mode" % env.GetBuildType())
+    print("Building in %s mode" % env["BUILD_TYPE"])
 
     return program
 
@@ -125,61 +120,67 @@ def ProcessProgramDeps(env):
     # process framework scripts
     env.BuildFrameworks(env.get("PIOFRAMEWORK"))
 
-    if "debug" in env.GetBuildType():
+    if "debug" in env["BUILD_TYPE"]:
         env.ConfigureDebugTarget()
-    if "test" in env.GetBuildType():
-        env.ConfigureTestTarget()
 
     # remove specified flags
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
-    if "compiledb" in COMMAND_LINE_TARGETS and env.get(
-        "COMPILATIONDB_INCLUDE_TOOLCHAIN"
-    ):
-        for scope, includes in env.DumpIntegrationIncludes().items():
-            if scope in ("toolchain",):
-                env.Append(CPPPATH=includes)
+    if "compiledb" in COMMAND_LINE_TARGETS:
+        # Resolve absolute path of toolchain
+        for cmd in ("CC", "CXX", "AS"):
+            if cmd not in env:
+                continue
+            if os.path.isabs(env[cmd]):
+                continue
+            env[cmd] = where_is_program(
+                env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")
+            )
+
+        if env.get("COMPILATIONDB_INCLUDE_TOOLCHAIN"):
+            for scope, includes in env.DumpIntegrationIncludes().items():
+                if scope in ("toolchain",):
+                    env.Append(CPPPATH=includes)
 
 
 def ProcessProjectDeps(env):
-    project_lib_builder = env.ConfigureProjectLibBuilder()
+    plb = env.ConfigureProjectLibBuilder()
 
     # prepend project libs to the beginning of list
-    env.Prepend(LIBS=project_lib_builder.build())
+    env.Prepend(LIBS=plb.build())
     # prepend extra linker related options from libs
     env.PrependUnique(
         **{
-            key: project_lib_builder.env.get(key)
+            key: plb.env.get(key)
             for key in ("LIBS", "LIBPATH", "LINKFLAGS")
-            if project_lib_builder.env.get(key)
+            if plb.env.get(key)
         }
     )
 
-    projenv = env.Clone()
-
-    # CPPPATH from dependencies
-    projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH"))
-    # extra build flags from `platformio.ini`
-    projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
-
-    if "test" in env.GetBuildType():
-        projenv.BuildSources(
+    if "test" in env["BUILD_TYPE"]:
+        build_files_before_nums = len(env.get("PIOBUILDFILES", []))
+        plb.env.BuildSources(
             "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
         )
-    if "test" not in env.GetBuildType() or env.GetProjectOption("test_build_src"):
-        projenv.BuildSources(
+        if len(env.get("PIOBUILDFILES", [])) - build_files_before_nums < 1:
+            sys.stderr.write(
+                "Error: Nothing to build. Please put your test suites "
+                "to the '%s' folder\n" % env.subst("$PROJECT_TEST_DIR")
+            )
+            env.Exit(1)
+
+    if "test" not in env["BUILD_TYPE"] or env.GetProjectOption("test_build_src"):
+        plb.env.BuildSources(
             "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER")
         )
 
     if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
         sys.stderr.write(
             "Error: Nothing to build. Please put your source code files "
-            "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
+            "to the '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
         )
         env.Exit(1)
 
-    Export("projenv")
-
 
 def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
     if not isinstance(flags, list):
@@ -209,13 +210,14 @@ def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
     # fix relative CPPPATH & LIBPATH
     for k in ("CPPPATH", "LIBPATH"):
         for i, p in enumerate(result.get(k, [])):
+            p = env.subst(p)
             if os.path.isdir(p):
                 result[k][i] = os.path.abspath(p)
 
     # fix relative path for "-include"
     for i, f in enumerate(result.get("CCFLAGS", [])):
         if isinstance(f, tuple) and f[0] == "-include":
-            result["CCFLAGS"][i] = (f[0], env.File(os.path.abspath(f[1].get_path())))
+            result["CCFLAGS"][i] = (f[0], env.subst(f[1].get_path()))
 
     return result
 
@@ -244,33 +246,30 @@ def ProcessUnFlags(env, flags):
     if not flags:
         return
     parsed = env.ParseFlagsExtended(flags)
-
-    # get all flags and copy them to each "*FLAGS" variable
-    all_flags = []
-    for key, unflags in parsed.items():
-        if key.endswith("FLAGS"):
-            all_flags.extend(unflags)
-    for key, unflags in parsed.items():
-        if key.endswith("FLAGS"):
-            parsed[key].extend(all_flags)
-
-    for key, unflags in parsed.items():
-        for unflag in unflags:
-            for current in env.get(key, []):
-                conditions = [
-                    unflag == current,
-                    isinstance(current, (tuple, list)) and unflag[0] == current[0],
-                ]
-                if any(conditions):
-                    env[key].remove(current)
+    unflag_scopes = tuple(set(["ASPPFLAGS"] + list(parsed.keys())))
+    for scope in unflag_scopes:
+        for unflags in parsed.values():
+            for unflag in unflags:
+                for current in list(env.get(scope, [])):
+                    conditions = [
+                        unflag == current,
+                        not isinstance(unflag, (tuple, list))
+                        and isinstance(current, (tuple, list))
+                        and unflag == current[0],
+                    ]
+                    if any(conditions):
+                        env[scope].remove(current)
 
 
-def MatchSourceFiles(env, src_dir, src_filter=None):
+def StringifyMacro(env, value):  # pylint: disable=unused-argument
+    return '\\"%s\\"' % value.replace('"', '\\\\\\"')
+
+
+def MatchSourceFiles(env, src_dir, src_filter=None, src_exts=None):
     src_filter = env.subst(src_filter) if src_filter else None
     src_filter = src_filter or SRC_FILTER_DEFAULT
-    return fs.match_src_files(
-        env.subst(src_dir), src_filter, SRC_BUILD_EXT + SRC_HEADER_EXT
-    )
+    src_exts = src_exts or (SRC_BUILD_EXT + SRC_HEADER_EXT)
+    return fs.match_src_files(env.subst(src_dir), src_filter, src_exts)
 
 
 def CollectBuildFiles(
@@ -283,7 +282,7 @@ def CollectBuildFiles(
     if src_dir.endswith(os.sep):
         src_dir = src_dir[:-1]
 
-    for item in env.MatchSourceFiles(src_dir, src_filter):
+    for item in env.MatchSourceFiles(src_dir, src_filter, SRC_BUILD_EXT):
         _reldir = os.path.dirname(item)
         _src_dir = os.path.join(src_dir, _reldir) if _reldir else src_dir
         _var_dir = os.path.join(variant_dir, _reldir) if _reldir else variant_dir
@@ -292,8 +291,7 @@ def CollectBuildFiles(
             variants.append(_var_dir)
             env.VariantDir(_var_dir, _src_dir, duplicate)
 
-        if fs.path_endswith_ext(item, SRC_BUILD_EXT):
-            sources.append(env.File(os.path.join(_var_dir, os.path.basename(item))))
+        sources.append(env.File(os.path.join(_var_dir, os.path.basename(item))))
 
     middlewares = env.get("__PIO_BUILD_MIDDLEWARES")
     if not middlewares:
@@ -305,7 +303,12 @@ def CollectBuildFiles(
         for callback, pattern in middlewares:
             if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern):
                 continue
-            new_node = callback(new_node)
+            if callback.__code__.co_argcount == 2:
+                new_node = callback(env, new_node)
+            else:
+                new_node = callback(new_node)
+            if not new_node:
+                break
         if new_node:
             new_sources.append(new_node)
 
@@ -327,25 +330,18 @@ def BuildFrameworks(env, frameworks):
         )
         env.Exit(1)
 
-    board_frameworks = env.BoardConfig().get("frameworks", [])
-    if frameworks == ["platformio"]:
-        if board_frameworks:
-            frameworks.insert(0, board_frameworks[0])
-        else:
-            sys.stderr.write("Error: Please specify `board` in `platformio.ini`\n")
-            env.Exit(1)
-
-    for f in frameworks:
-        if f == "arduino":
-            # Arduino IDE appends .o the end of filename
+    supported_frameworks = env.BoardConfig().get("frameworks", [])
+    for name in frameworks:
+        if name == "arduino":
+            # Arduino IDE appends .o to the end of filename
             Builder.match_splitext = scons_patched_match_splitext
             if "nobuild" not in COMMAND_LINE_TARGETS:
                 env.ConvertInoToCpp()
 
-        if f in board_frameworks:
-            SConscript(env.GetFrameworkScript(f), exports="env")
+        if name in supported_frameworks:
+            SConscript(env.GetFrameworkScript(name), exports="env")
         else:
-            sys.stderr.write("Error: This board doesn't support %s framework!\n" % f)
+            sys.stderr.write("Error: This board doesn't support %s framework!\n" % name)
             env.Exit(1)
 
 
@@ -356,6 +352,14 @@ def BuildLibrary(env, variant_dir, src_dir, src_filter=None, nodes=None):
 
 
 def BuildSources(env, variant_dir, src_dir, src_filter=None):
+    if env.get("PIOMAINPROG"):
+        sys.stderr.write(
+            "Error: The main program is already constructed and the inline "
+            "source files are not allowed. Please use `env.BuildLibrary(...)` "
+            "or PRE-type script instead."
+        )
+        env.Exit(1)
+
     nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter)
     DefaultEnvironment().Append(
         PIOBUILDFILES=[
@@ -376,6 +380,7 @@ def generate(env):
     env.AddMethod(ParseFlagsExtended)
     env.AddMethod(ProcessFlags)
     env.AddMethod(ProcessUnFlags)
+    env.AddMethod(StringifyMacro)
     env.AddMethod(MatchSourceFiles)
     env.AddMethod(CollectBuildFiles)
     env.AddMethod(AddBuildMiddleware)

platformio/builder/tools/piohooks.py

@@ -0,0 +1,50 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def AddActionWrapper(handler):
+    def wraps(env, files, action):
+        if not isinstance(files, (list, tuple, set)):
+            files = [files]
+        known_nodes = []
+        unknown_files = []
+        for item in files:
+            nodes = env.arg2nodes(item, env.fs.Entry)
+            if nodes and nodes[0].exists():
+                known_nodes.extend(nodes)
+            else:
+                unknown_files.append(item)
+        if unknown_files:
+            env.Append(**{"_PIO_DELAYED_ACTIONS": [(handler, unknown_files, action)]})
+        if known_nodes:
+            return handler(known_nodes, action)
+        return []
+
+    return wraps
+
+
+def ProcessDelayedActions(env):
+    for func, nodes, action in env.get("_PIO_DELAYED_ACTIONS", []):
+        func(nodes, action)
+
+
+def generate(env):
+    env.Replace(**{"_PIO_DELAYED_ACTIONS": []})
+    env.AddMethod(AddActionWrapper(env.AddPreAction), "AddPreAction")
+    env.AddMethod(AddActionWrapper(env.AddPostAction), "AddPostAction")
+    env.AddMethod(ProcessDelayedActions)
+
+
+def exists(_):
+    return True

platformio/builder/tools/pioino.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import atexit
 import glob
 import io
@@ -26,8 +24,7 @@ import click
 from platformio.compat import get_filesystem_encoding, get_locale_encoding
 
 
-class InoToCPPConverter(object):
-
+class InoToCPPConverter:
     PROTOTYPE_RE = re.compile(
         r"""^(
         (?:template\<.*\>\s*)?      # template
@@ -105,7 +102,7 @@ class InoToCPPConverter(object):
         return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
 
     def process(self, contents):
-        out_file = self._main_ino + ".cpp"
+        out_file = re.sub(r"[\"\'\;]+", "", self._main_ino) + ".cpp"
         assert self._gcc_preprocess(contents, out_file)
         contents = self.read_safe_contents(out_file)
         contents = self._join_multiline_strings(contents)
@@ -225,11 +222,15 @@ class InoToCPPConverter(object):
         return "\n".join(result)
 
 
-def ConvertInoToCpp(env):
+def FindInoNodes(env):
     src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
-    ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
+    return env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
         os.path.join(src_dir, "*.pde")
     )
+
+
+def ConvertInoToCpp(env):
+    ino_nodes = env.FindInoNodes()
     if not ino_nodes:
         return
     c = InoToCPPConverter(env)
@@ -247,6 +248,7 @@ def _delete_file(path):
 
 
 def generate(env):
+    env.AddMethod(FindInoNodes)
     env.AddMethod(ConvertInoToCpp)
 
 

platformio/builder/tools/piointegration.py

@@ -12,17 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import glob
 import os
 
 import SCons.Defaults  # pylint: disable=import-error
 import SCons.Subst  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 
 from platformio.proc import exec_command, where_is_program
 
 
+def IsIntegrationDump(_):
+    return set(["__idedata", "idedata"]) & set(COMMAND_LINE_TARGETS)
+
+
 def DumpIntegrationIncludes(env):
     result = dict(build=[], compatlib=[], toolchain=[])
 
@@ -60,7 +63,7 @@ def DumpIntegrationIncludes(env):
     return result
 
 
-def _get_gcc_defines(env):
+def get_gcc_defines(env):
     items = []
     try:
         sysenv = os.environ.copy()
@@ -83,13 +86,13 @@ def _get_gcc_defines(env):
     return items
 
 
-def _dump_defines(env):
+def dump_defines(env):
     defines = []
     # global symbols
     for item in SCons.Defaults.processDefines(env.get("CPPDEFINES", [])):
         item = item.strip()
         if item:
-            defines.append(env.subst(item).replace("\\", ""))
+            defines.append(env.subst(item).replace('\\"', '"'))
 
     # special symbol for Atmel AVR MCU
     if env["PIOPLATFORM"] == "atmelavr":
@@ -108,12 +111,12 @@ def _dump_defines(env):
 
     # built-in GCC marcos
     # if env.GetCompilerType() == "gcc":
-    #     defines.extend(_get_gcc_defines(env))
+    #     defines.extend(get_gcc_defines(env))
 
     return defines
 
 
-def _get_svd_path(env):
+def dump_svd_path(env):
     svd_path = env.GetProjectOption("debug_svd_path")
     if svd_path:
         return os.path.abspath(svd_path)
@@ -135,54 +138,45 @@ def _get_svd_path(env):
     return None
 
 
-def _subst_cmd(env, cmd):
-    args = env.subst_list(cmd, SCons.Subst.SUBST_CMD)[0]
-    return " ".join([SCons.Subst.quote_spaces(arg) for arg in args])
+def _split_flags_string(env, s):
+    args = env.subst_list(s, SCons.Subst.SUBST_CMD)[0]
+    return [str(arg) for arg in args]
 
 
-def DumpIntegrationData(env, globalenv):
-    """env here is `projenv`"""
-
+def DumpIntegrationData(*args):
+    projenv, globalenv = args[0:2]  # pylint: disable=unbalanced-tuple-unpacking
     data = {
-        "env_name": env["PIOENV"],
-        "libsource_dirs": [env.subst(item) for item in env.GetLibSourceDirs()],
-        "defines": _dump_defines(env),
-        "includes": env.DumpIntegrationIncludes(),
-        "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
-        "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
-        "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
-        "prog_path": env.subst("$PROG_PATH"),
-        "svd_path": _get_svd_path(env),
-        "compiler_type": env.GetCompilerType(),
+        "build_type": globalenv.GetBuildType(),
+        "env_name": globalenv["PIOENV"],
+        "libsource_dirs": [
+            globalenv.subst(item) for item in globalenv.GetLibSourceDirs()
+        ],
+        "defines": dump_defines(projenv),
+        "includes": projenv.DumpIntegrationIncludes(),
+        "cc_flags": _split_flags_string(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS"),
+        "cxx_flags": _split_flags_string(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS"),
+        "cc_path": where_is_program(
+            globalenv.subst("$CC"), globalenv.subst("${ENV['PATH']}")
+        ),
+        "cxx_path": where_is_program(
+            globalenv.subst("$CXX"), globalenv.subst("${ENV['PATH']}")
+        ),
+        "gdb_path": where_is_program(
+            globalenv.subst("$GDB"), globalenv.subst("${ENV['PATH']}")
+        ),
+        "prog_path": globalenv.subst("$PROGPATH"),
+        "svd_path": dump_svd_path(globalenv),
+        "compiler_type": globalenv.GetCompilerType(),
         "targets": globalenv.DumpTargets(),
         "extra": dict(
             flash_images=[
-                {"offset": item[0], "path": env.subst(item[1])}
-                for item in env.get("FLASH_EXTRA_IMAGES", [])
+                {"offset": item[0], "path": globalenv.subst(item[1])}
+                for item in globalenv.get("FLASH_EXTRA_IMAGES", [])
             ]
         ),
     }
-    data["extra"].update(env.get("IDE_EXTRA_DATA", {}))
-
-    env_ = env.Clone()
-    # https://github.com/platformio/platformio-atom-ide/issues/34
-    _new_defines = []
-    for item in SCons.Defaults.processDefines(env_.get("CPPDEFINES", [])):
-        item = item.replace('\\"', '"')
-        if " " in item:
-            _new_defines.append(item.replace(" ", "\\\\ "))
-        else:
-            _new_defines.append(item)
-    env_.Replace(CPPDEFINES=_new_defines)
-
-    # export C/C++ build flags
-    data.update(
-        {
-            "cc_flags": _subst_cmd(env_, "$CFLAGS $CCFLAGS $CPPFLAGS"),
-            "cxx_flags": _subst_cmd(env_, "$CXXFLAGS $CCFLAGS $CPPFLAGS"),
-        }
-    )
-
+    for key in ("IDE_EXTRA_DATA", "INTEGRATION_EXTRA_DATA"):
+        data["extra"].update(globalenv.get(key, {}))
     return data
 
 
@@ -191,6 +185,9 @@ def exists(_):
 
 
 def generate(env):
+    env["IDE_EXTRA_DATA"] = {}  # legacy support
+    env["INTEGRATION_EXTRA_DATA"] = {}
+    env.AddMethod(IsIntegrationDump)
     env.AddMethod(DumpIntegrationIncludes)
     env.AddMethod(DumpIntegrationData)
     return env

platformio/builder/tools/piolib.py

@@ -12,11 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=no-self-use, unused-argument, too-many-lines
 # pylint: disable=too-many-instance-attributes, too-many-public-methods
-# pylint: disable=assignment-from-no-return
-
-from __future__ import absolute_import
+# pylint: disable=assignment-from-no-return, unused-argument, too-many-lines
 
 import hashlib
 import io
@@ -29,10 +26,10 @@ import SCons.Scanner  # pylint: disable=import-error
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
-from platformio import exception, fs, util
-from platformio.builder.tools import platformio as piotool
-from platformio.clients.http import HTTPClientError, InternetIsOffline
+from platformio import exception, fs
+from platformio.builder.tools import piobuild
 from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
+from platformio.http import HTTPClientError, InternetConnectionError
 from platformio.package.exception import (
     MissingPackageManifestError,
     UnknownPackageError,
@@ -42,11 +39,11 @@ from platformio.package.manifest.parser import (
     ManifestParserError,
     ManifestParserFactory,
 )
-from platformio.package.meta import PackageItem
+from platformio.package.meta import PackageCompatibility, PackageItem
 from platformio.project.options import ProjectOptions
 
 
-class LibBuilderFactory(object):
+class LibBuilderFactory:
     @staticmethod
     def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))):
         clsname = "UnknownLibBuilder"
@@ -60,16 +57,16 @@ class LibBuilderFactory(object):
             elif used_frameworks:
                 clsname = "%sLibBuilder" % used_frameworks[0].capitalize()
 
-        obj = getattr(sys.modules[__name__], clsname)(env, path, verbose=verbose)
+        obj = globals()[clsname](env, path, verbose=verbose)
 
         # Handle PlatformIOLibBuilder.manifest.build.builder
         # pylint: disable=protected-access
         if isinstance(obj, PlatformIOLibBuilder) and obj._manifest.get("build", {}).get(
             "builder"
         ):
-            obj = getattr(
-                sys.modules[__name__], obj._manifest.get("build", {}).get("builder")
-            )(env, path, verbose=verbose)
+            obj = globals()[obj._manifest.get("build", {}).get("builder")](
+                env, path, verbose=verbose
+            )
 
         assert isinstance(obj, LibBuilderBase)
         return obj
@@ -95,7 +92,7 @@ class LibBuilderFactory(object):
                 return ["mbed"]
             for fname in files:
                 if not fs.path_endswith_ext(
-                    fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT
+                    fname, piobuild.SRC_BUILD_EXT + piobuild.SRC_HEADER_EXT
                 ):
                     continue
                 with io.open(
@@ -111,8 +108,7 @@ class LibBuilderFactory(object):
         return []
 
 
-class LibBuilderBase(object):
-
+class LibBuilderBase:
     CLASSIC_SCANNER = SCons.Scanner.C.CScanner()
     CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner()
     # Max depth of nested includes:
@@ -144,7 +140,11 @@ class LibBuilderBase(object):
 
         self._deps_are_processed = False
         self._circular_deps = []
-        self._processed_files = []
+        self._processed_search_files = []
+
+        # pass a macro to the projenv + libs
+        if "test" in env["BUILD_TYPE"]:
+            self.env.Append(CPPDEFINES=["PIO_UNIT_TESTING"])
 
         # reset source filter, could be overridden with extra script
         self.env["SRC_FILTER"] = ""
@@ -155,20 +155,27 @@ class LibBuilderBase(object):
     def __repr__(self):
         return "%s(%r)" % (self.__class__, self.path)
 
-    def __contains__(self, path):
-        p1 = self.path
-        p2 = path
+    def __contains__(self, child_path):
+        return self.is_common_builder(self.path, child_path)
+
+    def is_common_builder(self, root_path, child_path):
         if IS_WINDOWS:
-            p1 = p1.lower()
-            p2 = p2.lower()
-        if p1 == p2:
+            root_path = root_path.lower()
+            child_path = child_path.lower()
+        if root_path == child_path:
             return True
-        if os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep:
+        if (
+            os.path.commonprefix([root_path + os.path.sep, child_path])
+            == root_path + os.path.sep
+        ):
             return True
         # try to resolve paths
-        p1 = os.path.os.path.realpath(p1)
-        p2 = os.path.os.path.realpath(p2)
-        return os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep
+        root_path = os.path.realpath(root_path)
+        child_path = os.path.realpath(child_path)
+        return (
+            os.path.commonprefix([root_path + os.path.sep, child_path])
+            == root_path + os.path.sep
+        )
 
     @property
     def name(self):
@@ -189,7 +196,7 @@ class LibBuilderBase(object):
 
     @property
     def src_filter(self):
-        return piotool.SRC_FILTER_DEFAULT + [
+        return piobuild.SRC_FILTER_DEFAULT + [
             "-<example%s>" % os.sep,
             "-<examples%s>" % os.sep,
             "-<test%s>" % os.sep,
@@ -290,11 +297,12 @@ class LibBuilderBase(object):
         with fs.cd(self.path):
             self.env.ProcessFlags(self.build_flags)
             if self.extra_script:
-                self.env.SConscriptChdir(1)
+                self.env.SConscriptChdir(True)
                 self.env.SConscript(
                     os.path.abspath(self.extra_script),
                     exports={"env": self.env, "pio_lib_builder": self},
                 )
+                self.env.SConscriptChdir(False)
             self.env.ProcessUnFlags(self.build_unflags)
 
     def process_dependencies(self):
@@ -318,21 +326,14 @@ class LibBuilderBase(object):
                 )
 
     def get_search_files(self):
-        items = [
+        return [
             os.path.join(self.src_dir, item)
-            for item in self.env.MatchSourceFiles(self.src_dir, self.src_filter)
-        ]
-        include_dir = self.include_dir
-        if include_dir:
-            items.extend(
-                [
-                    os.path.join(include_dir, item)
-                    for item in self.env.MatchSourceFiles(include_dir)
-                ]
+            for item in self.env.MatchSourceFiles(
+                self.src_dir, self.src_filter, piobuild.SRC_BUILD_EXT
             )
-        return items
+        ]
 
-    def _get_found_includes(  # pylint: disable=too-many-branches
+    def get_implicit_includes(  # pylint: disable=too-many-branches
         self, search_files=None
     ):
         # all include directories
@@ -340,7 +341,7 @@ class LibBuilderBase(object):
             LibBuilderBase._INCLUDE_DIRS_CACHE = [
                 self.env.Dir(d)
                 for d in ProjectAsLibBuilder(
-                    self.envorigin, "$PROJECT_DIR"
+                    self.envorigin, "$PROJECT_DIR", export_projenv=False
                 ).get_include_dirs()
             ]
             for lb in self.env.GetLibBuilders():
@@ -353,53 +354,55 @@ class LibBuilderBase(object):
         include_dirs.extend(LibBuilderBase._INCLUDE_DIRS_CACHE)
 
         result = []
-        for path in search_files or []:
-            if path in self._processed_files:
+        search_files = search_files or []
+        while search_files:
+            node = self.env.File(search_files.pop(0))
+            if node.get_abspath() in self._processed_search_files:
                 continue
-            self._processed_files.append(path)
+            self._processed_search_files.append(node.get_abspath())
 
             try:
                 assert "+" in self.lib_ldf_mode
                 candidates = LibBuilderBase.CCONDITIONAL_SCANNER(
-                    self.env.File(path),
+                    node,
                     self.env,
                     tuple(include_dirs),
                     depth=self.CCONDITIONAL_SCANNER_DEPTH,
                 )
-                # mark candidates already processed via Conditional Scanner
-                self._processed_files.extend(
-                    [
-                        c.get_abspath()
-                        for c in candidates
-                        if c.get_abspath() not in self._processed_files
-                    ]
-                )
-            except Exception as e:  # pylint: disable=broad-except
+
+            except Exception as exc:  # pylint: disable=broad-except
                 if self.verbose and "+" in self.lib_ldf_mode:
                     sys.stderr.write(
                         "Warning! Classic Pre Processor is used for `%s`, "
-                        "advanced has failed with `%s`\n" % (path, e)
+                        "advanced has failed with `%s`\n" % (node.get_abspath(), exc)
                     )
                 candidates = LibBuilderBase.CLASSIC_SCANNER(
-                    self.env.File(path), self.env, tuple(include_dirs)
+                    node, self.env, tuple(include_dirs)
                 )
 
-            # print(path, [c.get_abspath() for c in candidates])
+            # print(node.get_abspath(), [c.get_abspath() for c in candidates])
             for item in candidates:
+                item_path = item.get_abspath()
+                # process internal files recursively
+                if (
+                    item_path not in self._processed_search_files
+                    and item_path not in search_files
+                    and item_path in self
+                ):
+                    search_files.append(item_path)
                 if item not in result:
                     result.append(item)
                 if not self.PARSE_SRC_BY_H_NAME:
                     continue
-                _h_path = item.get_abspath()
-                if not fs.path_endswith_ext(_h_path, piotool.SRC_HEADER_EXT):
+                if not fs.path_endswith_ext(item_path, piobuild.SRC_HEADER_EXT):
                     continue
-                _f_part = _h_path[: _h_path.rindex(".")]
-                for ext in piotool.SRC_C_EXT + piotool.SRC_CXX_EXT:
-                    if not os.path.isfile("%s.%s" % (_f_part, ext)):
+                item_fname = item_path[: item_path.rindex(".")]
+                for ext in piobuild.SRC_C_EXT + piobuild.SRC_CXX_EXT:
+                    if not os.path.isfile("%s.%s" % (item_fname, ext)):
                         continue
-                    _c_path = self.env.File("%s.%s" % (_f_part, ext))
-                    if _c_path not in result:
-                        result.append(_c_path)
+                    item_c_node = self.env.File("%s.%s" % (item_fname, ext))
+                    if item_c_node not in result:
+                        result.append(item_c_node)
 
         return result
 
@@ -414,12 +417,13 @@ class LibBuilderBase(object):
             search_files = self.get_search_files()
 
         lib_inc_map = {}
-        for inc in self._get_found_includes(search_files):
+        for inc in self.get_implicit_includes(search_files):
+            inc_path = inc.get_abspath()
             for lb in self.env.GetLibBuilders():
-                if inc.get_abspath() in lb:
+                if inc_path in lb:
                     if lb not in lib_inc_map:
                         lib_inc_map[lb] = []
-                    lib_inc_map[lb].append(inc.get_abspath())
+                    lib_inc_map[lb].append(inc_path)
                     break
 
         for lb, lb_search_files in lib_inc_map.items():
@@ -453,11 +457,17 @@ class LibBuilderBase(object):
 
     def build(self):
         libs = []
+        shared_scopes = ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS")
         for lb in self.depbuilders:
             libs.extend(lb.build())
             # copy shared information to self env
-            for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
-                self.env.PrependUnique(**{key: lb.env.get(key)})
+            self.env.PrependUnique(
+                **{
+                    scope: lb.env.get(scope)
+                    for scope in shared_scopes
+                    if lb.env.get(scope)
+                }
+            )
 
         for lb in self._circular_deps:
             self.env.PrependUnique(CPPPATH=lb.get_include_dirs())
@@ -472,8 +482,13 @@ class LibBuilderBase(object):
             for lb in self.env.GetLibBuilders():
                 if self == lb or not lb.is_built:
                     continue
-                for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
-                    self.env.PrependUnique(**{key: lb.env.get(key)})
+                self.env.PrependUnique(
+                    **{
+                        scope: lb.env.get(scope)
+                        for scope in shared_scopes
+                        if lb.env.get(scope)
+                    }
+                )
 
         do_not_archive = not self.lib_archive
         if not do_not_archive:
@@ -514,7 +529,7 @@ class ArduinoLibBuilder(LibBuilderBase):
         return os.path.join(self.path, "include")
 
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
+        include_dirs = super().get_include_dirs()
         if os.path.isdir(os.path.join(self.path, "src")):
             return include_dirs
         if os.path.isdir(os.path.join(self.path, "utility")):
@@ -543,7 +558,7 @@ class ArduinoLibBuilder(LibBuilderBase):
 
         src_filter = []
         is_utility = os.path.isdir(os.path.join(self.path, "utility"))
-        for ext in piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT:
+        for ext in piobuild.SRC_BUILD_EXT + piobuild.SRC_HEADER_EXT:
             # arduino ide ignores files with .asm or .ASM extensions
             if ext.lower() == "asm":
                 continue
@@ -574,10 +589,14 @@ class ArduinoLibBuilder(LibBuilderBase):
         return "chain+"
 
     def is_frameworks_compatible(self, frameworks):
-        return util.items_in_list(frameworks, ["arduino", "energia"])
+        return PackageCompatibility(frameworks=frameworks).is_compatible(
+            PackageCompatibility(frameworks=["arduino", "energia"])
+        )
 
     def is_platforms_compatible(self, platforms):
-        return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"])
+        return PackageCompatibility(platforms=platforms).is_compatible(
+            PackageCompatibility(platforms=self._manifest.get("platforms"))
+        )
 
     @property
     def build_flags(self):
@@ -612,7 +631,7 @@ class MbedLibBuilder(LibBuilderBase):
         return LibBuilderBase.src_dir.fget(self)  # pylint: disable=no-member
 
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
+        include_dirs = super().get_include_dirs()
         if self.path not in include_dirs:
             include_dirs.append(self.path)
 
@@ -632,7 +651,9 @@ class MbedLibBuilder(LibBuilderBase):
         return include_dirs
 
     def is_frameworks_compatible(self, frameworks):
-        return util.items_in_list(frameworks, ["mbed"])
+        return PackageCompatibility(frameworks=frameworks).is_compatible(
+            PackageCompatibility(frameworks=["mbed"])
+        )
 
     def process_extra_options(self):
         self._process_mbed_lib_confs()
@@ -757,6 +778,24 @@ class PlatformIOLibBuilder(LibBuilderBase):
                 return os.path.abspath(self._manifest.get("build").get("includeDir"))
         return LibBuilderBase.include_dir.fget(self)  # pylint: disable=no-member
 
+    def get_include_dirs(self):
+        include_dirs = super().get_include_dirs()
+
+        # backwards compatibility with PlatformIO 2.0
+        if (
+            "build" not in self._manifest
+            and self._has_arduino_manifest()
+            and not os.path.isdir(os.path.join(self.path, "src"))
+            and os.path.isdir(os.path.join(self.path, "utility"))
+        ):
+            include_dirs.append(os.path.join(self.path, "utility"))
+
+        for path in self.env.get("CPPPATH", []):
+            if path not in self.envorigin.get("CPPPATH", []):
+                include_dirs.append(self.env.subst(path))
+
+        return include_dirs
+
     @property
     def src_dir(self):
         if "srcDir" in self._manifest.get("build", {}):
@@ -827,36 +866,33 @@ class PlatformIOLibBuilder(LibBuilderBase):
         )
 
     def is_platforms_compatible(self, platforms):
-        return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"])
+        return PackageCompatibility(platforms=platforms).is_compatible(
+            PackageCompatibility(platforms=self._manifest.get("platforms"))
+        )
 
     def is_frameworks_compatible(self, frameworks):
-        return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"])
-
-    def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
-
-        # backwards compatibility with PlatformIO 2.0
-        if (
-            "build" not in self._manifest
-            and self._has_arduino_manifest()
-            and not os.path.isdir(os.path.join(self.path, "src"))
-            and os.path.isdir(os.path.join(self.path, "utility"))
-        ):
-            include_dirs.append(os.path.join(self.path, "utility"))
-
-        for path in self.env.get("CPPPATH", []):
-            if path not in self.envorigin.get("CPPPATH", []):
-                include_dirs.append(self.env.subst(path))
-
-        return include_dirs
+        return PackageCompatibility(frameworks=frameworks).is_compatible(
+            PackageCompatibility(frameworks=self._manifest.get("frameworks"))
+        )
 
 
 class ProjectAsLibBuilder(LibBuilderBase):
     def __init__(self, env, *args, **kwargs):
+        export_projenv = kwargs.get("export_projenv", True)
+        if "export_projenv" in kwargs:
+            del kwargs["export_projenv"]
         # backup original value, will be reset in base.__init__
         project_src_filter = env.get("SRC_FILTER")
         super().__init__(env, *args, **kwargs)
         self.env["SRC_FILTER"] = project_src_filter
+        if export_projenv:
+            env.Export(dict(projenv=self.env))
+
+    def __contains__(self, child_path):
+        for root_path in (self.include_dir, self.src_dir, self.test_dir):
+            if root_path and self.is_common_builder(root_path, child_path):
+                return True
+        return False
 
     @property
     def include_dir(self):
@@ -867,21 +903,18 @@ class ProjectAsLibBuilder(LibBuilderBase):
     def src_dir(self):
         return self.env.subst("$PROJECT_SRC_DIR")
 
-    def get_include_dirs(self):
-        include_dirs = []
-        project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
-        if os.path.isdir(project_include_dir):
-            include_dirs.append(project_include_dir)
-        for include_dir in LibBuilderBase.get_include_dirs(self):
-            if include_dir not in include_dirs:
-                include_dirs.append(include_dir)
-        return include_dirs
+    @property
+    def test_dir(self):
+        return self.env.subst("$PROJECT_TEST_DIR")
 
     def get_search_files(self):
+        items = []
+        build_type = self.env["BUILD_TYPE"]
         # project files
-        items = LibBuilderBase.get_search_files(self)
+        if "test" not in build_type or self.env.GetProjectOption("test_build_src"):
+            items.extend(super().get_search_files())
         # test files
-        if "test" in self.env.GetBuildType():
+        if "test" in build_type:
             items.extend(
                 [
                     os.path.join("$PROJECT_TEST_DIR", item)
@@ -949,8 +982,12 @@ class ProjectAsLibBuilder(LibBuilderBase):
             try:
                 lm.install(spec)
                 did_install = True
-            except (HTTPClientError, UnknownPackageError, InternetIsOffline) as e:
-                click.secho("Warning! %s" % e, fg="yellow")
+            except (
+                HTTPClientError,
+                UnknownPackageError,
+                InternetConnectionError,
+            ) as exc:
+                click.secho("Warning! %s" % exc, fg="yellow")
 
         # reset cache
         if did_install:
@@ -994,7 +1031,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
 
     def build(self):
         self.is_built = True  # do not build Project now
-        result = LibBuilderBase.build(self)
+        result = super().build()
         self.env.PrependUnique(CPPPATH=self.get_include_dirs())
         return result
 
@@ -1019,7 +1056,7 @@ def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)))
             sys.stderr.write("Platform incompatible library %s\n" % lb.path)
         return False
     if compat_mode in ("soft", "strict") and not lb.is_frameworks_compatible(
-        env.get("PIOFRAMEWORK", [])
+        env.get("PIOFRAMEWORK", "__noframework__")
     ):
         if verbose:
             sys.stderr.write("Framework incompatible library %s\n" % lb.path)
@@ -1027,14 +1064,15 @@ def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)))
     return True
 
 
-def GetLibBuilders(env):  # pylint: disable=too-many-branches
-    if DefaultEnvironment().get("__PIO_LIB_BUILDERS", None) is not None:
+def GetLibBuilders(_):  # pylint: disable=too-many-branches
+    env = DefaultEnvironment()
+    if env.get("__PIO_LIB_BUILDERS", None) is not None:
         return sorted(
-            DefaultEnvironment()["__PIO_LIB_BUILDERS"],
+            env["__PIO_LIB_BUILDERS"],
             key=lambda lb: 0 if lb.is_dependent else 1,
         )
 
-    DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=[])
+    env.Replace(__PIO_LIB_BUILDERS=[])
 
     verbose = int(ARGUMENTS.get("PIOVERBOSE", 0))
     found_incompat = False
@@ -1060,13 +1098,13 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
                     )
                 continue
             if env.IsCompatibleLibBuilder(lb):
-                DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
+                env.Append(__PIO_LIB_BUILDERS=[lb])
             else:
                 found_incompat = True
 
     for lb in env.get("EXTRA_LIB_BUILDERS", []):
         if env.IsCompatibleLibBuilder(lb):
-            DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
+            env.Append(__PIO_LIB_BUILDERS=[lb])
         else:
             found_incompat = True
 
@@ -1077,7 +1115,7 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             "ldf-compat-mode\n"
         )
 
-    return DefaultEnvironment()["__PIO_LIB_BUILDERS"]
+    return env["__PIO_LIB_BUILDERS"]
 
 
 def ConfigureProjectLibBuilder(env):
@@ -1123,10 +1161,14 @@ def ConfigureProjectLibBuilder(env):
                 click.echo("Path: %s" % lb.path, nl=False)
                 click.echo(")", nl=False)
             click.echo("")
-            if lb.depbuilders:
+            if lb.verbose and lb.depbuilders:
                 _print_deps_tree(lb, level + 1)
 
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
+
+    if "test" in env["BUILD_TYPE"]:
+        project.env.ConfigureTestTarget()
+
     ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project)  # pylint: disable=no-member
 
     click.echo("LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf")

platformio/builder/tools/piomaxlen.py

@@ -12,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import hashlib
 import os
 import re
 
 from SCons.Platform import TempFileMunge  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 from SCons.Subst import quote_spaces  # pylint: disable=import-error
 
 from platformio.compat import IS_WINDOWS, hashlib_encode_data
@@ -70,11 +69,13 @@ def _file_long_data(env, data):
     return tmp_file
 
 
-def exists(_):
-    return True
+def exists(env):
+    return "compiledb" not in COMMAND_LINE_TARGETS and not env.IsIntegrationDump()
 
 
 def generate(env):
+    if not exists(env):
+        return env
     kwargs = dict(
         _long_sources_hook=long_sources_hook,
         TEMPFILE=TempFileMunge,

platformio/builder/tools/piomisc.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import os
 import sys
 
@@ -114,7 +112,15 @@ def ConfigureDebugTarget(env):
     ]
 
     if optimization_flags:
-        env.AppendUnique(ASFLAGS=optimization_flags, LINKFLAGS=optimization_flags)
+        env.AppendUnique(
+            ASFLAGS=[
+                # skip -O flags for assembler
+                f
+                for f in optimization_flags
+                if f.startswith("-g")
+            ],
+            LINKFLAGS=optimization_flags,
+        )
 
 
 def GetExtraScripts(env, scope):

platformio/builder/tools/pioplatform.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import os
 import sys
 
@@ -35,9 +33,7 @@ from platformio.project.config import ProjectOptions
 @util.memoized()
 def _PioPlatform():
     env = DefaultEnvironment()
-    p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"]))
-    p.configure_project_packages(env["PIOENV"], COMMAND_LINE_TARGETS)
-    return p
+    return PlatformFactory.from_env(env["PIOENV"], targets=COMMAND_LINE_TARGETS)
 
 
 def PioPlatform(_):
@@ -51,8 +47,8 @@ def BoardConfig(env, board=None):
             board = board or env.get("BOARD")
             assert board, "BoardConfig: Board is not defined"
             return p.board_config(board)
-        except (AssertionError, UnknownBoard) as e:
-            sys.stderr.write("Error: %s\n" % str(e))
+        except (AssertionError, UnknownBoard) as exc:
+            sys.stderr.write("Error: %s\n" % str(exc))
             env.Exit(1)
     return None
 

platformio/builder/tools/pioproject.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 from platformio.compat import MISSING
 from platformio.project.config import ProjectConfig
 

platformio/builder/tools/piosize.py

@@ -14,8 +14,6 @@
 
 # pylint: disable=too-many-locals
 
-from __future__ import absolute_import
-
 import json
 import sys
 from os import environ, makedirs, remove
@@ -55,7 +53,7 @@ def _get_symbol_locations(env, elf_path, addrs):
     locations = [line for line in result["out"].split("\n") if line]
     assert len(addrs) == len(locations)
 
-    return dict(zip(addrs, [l.strip() for l in locations]))
+    return dict(zip(addrs, [loc.strip() for loc in locations]))
 
 
 def _get_demangled_names(env, mangled_names):
@@ -75,31 +73,7 @@ def _get_demangled_names(env, mangled_names):
     )
 
 
-def _determine_section(sections, symbol_addr):
-    for section, info in sections.items():
-        if not _is_flash_section(info) and not _is_ram_section(info):
-            continue
-        if symbol_addr in range(info["start_addr"], info["start_addr"] + info["size"]):
-            return section
-    return "unknown"
-
-
-def _is_ram_section(section):
-    return (
-        section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS")
-        and section.get("flags", "") == "WA"
-    )
-
-
-def _is_flash_section(section):
-    return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "")
-
-
-def _is_valid_symbol(symbol_name, symbol_type, symbol_address):
-    return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE"
-
-
-def _collect_sections_info(elffile):
+def _collect_sections_info(env, elffile):
     sections = {}
     for section in elffile.iter_sections():
         if section.is_null() or section.name.startswith(".debug"):
@@ -109,13 +83,18 @@ def _collect_sections_info(elffile):
         section_flags = describe_sh_flags(section["sh_flags"])
         section_size = section.data_size
 
-        sections[section.name] = {
+        section_data = {
+            "name": section.name,
             "size": section_size,
             "start_addr": section["sh_addr"],
             "type": section_type,
             "flags": section_flags,
         }
 
+        sections[section.name] = section_data
+        sections[section.name]["in_flash"] = env.pioSizeIsFlashSection(section_data)
+        sections[section.name]["in_ram"] = env.pioSizeIsRamSection(section_data)
+
     return sections
 
 
@@ -138,7 +117,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         symbol_size = s["st_size"]
         symbol_type = symbol_info["type"]
 
-        if not _is_valid_symbol(s.name, symbol_type, symbol_addr):
+        if not env.pioSizeIsValidSymbol(s.name, symbol_type, symbol_addr):
             continue
 
         symbol = {
@@ -147,7 +126,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
             "name": s.name,
             "type": symbol_type,
             "size": symbol_size,
-            "section": _determine_section(sections, symbol_addr),
+            "section": env.pioSizeDetermineSection(sections, symbol_addr),
         }
 
         if s.name.startswith("_Z"):
@@ -177,12 +156,36 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
     return symbols
 
 
-def _calculate_firmware_size(sections):
+def pioSizeDetermineSection(_, sections, symbol_addr):
+    for section, info in sections.items():
+        if not info.get("in_flash", False) and not info.get("in_ram", False):
+            continue
+        if symbol_addr in range(info["start_addr"], info["start_addr"] + info["size"]):
+            return section
+    return "unknown"
+
+
+def pioSizeIsValidSymbol(_, symbol_name, symbol_type, symbol_address):
+    return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE"
+
+
+def pioSizeIsRamSection(_, section):
+    return (
+        section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS")
+        and section.get("flags", "") == "WA"
+    )
+
+
+def pioSizeIsFlashSection(_, section):
+    return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "")
+
+
+def pioSizeCalculateFirmwareSize(_, sections):
     flash_size = ram_size = 0
     for section_info in sections.values():
-        if _is_flash_section(section_info):
+        if section_info.get("in_flash", False):
             flash_size += section_info.get("size", 0)
-        if _is_ram_section(section_info):
+        if section_info.get("in_ram", False):
             ram_size += section_info.get("size", 0)
 
     return ram_size, flash_size
@@ -212,8 +215,8 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
             sys.stderr.write("Elf file doesn't contain DWARF information")
             env.Exit(1)
 
-        sections = _collect_sections_info(elffile)
-        firmware_ram, firmware_flash = _calculate_firmware_size(sections)
+        sections = _collect_sections_info(env, elffile)
+        firmware_ram, firmware_flash = env.pioSizeCalculateFirmwareSize(sections)
         data["memory"]["total"] = {
             "ram_size": firmware_ram,
             "flash_size": firmware_flash,
@@ -228,9 +231,11 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
 
             symbol_size = symbol.get("size", 0)
             section = sections.get(symbol.get("section", ""), {})
-            if _is_ram_section(section):
+            if not section:
+                continue
+            if section.get("in_ram", False):
                 files[file_path]["ram_size"] += symbol_size
-            if _is_flash_section(section):
+            if section.get("in_flash", False):
                 files[file_path]["flash_size"] += symbol_size
 
             files[file_path]["symbols"].append(symbol)
@@ -252,5 +257,10 @@ def exists(_):
 
 
 def generate(env):
+    env.AddMethod(pioSizeIsRamSection)
+    env.AddMethod(pioSizeIsFlashSection)
+    env.AddMethod(pioSizeCalculateFirmwareSize)
+    env.AddMethod(pioSizeDetermineSection)
+    env.AddMethod(pioSizeIsValidSymbol)
     env.AddMethod(DumpSizeData)
     return env

platformio/builder/tools/piotarget.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import os
 
 from SCons.Action import Action  # pylint: disable=import-error
@@ -29,7 +27,11 @@ def VerboseAction(_, act, actstr):
     return Action(act, actstr)
 
 
-def PioClean(env, clean_all=False):
+def IsCleanTarget(env):
+    return env.GetOption("clean")
+
+
+def CleanProject(env, fullclean=False):
     def _relpath(path):
         if compat.IS_WINDOWS:
             prefix = os.getcwd()[:2].lower()
@@ -43,26 +45,18 @@ def PioClean(env, clean_all=False):
 
     def _clean_dir(path):
         clean_rel_path = _relpath(path)
-        for root, _, files in os.walk(path):
-            for f in files:
-                dst = os.path.join(root, f)
-                os.remove(dst)
-                print(
-                    "Removed %s"
-                    % (dst if not clean_rel_path.startswith(".") else _relpath(dst))
-                )
+        print(f"Removing {clean_rel_path}")
+        fs.rmtree(path)
 
     build_dir = env.subst("$BUILD_DIR")
-    libdeps_dir = env.subst("$PROJECT_LIBDEPS_DIR")
+    libdeps_dir = env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))
     if os.path.isdir(build_dir):
         _clean_dir(build_dir)
-        fs.rmtree(build_dir)
     else:
         print("Build environment is clean")
 
-    if clean_all and os.path.isdir(libdeps_dir):
+    if fullclean and os.path.isdir(libdeps_dir):
         _clean_dir(libdeps_dir)
-        fs.rmtree(libdeps_dir)
 
     print("Done cleaning")
 
@@ -104,19 +98,6 @@ def DumpTargets(env):
         t["group"] == "Platform" for t in targets.values()
     ):
         targets["upload"] = dict(name="upload", group="Platform", title="Upload")
-    targets["compiledb"] = dict(
-        name="compiledb",
-        title="Compilation Database",
-        description="Generate compilation database `compile_commands.json`",
-        group="Advanced",
-    )
-    targets["clean"] = dict(name="clean", title="Clean", group="General")
-    targets["cleanall"] = dict(
-        name="cleanall",
-        title="Clean All",
-        group="General",
-        description="Clean a build environment and installed library dependencies",
-    )
     return list(targets.values())
 
 
@@ -126,7 +107,8 @@ def exists(_):
 
 def generate(env):
     env.AddMethod(VerboseAction)
-    env.AddMethod(PioClean)
+    env.AddMethod(IsCleanTarget)
+    env.AddMethod(CleanProject)
     env.AddMethod(AddTarget)
     env.AddMethod(AddPlatformTarget)
     env.AddMethod(AddCustomTarget)

platformio/builder/tools/piotest.py

@@ -12,19 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import absolute_import
-
 import os
 
-from platformio.builder.tools import platformio as piotool
+from platformio.builder.tools import piobuild
 from platformio.test.result import TestSuite
 from platformio.test.runners.factory import TestRunnerFactory
 
 
 def ConfigureTestTarget(env):
     env.Append(
-        CPPDEFINES=["UNIT_TEST", "PIO_UNIT_TESTING"],
-        PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piotool.SRC_BUILD_EXT],
+        CPPDEFINES=["UNIT_TEST"],  # deprecated, use PIO_UNIT_TESTING
+        PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piobuild.SRC_BUILD_EXT],
     )
     env.Prepend(CPPPATH=["$PROJECT_TEST_DIR"])
 
@@ -38,7 +36,7 @@ def ConfigureTestTarget(env):
             env.Prepend(
                 PIOTEST_SRC_FILTER=[
                     f"+<{test_name}{os.path.sep}*.{ext}>"
-                    for ext in piotool.SRC_BUILD_EXT
+                    for ext in piobuild.SRC_BUILD_EXT
                 ],
                 CPPPATH=[os.path.join("$PROJECT_TEST_DIR", test_name)],
             )

platformio/builder/tools/pioupload.py

@@ -14,8 +14,6 @@
 
 # pylint: disable=unused-argument
 
-from __future__ import absolute_import
-
 import os
 import re
 import sys
@@ -26,8 +24,8 @@ from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from serial import Serial, SerialException
 
 from platformio import exception, fs
-from platformio.device.finder import find_mbed_disk, find_serial_port, is_pattern_port
-from platformio.device.list import list_serial_ports
+from platformio.device.finder import SerialPortFinder, find_mbed_disk, is_pattern_port
+from platformio.device.list.util import list_serial_ports
 from platformio.proc import exec_command
 
 
@@ -109,14 +107,15 @@ def AutodetectUploadPort(*args, **kwargs):
     else:
         try:
             fs.ensure_udev_rules()
-        except exception.InvalidUdevRules as e:
-            sys.stderr.write("\n%s\n\n" % e)
+        except exception.InvalidUdevRules as exc:
+            sys.stderr.write("\n%s\n\n" % exc)
         env.Replace(
-            UPLOAD_PORT=find_serial_port(
-                initial_port=initial_port,
+            UPLOAD_PORT=SerialPortFinder(
                 board_config=env.BoardConfig() if "BOARD" in env else None,
                 upload_protocol=upload_protocol,
-            )
+                prefer_gdb_port="blackmagic" in upload_protocol,
+                verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)),
+            ).find(initial_port)
         )
 
     if env.subst("$UPLOAD_PORT"):

platformio/cache.py

@@ -23,7 +23,7 @@ from platformio.package.lockfile import LockFile
 from platformio.project.helpers import get_project_cache_dir
 
 
-class ContentCache(object):
+class ContentCache:
     def __init__(self, namespace=None):
         self.cache_dir = os.path.join(get_project_cache_dir(), namespace or "content")
         self._db_path = os.path.join(self.cache_dir, "db.data")

platformio/check/cli.py

@@ -26,8 +26,8 @@ import click
 from tabulate import tabulate
 
 from platformio import app, exception, fs, util
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools import CheckToolFactory
+from platformio.check.defect import DefectItem
+from platformio.check.tools import CheckToolFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import find_project_dir_above, get_project_dir
 
@@ -38,18 +38,15 @@ from platformio.project.helpers import find_project_dir_above, get_project_dir
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(
-        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
-    ),
+    type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True),
 )
 @click.option(
     "-c",
     "--project-conf",
-    type=click.Path(
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
 )
-@click.option("--pattern", multiple=True)
+@click.option("--pattern", multiple=True, hidden=True)
+@click.option("-f", "--src-filters", multiple=True)
 @click.option("--flags", multiple=True)
 @click.option(
     "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values())
@@ -67,6 +64,7 @@ def cli(
     environment,
     project_dir,
     project_conf,
+    src_filters,
     pattern,
     flags,
     severity,
@@ -105,14 +103,24 @@ def cli(
                     "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)
                 )
 
-            default_patterns = [
-                config.get("platformio", "src_dir"),
-                config.get("platformio", "include_dir"),
+            default_src_filters = [
+                "+<%s>" % os.path.basename(config.get("platformio", "src_dir")),
+                "+<%s>" % os.path.basename(config.get("platformio", "include_dir")),
             ]
+
+            src_filters = (
+                src_filters
+                or pattern
+                or env_options.get(
+                    "check_src_filters",
+                    env_options.get("check_patterns", default_src_filters),
+                )
+            )
+
             tool_options = dict(
                 verbose=verbose,
                 silent=silent,
-                patterns=pattern or env_options.get("check_patterns", default_patterns),
+                src_filters=src_filters,
                 flags=flags or env_options.get("check_flags"),
                 severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]]
                 if silent
@@ -198,7 +206,7 @@ def print_processing_header(tool, envname, envdump):
         "Checking %s > %s (%s)"
         % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump))
     )
-    terminal_width, _ = shutil.get_terminal_size()
+    terminal_width = shutil.get_terminal_size().columns
     click.secho("-" * terminal_width, bold=True)
 
 
@@ -265,7 +273,7 @@ def print_defects_stats(results):
     tabular_data.append(total)
 
     headers = ["Component"]
-    headers.extend([l.upper() for l in severity_labels])
+    headers.extend([label.upper() for label in severity_labels])
     headers = [click.style(h, bold=True) for h in headers]
     click.echo(tabulate(tabular_data, headers=headers, numalign="center"))
     click.echo()

platformio/check/defect.py

@@ -16,14 +16,14 @@ import os
 
 import click
 
+from platformio.exception import PlatformioException
 from platformio.project.helpers import get_project_dir
 
 # pylint: disable=too-many-instance-attributes, redefined-builtin
 # pylint: disable=too-many-arguments
 
 
-class DefectItem(object):
-
+class DefectItem:
     SEVERITY_HIGH = 1
     SEVERITY_MEDIUM = 2
     SEVERITY_LOW = 4
@@ -79,7 +79,7 @@ class DefectItem(object):
         for key, value in DefectItem.SEVERITY_LABELS.items():
             if label == value:
                 return key
-        raise Exception("Unknown severity label -> %s" % label)
+        raise PlatformioException("Unknown severity label -> %s" % label)
 
     def as_dict(self):
         return {

platformio/check/tools/__init__.py

@@ -13,12 +13,12 @@
 # limitations under the License.
 
 from platformio import exception
-from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool
-from platformio.commands.check.tools.cppcheck import CppcheckCheckTool
-from platformio.commands.check.tools.pvsstudio import PvsStudioCheckTool
+from platformio.check.tools.clangtidy import ClangtidyCheckTool
+from platformio.check.tools.cppcheck import CppcheckCheckTool
+from platformio.check.tools.pvsstudio import PvsStudioCheckTool
 
 
-class CheckToolFactory(object):
+class CheckToolFactory:
     @staticmethod
     def new(tool, project_dir, config, envname, options):
         cls = None

platformio/check/tools/base.py

@@ -12,24 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import glob
 import os
 import tempfile
 
 import click
 
 from platformio import fs, proc
-from platformio.commands.check.defect import DefectItem
+from platformio.check.defect import DefectItem
 from platformio.package.manager.core import get_core_package_dir
 from platformio.package.meta import PackageSpec
 from platformio.project.helpers import load_build_metadata
 
 
-class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
+class CheckToolBase:  # pylint: disable=too-many-instance-attributes
     def __init__(self, project_dir, config, envname, options):
         self.config = config
         self.envname = envname
         self.options = options
+        self.project_dir = project_dir
         self.cc_flags = []
         self.cxx_flags = []
         self.cpp_includes = []
@@ -41,7 +41,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         self._defects = []
         self._on_defect_callback = None
         self._bad_input = False
-        self._load_cpp_data(project_dir)
+        self._load_cpp_data()
 
         # detect all defects by default
         if not self.options.get("severity"):
@@ -56,12 +56,12 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
             for s in self.options["severity"]
         ]
 
-    def _load_cpp_data(self, project_dir):
-        data = load_build_metadata(project_dir, self.envname)
+    def _load_cpp_data(self):
+        data = load_build_metadata(self.project_dir, self.envname)
         if not data:
             return
-        self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", ""))
-        self.cxx_flags = click.parser.split_arg_string(data.get("cxx_flags", ""))
+        self.cc_flags = data.get("cc_flags", [])
+        self.cxx_flags = data.get("cxx_flags", [])
         self.cpp_includes = self._dump_includes(data.get("includes", {}))
         self.cpp_defines = data.get("defines", [])
         self.cc_path = data.get("cc_path")
@@ -99,6 +99,13 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
                 includes_file,
             )
             result = proc.exec_command(cmd, shell=True)
+
+            if result["returncode"] != 0:
+                click.echo("Warning: Failed to extract toolchain defines!")
+                if self.options.get("verbose"):
+                    click.echo(result["out"])
+                    click.echo(result["err"])
+
             for line in result["out"].split("\n"):
                 tokens = line.strip().split(" ", 2)
                 if not tokens or tokens[0] != "#define":
@@ -201,7 +208,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         return result
 
     @staticmethod
-    def get_project_target_files(patterns):
+    def get_project_target_files(project_dir, src_filters):
         c_extension = (".c",)
         cpp_extensions = (".cc", ".cpp", ".cxx", ".ino")
         header_extensions = (".h", ".hh", ".hpp", ".hxx")
@@ -216,13 +223,9 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
             elif path.endswith(cpp_extensions):
                 result["c++"].append(os.path.abspath(path))
 
-        for pattern in patterns:
-            for item in glob.glob(pattern, recursive=True):
-                if not os.path.isdir(item):
-                    _add_file(item)
-                for root, _, files in os.walk(item, followlinks=True):
-                    for f in files:
-                        _add_file(os.path.join(root, f))
+        src_filters = normalize_src_filters(src_filters)
+        for f in fs.match_src_files(project_dir, src_filters):
+            _add_file(f)
 
         return result
 
@@ -243,3 +246,22 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         self.clean_up()
 
         return self._bad_input
+
+
+#
+# Helpers
+#
+
+
+def normalize_src_filters(src_filters):
+    def _normalize(src_filters):
+        return (
+            src_filters
+            if src_filters.startswith(("+<", "-<"))
+            else "+<%s>" % src_filters
+        )
+
+    if isinstance(src_filters, (list, tuple)):
+        return " ".join([_normalize(f) for f in src_filters])
+
+    return _normalize(src_filters)

platformio/check/tools/clangtidy.py

@@ -15,8 +15,8 @@
 import re
 from os.path import join
 
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 
 
 class ClangtidyCheckTool(CheckToolBase):
@@ -64,11 +64,13 @@ class ClangtidyCheckTool(CheckToolBase):
         ):
             cmd.append("--checks=*")
 
-        project_files = self.get_project_target_files(self.options["patterns"])
+        project_files = self.get_project_target_files(
+            self.project_dir, self.options["src_filters"]
+        )
 
         src_files = []
-        for scope in project_files:
-            src_files.extend(project_files[scope])
+        for items in project_files.values():
+            src_files.extend(items)
 
         cmd.extend(flags + src_files + ["--"])
         cmd.extend(

platformio/check/tools/cppcheck.py

@@ -17,8 +17,8 @@ import os
 import click
 
 from platformio import proc
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 
 
 class CppcheckCheckTool(CheckToolBase):
@@ -96,7 +96,7 @@ class CppcheckCheckTool(CheckToolBase):
                     )
                     click.echo()
                 self._bad_input = True
-                self._buffer = ""
+            self._buffer = ""
             return None
 
         self._buffer = ""
@@ -214,7 +214,9 @@ class CppcheckCheckTool(CheckToolBase):
         if not self.is_flag_set("--addon", self.get_flags("cppcheck")):
             return
 
-        for files in self.get_project_target_files(self.options["patterns"]).values():
+        for files in self.get_project_target_files(
+            self.project_dir, self.options["src_filters"]
+        ).values():
             for f in files:
                 dump_file = f + ".dump"
                 if os.path.isfile(dump_file):
@@ -243,7 +245,9 @@ class CppcheckCheckTool(CheckToolBase):
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
 
-        project_files = self.get_project_target_files(self.options["patterns"])
+        project_files = self.get_project_target_files(
+            self.project_dir, self.options["src_filters"]
+        )
         src_files_scope = ("c", "c++")
         if not any(project_files[t] for t in src_files_scope):
             click.echo("Error: Nothing to check.")

platformio/check/tools/pvsstudio.py

@@ -20,8 +20,8 @@ from xml.etree.ElementTree import fromstring
 import click
 
 from platformio import proc
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 from platformio.compat import IS_WINDOWS
 
 
@@ -227,7 +227,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
         for scope, files in self.get_project_target_files(
-            self.options["patterns"]
+            self.project_dir, self.options["src_filters"]
         ).items():
             if scope not in ("c", "c++"):
                 continue

platformio/cli.py

@@ -0,0 +1,110 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+from pathlib import Path
+
+import click
+
+
+class PlatformioCLI(click.MultiCommand):
+    leftover_args = []
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._pio_root_path = Path(__file__).parent
+        self._pio_cmd_aliases = dict(package="pkg")
+
+    def _find_pio_commands(self):
+        def _to_module_path(p):
+            return (
+                "platformio." + ".".join(p.relative_to(self._pio_root_path).parts)[:-3]
+            )
+
+        result = {}
+        for p in self._pio_root_path.rglob("cli.py"):
+            # skip this module
+            if p.parent == self._pio_root_path:
+                continue
+            cmd_name = p.parent.name
+            result[self._pio_cmd_aliases.get(cmd_name, cmd_name)] = _to_module_path(p)
+
+        # find legacy commands
+        for p in (self._pio_root_path / "commands").iterdir():
+            if p.name.startswith("_"):
+                continue
+            if (p / "command.py").is_file():
+                result[p.name] = _to_module_path(p / "command.py")
+            elif p.name.endswith(".py"):
+                result[p.name[:-3]] = _to_module_path(p)
+
+        return result
+
+    @staticmethod
+    def in_silence():
+        args = PlatformioCLI.leftover_args
+        return args and any(
+            [
+                args[0] == "debug" and "--interpreter" in " ".join(args),
+                args[0] == "upgrade",
+                "--json-output" in args,
+                "--version" in args,
+            ]
+        )
+
+    @classmethod
+    def reveal_cmd_path_args(cls, ctx):
+        result = []
+        group = ctx.command
+        args = cls.leftover_args[::]
+        while args:
+            cmd_name = args.pop(0)
+            next_group = group.get_command(ctx, cmd_name)
+            if next_group:
+                group = next_group
+                result.append(cmd_name)
+            if not hasattr(group, "get_command"):
+                break
+        return result
+
+    def invoke(self, ctx):
+        PlatformioCLI.leftover_args = ctx.args
+        if hasattr(ctx, "protected_args"):
+            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
+        return super().invoke(ctx)
+
+    def list_commands(self, ctx):
+        return sorted(list(self._find_pio_commands()))
+
+    def get_command(self, ctx, cmd_name):
+        commands = self._find_pio_commands()
+        if cmd_name not in commands:
+            return self._handle_obsolate_command(ctx, cmd_name)
+        module = importlib.import_module(commands[cmd_name])
+        return getattr(module, "cli")
+
+    @staticmethod
+    def _handle_obsolate_command(ctx, cmd_name):
+        # pylint: disable=import-outside-toplevel
+        if cmd_name == "init":
+            from platformio.project.commands.init import project_init_cmd
+
+            return project_init_cmd
+
+        if cmd_name == "package":
+            from platformio.package.cli import cli
+
+            return cli
+
+        raise click.UsageError('No such command "%s"' % cmd_name, ctx)

platformio/commands/__init__.py

@@ -11,76 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import os
-
-import click
-
-
-class PlatformioCLI(click.MultiCommand):
-
-    leftover_args = []
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._pio_cmds_dir = os.path.dirname(__file__)
-
-    @staticmethod
-    def in_silence():
-        args = PlatformioCLI.leftover_args
-        return args and any(
-            [
-                args[0] == "debug" and "--interpreter" in " ".join(args),
-                args[0] == "upgrade",
-                "--json-output" in args,
-                "--version" in args,
-            ]
-        )
-
-    def invoke(self, ctx):
-        PlatformioCLI.leftover_args = ctx.args
-        if hasattr(ctx, "protected_args"):
-            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
-        return super().invoke(ctx)
-
-    def list_commands(self, ctx):
-        cmds = []
-        for cmd_name in os.listdir(self._pio_cmds_dir):
-            if cmd_name.startswith("__init__"):
-                continue
-            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
-                cmds.append(cmd_name)
-            elif cmd_name.endswith(".py"):
-                cmds.append(cmd_name[:-3])
-        cmds.sort()
-        return cmds
-
-    def get_command(self, ctx, cmd_name):
-        mod = None
-        try:
-            mod_path = "platformio.commands." + cmd_name
-            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
-                mod_path = "platformio.commands.%s.command" % cmd_name
-            mod = __import__(mod_path, None, None, ["cli"])
-        except ImportError:
-            try:
-                return self._handle_obsolate_command(cmd_name)
-            except AttributeError:
-                pass
-            raise click.UsageError('No such command "%s"' % cmd_name, ctx)
-        return mod.cli
-
-    @staticmethod
-    def _handle_obsolate_command(name):
-        # pylint: disable=import-outside-toplevel
-        if name == "init":
-            from platformio.project.commands.init import project_init_cmd
-
-            return project_init_cmd
-
-        if name == "package":
-            from platformio.commands.pkg import cli
-
-            return cli
-
-        raise AttributeError()

platformio/commands/access.py

@@ -1,154 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import json
-import re
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.registry import RegistryClient
-from platformio.commands.account import validate_username
-from platformio.commands.team import validate_orgname_teamname
-
-
-def validate_client(value):
-    if ":" in value:
-        validate_orgname_teamname(value)
-    else:
-        validate_username(value)
-    return value
-
-
-@click.group("access", short_help="Manage resource access")
-def cli():
-    pass
-
-
-def validate_urn(value):
-    value = str(value).strip()
-    if not re.match(r"^prn:reg:pkg:(\d+):(\w+)$", value, flags=re.I):
-        raise click.BadParameter("Invalid URN format.")
-    return value
-
-
-@cli.command("public", short_help="Make resource public")
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_public(urn, urn_type):
-    client = RegistryClient()
-    client.update_resource(urn=urn, private=0)
-    return click.secho(
-        "The resource %s has been successfully updated." % urn,
-        fg="green",
-    )
-
-
-@cli.command("private", short_help="Make resource private")
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_private(urn, urn_type):
-    client = RegistryClient()
-    client.update_resource(urn=urn, private=1)
-    return click.secho(
-        "The resource %s has been successfully updated." % urn,
-        fg="green",
-    )
-
-
-@cli.command("grant", short_help="Grant access")
-@click.argument("level", type=click.Choice(["admin", "maintainer", "guest"]))
-@click.argument(
-    "client",
-    metavar="[<ORGNAME:TEAMNAME>|<USERNAME>]",
-    callback=lambda _, __, value: validate_client(value),
-)
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_grant(level, client, urn, urn_type):
-    reg_client = RegistryClient()
-    reg_client.grant_access_for_resource(urn=urn, client=client, level=level)
-    return click.secho(
-        "Access for resource %s has been granted for %s" % (urn, client),
-        fg="green",
-    )
-
-
-@cli.command("revoke", short_help="Revoke access")
-@click.argument(
-    "client",
-    metavar="[ORGNAME:TEAMNAME|USERNAME]",
-    callback=lambda _, __, value: validate_client(value),
-)
-@click.argument(
-    "urn",
-    callback=lambda _, __, value: validate_urn(value),
-)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-def access_revoke(client, urn, urn_type):
-    reg_client = RegistryClient()
-    reg_client.revoke_access_from_resource(urn=urn, client=client)
-    return click.secho(
-        "Access for resource %s has been revoked for %s" % (urn, client),
-        fg="green",
-    )
-
-
-@cli.command("list", short_help="List published resources")
-@click.argument("owner", required=False)
-@click.option("--urn-type", type=click.Choice(["prn:reg:pkg"]), default="prn:reg:pkg")
-@click.option("--json-output", is_flag=True)
-def access_list(owner, urn_type, json_output):
-    reg_client = RegistryClient()
-    resources = reg_client.list_resources(owner=owner)
-    if json_output:
-        return click.echo(json.dumps(resources))
-    if not resources:
-        return click.secho("You do not have any resources.", fg="yellow")
-    for resource in resources:
-        click.echo()
-        click.secho(resource.get("name"), fg="cyan")
-        click.echo("-" * len(resource.get("name")))
-        table_data = []
-        table_data.append(("URN:", resource.get("urn")))
-        table_data.append(("Owner:", resource.get("owner")))
-        table_data.append(
-            (
-                "Access:",
-                click.style("Private", fg="red")
-                if resource.get("private", False)
-                else "Public",
-            )
-        )
-        table_data.append(
-            (
-                "Access level(s):",
-                ", ".join(
-                    (level.capitalize() for level in resource.get("access_levels"))
-                ),
-            )
-        )
-        click.echo(tabulate(table_data, tablefmt="plain"))
-    return click.echo()

platformio/commands/account.py

@@ -1,292 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import json
-import re
-
-import click
-from tabulate import tabulate
-
-from platformio import util
-from platformio.clients.account import AccountClient, AccountNotAuthorized
-
-
-@click.group("account", short_help="Manage PlatformIO account")
-def cli():
-    pass
-
-
-def validate_username(value, field="username"):
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I):
-        raise click.BadParameter(
-            "Invalid %s format. "
-            "%s must contain only alphanumeric characters "
-            "or single hyphens, cannot begin or end with a hyphen, "
-            "and must not be longer than 38 characters."
-            % (field.lower(), field.capitalize())
-        )
-    return value
-
-
-def validate_email(value):
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d_.+-]+@[a-z\d\-]+\.[a-z\d\-.]+$", value, flags=re.I):
-        raise click.BadParameter("Invalid email address")
-    return value
-
-
-def validate_password(value):
-    value = str(value).strip()
-    if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
-        raise click.BadParameter(
-            "Invalid password format. "
-            "Password must contain at least 8 characters"
-            " including a number and a lowercase letter"
-        )
-    return value
-
-
-@cli.command("register", short_help="Create new PlatformIO Account")
-@click.option(
-    "-u",
-    "--username",
-    prompt=True,
-    callback=lambda _, __, value: validate_username(value),
-)
-@click.option(
-    "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value)
-)
-@click.option(
-    "-p",
-    "--password",
-    prompt=True,
-    hide_input=True,
-    confirmation_prompt=True,
-    callback=lambda _, __, value: validate_password(value),
-)
-@click.option("--firstname", prompt=True)
-@click.option("--lastname", prompt=True)
-def account_register(username, email, password, firstname, lastname):
-    client = AccountClient()
-    client.registration(username, email, password, firstname, lastname)
-    return click.secho(
-        "An account has been successfully created. "
-        "Please check your mail to activate your account and verify your email address.",
-        fg="green",
-    )
-
-
-@cli.command("login", short_help="Log in to PlatformIO Account")
-@click.option("-u", "--username", prompt="Username or email")
-@click.option("-p", "--password", prompt=True, hide_input=True)
-def account_login(username, password):
-    client = AccountClient()
-    client.login(username, password)
-    return click.secho("Successfully logged in!", fg="green")
-
-
-@cli.command("logout", short_help="Log out of PlatformIO Account")
-def account_logout():
-    client = AccountClient()
-    client.logout()
-    return click.secho("Successfully logged out!", fg="green")
-
-
-@cli.command("password", short_help="Change password")
-@click.option("--old-password", prompt=True, hide_input=True)
-@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
-def account_password(old_password, new_password):
-    client = AccountClient()
-    client.change_password(old_password, new_password)
-    return click.secho("Password successfully changed!", fg="green")
-
-
-@cli.command("token", short_help="Get or regenerate Authentication Token")
-@click.option("-p", "--password", prompt=True, hide_input=True)
-@click.option("--regenerate", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_token(password, regenerate, json_output):
-    client = AccountClient()
-    auth_token = client.auth_token(password, regenerate)
-    if json_output:
-        return click.echo(json.dumps({"status": "success", "result": auth_token}))
-    return click.secho("Personal Authentication Token: %s" % auth_token, fg="green")
-
-
-@cli.command("forgot", short_help="Forgot password")
-@click.option("--username", prompt="Username or email")
-def account_forgot(username):
-    client = AccountClient()
-    client.forgot_password(username)
-    return click.secho(
-        "If this account is registered, we will send the "
-        "further instructions to your email.",
-        fg="green",
-    )
-
-
-@cli.command("update", short_help="Update profile information")
-@click.option("--current-password", prompt=True, hide_input=True)
-@click.option("--username")
-@click.option("--email")
-@click.option("--firstname")
-@click.option("--lastname")
-def account_update(current_password, **kwargs):
-    client = AccountClient()
-    profile = client.get_profile()
-    new_profile = profile.copy()
-    if not any(kwargs.values()):
-        for field in profile:
-            new_profile[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=profile[field]
-            )
-            if field == "email":
-                validate_email(new_profile[field])
-            if field == "username":
-                validate_username(new_profile[field])
-    else:
-        new_profile.update({key: value for key, value in kwargs.items() if value})
-    client.update_profile(new_profile, current_password)
-    click.secho("Profile successfully updated!", fg="green")
-    username_changed = new_profile["username"] != profile["username"]
-    email_changed = new_profile["email"] != profile["email"]
-    if not username_changed and not email_changed:
-        return None
-    try:
-        client.logout()
-    except AccountNotAuthorized:
-        pass
-    if email_changed:
-        return click.secho(
-            "Please check your mail to verify your new email address and re-login. ",
-            fg="yellow",
-        )
-    return click.secho("Please re-login.", fg="yellow")
-
-
-@cli.command("destroy", short_help="Destroy account")
-def account_destroy():
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the %s user account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % client.get_logged_username(),
-        abort=True,
-    )
-    client.destroy_account()
-    try:
-        client.logout()
-    except AccountNotAuthorized:
-        pass
-    return click.secho(
-        "User account has been destroyed.",
-        fg="green",
-    )
-
-
-@cli.command("show", short_help="PlatformIO Account information")
-@click.option("--offline", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_show(offline, json_output):
-    client = AccountClient()
-    info = client.get_account_info(offline)
-    if json_output:
-        return click.echo(json.dumps(info))
-    click.echo()
-    if info.get("profile"):
-        print_profile(info["profile"])
-    if info.get("packages"):
-        print_packages(info["packages"])
-    if info.get("subscriptions"):
-        print_subscriptions(info["subscriptions"])
-    return click.echo()
-
-
-def print_profile(profile):
-    click.secho("Profile", fg="cyan", bold=True)
-    click.echo("=" * len("Profile"))
-    data = []
-    if profile.get("username"):
-        data.append(("Username:", profile["username"]))
-    if profile.get("email"):
-        data.append(("Email:", profile["email"]))
-    if profile.get("firstname"):
-        data.append(("First name:", profile["firstname"]))
-    if profile.get("lastname"):
-        data.append(("Last name:", profile["lastname"]))
-    click.echo(tabulate(data, tablefmt="plain"))
-
-
-def print_packages(packages):
-    click.echo()
-    click.secho("Packages", fg="cyan")
-    click.echo("=" * len("Packages"))
-    for package in packages:
-        click.echo()
-        click.secho(package.get("name"), bold=True)
-        click.echo("-" * len(package.get("name")))
-        if package.get("description"):
-            click.echo(package.get("description"))
-        data = []
-        expire = "-"
-        if "subscription" in package:
-            expire = util.parse_datetime(
-                package["subscription"].get("end_at")
-                or package["subscription"].get("next_bill_at")
-            ).strftime("%Y-%m-%d")
-        data.append(("Expire:", expire))
-        services = []
-        for key in package:
-            if not key.startswith("service."):
-                continue
-            if isinstance(package[key], dict):
-                services.append(package[key].get("title"))
-            else:
-                services.append(package[key])
-        if services:
-            data.append(("Services:", ", ".join(services)))
-        click.echo(tabulate(data, tablefmt="plain"))
-
-
-def print_subscriptions(subscriptions):
-    click.echo()
-    click.secho("Subscriptions", fg="cyan")
-    click.echo("=" * len("Subscriptions"))
-    for subscription in subscriptions:
-        click.echo()
-        click.secho(subscription.get("product_name"), bold=True)
-        click.echo("-" * len(subscription.get("product_name")))
-        data = [("State:", subscription.get("status"))]
-        begin_at = util.parse_datetime(subscription.get("begin_at")).strftime("%c")
-        data.append(("Start date:", begin_at or "-"))
-        end_at = subscription.get("end_at")
-        if end_at:
-            end_at = util.parse_datetime(subscription.get("end_at")).strftime("%c")
-        data.append(("End date:", end_at or "-"))
-        next_bill_at = subscription.get("next_bill_at")
-        if next_bill_at:
-            next_bill_at = util.parse_datetime(
-                subscription.get("next_bill_at")
-            ).strftime("%c")
-        data.append(("Next payment:", next_bill_at or "-"))
-        data.append(
-            ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")
-        )
-        data.append(
-            ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-")
-        )
-        click.echo(tabulate(data, tablefmt="plain"))

platformio/commands/boards.py

@@ -41,8 +41,8 @@ def cli(query, installed, json_output):  # pylint: disable=R0912
             grpboards[board["platform"]] = []
         grpboards[board["platform"]].append(board)
 
-    terminal_width, _ = shutil.get_terminal_size()
-    for (platform, boards) in sorted(grpboards.items()):
+    terminal_width = shutil.get_terminal_size().columns
+    for platform, boards in sorted(grpboards.items()):
         click.echo("")
         click.echo("Platform: ", nl=False)
         click.secho(platform, bold=True)

platformio/commands/ci.py

@@ -19,11 +19,11 @@ import tempfile
 
 import click
 
-from platformio import app, fs
-from platformio.commands.run.command import cli as cmd_run
+from platformio import fs
 from platformio.exception import CIBuildEnvsEmpty
 from platformio.project.commands.init import project_init_cmd, validate_boards
 from platformio.project.config import ProjectConfig
+from platformio.run.cli import cli as cmd_run
 
 
 def validate_path(ctx, param, value):  # pylint: disable=unused-argument
@@ -39,8 +39,8 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
     try:
         assert invalid_path is None
         return value
-    except AssertionError:
-        raise click.BadParameter("Found invalid path: %s" % invalid_path)
+    except AssertionError as exc:
+        raise click.BadParameter("Found invalid path: %s" % invalid_path) from exc
 
 
 @click.command("ci", short_help="Continuous Integration")
@@ -51,17 +51,16 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
 @click.option(
     "--build-dir",
     default=tempfile.mkdtemp,
-    type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
+    type=click.Path(file_okay=False, dir_okay=True, writable=True),
 )
 @click.option("--keep-build-dir", is_flag=True)
 @click.option(
     "-c",
     "--project-conf",
-    type=click.Path(
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
 )
 @click.option("-O", "--project-option", multiple=True)
+@click.option("-e", "--environment", "environments", multiple=True)
 @click.option("-v", "--verbose", is_flag=True)
 @click.pass_context
 def cli(  # pylint: disable=too-many-arguments, too-many-branches
@@ -74,17 +73,15 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
     keep_build_dir,
     project_conf,
     project_option,
+    environments,
     verbose,
 ):
-
     if not src and os.getenv("PLATFORMIO_CI_SRC"):
         src = validate_path(ctx, None, os.getenv("PLATFORMIO_CI_SRC").split(":"))
     if not src:
         raise click.BadParameter("Missing argument 'src'")
 
     try:
-        app.set_session_var("force_option", True)
-
         if not keep_build_dir and os.path.isdir(build_dir):
             fs.rmtree(build_dir)
         if not os.path.isdir(build_dir):
@@ -110,12 +107,14 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
         ctx.invoke(
             project_init_cmd,
             project_dir=build_dir,
-            board=board,
-            project_option=project_option,
+            boards=board,
+            project_options=project_option,
         )
 
         # process project
-        ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose)
+        ctx.invoke(
+            cmd_run, project_dir=build_dir, environment=environments, verbose=verbose
+        )
     finally:
         if not keep_build_dir:
             fs.rmtree(build_dir)

platformio/commands/device/__init__.py

@@ -13,6 +13,6 @@
 # limitations under the License.
 
 # pylint: disable=unused-import
-from platformio.device.filters.base import (
+from platformio.device.monitor.filters.base import (
     DeviceMonitorFilterBase as DeviceMonitorFilter,
 )

platformio/commands/home/helpers.py

@@ -1,60 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import socket
-
-import requests
-from starlette.concurrency import run_in_threadpool
-
-from platformio import util
-from platformio.compat import IS_WINDOWS
-from platformio.proc import where_is_program
-
-
-class AsyncSession(requests.Session):
-    async def request(  # pylint: disable=signature-differs,invalid-overridden-method
-        self, *args, **kwargs
-    ):
-        func = super().request
-        return await run_in_threadpool(func, *args, **kwargs)
-
-
-@util.memoized(expire="60s")
-def requests_session():
-    return AsyncSession()
-
-
-@util.memoized(expire="60s")
-def get_core_fullpath():
-    return where_is_program("platformio" + (".exe" if IS_WINDOWS else ""))
-
-
-def is_port_used(host, port):
-    socket.setdefaulttimeout(1)
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    if IS_WINDOWS:
-        try:
-            s.bind((host, port))
-            s.close()
-            return False
-        except (OSError, socket.error):
-            pass
-    else:
-        try:
-            s.connect((host, port))
-            s.close()
-        except socket.error:
-            return False
-
-    return True

platformio/commands/lib.py

@@ -17,16 +17,18 @@
 import json
 import logging
 import os
-import time
-from urllib.parse import quote
 
 import click
-from tabulate import tabulate
 
-from platformio import exception, fs, util
-from platformio.commands import PlatformioCLI
-from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps
-from platformio.package.exception import NotGlobalLibDir, UnknownPackageError
+from platformio import exception, fs
+from platformio.cli import PlatformioCLI
+from platformio.package.commands.install import package_install_cmd
+from platformio.package.commands.list import package_list_cmd
+from platformio.package.commands.search import package_search_cmd
+from platformio.package.commands.show import package_show_cmd
+from platformio.package.commands.uninstall import package_uninstall_cmd
+from platformio.package.commands.update import package_update_cmd
+from platformio.package.exception import NotGlobalLibDir
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.meta import PackageItem, PackageSpec
 from platformio.proc import is_ci
@@ -43,15 +45,27 @@ def get_project_global_lib_dir():
     return ProjectConfig.get_instance().get("platformio", "globallib_dir")
 
 
+def invoke_command(ctx, cmd, **kwargs):
+    input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
+    project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
+    for input_dir in input_dirs:
+        cmd_kwargs = kwargs.copy()
+        if is_platformio_project(input_dir):
+            cmd_kwargs["project_dir"] = input_dir
+            cmd_kwargs["environments"] = project_environments
+        else:
+            cmd_kwargs["global"] = True
+            cmd_kwargs["storage_dir"] = input_dir
+        ctx.invoke(cmd, **cmd_kwargs)
+
+
 @click.group(short_help="Library manager", hidden=True)
 @click.option(
     "-d",
     "--storage-dir",
     multiple=True,
     default=None,
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True),
     help="Manage custom library storage",
 )
 @click.option(
@@ -69,13 +83,6 @@ def get_project_global_lib_dir():
 @click.pass_context
 def cli(ctx, **options):
     in_silence = PlatformioCLI.in_silence()
-    if not in_silence:
-        click.secho(
-            "\nWARNING!!! This command is deprecated and will be removed in "
-            "the next releases. \nPlease use `pio pkg` instead.\n",
-            fg="yellow",
-        )
-
     storage_cmds = ("install", "uninstall", "update", "list")
     # skip commands that don't need storage folder
     if ctx.invoked_subcommand not in storage_cmds or (
@@ -148,55 +155,19 @@ def cli(ctx, **options):
 def lib_install(  # pylint: disable=too-many-arguments,unused-argument
     ctx, libraries, save, silent, interactive, force
 ):
-    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
-    storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, [])
-
-    installed_pkgs = {}
-    for storage_dir in storage_dirs:
-        if not silent and (libraries or storage_dir in storage_libdeps):
-            print_storage_header(storage_dirs, storage_dir)
-        lm = LibraryPackageManager(storage_dir)
-        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
-
-        if libraries:
-            installed_pkgs = {
-                library: lm.install(library, force=force) for library in libraries
-            }
-
-        elif storage_dir in storage_libdeps:
-            for library in storage_libdeps[storage_dir]:
-                lm.install(library, force=force)
-
-    if save and installed_pkgs:
-        _save_deps(ctx, installed_pkgs)
-
-
-def _save_deps(ctx, pkgs, action="add"):
-    specs = []
-    for library, pkg in pkgs.items():
-        spec = PackageSpec(library)
-        if spec.external:
-            specs.append(spec)
-        else:
-            specs.append(
-                PackageSpec(
-                    owner=pkg.metadata.spec.owner,
-                    name=pkg.metadata.spec.name,
-                    requirements=spec.requirements
-                    or (
-                        ("^%s" % pkg.metadata.version)
-                        if not pkg.metadata.version.build
-                        else pkg.metadata.version
-                    ),
-                )
-            )
-
-    input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
-    project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
-    for input_dir in input_dirs:
-        if not is_platformio_project(input_dir):
-            continue
-        save_project_libdeps(input_dir, specs, project_environments, action=action)
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg install` instead.\n",
+        fg="yellow",
+    )
+    return invoke_command(
+        ctx,
+        package_install_cmd,
+        libraries=libraries,
+        no_save=not save,
+        force=force,
+        silent=silent,
+    )
 
 
 @cli.command("uninstall", short_help="Remove libraries")
@@ -211,16 +182,18 @@ def _save_deps(ctx, pkgs, action="add"):
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.pass_context
 def lib_uninstall(ctx, libraries, save, silent):
-    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
-    uninstalled_pkgs = {}
-    for storage_dir in storage_dirs:
-        print_storage_header(storage_dirs, storage_dir)
-        lm = LibraryPackageManager(storage_dir)
-        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
-        uninstalled_pkgs = {library: lm.uninstall(library) for library in libraries}
-
-    if save and uninstalled_pkgs:
-        _save_deps(ctx, uninstalled_pkgs, action="remove")
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg uninstall` instead.\n",
+        fg="yellow",
+    )
+    invoke_command(
+        ctx,
+        package_uninstall_cmd,
+        libraries=libraries,
+        no_save=not save,
+        silent=silent,
+    )
 
 
 @cli.command("update", short_help="Update installed libraries")
@@ -246,88 +219,81 @@ def lib_update(  # pylint: disable=too-many-arguments
             "This command is deprecated, please use `pio pkg outdated` instead"
         )
 
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg update` instead.\n",
+            fg="yellow",
+        )
+        return invoke_command(
+            ctx,
+            package_update_cmd,
+            libraries=libraries,
+            silent=silent,
+        )
+
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     json_result = {}
     for storage_dir in storage_dirs:
-        if not json_output:
-            print_storage_header(storage_dirs, storage_dir)
         lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, [])
         lm = LibraryPackageManager(storage_dir)
         lm.set_log_level(logging.WARN if silent else logging.DEBUG)
         _libraries = libraries or lib_deps or lm.get_installed()
 
-        if only_check and json_output:
-            result = []
-            for library in _libraries:
-                spec = None
-                pkg = None
-                if isinstance(library, PackageItem):
-                    pkg = library
-                else:
-                    spec = PackageSpec(library)
-                    pkg = lm.get_package(spec)
-                if not pkg:
-                    continue
-                outdated = lm.outdated(pkg, spec)
-                if not outdated.is_outdated(allow_incompatible=True):
-                    continue
-                manifest = lm.legacy_load_manifest(pkg)
-                manifest["versionWanted"] = (
-                    str(outdated.wanted) if outdated.wanted else None
-                )
-                manifest["versionLatest"] = (
-                    str(outdated.latest) if outdated.latest else None
-                )
-                result.append(manifest)
-            json_result[storage_dir] = result
-        else:
-            for library in _libraries:
-                to_spec = (
-                    None if isinstance(library, PackageItem) else PackageSpec(library)
-                )
-                try:
-                    lm.update(library, to_spec=to_spec)
-                except UnknownPackageError as e:
-                    if library not in lib_deps:
-                        raise e
-
-    if json_output:
-        return click.echo(
-            json.dumps(
-                json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
+        result = []
+        for library in _libraries:
+            spec = None
+            pkg = None
+            if isinstance(library, PackageItem):
+                pkg = library
+            else:
+                spec = PackageSpec(library)
+                pkg = lm.get_package(spec)
+            if not pkg:
+                continue
+            outdated = lm.outdated(pkg, spec)
+            if not outdated.is_outdated(allow_incompatible=True):
+                continue
+            manifest = lm.legacy_load_manifest(pkg)
+            manifest["versionWanted"] = (
+                str(outdated.wanted) if outdated.wanted else None
             )
-        )
+            manifest["versionLatest"] = (
+                str(outdated.latest) if outdated.latest else None
+            )
+            result.append(manifest)
 
-    return True
+        json_result[storage_dir] = result
+
+    return click.echo(
+        json.dumps(
+            json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
+        )
+    )
 
 
 @cli.command("list", short_help="List installed libraries")
 @click.option("--json-output", is_flag=True)
 @click.pass_context
 def lib_list(ctx, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg list` instead.\n",
+            fg="yellow",
+        )
+        return invoke_command(ctx, package_list_cmd, only_libraries=True)
+
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     json_result = {}
     for storage_dir in storage_dirs:
-        if not json_output:
-            print_storage_header(storage_dirs, storage_dir)
         lm = LibraryPackageManager(storage_dir)
-        items = lm.legacy_get_installed()
-        if json_output:
-            json_result[storage_dir] = items
-        elif items:
-            for item in sorted(items, key=lambda i: i["name"]):
-                print_lib_item(item)
-        else:
-            click.echo("No items found")
-
-    if json_output:
-        return click.echo(
-            json.dumps(
-                json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
-            )
+        json_result[storage_dir] = lm.legacy_get_installed()
+    return click.echo(
+        json.dumps(
+            json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
         )
-
-    return True
+    )
 
 
 @cli.command("search", short_help="Search for a library")
@@ -347,8 +313,10 @@ def lib_list(ctx, json_output):
     is_flag=True,
     help="Do not prompt, automatically paginate with delay",
 )
-def lib_search(query, json_output, page, noninteractive, **filters):
-    regclient = LibraryPackageManager().get_registry_client_instance()
+@click.pass_context
+def lib_search(  # pylint: disable=unused-argument
+    ctx, query, json_output, page, noninteractive, **filters
+):
     if not query:
         query = []
     if not isinstance(query, list):
@@ -358,72 +326,30 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         for value in values:
             query.append('%s:"%s"' % (key, value))
 
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg search` instead.\n",
+            fg="yellow",
+        )
+        query.append("type:library")
+        return ctx.invoke(package_search_cmd, query=" ".join(query), page=page)
+
+    regclient = LibraryPackageManager().get_registry_client_instance()
     result = regclient.fetch_json_data(
         "get",
         "/v2/lib/search",
         params=dict(query=" ".join(query), page=page),
         x_cache_valid="1d",
     )
-
-    if json_output:
-        click.echo(json.dumps(result))
-        return
-
-    if result["total"] == 0:
-        click.secho(
-            "Nothing has been found by your request\n"
-            "Try a less-specific search or use truncation (or wildcard) "
-            "operator",
-            fg="yellow",
-            nl=False,
-        )
-        click.secho(" *", fg="green")
-        click.secho("For example: DS*, PCA*, DHT* and etc.\n", fg="yellow")
-        click.echo(
-            "For more examples and advanced search syntax, please use documentation:"
-        )
-        click.secho(
-            "https://docs.platformio.org/page/userguide/lib/cmd_search.html\n",
-            fg="cyan",
-        )
-        return
-
-    click.secho(
-        "Found %d libraries:\n" % result["total"],
-        fg="green" if result["total"] else "yellow",
-    )
-
-    while True:
-        for item in result["items"]:
-            print_lib_item(item)
-
-        if int(result["page"]) * int(result["perpage"]) >= int(result["total"]):
-            break
-
-        if noninteractive:
-            click.echo()
-            click.secho(
-                "Loading next %d libraries... Press Ctrl+C to stop!"
-                % result["perpage"],
-                fg="yellow",
-            )
-            click.echo()
-            time.sleep(5)
-        elif not click.confirm("Show next libraries?"):
-            break
-        result = regclient.fetch_json_data(
-            "get",
-            "/v2/lib/search",
-            params=dict(query=" ".join(query), page=int(result["page"]) + 1),
-            x_cache_valid="1d",
-        )
+    return click.echo(json.dumps(result))
 
 
 @cli.command("builtin", short_help="List built-in libraries")
 @click.option("--storage", multiple=True)
 @click.option("--json-output", is_flag=True)
 def lib_builtin(storage, json_output):
-    items = get_builtin_libs(storage)
+    items = LibraryPackageManager.get_builtin_libs(storage)
     if json_output:
         return click.echo(json.dumps(items))
 
@@ -443,7 +369,16 @@ def lib_builtin(storage, json_output):
 @cli.command("show", short_help="Show detailed info about a library")
 @click.argument("library", metavar="[LIBRARY]")
 @click.option("--json-output", is_flag=True)
-def lib_show(library, json_output):
+@click.pass_context
+def lib_show(ctx, library, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg show` instead.\n",
+            fg="yellow",
+        )
+        return ctx.invoke(package_show_cmd, pkg_type="library", spec=library)
+
     lm = LibraryPackageManager()
     lm.set_log_level(logging.ERROR if json_output else logging.DEBUG)
     lib_id = lm.reveal_registry_package_id(library)
@@ -451,86 +386,7 @@ def lib_show(library, json_output):
     lib = regclient.fetch_json_data(
         "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h"
     )
-    if json_output:
-        return click.echo(json.dumps(lib))
-
-    title = "{ownername}/{name}".format(**lib)
-    click.secho(title, fg="cyan")
-    click.echo("=" * len(title))
-    click.echo(lib["description"])
-    click.echo()
-
-    click.secho("ID: %d" % lib["id"])
-    click.echo(
-        "Version: %s, released %s"
-        % (
-            lib["version"]["name"],
-            util.parse_datetime(lib["version"]["released"]).strftime("%c"),
-        )
-    )
-    click.echo("Manifest: %s" % lib["confurl"])
-    for key in ("homepage", "repository", "license"):
-        if key not in lib or not lib[key]:
-            continue
-        if isinstance(lib[key], list):
-            click.echo("%s: %s" % (key.capitalize(), ", ".join(lib[key])))
-        else:
-            click.echo("%s: %s" % (key.capitalize(), lib[key]))
-
-    blocks = []
-
-    _authors = []
-    for author in lib.get("authors", []):
-        _data = []
-        for key in ("name", "email", "url", "maintainer"):
-            if not author.get(key):
-                continue
-            if key == "email":
-                _data.append("<%s>" % author[key])
-            elif key == "maintainer":
-                _data.append("(maintainer)")
-            else:
-                _data.append(author[key])
-        _authors.append(" ".join(_data))
-    if _authors:
-        blocks.append(("Authors", _authors))
-
-    blocks.append(("Keywords", lib["keywords"]))
-    for key in ("frameworks", "platforms"):
-        if key not in lib or not lib[key]:
-            continue
-        blocks.append(("Compatible %s" % key, [i["title"] for i in lib[key]]))
-    blocks.append(("Headers", lib["headers"]))
-    blocks.append(("Examples", lib["examples"]))
-    blocks.append(
-        (
-            "Versions",
-            [
-                "%s, released %s"
-                % (v["name"], util.parse_datetime(v["released"]).strftime("%c"))
-                for v in lib["versions"]
-            ],
-        )
-    )
-    blocks.append(
-        (
-            "Unique Downloads",
-            [
-                "Today: %s" % lib["dlstats"]["day"],
-                "Week: %s" % lib["dlstats"]["week"],
-                "Month: %s" % lib["dlstats"]["month"],
-            ],
-        )
-    )
-
-    for (title, rows) in blocks:
-        click.echo()
-        click.secho(title, bold=True)
-        click.echo("-" * len(title))
-        for row in rows:
-            click.echo(row)
-
-    return True
+    return click.echo(json.dumps(lib))
 
 
 @cli.command("register", short_help="Deprecated")
@@ -544,76 +400,18 @@ def lib_register(config_url):  # pylint: disable=unused-argument
 @cli.command("stats", short_help="Library Registry Statistics")
 @click.option("--json-output", is_flag=True)
 def lib_stats(json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease visit "
+            "https://registry.platformio.org\n",
+            fg="yellow",
+        )
+        return None
+
     regclient = LibraryPackageManager().get_registry_client_instance()
     result = regclient.fetch_json_data("get", "/v2/lib/stats", x_cache_valid="1h")
-
-    if json_output:
-        return click.echo(json.dumps(result))
-
-    for key in ("updated", "added"):
-        tabular_data = [
-            (
-                click.style(item["name"], fg="cyan"),
-                util.parse_datetime(item["date"]).strftime("%c"),
-                "https://platformio.org/lib/show/%s/%s"
-                % (item["id"], quote(item["name"])),
-            )
-            for item in result.get(key, [])
-        ]
-        table = tabulate(
-            tabular_data,
-            headers=[click.style("RECENTLY " + key.upper(), bold=True), "Date", "URL"],
-        )
-        click.echo(table)
-        click.echo()
-
-    for key in ("lastkeywords", "topkeywords"):
-        tabular_data = [
-            (
-                click.style(name, fg="cyan"),
-                "https://platformio.org/lib/search?query=" + quote("keyword:%s" % name),
-            )
-            for name in result.get(key, [])
-        ]
-        table = tabulate(
-            tabular_data,
-            headers=[
-                click.style(
-                    ("RECENT" if key == "lastkeywords" else "POPULAR") + " KEYWORDS",
-                    bold=True,
-                ),
-                "URL",
-            ],
-        )
-        click.echo(table)
-        click.echo()
-
-    for key, title in (("dlday", "Today"), ("dlweek", "Week"), ("dlmonth", "Month")):
-        tabular_data = [
-            (
-                click.style(item["name"], fg="cyan"),
-                "https://platformio.org/lib/show/%s/%s"
-                % (item["id"], quote(item["name"])),
-            )
-            for item in result.get(key, [])
-        ]
-        table = tabulate(
-            tabular_data,
-            headers=[click.style("FEATURED: " + title.upper(), bold=True), "URL"],
-        )
-        click.echo(table)
-        click.echo()
-
-    return True
-
-
-def print_storage_header(storage_dirs, storage_dir):
-    if storage_dirs and storage_dirs[0] != storage_dir:
-        click.echo("")
-    click.echo(
-        click.style("Library Storage: ", bold=True)
-        + click.style(storage_dir, fg="blue")
-    )
+    return click.echo(json.dumps(result))
 
 
 def print_lib_item(item):

platformio/commands/lib/helpers.py

@@ -1,104 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-from platformio import util
-from platformio.compat import ci_strings_are_equal
-from platformio.package.manager.platform import PlatformPackageManager
-from platformio.package.meta import PackageSpec
-from platformio.platform.factory import PlatformFactory
-from platformio.project.config import ProjectConfig
-from platformio.project.exception import InvalidProjectConfError
-
-
-@util.memoized(expire="60s")
-def get_builtin_libs(storage_names=None):
-    # pylint: disable=import-outside-toplevel
-    from platformio.package.manager.library import LibraryPackageManager
-
-    items = []
-    storage_names = storage_names or []
-    pm = PlatformPackageManager()
-    for pkg in pm.get_installed():
-        p = PlatformFactory.new(pkg)
-        for storage in p.get_lib_storages():
-            if storage_names and storage["name"] not in storage_names:
-                continue
-            lm = LibraryPackageManager(storage["path"])
-            items.append(
-                {
-                    "name": storage["name"],
-                    "path": storage["path"],
-                    "items": lm.legacy_get_installed(),
-                }
-            )
-    return items
-
-
-def is_builtin_lib(name):
-    for storage in get_builtin_libs():
-        for lib in storage["items"]:
-            if lib.get("name") == name:
-                return True
-    return False
-
-
-def ignore_deps_by_specs(deps, specs):
-    result = []
-    for dep in deps:
-        depspec = PackageSpec(dep)
-        if depspec.external:
-            result.append(dep)
-            continue
-        ignore_conditions = []
-        for spec in specs:
-            if depspec.owner:
-                ignore_conditions.append(
-                    ci_strings_are_equal(depspec.owner, spec.owner)
-                    and ci_strings_are_equal(depspec.name, spec.name)
-                )
-            else:
-                ignore_conditions.append(ci_strings_are_equal(depspec.name, spec.name))
-        if not any(ignore_conditions):
-            result.append(dep)
-    return result
-
-
-def save_project_libdeps(project_dir, specs, environments=None, action="add"):
-    config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini"))
-    config.validate(environments)
-    for env in config.envs():
-        if environments and env not in environments:
-            continue
-        config.expand_interpolations = False
-        candidates = []
-        try:
-            candidates = ignore_deps_by_specs(
-                config.get("env:" + env, "lib_deps"), specs
-            )
-        except InvalidProjectConfError:
-            pass
-        if action == "add":
-            candidates.extend(spec.as_dependency() for spec in specs)
-        if candidates:
-            result = []
-            for item in candidates:
-                item = item.strip()
-                if item and item not in result:
-                    result.append(item)
-            config.set("env:" + env, "lib_deps", result)
-        elif config.has_option("env:" + env, "lib_deps"):
-            config.remove_option("env:" + env, "lib_deps")
-    config.save()

platformio/commands/org.py

@@ -1,165 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import json
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.account import AccountClient
-from platformio.commands.account import validate_email, validate_username
-
-
-@click.group("org", short_help="Manage organizations")
-def cli():
-    pass
-
-
-def validate_orgname(value):
-    return validate_username(value, "Organization name")
-
-
-@cli.command("create", short_help="Create a new organization")
-@click.argument(
-    "orgname",
-    callback=lambda _, __, value: validate_orgname(value),
-)
-@click.option(
-    "--email", callback=lambda _, __, value: validate_email(value) if value else value
-)
-@click.option(
-    "--displayname",
-)
-def org_create(orgname, email, displayname):
-    client = AccountClient()
-    client.create_org(orgname, email, displayname)
-    return click.secho(
-        "The organization `%s` has been successfully created." % orgname,
-        fg="green",
-    )
-
-
-@cli.command("list", short_help="List organizations and their members")
-@click.option("--json-output", is_flag=True)
-def org_list(json_output):
-    client = AccountClient()
-    orgs = client.list_orgs()
-    if json_output:
-        return click.echo(json.dumps(orgs))
-    if not orgs:
-        return click.echo("You do not have any organization")
-    for org in orgs:
-        click.echo()
-        click.secho(org.get("orgname"), fg="cyan")
-        click.echo("-" * len(org.get("orgname")))
-        data = []
-        if org.get("displayname"):
-            data.append(("Display Name:", org.get("displayname")))
-        if org.get("email"):
-            data.append(("Email:", org.get("email")))
-        data.append(
-            (
-                "Owners:",
-                ", ".join((owner.get("username") for owner in org.get("owners"))),
-            )
-        )
-        click.echo(tabulate(data, tablefmt="plain"))
-    return click.echo()
-
-
-@cli.command("update", short_help="Update organization")
-@click.argument("cur_orgname")
-@click.option(
-    "--orgname",
-    callback=lambda _, __, value: validate_orgname(value),
-    help="A new orgname",
-)
-@click.option("--email")
-@click.option("--displayname")
-def org_update(cur_orgname, **kwargs):
-    client = AccountClient()
-    org = client.get_org(cur_orgname)
-    del org["owners"]
-    new_org = org.copy()
-    if not any(kwargs.values()):
-        for field in org:
-            new_org[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=org[field]
-            )
-            if field == "email":
-                validate_email(new_org[field])
-            if field == "orgname":
-                validate_orgname(new_org[field])
-    else:
-        new_org.update(
-            {key.replace("new_", ""): value for key, value in kwargs.items() if value}
-        )
-    client.update_org(cur_orgname, new_org)
-    return click.secho(
-        "The organization `%s` has been successfully updated." % cur_orgname,
-        fg="green",
-    )
-
-
-@cli.command("destroy", short_help="Destroy organization")
-@click.argument("orgname")
-def account_destroy(orgname):
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the `%s` organization account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % orgname,
-        abort=True,
-    )
-    client.destroy_org(orgname)
-    return click.secho(
-        "Organization `%s` has been destroyed." % orgname,
-        fg="green",
-    )
-
-
-@cli.command("add", short_help="Add a new owner to organization")
-@click.argument(
-    "orgname",
-)
-@click.argument(
-    "username",
-)
-def org_add_owner(orgname, username):
-    client = AccountClient()
-    client.add_org_owner(orgname, username)
-    return click.secho(
-        "The new owner `%s` has been successfully added to the `%s` organization."
-        % (username, orgname),
-        fg="green",
-    )
-
-
-@cli.command("remove", short_help="Remove an owner from organization")
-@click.argument(
-    "orgname",
-)
-@click.argument(
-    "username",
-)
-def org_remove_owner(orgname, username):
-    client = AccountClient()
-    client.remove_org_owner(orgname, username)
-    return click.secho(
-        "The `%s` owner has been successfully removed from the `%s` organization."
-        % (username, orgname),
-        fg="green",
-    )

platformio/commands/platform.py

@@ -18,10 +18,13 @@ import os
 
 import click
 
-from platformio.commands import PlatformioCLI
-from platformio.commands.boards import print_boards
 from platformio.exception import UserSideException
-from platformio.package.exception import UnknownPackageError
+from platformio.package.commands.install import package_install_cmd
+from platformio.package.commands.list import package_list_cmd
+from platformio.package.commands.search import package_search_cmd
+from platformio.package.commands.show import package_show_cmd
+from platformio.package.commands.uninstall import package_uninstall_cmd
+from platformio.package.commands.update import package_update_cmd
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.meta import PackageItem, PackageSpec
 from platformio.package.version import get_original_version
@@ -31,18 +34,23 @@ from platformio.platform.factory import PlatformFactory
 
 @click.group(short_help="Platform manager", hidden=True)
 def cli():
-    if not PlatformioCLI.in_silence():
-        click.secho(
-            "\nWARNING!!! This command is deprecated and will be removed in "
-            "the next releases. \nPlease use `pio pkg` instead.\n",
-            fg="yellow",
-        )
+    pass
 
 
 @cli.command("search", short_help="Search for development platform")
 @click.argument("query", required=False)
 @click.option("--json-output", is_flag=True)
-def platform_search(query, json_output):
+@click.pass_context
+def platform_search(ctx, query, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg search` instead.\n",
+            fg="yellow",
+        )
+        query = query or ""
+        return ctx.invoke(package_search_cmd, query=f"type:platform {query}".strip())
+
     platforms = []
     for platform in _get_registry_platforms():
         if query == "all":
@@ -55,17 +63,23 @@ def platform_search(query, json_output):
                 platform["name"], with_boards=False, expose_packages=False
             )
         )
-
-    if json_output:
-        click.echo(json.dumps(platforms))
-    else:
-        _print_platforms(platforms)
+    click.echo(json.dumps(platforms))
+    return None
 
 
 @cli.command("frameworks", short_help="List supported frameworks, SDKs")
 @click.argument("query", required=False)
 @click.option("--json-output", is_flag=True)
 def platform_frameworks(query, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease visit https://docs.platformio.org"
+            "/en/latest/frameworks/index.html\n",
+            fg="yellow",
+        )
+        return
+
     regclient = PlatformPackageManager().get_registry_client_instance()
     frameworks = []
     for framework in regclient.fetch_json_data(
@@ -85,15 +99,21 @@ def platform_frameworks(query, json_output):
         frameworks.append(framework)
 
     frameworks = sorted(frameworks, key=lambda manifest: manifest["name"])
-    if json_output:
-        click.echo(json.dumps(frameworks))
-    else:
-        _print_platforms(frameworks)
+    click.echo(json.dumps(frameworks))
 
 
 @cli.command("list", short_help="List installed development platforms")
 @click.option("--json-output", is_flag=True)
-def platform_list(json_output):
+@click.pass_context
+def platform_list(ctx, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg list` instead.\n",
+            fg="yellow",
+        )
+        return ctx.invoke(package_list_cmd, **{"global": True, "only_platforms": True})
+
     platforms = []
     pm = PlatformPackageManager()
     for pkg in pm.get_installed():
@@ -102,75 +122,27 @@ def platform_list(json_output):
         )
 
     platforms = sorted(platforms, key=lambda manifest: manifest["name"])
-    if json_output:
-        click.echo(json.dumps(platforms))
-    else:
-        _print_platforms(platforms)
+    click.echo(json.dumps(platforms))
+    return None
 
 
 @cli.command("show", short_help="Show details about development platform")
 @click.argument("platform")
 @click.option("--json-output", is_flag=True)
-def platform_show(platform, json_output):  # pylint: disable=too-many-branches
+@click.pass_context
+def platform_show(ctx, platform, json_output):  # pylint: disable=too-many-branches
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg show` instead.\n",
+            fg="yellow",
+        )
+        return ctx.invoke(package_show_cmd, pkg_type="platform", spec=platform)
+
     data = _get_platform_data(platform)
     if not data:
         raise UnknownPlatform(platform)
-    if json_output:
-        return click.echo(json.dumps(data))
-
-    dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"]
-    click.echo(
-        "{dep} ~ {title}".format(dep=click.style(dep, fg="cyan"), title=data["title"])
-    )
-    click.echo("=" * (3 + len(dep + data["title"])))
-    click.echo(data["description"])
-    click.echo()
-    if "version" in data:
-        click.echo("Version: %s" % data["version"])
-    if data["homepage"]:
-        click.echo("Home: %s" % data["homepage"])
-    if data["repository"]:
-        click.echo("Repository: %s" % data["repository"])
-    if data["url"]:
-        click.echo("Vendor: %s" % data["url"])
-    if data["license"]:
-        click.echo("License: %s" % data["license"])
-    if data["frameworks"]:
-        click.echo("Frameworks: %s" % ", ".join(data["frameworks"]))
-
-    if not data["packages"]:
-        return None
-
-    if not isinstance(data["packages"][0], dict):
-        click.echo("Packages: %s" % ", ".join(data["packages"]))
-    else:
-        click.echo()
-        click.secho("Packages", bold=True)
-        click.echo("--------")
-        for item in data["packages"]:
-            click.echo()
-            click.echo("Package %s" % click.style(item["name"], fg="yellow"))
-            click.echo("-" * (8 + len(item["name"])))
-            if item["type"]:
-                click.echo("Type: %s" % item["type"])
-            click.echo("Requirements: %s" % item["requirements"])
-            click.echo(
-                "Installed: %s" % ("Yes" if item.get("version") else "No (optional)")
-            )
-            if "version" in item:
-                click.echo("Version: %s" % item["version"])
-            if "originalVersion" in item:
-                click.echo("Original version: %s" % item["originalVersion"])
-            if "description" in item:
-                click.echo("Description: %s" % item["description"])
-
-    if data["boards"]:
-        click.echo()
-        click.secho("Boards", bold=True)
-        click.echo("------")
-        print_boards(data["boards"])
-
-    return True
+    return click.echo(json.dumps(data))
 
 
 @cli.command("install", short_help="Install new development platform")
@@ -186,7 +158,9 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     is_flag=True,
     help="Reinstall/redownload dev/platform and its packages if exist",
 )
-def platform_install(  # pylint: disable=too-many-arguments,too-many-locals
+@click.pass_context
+def platform_install(  # pylint: disable=too-many-arguments
+    ctx,
     platforms,
     with_package,
     without_package,
@@ -195,70 +169,42 @@ def platform_install(  # pylint: disable=too-many-arguments,too-many-locals
     silent,
     force,
 ):
-    def _find_pkg_names(p, candidates):
-        result = []
-        for candidate in candidates:
-            found = False
-            # lookup by package types
-            for _name, _opts in p.packages.items():
-                if _opts.get("type") == candidate:
-                    result.append(_name)
-                    found = True
-            if (
-                p.frameworks
-                and candidate.startswith("framework-")
-                and candidate[10:] in p.frameworks
-            ):
-                result.append(p.frameworks[candidate[10:]]["package"])
-                found = True
-            if not found:
-                result.append(candidate)
-        return result
-
-    pm = PlatformPackageManager()
-    pm.set_log_level(logging.WARN if silent else logging.DEBUG)
-    for platform in platforms:
-        if with_package or without_package or with_all_packages:
-            pkg = pm.install(platform, skip_dependencies=True)
-            p = PlatformFactory.new(pkg)
-            if with_all_packages:
-                with_package = list(p.packages)
-            with_package = set(_find_pkg_names(p, with_package or []))
-            without_package = set(_find_pkg_names(p, without_package or []))
-            upkgs = with_package | without_package
-            ppkgs = set(p.packages)
-            if not upkgs.issubset(ppkgs):
-                raise UnknownPackageError(", ".join(upkgs - ppkgs))
-            for name, options in p.packages.items():
-                if name in without_package:
-                    continue
-                if name in with_package or not (
-                    skip_default_package or options.get("optional", False)
-                ):
-                    p.pm.install(p.get_package_spec(name), force=force)
-        else:
-            pkg = pm.install(platform, skip_dependencies=skip_default_package)
-
-        if pkg and not silent:
-            click.secho(
-                "The platform '%s' has been successfully installed!\n"
-                "The rest of the packages will be installed later "
-                "depending on your build environment." % platform,
-                fg="green",
-            )
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg install` instead.\n",
+        fg="yellow",
+    )
+    ctx.invoke(
+        package_install_cmd,
+        **{
+            "global": True,
+            "platforms": platforms,
+            "skip_dependencies": (
+                not with_all_packages
+                and (with_package or without_package or skip_default_package)
+            ),
+            "silent": silent,
+            "force": force,
+        },
+    )
 
 
 @cli.command("uninstall", short_help="Uninstall development platform")
 @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]")
-def platform_uninstall(platforms):
-    pm = PlatformPackageManager()
-    pm.set_log_level(logging.DEBUG)
-    for platform in platforms:
-        if pm.uninstall(platform):
-            click.secho(
-                "The platform '%s' has been successfully removed!" % platform,
-                fg="green",
-            )
+@click.pass_context
+def platform_uninstall(ctx, platforms):
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg uninstall` instead.\n",
+        fg="yellow",
+    )
+    ctx.invoke(
+        package_uninstall_cmd,
+        **{
+            "global": True,
+            "platforms": platforms,
+        },
+    )
 
 
 @cli.command("update", short_help="Update installed development platforms")
@@ -277,61 +223,59 @@ def platform_uninstall(platforms):
 )
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option("--json-output", is_flag=True)
+@click.pass_context
 def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
-    platforms, only_check, dry_run, silent, json_output, **_
+    ctx, platforms, only_check, dry_run, silent, json_output, **_
 ):
+    only_check = dry_run or only_check
+
     if only_check and not json_output:
         raise UserSideException(
             "This command is deprecated, please use `pio pkg outdated` instead"
         )
 
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg update` instead.\n",
+            fg="yellow",
+        )
+        return ctx.invoke(
+            package_update_cmd,
+            **{
+                "global": True,
+                "platforms": platforms,
+                "silent": silent,
+            },
+        )
+
     pm = PlatformPackageManager()
     pm.set_log_level(logging.WARN if silent else logging.DEBUG)
     platforms = platforms or pm.get_installed()
-    only_check = dry_run or only_check
-
-    if only_check and json_output:
-        result = []
-        for platform in platforms:
-            spec = None
-            pkg = None
-            if isinstance(platform, PackageItem):
-                pkg = platform
-            else:
-                spec = PackageSpec(platform)
-                pkg = pm.get_package(spec)
-            if not pkg:
-                continue
-            outdated = pm.outdated(pkg, spec)
-            if (
-                not outdated.is_outdated(allow_incompatible=True)
-                and not PlatformFactory.new(pkg).are_outdated_packages()
-            ):
-                continue
-            data = _get_installed_platform_data(
-                pkg, with_boards=False, expose_packages=False
-            )
-            if outdated.is_outdated(allow_incompatible=True):
-                data["versionLatest"] = (
-                    str(outdated.latest) if outdated.latest else None
-                )
-            result.append(data)
-        return click.echo(json.dumps(result))
-
+    result = []
     for platform in platforms:
-        click.echo(
-            "Platform %s"
-            % click.style(
-                platform.metadata.name
-                if isinstance(platform, PackageItem)
-                else platform,
-                fg="cyan",
-            )
+        spec = None
+        pkg = None
+        if isinstance(platform, PackageItem):
+            pkg = platform
+        else:
+            spec = PackageSpec(platform)
+            pkg = pm.get_package(spec)
+        if not pkg:
+            continue
+        outdated = pm.outdated(pkg, spec)
+        if (
+            not outdated.is_outdated(allow_incompatible=True)
+            and not PlatformFactory.new(pkg).are_outdated_packages()
+        ):
+            continue
+        data = _get_installed_platform_data(
+            pkg, with_boards=False, expose_packages=False
         )
-        click.echo("--------")
-        pm.update(platform)
-        click.echo()
-
+        if outdated.is_outdated(allow_incompatible=True):
+            data["versionLatest"] = str(outdated.latest) if outdated.latest else None
+        result.append(data)
+    click.echo(json.dumps(result))
     return True
 
 
@@ -340,32 +284,6 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
 #
 
 
-def _print_platforms(platforms):
-    for platform in platforms:
-        click.echo(
-            "{name} ~ {title}".format(
-                name=click.style(platform["name"], fg="cyan"), title=platform["title"]
-            )
-        )
-        click.echo("=" * (3 + len(platform["name"] + platform["title"])))
-        click.echo(platform["description"])
-        click.echo()
-        if "homepage" in platform:
-            click.echo("Home: %s" % platform["homepage"])
-        if "frameworks" in platform and platform["frameworks"]:
-            click.echo("Frameworks: %s" % ", ".join(platform["frameworks"]))
-        if "packages" in platform:
-            click.echo("Packages: %s" % ", ".join(platform["packages"]))
-        if "version" in platform:
-            if "__src_url" in platform:
-                click.echo(
-                    "Version: %s (%s)" % (platform["version"], platform["__src_url"])
-                )
-            else:
-                click.echo("Version: " + platform["version"])
-        click.echo()
-
-
 def _get_registry_platforms():
     regclient = PlatformPackageManager().get_registry_client_instance()
     return regclient.fetch_json_data("get", "/v2/platforms", x_cache_valid="1d")

platformio/commands/system/command.py

@@ -1,191 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import platform
-import sys
-
-import click
-from tabulate import tabulate
-
-from platformio import __version__, compat, fs, proc, util
-from platformio.commands.system.completion import (
-    ShellType,
-    get_completion_install_path,
-    install_completion_code,
-    uninstall_completion_code,
-)
-from platformio.commands.system.prune import (
-    prune_cached_data,
-    prune_core_packages,
-    prune_platform_packages,
-)
-from platformio.package.manager.library import LibraryPackageManager
-from platformio.package.manager.platform import PlatformPackageManager
-from platformio.package.manager.tool import ToolPackageManager
-from platformio.project.config import ProjectConfig
-
-
-@click.group("system", short_help="Miscellaneous system commands")
-def cli():
-    pass
-
-
-@cli.command("info", short_help="Display system-wide information")
-@click.option("--json-output", is_flag=True)
-def system_info(json_output):
-    project_config = ProjectConfig()
-    data = {}
-    data["core_version"] = {"title": "PlatformIO Core", "value": __version__}
-    data["python_version"] = {
-        "title": "Python",
-        "value": "{0}.{1}.{2}-{3}.{4}".format(*list(sys.version_info)),
-    }
-    data["system"] = {"title": "System Type", "value": util.get_systype()}
-    data["platform"] = {"title": "Platform", "value": platform.platform(terse=True)}
-    data["filesystem_encoding"] = {
-        "title": "File System Encoding",
-        "value": compat.get_filesystem_encoding(),
-    }
-    data["locale_encoding"] = {
-        "title": "Locale Encoding",
-        "value": compat.get_locale_encoding(),
-    }
-    data["core_dir"] = {
-        "title": "PlatformIO Core Directory",
-        "value": project_config.get("platformio", "core_dir"),
-    }
-    data["platformio_exe"] = {
-        "title": "PlatformIO Core Executable",
-        "value": proc.where_is_program(
-            "platformio.exe" if compat.IS_WINDOWS else "platformio"
-        ),
-    }
-    data["python_exe"] = {
-        "title": "Python Executable",
-        "value": proc.get_pythonexe_path(),
-    }
-    data["global_lib_nums"] = {
-        "title": "Global Libraries",
-        "value": len(LibraryPackageManager().get_installed()),
-    }
-    data["dev_platform_nums"] = {
-        "title": "Development Platforms",
-        "value": len(PlatformPackageManager().get_installed()),
-    }
-    data["package_tool_nums"] = {
-        "title": "Tools & Toolchains",
-        "value": len(
-            ToolPackageManager(
-                project_config.get("platformio", "packages_dir")
-            ).get_installed()
-        ),
-    }
-
-    click.echo(
-        json.dumps(data)
-        if json_output
-        else tabulate([(item["title"], item["value"]) for item in data.values()])
-    )
-
-
-@cli.command("prune", short_help="Remove unused data")
-@click.option("--force", "-f", is_flag=True, help="Do not prompt for confirmation")
-@click.option(
-    "--dry-run", is_flag=True, help="Do not prune, only show data that will be removed"
-)
-@click.option("--cache", is_flag=True, help="Prune only cached data")
-@click.option(
-    "--core-packages", is_flag=True, help="Prune only unnecessary core packages"
-)
-@click.option(
-    "--platform-packages",
-    is_flag=True,
-    help="Prune only unnecessary development platform packages",
-)
-def system_prune(force, dry_run, cache, core_packages, platform_packages):
-    if dry_run:
-        click.secho(
-            "Dry run mode (do not prune, only show data that will be removed)",
-            fg="yellow",
-        )
-        click.echo()
-
-    reclaimed_cache = 0
-    reclaimed_core_packages = 0
-    reclaimed_platform_packages = 0
-    prune_all = not any([cache, core_packages, platform_packages])
-
-    if cache or prune_all:
-        reclaimed_cache = prune_cached_data(force, dry_run)
-        click.echo()
-
-    if core_packages or prune_all:
-        reclaimed_core_packages = prune_core_packages(force, dry_run)
-        click.echo()
-
-    if platform_packages or prune_all:
-        reclaimed_platform_packages = prune_platform_packages(force, dry_run)
-        click.echo()
-
-    click.secho(
-        "Total reclaimed space: %s"
-        % fs.humanize_file_size(
-            reclaimed_cache + reclaimed_core_packages + reclaimed_platform_packages
-        ),
-        fg="green",
-    )
-
-
-@cli.group("completion", short_help="Shell completion support")
-def completion():
-    pass
-
-
-@completion.command("install", short_help="Install shell completion files/code")
-@click.argument("shell", type=click.Choice([t.value for t in ShellType]))
-@click.option(
-    "--path",
-    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
-    help="Custom installation path of the code to be evaluated by the shell. "
-    "The standard installation path is used by default.",
-)
-def completion_install(shell, path):
-    shell = ShellType(shell)
-    path = path or get_completion_install_path(shell)
-    install_completion_code(shell, path)
-    click.echo(
-        "PlatformIO CLI completion has been installed for %s shell to %s \n"
-        "Please restart a current shell session."
-        % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue"))
-    )
-
-
-@completion.command("uninstall", short_help="Uninstall shell completion files/code")
-@click.argument("shell", type=click.Choice([t.value for t in ShellType]))
-@click.option(
-    "--path",
-    type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
-    help="Custom installation path of the code to be evaluated by the shell. "
-    "The standard installation path is used by default.",
-)
-def completion_uninstall(shell, path):
-    shell = ShellType(shell)
-    path = path or get_completion_install_path(shell)
-    uninstall_completion_code(shell, path)
-    click.echo(
-        "PlatformIO CLI completion has been uninstalled for %s shell from %s \n"
-        "Please restart a current shell session."
-        % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue"))
-    )

platformio/commands/team.py

@@ -1,212 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import json
-import re
-
-import click
-from tabulate import tabulate
-
-from platformio.clients.account import AccountClient
-
-
-def validate_orgname_teamname(value, teamname_validate=False):
-    if ":" not in value:
-        raise click.BadParameter(
-            "Please specify organization and team name in the next"
-            " format - orgname:teamname. For example, mycompany:DreamTeam"
-        )
-    teamname = str(value.strip().split(":", 1)[1])
-    if teamname_validate:
-        validate_teamname(teamname)
-    return value
-
-
-def validate_teamname(value):
-    if not value:
-        return value
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I):
-        raise click.BadParameter(
-            "Invalid team name format. "
-            "Team name must only contain alphanumeric characters, "
-            "single hyphens, underscores, spaces. It can not "
-            "begin or end with a hyphen or a underscore and must"
-            " not be longer than 20 characters."
-        )
-    return value
-
-
-@click.group("team", short_help="Manage organization teams")
-def cli():
-    pass
-
-
-@cli.command("create", short_help="Create a new team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(
-        value, teamname_validate=True
-    ),
-)
-@click.option(
-    "--description",
-)
-def team_create(orgname_teamname, description):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.create_team(orgname, teamname, description)
-    return click.secho(
-        "The team %s has been successfully created." % teamname,
-        fg="green",
-    )
-
-
-@cli.command("list", short_help="List teams")
-@click.argument("orgname", required=False)
-@click.option("--json-output", is_flag=True)
-def team_list(orgname, json_output):
-    client = AccountClient()
-    data = {}
-    if not orgname:
-        for item in client.list_orgs():
-            teams = client.list_teams(item.get("orgname"))
-            data[item.get("orgname")] = teams
-    else:
-        teams = client.list_teams(orgname)
-        data[orgname] = teams
-    if json_output:
-        return click.echo(json.dumps(data[orgname] if orgname else data))
-    if not any(data.values()):
-        return click.secho("You do not have any teams.", fg="yellow")
-    for org_name in data:
-        for team in data[org_name]:
-            click.echo()
-            click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan")
-            click.echo("-" * len("%s:%s" % (org_name, team.get("name"))))
-            table_data = []
-            if team.get("description"):
-                table_data.append(("Description:", team.get("description")))
-            table_data.append(
-                (
-                    "Members:",
-                    ", ".join(
-                        (member.get("username") for member in team.get("members"))
-                    )
-                    if team.get("members")
-                    else "-",
-                )
-            )
-            click.echo(tabulate(table_data, tablefmt="plain"))
-    return click.echo()
-
-
-@cli.command("update", short_help="Update team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.option(
-    "--name",
-    callback=lambda _, __, value: validate_teamname(value),
-    help="A new team name",
-)
-@click.option(
-    "--description",
-)
-def team_update(orgname_teamname, **kwargs):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    team = client.get_team(orgname, teamname)
-    del team["id"]
-    del team["members"]
-    new_team = team.copy()
-    if not any(kwargs.values()):
-        for field in team:
-            new_team[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=team[field]
-            )
-            if field == "name":
-                validate_teamname(new_team[field])
-    else:
-        new_team.update({key: value for key, value in kwargs.items() if value})
-    client.update_team(orgname, teamname, new_team)
-    return click.secho(
-        "The team %s has been successfully updated." % teamname,
-        fg="green",
-    )
-
-
-@cli.command("destroy", short_help="Destroy a team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-def team_destroy(orgname_teamname):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    click.confirm(
-        click.style(
-            "Are you sure you want to destroy the %s team?" % teamname, fg="yellow"
-        ),
-        abort=True,
-    )
-    client = AccountClient()
-    client.destroy_team(orgname, teamname)
-    return click.secho(
-        "The team %s has been successfully destroyed." % teamname,
-        fg="green",
-    )
-
-
-@cli.command("add", short_help="Add a new member to team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.argument(
-    "username",
-)
-def team_add_member(orgname_teamname, username):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.add_team_member(orgname, teamname, username)
-    return click.secho(
-        "The new member %s has been successfully added to the %s team."
-        % (username, teamname),
-        fg="green",
-    )
-
-
-@cli.command("remove", short_help="Remove a member from team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.argument("username")
-def team_remove_owner(orgname_teamname, username):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.remove_team_member(orgname, teamname, username)
-    return click.secho(
-        "The %s member has been successfully removed from the %s team."
-        % (username, teamname),
-        fg="green",
-    )

platformio/commands/upgrade.py

@@ -13,24 +13,34 @@
 # limitations under the License.
 
 import json
-import os
 import re
-from zipfile import ZipFile
+import subprocess
 
 import click
 
-from platformio import VERSION, __version__, app, exception
-from platformio.clients.http import fetch_remote_content
-from platformio.compat import IS_WINDOWS
+from platformio import VERSION, __install_requires__, __version__, app, exception
+from platformio.http import fetch_remote_content
 from platformio.package.manager.core import update_core_packages
-from platformio.proc import exec_command, get_pythonexe_path
-from platformio.project.helpers import get_project_cache_dir
+from platformio.proc import get_pythonexe_path
+
+PYPI_JSON_URL = "https://pypi.org/pypi/platformio/json"
+DEVELOP_ZIP_URL = "https://github.com/platformio/platformio-core/archive/develop.zip"
+DEVELOP_INIT_SCRIPT_URL = (
+    "https://raw.githubusercontent.com/platformio/platformio-core"
+    "/develop/platformio/__init__.py"
+)
 
 
 @click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version")
 @click.option("--dev", is_flag=True, help="Use development branch")
-def cli(dev):
+@click.option("--only-dependencies", is_flag=True)
+@click.option("--verbose", "-v", is_flag=True)
+def cli(dev, only_dependencies, verbose):
+    if only_dependencies:
+        return upgrade_pypi_dependencies(verbose)
+
     update_core_packages()
+
     if not dev and __version__ == get_latest_version():
         return click.secho(
             "You're up-to-date!\nPlatformIO %s is currently the "
@@ -38,29 +48,35 @@ def cli(dev):
             fg="green",
         )
 
-    click.secho("Please wait while upgrading PlatformIO ...", fg="yellow")
+    click.secho("Please wait while upgrading PlatformIO Core ...", fg="yellow")
 
+    python_exe = get_pythonexe_path()
     to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".")
-    cmds = (
-        ["pip", "install", "--upgrade", download_dist_package(to_develop)],
-        ["platformio", "--version"],
-    )
+    pkg_spec = DEVELOP_ZIP_URL if to_develop else "platformio"
 
-    cmd = None
-    r = {}
     try:
-        for cmd in cmds:
-            cmd = [get_pythonexe_path(), "-m"] + cmd
-            r = exec_command(cmd)
+        # PIO Core
+        subprocess.run(
+            [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec],
+            check=True,
+            stdout=subprocess.PIPE if not verbose else None,
+        )
 
-            # try pip with disabled cache
-            if r["returncode"] != 0 and cmd[2] == "pip":
-                cmd.insert(3, "--no-cache-dir")
-                r = exec_command(cmd)
+        # PyPI dependencies
+        subprocess.run(
+            [python_exe, "-m", "platformio", "upgrade", "--only-dependencies"],
+            check=False,
+            stdout=subprocess.PIPE,
+        )
 
-            assert r["returncode"] == 0
-        assert "version" in r["out"]
-        actual_version = r["out"].strip().split("version", 1)[1].strip()
+        # Check version
+        output = subprocess.run(
+            [python_exe, "-m", "platformio", "--version"],
+            check=True,
+            stdout=subprocess.PIPE,
+        ).stdout.decode()
+        assert "version" in output
+        actual_version = output.split("version", 1)[1].strip()
         click.secho(
             "PlatformIO has been successfully upgraded to %s" % actual_version,
             fg="green",
@@ -71,52 +87,35 @@ def cli(dev):
             click.secho(
                 "Warning! Please restart IDE to affect PIO Home changes", fg="yellow"
             )
-    except Exception as e:  # pylint: disable=broad-except
-        if not r:
-            raise exception.UpgradeError("\n".join([str(cmd), str(e)]))
-        permission_errors = ("permission denied", "not permitted")
-        if any(m in r["err"].lower() for m in permission_errors) and not IS_WINDOWS:
-            click.secho(
-                """
------------------
-Permission denied
------------------
-You need the `sudo` permission to install Python packages. Try
-
-> sudo pip install -U platformio
-
-WARNING! Don't use `sudo` for the rest PlatformIO commands.
-""",
-                fg="yellow",
-                err=True,
-            )
-            raise exception.ReturnErrorCode(1)
-        raise exception.UpgradeError("\n".join([str(cmd), r["out"], r["err"]]))
+    except (AssertionError, subprocess.CalledProcessError) as exc:
+        click.secho(
+            "\nWarning!!! Could not automatically upgrade the PlatformIO Core.",
+            fg="red",
+        )
+        click.secho(
+            "Please upgrade it manually using the following command:\n",
+            fg="red",
+        )
+        click.secho(f'"{python_exe}" -m pip install -U {pkg_spec}\n', fg="cyan")
+        raise exception.ReturnErrorCode(1) from exc
 
     return True
 
 
-def download_dist_package(to_develop):
-    if not to_develop:
-        return "platformio"
-    dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip"
-    cache_dir = get_project_cache_dir()
-    if not os.path.isdir(cache_dir):
-        os.makedirs(cache_dir)
-    pkg_name = os.path.join(cache_dir, "piocoredevelop.zip")
-    try:
-        with open(pkg_name, "wb") as fp:
-            r = exec_command(
-                ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True
-            )
-            assert r["returncode"] == 0
-        # check ZIP structure
-        with ZipFile(pkg_name) as zp:
-            assert zp.testzip() is None
-        return pkg_name
-    except:  # pylint: disable=bare-except
-        pass
-    return dl_url
+def upgrade_pypi_dependencies(verbose):
+    subprocess.run(
+        [
+            get_pythonexe_path(),
+            "-m",
+            "pip",
+            "install",
+            "--upgrade",
+            "pip",
+            *__install_requires__,
+        ],
+        check=True,
+        stdout=subprocess.PIPE if not verbose else None,
+    )
 
 
 def get_latest_version():
@@ -127,16 +126,13 @@ def get_latest_version():
             except:  # pylint: disable=bare-except
                 pass
         return get_pypi_latest_version()
-    except:
-        raise exception.GetLatestVersionError()
+    except Exception as exc:
+        raise exception.GetLatestVersionError() from exc
 
 
 def get_develop_latest_version():
     version = None
-    content = fetch_remote_content(
-        "https://raw.githubusercontent.com/platformio/platformio"
-        "/develop/platformio/__init__.py"
-    )
+    content = fetch_remote_content(DEVELOP_INIT_SCRIPT_URL)
     for line in content.split("\n"):
         line = line.strip()
         if not line.startswith("VERSION"):
@@ -153,5 +149,5 @@ def get_develop_latest_version():
 
 
 def get_pypi_latest_version():
-    content = fetch_remote_content("https://pypi.org/pypi/platformio/json")
+    content = fetch_remote_content(PYPI_JSON_URL)
     return json.loads(content)["info"]["version"]

platformio/compat.py

@@ -17,6 +17,7 @@
 import importlib.util
 import inspect
 import locale
+import shlex
 import sys
 
 from platformio.exception import UserSideException
@@ -29,6 +30,20 @@ else:
     from asyncio import get_event_loop as aio_get_running_loop
 
 
+if sys.version_info >= (3, 8):
+    from shlex import join as shlex_join
+else:
+
+    def shlex_join(split_command):
+        return " ".join(shlex.quote(arg) for arg in split_command)
+
+
+if sys.version_info >= (3, 9):
+    from asyncio import to_thread as aio_to_thread
+else:
+    from starlette.concurrency import run_in_threadpool as aio_to_thread
+
+
 PY2 = sys.version_info[0] == 2  # DO NOT REMOVE IT. ESP8266/ESP32 depend on it
 IS_CYGWIN = sys.platform.startswith("cygwin")
 IS_WINDOWS = WINDOWS = sys.platform.startswith("win")
@@ -41,6 +56,22 @@ def is_bytes(x):
     return isinstance(x, (bytes, memoryview, bytearray))
 
 
+def isascii(text):
+    if sys.version_info >= (3, 7):
+        return text.isascii()
+    for c in text or "":
+        if ord(c) > 127:
+            return False
+    return True
+
+
+def is_terminal():
+    try:
+        return sys.stdout.isatty()
+    except Exception:  # pylint: disable=broad-except
+        return False
+
+
 def ci_strings_are_equal(a, b):
     if a == b:
         return True
@@ -69,10 +100,7 @@ def get_filesystem_encoding():
 
 
 def get_locale_encoding():
-    try:
-        return locale.getdefaultlocale()[1]
-    except ValueError:
-        return None
+    return locale.getpreferredencoding()
 
 
 def get_object_members(obj, ignore_private=True):

platformio/debug/cli.py

@@ -28,9 +28,9 @@ from platformio.debug import helpers
 from platformio.debug.config.factory import DebugConfigFactory
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.debug.process.gdb import GDBClientProcess
+from platformio.exception import ReturnErrorCode
 from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
-from platformio.project.exception import ProjectEnvsNotAvailableError
 from platformio.project.helpers import is_platformio_project
 from platformio.project.options import ProjectOptions
 
@@ -44,16 +44,12 @@ from platformio.project.options import ProjectOptions
     "-d",
     "--project-dir",
     default=os.getcwd,
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True),
 )
 @click.option(
     "-c",
     "--project-conf",
-    type=click.Path(
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
+    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
 )
 @click.option("--environment", "-e", metavar="<environment>")
 @click.option("--load-mode", type=ProjectOptions["env.debug_load_mode"].type)
@@ -61,7 +57,7 @@ from platformio.project.options import ProjectOptions
 @click.option("--interface", type=click.Choice(["gdb"]))
 @click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
 @click.pass_context
-def debug_cmd(
+def cli(
     ctx,
     project_dir,
     project_conf,
@@ -81,61 +77,55 @@ def debug_cmd(
             project_dir = os.getenv(name)
 
     with fs.cd(project_dir):
-        return _debug_in_project_dir(
+        project_config = ProjectConfig.get_instance(project_conf)
+        project_config.validate(envs=[environment] if environment else None)
+        env_name = environment or helpers.get_default_debug_env(project_config)
+
+        if not interface:
+            return helpers.predebug_project(
+                ctx, project_dir, project_config, env_name, False, verbose
+            )
+
+        configure_args = (
             ctx,
-            project_dir,
-            project_conf,
-            environment,
+            project_config,
+            env_name,
             load_mode,
             verbose,
-            interface,
             __unprocessed,
         )
+        if helpers.is_gdbmi_mode():
+            os.environ["PLATFORMIO_DISABLE_PROGRESSBAR"] = "true"
+            stream = helpers.GDBMIConsoleStream()
+            with proc.capture_std_streams(stream):
+                debug_config = _configure(*configure_args)
+            stream.close()
+        else:
+            debug_config = _configure(*configure_args)
+
+        _run(project_dir, debug_config, __unprocessed)
+
+    return None
 
 
-def _debug_in_project_dir(
-    ctx,
-    project_dir,
-    project_conf,
-    environment,
-    load_mode,
-    verbose,
-    interface,
-    __unprocessed,
-):
-    project_config = ProjectConfig.get_instance(project_conf)
-    project_config.validate(envs=[environment] if environment else None)
-    env_name = environment or helpers.get_default_debug_env(project_config)
-
-    if not interface:
-        return helpers.predebug_project(
-            ctx, project_dir, project_config, env_name, False, verbose
-        )
-
-    env_options = project_config.items(env=env_name, as_dict=True)
-    if "platform" not in env_options:
-        raise ProjectEnvsNotAvailableError()
-
+def _configure(ctx, project_config, env_name, load_mode, verbose, __unprocessed):
+    platform = PlatformFactory.from_env(env_name, autoinstall=True)
     debug_config = DebugConfigFactory.new(
-        PlatformFactory.new(env_options["platform"], autoinstall=True),
+        platform,
         project_config,
         env_name,
     )
-
     if "--version" in __unprocessed:
-        return subprocess.run(
-            [debug_config.client_executable_path, "--version"], check=True
+        raise ReturnErrorCode(
+            subprocess.run(
+                [debug_config.client_executable_path, "--version"], check=True
+            ).returncode
         )
 
     try:
         fs.ensure_udev_rules()
-    except exception.InvalidUdevRules as e:
-        click.echo(
-            helpers.escape_gdbmi_stream("~", str(e) + "\n")
-            if helpers.is_gdbmi_mode()
-            else str(e) + "\n",
-            nl=False,
-        )
+    except exception.InvalidUdevRules as exc:
+        click.echo(str(exc))
 
     rebuild_prog = False
     preload = debug_config.load_cmds == ["preload"]
@@ -157,25 +147,10 @@ def _debug_in_project_dir(
         debug_config.load_cmds = []
 
     if rebuild_prog:
-        if helpers.is_gdbmi_mode():
-            click.echo(
-                helpers.escape_gdbmi_stream(
-                    "~", "Preparing firmware for debugging...\n"
-                ),
-                nl=False,
-            )
-            stream = helpers.GDBMIConsoleStream()
-            with proc.capture_std_streams(stream):
-                helpers.predebug_project(
-                    ctx, project_dir, project_config, env_name, preload, verbose
-                )
-            stream.close()
-        else:
-            click.echo("Preparing firmware for debugging...")
-            helpers.predebug_project(
-                ctx, project_dir, project_config, env_name, preload, verbose
-            )
-
+        click.echo("Preparing firmware for debugging...")
+        helpers.predebug_project(
+            ctx, os.getcwd(), project_config, env_name, preload, verbose
+        )
         # save SHA sum of newly created prog
         if load_mode == "modified":
             helpers.is_prog_obsolete(debug_config.program_path)
@@ -183,6 +158,10 @@ def _debug_in_project_dir(
     if not os.path.isfile(debug_config.program_path):
         raise DebugInvalidOptionsError("Program/firmware is missed")
 
+    return debug_config
+
+
+def _run(project_dir, debug_config, __unprocessed):
     loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop()
     asyncio.set_event_loop(loop)
 
@@ -199,5 +178,3 @@ def _debug_in_project_dir(
     finally:
         client.close()
         loop.close()
-
-    return True

platformio/debug/config/base.py

@@ -18,14 +18,13 @@ import os
 from platformio import fs, proc, util
 from platformio.compat import string_types
 from platformio.debug.exception import DebugInvalidOptionsError
-from platformio.debug.helpers import reveal_debug_port
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import load_build_metadata
 from platformio.project.options import ProjectOptions
 
 
 class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
-    def __init__(self, platform, project_config, env_name):
+    def __init__(self, platform, project_config, env_name, port=None):
         self.platform = platform
         self.project_config = project_config
         self.env_name = env_name
@@ -49,6 +48,7 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
         self._load_cmds = None
         self._port = None
 
+        self.port = port
         self.server = self._configure_server()
 
         try:
@@ -119,11 +119,9 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
 
     @property
     def port(self):
-        return reveal_debug_port(
+        return (
             self.env_options.get("debug_port", self.tool_settings.get("port"))
-            or self._port,
-            self.tool_name,
-            self.tool_settings,
+            or self._port
         )
 
     @port.setter
@@ -147,10 +145,10 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
         )
 
     def _load_build_data(self):
-        data = load_build_metadata(os.getcwd(), self.env_name, cache=True)
-        if data:
-            return data
-        raise DebugInvalidOptionsError("Could not load a build configuration")
+        data = load_build_metadata(os.getcwd(), self.env_name, cache=True, debug=True)
+        if not data:
+            raise DebugInvalidOptionsError("Could not load a build configuration")
+        return data
 
     def _configure_server(self):
         # user disabled server in platformio.ini
@@ -205,8 +203,8 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
     def get_init_script(self, debugger):
         try:
             return getattr(self, "%s_INIT_SCRIPT" % debugger.upper())
-        except AttributeError:
-            raise NotImplementedError
+        except AttributeError as exc:
+            raise NotImplementedError from exc
 
     def reveal_patterns(self, source, recursive=True):
         program_path = self.program_path or ""

platformio/debug/config/blackmagic.py

@@ -13,10 +13,11 @@
 # limitations under the License.
 
 from platformio.debug.config.base import DebugConfigBase
+from platformio.debug.exception import DebugInvalidOptionsError
+from platformio.device.finder import SerialPortFinder, is_pattern_port
 
 
 class BlackmagicDebugConfig(DebugConfigBase):
-
     GDB_INIT_SCRIPT = """
 define pio_reset_halt_target
     set language c
@@ -47,3 +48,24 @@ while ($busy)
 end
 set language auto
 """
+
+    @property
+    def port(self):
+        # pylint: disable=assignment-from-no-return
+        initial_port = DebugConfigBase.port.fget(self)
+        if initial_port and not is_pattern_port(initial_port):
+            return initial_port
+        port = SerialPortFinder(
+            board_config=self.board_config,
+            upload_protocol=self.tool_name,
+            prefer_gdb_port=True,
+        ).find(initial_port)
+        if port:
+            return port
+        raise DebugInvalidOptionsError(
+            "Please specify `debug_port` for the working environment"
+        )
+
+    @port.setter
+    def port(self, value):
+        self._port = value

platformio/debug/config/factory.py

@@ -19,7 +19,7 @@ from platformio.debug.config.generic import GenericDebugConfig
 from platformio.debug.config.native import NativeDebugConfig
 
 
-class DebugConfigFactory(object):
+class DebugConfigFactory:
     @staticmethod
     def get_clsname(name):
         name = re.sub(r"[^\da-z\_\-]+", "", name, flags=re.I)

platformio/debug/config/generic.py

@@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase
 
 
 class GenericDebugConfig(DebugConfigBase):
-
     GDB_INIT_SCRIPT = """
 define pio_reset_halt_target
     monitor reset halt
@@ -34,5 +33,6 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
+        if "port" not in kwargs:
+            kwargs["port"] = ":3333"
         super().__init__(*args, **kwargs)
-        self.port = ":3333"

platformio/debug/config/jlink.py

@@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase
 
 
 class JlinkDebugConfig(DebugConfigBase):
-
     GDB_INIT_SCRIPT = """
 define pio_reset_halt_target
     monitor reset
@@ -38,8 +37,9 @@ $INIT_BREAK
 """
 
     def __init__(self, *args, **kwargs):
+        if "port" not in kwargs:
+            kwargs["port"] = ":2331"
         super().__init__(*args, **kwargs)
-        self.port = ":2331"
 
     @property
     def server_ready_pattern(self):