Merge branch 'release/v6.0.2'

* Fix filter argument used in remote agent

* Linting

* Renamed reserved variable name to filter_

.github/workflows/core.yml

@@ -1,6 +1,6 @@
 name: Core
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:
@@ -8,27 +8,35 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [2.7, 3.7]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        exclude:
+          - os: macos-latest
+            python-version: "3.6"
+          - os: windows-latest
+            python-version: "3.10"
+
     runs-on: ${{ matrix.os }}
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
-          git submodule update --init --recursive
           python -m pip install --upgrade pip
           pip install tox
 
       - name: Python Lint
         run: |
           tox -e lint
+
       - name: Integration Tests
-        env:
-          PLATFORMIO_TEST_ACCOUNT_LOGIN: ${{ secrets.PLATFORMIO_TEST_ACCOUNT_LOGIN }}
-          PLATFORMIO_TEST_ACCOUNT_PASSWORD: ${{ secrets.PLATFORMIO_TEST_ACCOUNT_PASSWORD }}
         run: |
           tox -e testcore
 
@@ -40,3 +48,4 @@ jobs:
           job_name: '*Core*'
           commit: true
           url: ${{ secrets.SLACK_BUILD_WEBHOOK }}
+          token: ${{ secrets.SLACK_GITHUB_TOKEN }}

.github/workflows/deployment.yml

@@ -0,0 +1,42 @@
+name: Deployment
+
+on:
+  push:
+    branches:
+      - master
+      - "release/**"
+
+jobs:
+  deployment:
+    runs-on: ubuntu-latest
+    environment: production
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+
+      - name: Deployment Tests
+        env:
+          TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }}
+          TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }}
+          TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }}
+        run: |
+          tox -e testcore
+
+      - name: Publish package to PyPI
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/docs.yml

@@ -1,19 +1,21 @@
 name: Docs
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:
+    name: Build Docs
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          git submodule update --init --recursive
           python -m pip install --upgrade pip
           pip install tox
 
@@ -28,4 +30,80 @@ jobs:
           type: ${{ job.status }}
           job_name: '*Docs*'
           commit: true
-          url: ${{ secrets.SLACK_BUILD_WEBHOOK }}
+          url: ${{ secrets.SLACK_BUILD_WEBHOOK }}
+          token: ${{ secrets.SLACK_GITHUB_TOKEN }}
+
+      - name: Preserve Docs
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          tar -czvf docs.tar.gz -C docs/_build html rtdpage
+
+      - name: Save artifact
+        if: ${{ github.event_name == 'push' }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: docs
+          path: ./docs.tar.gz
+
+  deploy:
+    name: Deploy Docs
+    needs: build
+    runs-on: ubuntu-latest
+    env:
+      DOCS_REPO: platformio/platformio-docs
+      DOCS_DIR: platformio-docs
+      LATEST_DOCS_DIR: latest-docs
+      RELEASE_BUILD: ${{ startsWith(github.ref, 'refs/tags/v') }}
+    if: ${{ github.event_name == 'push' }}
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: docs
+      - name: Unpack artifact
+        run: |
+          mkdir ./${{ env.LATEST_DOCS_DIR }}
+          tar -xzf ./docs.tar.gz -C ./${{ env.LATEST_DOCS_DIR }}
+      - name: Delete Artifact
+        uses: geekyeggo/delete-artifact@v1
+        with:
+          name: docs
+      - name: Select Docs type
+        id: get-destination-dir
+        run: |
+          if [[ ${{ env.RELEASE_BUILD }} == true ]]; then
+            echo "::set-output name=dst_dir::stable"
+          else
+            echo "::set-output name=dst_dir::latest"
+          fi
+      - name: Checkout latest Docs
+        continue-on-error: true
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ env.DOCS_REPO }}
+          path: ${{ env.DOCS_DIR }}
+          ref: gh-pages
+      - name: Synchronize Docs
+        run: |
+          rm -rf ${{ env.DOCS_DIR }}/.git
+          rm -rf ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }}
+          mkdir -p ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }}
+          cp -rf ${{ env.LATEST_DOCS_DIR }}/html/* ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }}
+          if [[ ${{ env.RELEASE_BUILD }} == false ]]; then
+            rm -rf ${{ env.DOCS_DIR }}/page
+            mkdir -p ${{ env.DOCS_DIR }}/page
+            cp -rf ${{ env.LATEST_DOCS_DIR }}/rtdpage/* ${{ env.DOCS_DIR }}/page
+          fi
+      - name: Validate Docs
+        run: |
+          if [ -z "$(ls -A ${{ env.DOCS_DIR }})" ]; then
+             echo "Docs folder is empty. Aborting!"
+             exit 1
+          fi
+      - name: Deploy to Github Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          personal_token: ${{ secrets.DEPLOY_GH_DOCS_TOKEN }}
+          external_repository: ${{ env.DOCS_REPO }}
+          publish_dir: ./${{ env.DOCS_DIR }}
+          commit_message: Sync Docs

.github/workflows/examples.yml

@@ -1,45 +1,45 @@
 name: Examples
 
-on: [push]
+on: [push, pull_request]
+
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-16.04, windows-latest, macos-latest]
-        python-version: [2.7, 3.7]
+        os: [ubuntu-latest, windows-latest, macos-latest]
     runs-on: ${{ matrix.os }}
+    env:
+      PIO_INSTALL_DEVPLATFORM_OWNERNAMES: "platformio"
+      PIO_INSTALL_DEVPLATFORM_NAMES: "aceinna_imu,atmelavr,atmelmegaavr,atmelsam,espressif32,espressif8266,nordicnrf52,raspberrypi,ststm32,teensy"
+
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
         with:
-          python-version: ${{ matrix.python-version }}
+          submodules: "recursive"
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+
       - name: Install dependencies
         run: |
-          git submodule update --init --recursive
           python -m pip install --upgrade pip
           pip install tox
 
       - name: Run on Linux
         if: startsWith(matrix.os, 'ubuntu')
-        env:
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,intel_mcs51,aceinna_imu"
         run: |
-          # ChipKIT issue: install 32-bit support for GCC PIC32
-          sudo apt-get install libc6-i386
           # Free space
           sudo apt clean
           docker rmi $(docker image ls -aq)
           df -h
-          # Run
           tox -e testexamples
 
       - name: Run on macOS
         if: startsWith(matrix.os, 'macos')
-        env:
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,microchippic32,gd32v,nuclei"
         run: |
           df -h
           tox -e testexamples
@@ -48,7 +48,7 @@ jobs:
         if: startsWith(matrix.os, 'windows')
         env:
           PLATFORMIO_CORE_DIR: C:/pio
-          PIO_INSTALL_DEVPLATFORMS_IGNORE: "ststm8,infineonxmc,riscv_gap"
+          PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH
         run: |
           tox -e testexamples
 
@@ -60,3 +60,4 @@ jobs:
           job_name: '*Examples*'
           commit: true
           url: ${{ secrets.SLACK_BUILD_WEBHOOK }}
+          token: ${{ secrets.SLACK_GITHUB_TOKEN }}

.github/workflows/projects.yml

@@ -0,0 +1,69 @@
+name: Projects
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        project:
+          - marlin:
+            repository: "MarlinFirmware/Marlin"
+            folder: "Marlin"
+            config_dir: "Marlin"
+            env_name: "mega2560"
+          - esphome:
+            repository: "esphome/esphome"
+            folder: "esphome"
+            config_dir: "esphome"
+            env_name: "esp32-arduino"
+          - smartknob:
+            repository: "scottbez1/smartknob"
+            folder: "smartknob"
+            config_dir: "smartknob/firmware"
+            env_name: "view"
+          - espurna:
+            repository: "xoseperez/espurna"
+            folder: "espurna"
+            config_dir: "espurna/code"
+            env_name: "nodemcu-lolin"
+          - OpenMQTTGateway:
+            repository: "1technophile/OpenMQTTGateway"
+            folder: "OpenMQTTGateway"
+            config_dir: "OpenMQTTGateway"
+            env_name: "esp32-m5atom"
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        exclude:
+        - os: windows-latest
+          project: {"esphome": "", "repository": "esphome/esphome", "folder": "esphome", "config_dir": "esphome", "env_name": "esp32-arduino"}
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.9
+
+      - name: Install PlatformIO
+        run: pip install -U .
+
+      - name: Check out ${{ matrix.project.repository }}
+        uses: actions/checkout@v2
+        with:
+          submodules: "recursive"
+          repository: ${{ matrix.project.repository }}
+          path: ${{ matrix.project.folder }}
+
+      - name: Install ESPHome dependencies
+        # Requires esptool package as it's used in a custom prescript
+        if: ${{ contains(matrix.project.repository, 'esphome') }}
+        run: pip install esptool==3.*
+
+      - name: Compile ${{ matrix.project.repository }}
+        run: pio run -d ${{ matrix.project.config_dir }} -e ${{ matrix.project.env_name }}
+

.gitignore

@@ -1,6 +1,6 @@
 *.egg-info
 *.pyc
-.pioenvs
+__pycache__
 .tox
 docs/_build
 dist

.isort.cfg

@@ -1,3 +0,0 @@
-[settings]
-line_length=88
-known_third_party=OpenSSL, SCons, autobahn, jsonrpc, twisted, zope

.pylintrc

@@ -1,3 +1,6 @@
+[REPORTS]
+output-format=colorized
+
 [MESSAGES CONTROL]
 disable=
     bad-continuation,
@@ -11,5 +14,10 @@ disable=
     too-few-public-methods,
     useless-object-inheritance,
     useless-import-alias,
-    fixme,
-    bad-option-value
+    bad-option-value,
+    consider-using-dict-items,
+    consider-using-f-string,
+
+    ; PY2 Compat
+    super-with-arguments,
+    raise-missing-from

.readthedocs.yml

@@ -1,12 +0,0 @@
-# See https://docs.readthedocs.io/en/stable/config-file/index.html
-
-version: 2
-
-sphinx:
-  configuration: docs/conf.py
-
-formats:
-  - pdf
-
-submodules:
-  include: all

CONTRIBUTING.md

@@ -1,21 +1,21 @@
 Contributing
 ------------
 
-To get started, <a href="https://www.clahub.com/agreements/platformio/platformio-core">sign the Contributor License Agreement</a>.
+To get started, <a href="https://cla-assistant.io/platformio/platformio-core">sign the Contributor License Agreement</a>.
 
-1. Fork the repository on GitHub.
+1. Fork the repository on GitHub
 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git`
 3. Run `pip install tox`
-4. Go to the root of project where is located `tox.ini` and run `tox -e py27`
+4. Go to the root of project where is located `tox.ini` and run `tox -e py37`
 5. Activate current development environment:
 
-   * Windows: `.tox\py27\Scripts\activate`
-   * Bash/ZSH: `source .tox/py27/bin/activate`
-   * Fish: `source .tox/py27/bin/activate.fish`
+   * Windows: `.tox\py37\Scripts\activate`
+   * Bash/ZSH: `source .tox/py37/bin/activate`
+   * Fish: `source .tox/py37/bin/activate.fish`
 
 6. Make changes to code, documentation, etc.
-7. Lint source code `make lint`
+7. Lint source code `make before-commit`
 8. Run the tests `make test`
 9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html)
 10. Commit changes to your forked repository
-11. Submit a Pull Request on GitHub.
+11. Submit a Pull Request on GitHub

Makefile

@@ -1,16 +1,17 @@
 lint:
-	pylint --rcfile=./.pylintrc ./platformio
+	pylint -j 6 --rcfile=./.pylintrc ./tests
+	pylint -j 6 --rcfile=./.pylintrc ./platformio
 
 isort:
-	isort -rc ./platformio
-	isort -rc ./tests
+	isort ./platformio
+	isort ./tests
 
 format:
-	black --target-version py27 ./platformio
-	black --target-version py27 ./tests
+	black ./platformio
+	black ./tests
 
 test:
-	py.test --verbose --capture=no --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
+	py.test --verbose --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
 
 before-commit: isort format lint
 
@@ -27,8 +28,11 @@ clean: clean-docs
 
 profile:
 	# Usage $ > make PIOARGS="boards" profile
-	python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS}
+	python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS}
 	snakeviz .tox/.tmp/cprofile.prof
 
+pack:
+	python setup.py sdist
+
 publish:
 	python setup.py sdist upload

README.rst

@@ -1,136 +1,83 @@
-PlatformIO
-==========
+.. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg
+    :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md
+    :alt:  SWUbanner
+
+PlatformIO Core
+===============
 
 .. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg
-    :target: https://docs.platformio.org/page/core/index.html
+    :target: https://docs.platformio.org/en/latest/core/index.html
     :alt:  CI Build for PlatformIO Core
-.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
-    :target: https://github.com/platformio/platformio-examples
-    :alt:  CI Build for dev-platform examples
 .. image:: https://github.com/platformio/platformio-core/workflows/Docs/badge.svg
     :target: https://docs.platformio.org?utm_source=github&utm_medium=core
     :alt:  CI Build for Docs
+.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
+    :target: https://github.com/platformio/platformio-examples
+    :alt:  CI Build for dev-platform examples
+.. image:: https://github.com/platformio/platformio-core/workflows/Projects/badge.svg
+    :target: https://docs.platformio.org/en/latest/tutorials/index.html#projects
+    :alt:  CI Build for the Community Projects
 .. image:: https://img.shields.io/pypi/v/platformio.svg
     :target: https://pypi.python.org/pypi/platformio/
     :alt: Latest Version
-.. image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
-    :target: https://pypi.python.org/pypi/platformio/
-    :alt:  License
-.. image:: https://img.shields.io/badge/PlatformIO-Community-orange.svg
-   :alt: Community Forums
-   :target: https://community.platformio.org?utm_source=github&utm_medium=core
+.. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg
+   :alt: PlatformIO Labs
+   :target: https://piolabs.com/?utm_source=github&utm_medium=core
 
-**Quick Links:** `Web <https://platformio.org?utm_source=github&utm_medium=core>`_ |
+**Quick Links:** `Homepage <https://platformio.org?utm_source=github&utm_medium=core>`_ |
 `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_ |
+`Registry <https://registry.platformio.org?utm_source=github&utm_medium=core>`_ |
 `Project Examples <https://github.com/platformio/platformio-examples/>`__ |
 `Docs <https://docs.platformio.org?utm_source=github&utm_medium=core>`_ |
 `Donate <https://platformio.org/donate?utm_source=github&utm_medium=core>`_ |
-`Contact Us <https://platformio.org/contact?utm_source=github&utm_medium=core>`_
+`Contact Us <https://piolabs.com/?utm_source=github&utm_medium=core>`_
 
-**Social:** `Twitter <https://twitter.com/PlatformIO_Org>`_ |
-`LinkedIn <https://www.linkedin.com/company/platformio/>`_ |
+**Social:** `LinkedIn <https://www.linkedin.com/company/platformio/>`_ |
+`Twitter <https://twitter.com/PlatformIO_Org>`_ |
 `Facebook <https://www.facebook.com/platformio>`_ |
-`Hackaday <https://hackaday.io/project/7980-platformio>`_ |
-`Bintray <https://bintray.com/platformio>`_ |
-`Community <https://community.platformio.org?utm_source=github&utm_medium=core>`_
+`Community Forums <https://community.platformio.org?utm_source=github&utm_medium=core>`_
 
 .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png
     :target: https://platformio.org?utm_source=github&utm_medium=core
 
-`PlatformIO <https://platformio.org?utm_source=github&utm_medium=core>`_ a new generation ecosystem for embedded development
+`PlatformIO <https://platformio.org>`_ is a professional collaborative platform for embedded development.
+
+**A place where Developers and Teams have true Freedom! No more vendor lock-in!**
 
 * Open source, maximum permissive Apache 2.0 license
 * Cross-platform IDE and Unified Debugger
 * Static Code Analyzer and Remote Unit Testing
 * Multi-platform and Multi-architecture Build System
-* Firmware File Explorer and Memory Inspection.
+* Firmware File Explorer and Memory Inspection
 
 Get Started
 -----------
 
-* `What is PlatformIO? <https://docs.platformio.org/page/what-is-platformio.html?utm_source=github&utm_medium=core>`_
-
-Instruments
------------
-
+* `What is PlatformIO? <https://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
 * `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_
-* `PlatformIO Core (CLI) <https://docs.platformio.org/page/core.html?utm_source=github&utm_medium=core>`_
-* `Library Management <https://docs.platformio.org/page/librarymanager/index.html?utm_source=github&utm_medium=core>`_
+* `PlatformIO Core (CLI) <https://docs.platformio.org/en/latest/core.html?utm_source=github&utm_medium=core>`_
 * `Project Examples <https://github.com/platformio/platformio-examples?utm_source=github&utm_medium=core>`__
-* `Desktop IDEs Integration <https://docs.platformio.org/page/ide.html?utm_source=github&utm_medium=core>`_
-* `Continuous Integration <https://docs.platformio.org/page/ci/index.html?utm_source=github&utm_medium=core>`_
-* `Advanced Scripting API <https://docs.platformio.org/page/projectconf/advanced_scripting.html?utm_source=github&utm_medium=core>`_
 
-Professional
-------------
+Solutions
+---------
 
-* `PIO Check <https://docs.platformio.org/page/plus/pio-check.html?utm_source=github&utm_medium=core>`_
-* `PIO Remote <https://docs.platformio.org/page/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
-* `PIO Unified Debugger <https://docs.platformio.org/page/plus/debugging.html?utm_source=github&utm_medium=core>`_
-* `PIO Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
+* `Library Management <https://docs.platformio.org/en/latest/librarymanager/index.html?utm_source=github&utm_medium=core>`_
+* `Desktop IDEs Integration <https://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core>`_
+* `Continuous Integration <https://docs.platformio.org/en/latest/ci/index.html?utm_source=github&utm_medium=core>`_
+
+**Advanced**
+
+* `Debugging <https://docs.platformio.org/en/latest/plus/debugging.html?utm_source=github&utm_medium=core>`_
+* `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html?utm_source=github&utm_medium=core>`_
+* `Static Code Analysis <https://docs.platformio.org/en/latest/plus/pio-check.html?utm_source=github&utm_medium=core>`_
+* `Remote Development <https://docs.platformio.org/en/latest/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
 
 Registry
 --------
 
-* `Libraries <https://platformio.org/lib?utm_source=github&utm_medium=core>`_
-* `Development Platforms <https://platformio.org/platforms?utm_source=github&utm_medium=core>`_
-* `Frameworks <https://platformio.org/frameworks?utm_source=github&utm_medium=core>`_
-* `Embedded Boards <https://platformio.org/boards?utm_source=github&utm_medium=core>`_
-
-Development Platforms
----------------------
-
-* `Aceinna IMU <https://platformio.org/platforms/aceinna_imu?utm_source=github&utm_medium=core>`_
-* `Atmel AVR <https://platformio.org/platforms/atmelavr?utm_source=github&utm_medium=core>`_
-* `Atmel SAM <https://platformio.org/platforms/atmelsam?utm_source=github&utm_medium=core>`_
-* `Espressif 32 <https://platformio.org/platforms/espressif32?utm_source=github&utm_medium=core>`_
-* `Espressif 8266 <https://platformio.org/platforms/espressif8266?utm_source=github&utm_medium=core>`_
-* `Freescale Kinetis <https://platformio.org/platforms/freescalekinetis?utm_source=github&utm_medium=core>`_
-* `Infineon XMC <https://platformio.org/platforms/infineonxmc?utm_source=github&utm_medium=core>`_
-* `Intel ARC32 <https://platformio.org/platforms/intel_arc32?utm_source=github&utm_medium=core>`_
-* `Intel MCS-51 (8051) <https://platformio.org/platforms/intel_mcs51?utm_source=github&utm_medium=core>`_
-* `Kendryte K210 <https://platformio.org/platforms/kendryte210?utm_source=github&utm_medium=core>`_
-* `Lattice iCE40 <https://platformio.org/platforms/lattice_ice40?utm_source=github&utm_medium=core>`_
-* `Maxim 32 <https://platformio.org/platforms/maxim32?utm_source=github&utm_medium=core>`_
-* `Microchip PIC32 <https://platformio.org/platforms/microchippic32?utm_source=github&utm_medium=core>`_
-* `Nordic nRF51 <https://platformio.org/platforms/nordicnrf51?utm_source=github&utm_medium=core>`_
-* `Nordic nRF52 <https://platformio.org/platforms/nordicnrf52?utm_source=github&utm_medium=core>`_
-* `Nuclei <https://platformio.org/platforms/nuclei?utm_source=github&utm_medium=core>`_
-* `NXP LPC <https://platformio.org/platforms/nxplpc?utm_source=github&utm_medium=core>`_
-* `RISC-V <https://platformio.org/platforms/riscv?utm_source=github&utm_medium=core>`_
-* `RISC-V GAP <https://platformio.org/platforms/riscv_gap?utm_source=github&utm_medium=core>`_
-* `Shakti <https://platformio.org/platforms/shakti?utm_source=github&utm_medium=core>`_
-* `Silicon Labs EFM32 <https://platformio.org/platforms/siliconlabsefm32?utm_source=github&utm_medium=core>`_
-* `ST STM32 <https://platformio.org/platforms/ststm32?utm_source=github&utm_medium=core>`_
-* `ST STM8 <https://platformio.org/platforms/ststm8?utm_source=github&utm_medium=core>`_
-* `Teensy <https://platformio.org/platforms/teensy?utm_source=github&utm_medium=core>`_
-* `TI MSP430 <https://platformio.org/platforms/timsp430?utm_source=github&utm_medium=core>`_
-* `TI Tiva <https://platformio.org/platforms/titiva?utm_source=github&utm_medium=core>`_
-* `WIZNet W7500 <https://platformio.org/platforms/wiznet7500?utm_source=github&utm_medium=core>`_
-
-Frameworks
-----------
-
-* `Arduino <https://platformio.org/frameworks/arduino?utm_source=github&utm_medium=core>`_
-* `CMSIS <https://platformio.org/frameworks/cmsis?utm_source=github&utm_medium=core>`_
-* `ESP-IDF <https://platformio.org/frameworks/espidf?utm_source=github&utm_medium=core>`_
-* `ESP8266 Non-OS SDK <https://platformio.org/frameworks/esp8266-nonos-sdk?utm_source=github&utm_medium=core>`_
-* `ESP8266 RTOS SDK <https://platformio.org/frameworks/esp8266-rtos-sdk?utm_source=github&utm_medium=core>`_
-* `Freedom E SDK <https://platformio.org/frameworks/freedom-e-sdk?utm_source=github&utm_medium=core>`_
-* `GigaDevice GD32V SDK <https://platformio.org/frameworks/gd32vf103-sdk?utm_source=github&utm_medium=core>`_
-* `Kendryte Standalone SDK <https://platformio.org/frameworks/kendryte-standalone-sdk?utm_source=github&utm_medium=core>`_
-* `Kendryte FreeRTOS SDK <https://platformio.org/frameworks/kendryte-freertos-sdk?utm_source=github&utm_medium=core>`_
-* `libOpenCM3 <https://platformio.org/frameworks/libopencm3?utm_source=github&utm_medium=core>`_
-* `Mbed <https://platformio.org/frameworks/mbed?utm_source=github&utm_medium=core>`_
-* `Nuclei SDK <https://platformio.org/frameworks/nuclei-sdk?utm_source=github&utm_medium=core>`_
-* `PULP OS <https://platformio.org/frameworks/pulp-os?utm_source=github&utm_medium=core>`_
-* `Pumbaa <https://platformio.org/frameworks/pumbaa?utm_source=github&utm_medium=core>`_
-* `Shakti SDK <https://platformio.org/frameworks/shakti-sdk?utm_source=github&utm_medium=core>`_
-* `Simba <https://platformio.org/frameworks/simba?utm_source=github&utm_medium=core>`_
-* `SPL <https://platformio.org/frameworks/spl?utm_source=github&utm_medium=core>`_
-* `STM32Cube <https://platformio.org/frameworks/stm32cube?utm_source=github&utm_medium=core>`_
-* `WiringPi <https://platformio.org/frameworks/wiringpi?utm_source=github&utm_medium=core>`_
-* `Zephyr <https://platformio.org/frameworks/zephyr?utm_source=github&utm_medium=core>`_
+* `Libraries <https://registry.platformio.org/search?t=library&utm_source=github&utm_medium=core>`_
+* `Development Platforms <https://registry.platformio.org/search?t=platform&utm_source=github&utm_medium=core>`_
+* `Development Tools <https://registry.platformio.org/search?t=tool&utm_source=github&utm_medium=core>`_
 
 Contributing
 ------------
@@ -143,8 +90,7 @@ Telemetry / Privacy Policy
 Share minimal diagnostics and usage information to help us make PlatformIO better.
 It is enabled by default. For more information see:
 
-* `Telemetry Setting <https://docs.platformio.org/page/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
-* `SSL Setting <https://docs.platformio.org/page/userguide/cmd_settings.html?utm_source=github&utm_medium=core#strict-ssl>`_
+* `Telemetry Setting <https://docs.platformio.org/en/latest/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
 
 License
 -------

platformio/__init__.py

@@ -12,27 +12,51 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 3, 2)
+import sys
+
+VERSION = (6, 0, 2)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
 __description__ = (
-    "A new generation ecosystem for embedded development. "
+    "A professional collaborative platform for embedded development. "
     "Cross-platform IDE and Unified Debugger. "
     "Static Code Analyzer and Remote Unit Testing. "
     "Multi-platform and Multi-architecture Build System. "
     "Firmware File Explorer and Memory Inspection. "
-    "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, "
-    "RISC-V, FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3"
+    "IoT, Arduino, CMSIS, ESP-IDF, FreeRTOS, libOpenCM3, mbedOS, Pulp OS, SPL, "
+    "STM32Cube, Zephyr RTOS, ARM, AVR, Espressif (ESP8266/ESP32), FPGA, "
+    "MCS-51 (8051), MSP430, Nordic (nRF51/nRF52), NXP i.MX RT, PIC32, RISC-V, "
+    "STMicroelectronics (STM8/STM32), Teensy"
 )
 __url__ = "https://platformio.org"
 
-__author__ = "PlatformIO"
-__email__ = "contact@platformio.org"
+__author__ = "PlatformIO Labs"
+__email__ = "contact@piolabs.com"
 
 __license__ = "Apache Software License"
-__copyright__ = "Copyright 2014-present PlatformIO"
+__copyright__ = "Copyright 2014-present PlatformIO Labs"
 
-__apiurl__ = "https://api.platformio.org"
-__pioaccount_api__ = "https://api.accounts.platformio.org"
+__accounts_api__ = "https://api.accounts.platformio.org"
+__registry_mirror_hosts__ = [
+    "registry.platformio.org",
+    "registry.nm1.platformio.org",
+]
 __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
+
+__default_requests_timeout__ = (10, None)  # (connect, read)
+
+__core_packages__ = {
+    "contrib-piohome": "~3.4.1",
+    "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
+    "tool-scons": "~4.40300.0",
+    "tool-cppcheck": "~1.270.0",
+    "tool-clangtidy": "~1.120001.0",
+    "tool-pvs-studio": "~7.18.0",
+}
+
+__check_internet_hosts__ = [
+    "185.199.110.153",  # Github.com
+    "88.198.170.159",  # platformio.org
+    "github.com",
+] + __registry_mirror_hosts__

platformio/__main__.py

@@ -18,16 +18,16 @@ from traceback import format_exc
 
 import click
 
-from platformio import __version__, exception, maintenance, util
-from platformio.commands import PlatformioCLI
-from platformio.compat import CYGWIN
+from platformio import __version__, exception, maintenance
+from platformio.cli import PlatformioCLI
+from platformio.compat import IS_CYGWIN, ensure_python3
 
 
 @click.command(
     cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])
 )
-@click.version_option(__version__, prog_name="PlatformIO")
-@click.option("--force", "-f", is_flag=True, help="DEPRECATE")
+@click.version_option(__version__, prog_name="PlatformIO Core")
+@click.option("--force", "-f", is_flag=True, help="DEPRECATED", hidden=True)
 @click.option("--caller", "-c", help="Caller ID (service)")
 @click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters")
 @click.pass_context
@@ -56,15 +56,14 @@ def cli(ctx, force, caller, no_ansi):
     maintenance.on_platformio_start(ctx, force, caller)
 
 
-@cli.resultcallback()
+@cli.result_callback()
 @click.pass_context
 def process_result(ctx, result, *_, **__):
     maintenance.on_platformio_end(ctx, result)
 
 
-@util.memoized()
 def configure():
-    if CYGWIN:
+    if IS_CYGWIN:
         raise exception.CygwinEnvDetected()
 
     # https://urllib3.readthedocs.org
@@ -98,6 +97,7 @@ def main(argv=None):
         assert isinstance(argv, list)
         sys.argv = argv
     try:
+        ensure_python3(raise_exception=True)
         configure()
         cli()  # pylint: disable=no-value-for-parameter
     except SystemExit as e:
@@ -120,7 +120,7 @@ An unexpected error occurred. Further steps:
   `pip install -U platformio` command
 
 * Try to find answer in FAQ Troubleshooting section
-  https://docs.platformio.org/page/faq.html
+  https://docs.platformio.org/page/faq/index.html
 
 * Report this problem to the developers
   https://github.com/platformio/platformio-core/issues

platformio/account/cli.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.commands.destroy import account_destroy_cmd
+from platformio.account.commands.forgot import account_forgot_cmd
+from platformio.account.commands.login import account_login_cmd
+from platformio.account.commands.logout import account_logout_cmd
+from platformio.account.commands.password import account_password_cmd
+from platformio.account.commands.register import account_register_cmd
+from platformio.account.commands.show import account_show_cmd
+from platformio.account.commands.token import account_token_cmd
+from platformio.account.commands.update import account_update_cmd
+
+
+@click.group(
+    "account",
+    commands=[
+        account_destroy_cmd,
+        account_forgot_cmd,
+        account_login_cmd,
+        account_logout_cmd,
+        account_password_cmd,
+        account_register_cmd,
+        account_show_cmd,
+        account_token_cmd,
+        account_update_cmd,
+    ],
+    short_help="Manage PlatformIO account",
+)
+def cli():
+    pass

platformio/account/client.py

@@ -0,0 +1,356 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+
+from platformio import __accounts_api__, app
+from platformio.exception import PlatformioException
+from platformio.http import HTTPClient, HTTPClientError
+
+
+class AccountError(PlatformioException):
+
+    MESSAGE = "{0}"
+
+
+class AccountNotAuthorized(AccountError):
+
+    MESSAGE = "You are not authorized! Please log in to PlatformIO Account."
+
+
+class AccountAlreadyAuthorized(AccountError):
+
+    MESSAGE = "You are already authorized with {0} account."
+
+
+class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
+
+    SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
+
+    def __init__(self):
+        super().__init__(__accounts_api__)
+
+    @staticmethod
+    def get_refresh_token():
+        try:
+            return app.get_state_item("account").get("auth").get("refresh_token")
+        except:  # pylint:disable=bare-except
+            raise AccountNotAuthorized()
+
+    @staticmethod
+    def delete_local_session():
+        app.delete_state_item("account")
+
+    @staticmethod
+    def delete_local_state(key):
+        account = app.get_state_item("account")
+        if not account or key not in account:
+            return
+        del account[key]
+        app.set_state_item("account", account)
+
+    def fetch_json_data(self, *args, **kwargs):
+        try:
+            return super().fetch_json_data(*args, **kwargs)
+        except HTTPClientError as exc:
+            raise AccountError(exc) from exc
+
+    def fetch_authentication_token(self):
+        if os.environ.get("PLATFORMIO_AUTH_TOKEN"):
+            return os.environ.get("PLATFORMIO_AUTH_TOKEN")
+        auth = app.get_state_item("account", {}).get("auth", {})
+        if auth.get("access_token") and auth.get("access_token_expire"):
+            if auth.get("access_token_expire") > time.time():
+                return auth.get("access_token")
+            if auth.get("refresh_token"):
+                try:
+                    data = self.fetch_json_data(
+                        "post",
+                        "/v1/login",
+                        headers={
+                            "Authorization": "Bearer %s" % auth.get("refresh_token")
+                        },
+                    )
+                    app.set_state_item("account", data)
+                    return data.get("auth").get("access_token")
+                except AccountError:
+                    self.delete_local_session()
+        raise AccountNotAuthorized()
+
+    def login(self, username, password):
+        try:
+            self.fetch_authentication_token()
+        except:  # pylint:disable=bare-except
+            pass
+        else:
+            raise AccountAlreadyAuthorized(
+                app.get_state_item("account", {}).get("email", "")
+            )
+
+        data = self.fetch_json_data(
+            "post",
+            "/v1/login",
+            data={"username": username, "password": password},
+        )
+        app.set_state_item("account", data)
+        return data
+
+    def login_with_code(self, client_id, code, redirect_uri):
+        try:
+            self.fetch_authentication_token()
+        except:  # pylint:disable=bare-except
+            pass
+        else:
+            raise AccountAlreadyAuthorized(
+                app.get_state_item("account", {}).get("email", "")
+            )
+
+        result = self.fetch_json_data(
+            "post",
+            "/v1/login/code",
+            data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri},
+        )
+        app.set_state_item("account", result)
+        return result
+
+    def logout(self):
+        refresh_token = self.get_refresh_token()
+        self.delete_local_session()
+        try:
+            self.fetch_json_data(
+                "post",
+                "/v1/logout",
+                data={"refresh_token": refresh_token},
+            )
+        except AccountError:
+            pass
+        return True
+
+    def change_password(self, old_password, new_password):
+        return self.fetch_json_data(
+            "post",
+            "/v1/password",
+            data={"old_password": old_password, "new_password": new_password},
+            x_with_authorization=True,
+        )
+
+    def registration(
+        self, username, email, password, firstname, lastname
+    ):  # pylint:disable=too-many-arguments
+        try:
+            self.fetch_authentication_token()
+        except:  # pylint:disable=bare-except
+            pass
+        else:
+            raise AccountAlreadyAuthorized(
+                app.get_state_item("account", {}).get("email", "")
+            )
+
+        return self.fetch_json_data(
+            "post",
+            "/v1/registration",
+            data={
+                "username": username,
+                "email": email,
+                "password": password,
+                "firstname": firstname,
+                "lastname": lastname,
+            },
+        )
+
+    def auth_token(self, password, regenerate):
+        return self.fetch_json_data(
+            "post",
+            "/v1/token",
+            data={"password": password, "regenerate": 1 if regenerate else 0},
+            x_with_authorization=True,
+        ).get("auth_token")
+
+    def forgot_password(self, username):
+        return self.fetch_json_data(
+            "post",
+            "/v1/forgot",
+            data={"username": username},
+        )
+
+    def get_profile(self):
+        return self.fetch_json_data(
+            "get",
+            "/v1/profile",
+            x_with_authorization=True,
+        )
+
+    def update_profile(self, profile, current_password):
+        profile["current_password"] = current_password
+        self.delete_local_state("summary")
+        response = self.fetch_json_data(
+            "put",
+            "/v1/profile",
+            data=profile,
+            x_with_authorization=True,
+        )
+        return response
+
+    def get_account_info(self, offline=False):
+        account = app.get_state_item("account") or {}
+        if (
+            account.get("summary")
+            and account["summary"].get("expire_at", 0) > time.time()
+        ):
+            return account["summary"]
+        if offline and account.get("email"):
+            return {
+                "profile": {
+                    "email": account.get("email"),
+                    "username": account.get("username"),
+                }
+            }
+        result = self.fetch_json_data(
+            "get",
+            "/v1/summary",
+            x_with_authorization=True,
+        )
+        account["summary"] = dict(
+            profile=result.get("profile"),
+            packages=result.get("packages"),
+            subscriptions=result.get("subscriptions"),
+            user_id=result.get("user_id"),
+            expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL,
+        )
+        app.set_state_item("account", account)
+        return result
+
+    def get_logged_username(self):
+        return self.get_account_info(offline=True).get("profile").get("username")
+
+    def destroy_account(self):
+        return self.fetch_json_data(
+            "delete",
+            "/v1/account",
+            x_with_authorization=True,
+        )
+
+    def create_org(self, orgname, email, displayname):
+        return self.fetch_json_data(
+            "post",
+            "/v1/orgs",
+            data={"orgname": orgname, "email": email, "displayname": displayname},
+            x_with_authorization=True,
+        )
+
+    def get_org(self, orgname):
+        return self.fetch_json_data(
+            "get",
+            "/v1/orgs/%s" % orgname,
+            x_with_authorization=True,
+        )
+
+    def list_orgs(self):
+        return self.fetch_json_data(
+            "get",
+            "/v1/orgs",
+            x_with_authorization=True,
+        )
+
+    def update_org(self, orgname, data):
+        return self.fetch_json_data(
+            "put",
+            "/v1/orgs/%s" % orgname,
+            data={k: v for k, v in data.items() if v},
+            x_with_authorization=True,
+        )
+
+    def destroy_org(self, orgname):
+        return self.fetch_json_data(
+            "delete",
+            "/v1/orgs/%s" % orgname,
+            x_with_authorization=True,
+        )
+
+    def add_org_owner(self, orgname, username):
+        return self.fetch_json_data(
+            "post",
+            "/v1/orgs/%s/owners" % orgname,
+            data={"username": username},
+            x_with_authorization=True,
+        )
+
+    def list_org_owners(self, orgname):
+        return self.fetch_json_data(
+            "get",
+            "/v1/orgs/%s/owners" % orgname,
+            x_with_authorization=True,
+        )
+
+    def remove_org_owner(self, orgname, username):
+        return self.fetch_json_data(
+            "delete",
+            "/v1/orgs/%s/owners" % orgname,
+            data={"username": username},
+            x_with_authorization=True,
+        )
+
+    def create_team(self, orgname, teamname, description):
+        return self.fetch_json_data(
+            "post",
+            "/v1/orgs/%s/teams" % orgname,
+            data={"name": teamname, "description": description},
+            x_with_authorization=True,
+        )
+
+    def destroy_team(self, orgname, teamname):
+        return self.fetch_json_data(
+            "delete",
+            "/v1/orgs/%s/teams/%s" % (orgname, teamname),
+            x_with_authorization=True,
+        )
+
+    def get_team(self, orgname, teamname):
+        return self.fetch_json_data(
+            "get",
+            "/v1/orgs/%s/teams/%s" % (orgname, teamname),
+            x_with_authorization=True,
+        )
+
+    def list_teams(self, orgname):
+        return self.fetch_json_data(
+            "get",
+            "/v1/orgs/%s/teams" % orgname,
+            x_with_authorization=True,
+        )
+
+    def update_team(self, orgname, teamname, data):
+        return self.fetch_json_data(
+            "put",
+            "/v1/orgs/%s/teams/%s" % (orgname, teamname),
+            data={k: v for k, v in data.items() if v},
+            x_with_authorization=True,
+        )
+
+    def add_team_member(self, orgname, teamname, username):
+        return self.fetch_json_data(
+            "post",
+            "/v1/orgs/%s/teams/%s/members" % (orgname, teamname),
+            data={"username": username},
+            x_with_authorization=True,
+        )
+
+    def remove_team_member(self, orgname, teamname, username):
+        return self.fetch_json_data(
+            "delete",
+            "/v1/orgs/%s/teams/%s/members" % (orgname, teamname),
+            data={"username": username},
+            x_with_authorization=True,
+        )

platformio/account/commands/destroy.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient, AccountNotAuthorized
+
+
+@click.command("destroy", short_help="Destroy account")
+def account_destroy_cmd():
+    client = AccountClient()
+    click.confirm(
+        "Are you sure you want to delete the %s user account?\n"
+        "Warning! All linked data will be permanently removed and can not be restored."
+        % client.get_logged_username(),
+        abort=True,
+    )
+    client.destroy_account()
+    try:
+        client.logout()
+    except AccountNotAuthorized:
+        pass
+    click.secho(
+        "User account has been destroyed.",
+        fg="green",
+    )

platformio/account/commands/forgot.py

@@ -12,14 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio.commands.update import cli as cmd_update
+import click
+
+from platformio.account.client import AccountClient
 
 
-def test_update(clirunner, validate_cliresult):
-    matches = ("Platform Manager", "Up-to-date", "Library Manager")
-    result = clirunner.invoke(cmd_update, ["--only-check"])
-    validate_cliresult(result)
-    assert all([m in result.output for m in matches])
-    result = clirunner.invoke(cmd_update)
-    validate_cliresult(result)
-    assert all([m in result.output for m in matches])
+@click.command("forgot", short_help="Forgot password")
+@click.option("--username", prompt="Username or email")
+def account_forgot_cmd(username):
+    client = AccountClient()
+    client.forgot_password(username)
+    click.secho(
+        "If this account is registered, we will send the "
+        "further instructions to your email.",
+        fg="green",
+    )

platformio/account/commands/login.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("login", short_help="Log in to PlatformIO Account")
+@click.option("-u", "--username", prompt="Username or email")
+@click.option("-p", "--password", prompt=True, hide_input=True)
+def account_login_cmd(username, password):
+    client = AccountClient()
+    client.login(username, password)
+    click.secho("Successfully logged in!", fg="green")

platformio/account/commands/logout.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("logout", short_help="Log out of PlatformIO Account")
+def account_logout_cmd():
+    client = AccountClient()
+    client.logout()
+    click.secho("Successfully logged out!", fg="green")

platformio/account/commands/password.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("password", short_help="Change password")
+@click.option("--old-password", prompt=True, hide_input=True)
+@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
+def account_password_cmd(old_password, new_password):
+    client = AccountClient()
+    client.change_password(old_password, new_password)
+    click.secho("Password successfully changed!", fg="green")

platformio/account/commands/register.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import (
+    validate_email,
+    validate_password,
+    validate_username,
+)
+
+
+@click.command("register", short_help="Create new PlatformIO Account")
+@click.option(
+    "-u",
+    "--username",
+    prompt=True,
+    callback=lambda _, __, value: validate_username(value),
+)
+@click.option(
+    "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value)
+)
+@click.option(
+    "-p",
+    "--password",
+    prompt=True,
+    hide_input=True,
+    confirmation_prompt=True,
+    callback=lambda _, __, value: validate_password(value),
+)
+@click.option("--firstname", prompt=True)
+@click.option("--lastname", prompt=True)
+def account_register_cmd(username, email, password, firstname, lastname):
+    client = AccountClient()
+    client.registration(username, email, password, firstname, lastname)
+    click.secho(
+        "An account has been successfully created. "
+        "Please check your mail to activate your account and verify your email address.",
+        fg="green",
+    )

platformio/account/commands/show.py

@@ -0,0 +1,116 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio import util
+from platformio.account.client import AccountClient
+
+
+@click.command("show", short_help="PlatformIO Account information")
+@click.option("--offline", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_show_cmd(offline, json_output):
+    client = AccountClient()
+    info = client.get_account_info(offline)
+    if json_output:
+        click.echo(json.dumps(info))
+        return
+    click.echo()
+    if info.get("profile"):
+        print_profile(info["profile"])
+    if info.get("packages"):
+        print_packages(info["packages"])
+    if info.get("subscriptions"):
+        print_subscriptions(info["subscriptions"])
+    click.echo()
+
+
+def print_profile(profile):
+    click.secho("Profile", fg="cyan", bold=True)
+    click.echo("=" * len("Profile"))
+    data = []
+    if profile.get("username"):
+        data.append(("Username:", profile["username"]))
+    if profile.get("email"):
+        data.append(("Email:", profile["email"]))
+    if profile.get("firstname"):
+        data.append(("First name:", profile["firstname"]))
+    if profile.get("lastname"):
+        data.append(("Last name:", profile["lastname"]))
+    click.echo(tabulate(data, tablefmt="plain"))
+
+
+def print_packages(packages):
+    click.echo()
+    click.secho("Packages", fg="cyan")
+    click.echo("=" * len("Packages"))
+    for package in packages:
+        click.echo()
+        click.secho(package.get("name"), bold=True)
+        click.echo("-" * len(package.get("name")))
+        if package.get("description"):
+            click.echo(package.get("description"))
+        data = []
+        expire = "-"
+        if "subscription" in package:
+            expire = util.parse_datetime(
+                package["subscription"].get("end_at")
+                or package["subscription"].get("next_bill_at")
+            ).strftime("%Y-%m-%d")
+        data.append(("Expire:", expire))
+        services = []
+        for key in package:
+            if not key.startswith("service."):
+                continue
+            if isinstance(package[key], dict):
+                services.append(package[key].get("title"))
+            else:
+                services.append(package[key])
+        if services:
+            data.append(("Services:", ", ".join(services)))
+        click.echo(tabulate(data, tablefmt="plain"))
+
+
+def print_subscriptions(subscriptions):
+    click.echo()
+    click.secho("Subscriptions", fg="cyan")
+    click.echo("=" * len("Subscriptions"))
+    for subscription in subscriptions:
+        click.echo()
+        click.secho(subscription.get("product_name"), bold=True)
+        click.echo("-" * len(subscription.get("product_name")))
+        data = [("State:", subscription.get("status"))]
+        begin_at = util.parse_datetime(subscription.get("begin_at")).strftime("%c")
+        data.append(("Start date:", begin_at or "-"))
+        end_at = subscription.get("end_at")
+        if end_at:
+            end_at = util.parse_datetime(subscription.get("end_at")).strftime("%c")
+        data.append(("End date:", end_at or "-"))
+        next_bill_at = subscription.get("next_bill_at")
+        if next_bill_at:
+            next_bill_at = util.parse_datetime(
+                subscription.get("next_bill_at")
+            ).strftime("%c")
+        data.append(("Next payment:", next_bill_at or "-"))
+        data.append(
+            ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")
+        )
+        data.append(
+            ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-")
+        )
+        click.echo(tabulate(data, tablefmt="plain"))

platformio/account/commands/token.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("token", short_help="Get or regenerate Authentication Token")
+@click.option("-p", "--password", prompt=True, hide_input=True)
+@click.option("--regenerate", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_token_cmd(password, regenerate, json_output):
+    client = AccountClient()
+    auth_token = client.auth_token(password, regenerate)
+    if json_output:
+        click.echo(json.dumps({"status": "success", "result": auth_token}))
+        return
+    click.secho("Personal Authentication Token: %s" % auth_token, fg="green")

platformio/account/commands/update.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient, AccountNotAuthorized
+from platformio.account.validate import validate_email, validate_username
+
+
+@click.command("update", short_help="Update profile information")
+@click.option("--current-password", prompt=True, hide_input=True)
+@click.option("--username")
+@click.option("--email")
+@click.option("--firstname")
+@click.option("--lastname")
+def account_update_cmd(current_password, **kwargs):
+    client = AccountClient()
+    profile = client.get_profile()
+    new_profile = profile.copy()
+    if not any(kwargs.values()):
+        for field in profile:
+            new_profile[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=profile[field]
+            )
+            if field == "email":
+                validate_email(new_profile[field])
+            if field == "username":
+                validate_username(new_profile[field])
+    else:
+        new_profile.update({key: value for key, value in kwargs.items() if value})
+    client.update_profile(new_profile, current_password)
+    click.secho("Profile successfully updated!", fg="green")
+    username_changed = new_profile["username"] != profile["username"]
+    email_changed = new_profile["email"] != profile["email"]
+    if not username_changed and not email_changed:
+        return None
+    try:
+        client.logout()
+    except AccountNotAuthorized:
+        pass
+    if email_changed:
+        click.secho(
+            "Please check your mail to verify your new email address and re-login. ",
+            fg="yellow",
+        )
+        return None
+    click.secho("Please re-login.", fg="yellow")
+    return None

platformio/account/org/cli.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.org.commands.add import org_add_cmd
+from platformio.account.org.commands.create import org_create_cmd
+from platformio.account.org.commands.destroy import org_destroy_cmd
+from platformio.account.org.commands.list import org_list_cmd
+from platformio.account.org.commands.remove import org_remove_cmd
+from platformio.account.org.commands.update import org_update_cmd
+
+
+@click.group(
+    "account",
+    commands=[
+        org_add_cmd,
+        org_create_cmd,
+        org_destroy_cmd,
+        org_list_cmd,
+        org_remove_cmd,
+        org_update_cmd,
+    ],
+    short_help="Manage organizations",
+)
+def cli():
+    pass

platformio/account/org/commands/add.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("add", short_help="Add a new owner to organization")
+@click.argument(
+    "orgname",
+)
+@click.argument(
+    "username",
+)
+def org_add_cmd(orgname, username):
+    client = AccountClient()
+    client.add_org_owner(orgname, username)
+    return click.secho(
+        "The new owner `%s` has been successfully added to the `%s` organization."
+        % (username, orgname),
+        fg="green",
+    )

platformio/account/org/commands/create.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_email, validate_orgname
+
+
+@click.command("create", short_help="Create a new organization")
+@click.argument(
+    "orgname",
+    callback=lambda _, __, value: validate_orgname(value),
+)
+@click.option(
+    "--email", callback=lambda _, __, value: validate_email(value) if value else value
+)
+@click.option(
+    "--displayname",
+)
+def org_create_cmd(orgname, email, displayname):
+    client = AccountClient()
+    client.create_org(orgname, email, displayname)
+    return click.secho(
+        "The organization `%s` has been successfully created." % orgname,
+        fg="green",
+    )

platformio/account/org/commands/destroy.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("destroy", short_help="Destroy organization")
+@click.argument("orgname")
+def org_destroy_cmd(orgname):
+    client = AccountClient()
+    click.confirm(
+        "Are you sure you want to delete the `%s` organization account?\n"
+        "Warning! All linked data will be permanently removed and can not be restored."
+        % orgname,
+        abort=True,
+    )
+    client.destroy_org(orgname)
+    return click.secho(
+        "Organization `%s` has been destroyed." % orgname,
+        fg="green",
+    )

platformio/account/org/commands/list.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio.account.client import AccountClient
+
+
+@click.command("list", short_help="List organizations and their members")
+@click.option("--json-output", is_flag=True)
+def org_list_cmd(json_output):
+    client = AccountClient()
+    orgs = client.list_orgs()
+    if json_output:
+        return click.echo(json.dumps(orgs))
+    if not orgs:
+        return click.echo("You do not have any organization")
+    for org in orgs:
+        click.echo()
+        click.secho(org.get("orgname"), fg="cyan")
+        click.echo("-" * len(org.get("orgname")))
+        data = []
+        if org.get("displayname"):
+            data.append(("Display Name:", org.get("displayname")))
+        if org.get("email"):
+            data.append(("Email:", org.get("email")))
+        data.append(
+            (
+                "Owners:",
+                ", ".join((owner.get("username") for owner in org.get("owners"))),
+            )
+        )
+        click.echo(tabulate(data, tablefmt="plain"))
+    return click.echo()

platformio/account/org/commands/remove.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+
+
+@click.command("remove", short_help="Remove an owner from organization")
+@click.argument(
+    "orgname",
+)
+@click.argument(
+    "username",
+)
+def org_remove_cmd(orgname, username):
+    client = AccountClient()
+    client.remove_org_owner(orgname, username)
+    return click.secho(
+        "The `%s` owner has been successfully removed from the `%s` organization."
+        % (username, orgname),
+        fg="green",
+    )

platformio/account/org/commands/update.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_email, validate_orgname
+
+
+@click.command("update", short_help="Update organization")
+@click.argument("cur_orgname")
+@click.option(
+    "--orgname",
+    callback=lambda _, __, value: validate_orgname(value),
+    help="A new orgname",
+)
+@click.option("--email")
+@click.option("--displayname")
+def org_update_cmd(cur_orgname, **kwargs):
+    client = AccountClient()
+    org = client.get_org(cur_orgname)
+    del org["owners"]
+    new_org = org.copy()
+    if not any(kwargs.values()):
+        for field in org:
+            new_org[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=org[field]
+            )
+            if field == "email":
+                validate_email(new_org[field])
+            if field == "orgname":
+                validate_orgname(new_org[field])
+    else:
+        new_org.update(
+            {key.replace("new_", ""): value for key, value in kwargs.items() if value}
+        )
+    client.update_org(cur_orgname, new_org)
+    return click.secho(
+        "The organization `%s` has been successfully updated." % cur_orgname,
+        fg="green",
+    )

platformio/account/team/cli.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.team.commands.add import team_add_cmd
+from platformio.account.team.commands.create import team_create_cmd
+from platformio.account.team.commands.destroy import team_destroy_cmd
+from platformio.account.team.commands.list import team_list_cmd
+from platformio.account.team.commands.remove import team_remove_cmd
+from platformio.account.team.commands.update import team_update_cmd
+
+
+@click.group(
+    "team",
+    commands=[
+        team_add_cmd,
+        team_create_cmd,
+        team_destroy_cmd,
+        team_list_cmd,
+        team_remove_cmd,
+        team_update_cmd,
+    ],
+    short_help="Manage organization teams",
+)
+def cli():
+    pass

platformio/account/team/commands/add.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("add", short_help="Add a new member to team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.argument(
+    "username",
+)
+def team_add_cmd(orgname_teamname, username):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.add_team_member(orgname, teamname, username)
+    return click.secho(
+        "The new member %s has been successfully added to the %s team."
+        % (username, teamname),
+        fg="green",
+    )

platformio/account/team/commands/create.py

@@ -0,0 +1,39 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("create", short_help="Create a new team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(
+        value, teamname_validate=True
+    ),
+)
+@click.option(
+    "--description",
+)
+def team_create_cmd(orgname_teamname, description):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.create_team(orgname, teamname, description)
+    return click.secho(
+        "The team %s has been successfully created." % teamname,
+        fg="green",
+    )

platformio/account/team/commands/destroy.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("destroy", short_help="Destroy a team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+def team_destroy_cmd(orgname_teamname):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    click.confirm(
+        click.style(
+            "Are you sure you want to destroy the %s team?" % teamname, fg="yellow"
+        ),
+        abort=True,
+    )
+    client = AccountClient()
+    client.destroy_team(orgname, teamname)
+    return click.secho(
+        "The team %s has been successfully destroyed." % teamname,
+        fg="green",
+    )

platformio/account/team/commands/list.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import click
+from tabulate import tabulate
+
+from platformio.account.client import AccountClient
+
+
+@click.command("list", short_help="List teams")
+@click.argument("orgname", required=False)
+@click.option("--json-output", is_flag=True)
+def team_list_cmd(orgname, json_output):
+    client = AccountClient()
+    data = {}
+    if not orgname:
+        for item in client.list_orgs():
+            teams = client.list_teams(item.get("orgname"))
+            data[item.get("orgname")] = teams
+    else:
+        teams = client.list_teams(orgname)
+        data[orgname] = teams
+    if json_output:
+        return click.echo(json.dumps(data[orgname] if orgname else data))
+    if not any(data.values()):
+        return click.secho("You do not have any teams.", fg="yellow")
+    for org_name in data:
+        for team in data[org_name]:
+            click.echo()
+            click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan")
+            click.echo("-" * len("%s:%s" % (org_name, team.get("name"))))
+            table_data = []
+            if team.get("description"):
+                table_data.append(("Description:", team.get("description")))
+            table_data.append(
+                (
+                    "Members:",
+                    ", ".join(
+                        (member.get("username") for member in team.get("members"))
+                    )
+                    if team.get("members")
+                    else "-",
+                )
+            )
+            click.echo(tabulate(table_data, tablefmt="plain"))
+    return click.echo()

platformio/account/team/commands/remove.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname
+
+
+@click.command("remove", short_help="Remove a member from team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.argument("username")
+def team_remove_cmd(orgname_teamname, username):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    client.remove_team_member(orgname, teamname, username)
+    return click.secho(
+        "The %s member has been successfully removed from the %s team."
+        % (username, teamname),
+        fg="green",
+    )

platformio/account/team/commands/update.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from platformio.account.client import AccountClient
+from platformio.account.validate import validate_orgname_teamname, validate_teamname
+
+
+@click.command("update", short_help="Update team")
+@click.argument(
+    "orgname_teamname",
+    metavar="ORGNAME:TEAMNAME",
+    callback=lambda _, __, value: validate_orgname_teamname(value),
+)
+@click.option(
+    "--name",
+    callback=lambda _, __, value: validate_teamname(value),
+    help="A new team name",
+)
+@click.option(
+    "--description",
+)
+def team_update_cmd(orgname_teamname, **kwargs):
+    orgname, teamname = orgname_teamname.split(":", 1)
+    client = AccountClient()
+    team = client.get_team(orgname, teamname)
+    del team["id"]
+    del team["members"]
+    new_team = team.copy()
+    if not any(kwargs.values()):
+        for field in team:
+            new_team[field] = click.prompt(
+                field.replace("_", " ").capitalize(), default=team[field]
+            )
+            if field == "name":
+                validate_teamname(new_team[field])
+    else:
+        new_team.update({key: value for key, value in kwargs.items() if value})
+    client.update_team(orgname, teamname, new_team)
+    return click.secho(
+        "The team %s has been successfully updated." % teamname,
+        fg="green",
+    )

platformio/account/validate.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import click
+
+
+def validate_username(value, field="username"):
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I):
+        raise click.BadParameter(
+            "Invalid %s format. "
+            "%s must contain only alphanumeric characters "
+            "or single hyphens, cannot begin or end with a hyphen, "
+            "and must not be longer than 38 characters."
+            % (field.lower(), field.capitalize())
+        )
+    return value
+
+
+def validate_email(value):
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d_.+-]+@[a-z\d\-]+\.[a-z\d\-.]+$", value, flags=re.I):
+        raise click.BadParameter("Invalid email address")
+    return value
+
+
+def validate_password(value):
+    value = str(value).strip()
+    if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
+        raise click.BadParameter(
+            "Invalid password format. "
+            "Password must contain at least 8 characters"
+            " including a number and a lowercase letter"
+        )
+    return value
+
+
+def validate_orgname(value):
+    return validate_username(value, "Organization name")
+
+
+def validate_orgname_teamname(value, teamname_validate=False):
+    if ":" not in value:
+        raise click.BadParameter(
+            "Please specify organization and team name in the next"
+            " format - orgname:teamname. For example, mycompany:DreamTeam"
+        )
+    teamname = str(value.strip().split(":", 1)[1])
+    if teamname_validate:
+        validate_teamname(teamname)
+    return value
+
+
+def validate_teamname(value):
+    if not value:
+        return value
+    value = str(value).strip()
+    if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I):
+        raise click.BadParameter(
+            "Invalid team name format. "
+            "Team name must only contain alphanumeric characters, "
+            "single hyphens, underscores, spaces. It can not "
+            "begin or end with a hyphen or a underscore and must"
+            " not be longer than 20 characters."
+        )
+    return value

platformio/app.py

@@ -12,62 +12,43 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import codecs
+from __future__ import absolute_import
+
 import getpass
 import hashlib
+import json
 import os
 import platform
 import socket
 import uuid
-from os import environ, getenv, listdir, remove
-from os.path import dirname, isdir, isfile, join, realpath
-from time import time
 
-import requests
-
-from platformio import __version__, exception, fs, lockfile
-from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data
-from platformio.proc import is_ci
-from platformio.project.helpers import (
-    get_default_projects_dir,
-    get_project_cache_dir,
-    get_project_core_dir,
-)
+from platformio import __version__, exception, fs, proc
+from platformio.compat import IS_WINDOWS, hashlib_encode_data
+from platformio.package.lockfile import LockFile
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import get_default_projects_dir
 
 
 def projects_dir_validate(projects_dir):
-    assert isdir(projects_dir)
-    return realpath(projects_dir)
+    assert os.path.isdir(projects_dir)
+    return os.path.abspath(projects_dir)
 
 
 DEFAULT_SETTINGS = {
-    "auto_update_libraries": {
-        "description": "Automatically update libraries (Yes/No)",
-        "value": False,
-    },
-    "auto_update_platforms": {
-        "description": "Automatically update platforms (Yes/No)",
-        "value": False,
-    },
-    "check_libraries_interval": {
-        "description": "Check for the library updates interval (days)",
-        "value": 7,
-    },
     "check_platformio_interval": {
-        "description": "Check for the new PlatformIO interval (days)",
-        "value": 3,
-    },
-    "check_platforms_interval": {
-        "description": "Check for the platform updates interval (days)",
+        "description": "Check for the new PlatformIO Core interval (days)",
         "value": 7,
     },
+    "check_prune_system_threshold": {
+        "description": "Check for pruning unnecessary data threshold (megabytes)",
+        "value": 1024,
+    },
     "enable_cache": {
-        "description": "Enable caching for API requests and Library Manager",
+        "description": "Enable caching for HTTP API requests",
         "value": True,
     },
-    "strict_ssl": {"description": "Strict SSL for PlatformIO Services", "value": False},
     "enable_telemetry": {
-        "description": ("Telemetry service <http://bit.ly/pio-telemetry> (Yes/No)"),
+        "description": ("Telemetry service <https://bit.ly/pio-telemetry> (Yes/No)"),
         "value": True,
     },
     "force_verbose": {
@@ -75,7 +56,7 @@ DEFAULT_SETTINGS = {
         "value": False,
     },
     "projects_dir": {
-        "description": "Default location for PlatformIO projects (PIO Home)",
+        "description": "Default location for PlatformIO projects (PlatformIO Home)",
         "value": get_default_projects_dir(),
         "validator": projects_dir_validate,
     },
@@ -94,7 +75,10 @@ class State(object):
         self.path = path
         self.lock = lock
         if not self.path:
-            self.path = join(get_project_core_dir(), "appstate.json")
+            core_dir = ProjectConfig.get_instance().get("platformio", "core_dir")
+            if not os.path.isdir(core_dir):
+                os.makedirs(core_dir)
+            self.path = os.path.join(core_dir, "appstate.json")
         self._storage = {}
         self._lockfile = None
         self.modified = False
@@ -102,7 +86,7 @@ class State(object):
     def __enter__(self):
         try:
             self._lock_state_file()
-            if isfile(self.path):
+            if os.path.isfile(self.path):
                 self._storage = fs.load_json(self.path)
             assert isinstance(self._storage, dict)
         except (
@@ -117,20 +101,20 @@ class State(object):
     def __exit__(self, type_, value, traceback):
         if self.modified:
             try:
-                with open(self.path, "w") as fp:
-                    fp.write(dump_json_to_unicode(self._storage))
+                with open(self.path, mode="w", encoding="utf8") as fp:
+                    fp.write(json.dumps(self._storage))
             except IOError:
-                raise exception.HomeDirPermissionsError(get_project_core_dir())
+                raise exception.HomeDirPermissionsError(os.path.dirname(self.path))
         self._unlock_state_file()
 
     def _lock_state_file(self):
         if not self.lock:
             return
-        self._lockfile = lockfile.LockFile(self.path)
+        self._lockfile = LockFile(self.path)
         try:
             self._lockfile.acquire()
         except IOError:
-            raise exception.HomeDirPermissionsError(dirname(self.path))
+            raise exception.HomeDirPermissionsError(os.path.dirname(self.path))
 
     def _unlock_state_file(self):
         if hasattr(self, "_lockfile") and self._lockfile:
@@ -144,6 +128,9 @@ class State(object):
     def as_dict(self):
         return self._storage
 
+    def keys(self):
+        return self._storage.keys()
+
     def get(self, key, default=True):
         return self._storage.get(key, default)
 
@@ -169,146 +156,6 @@ class State(object):
         return item in self._storage
 
 
-class ContentCache(object):
-    def __init__(self, cache_dir=None):
-        self.cache_dir = None
-        self._db_path = None
-        self._lockfile = None
-
-        self.cache_dir = cache_dir or get_project_cache_dir()
-        self._db_path = join(self.cache_dir, "db.data")
-
-    def __enter__(self):
-        self.delete()
-        return self
-
-    def __exit__(self, type_, value, traceback):
-        pass
-
-    def _lock_dbindex(self):
-        if not self.cache_dir:
-            os.makedirs(self.cache_dir)
-        self._lockfile = lockfile.LockFile(self.cache_dir)
-        try:
-            self._lockfile.acquire()
-        except:  # pylint: disable=bare-except
-            return False
-
-        return True
-
-    def _unlock_dbindex(self):
-        if self._lockfile:
-            self._lockfile.release()
-        return True
-
-    def get_cache_path(self, key):
-        assert "/" not in key and "\\" not in key
-        key = str(key)
-        assert len(key) > 3
-        return join(self.cache_dir, key[-2:], key)
-
-    @staticmethod
-    def key_from_args(*args):
-        h = hashlib.md5()
-        for arg in args:
-            if arg:
-                h.update(hashlib_encode_data(arg))
-        return h.hexdigest()
-
-    def get(self, key):
-        cache_path = self.get_cache_path(key)
-        if not isfile(cache_path):
-            return None
-        with codecs.open(cache_path, "rb", encoding="utf8") as fp:
-            return fp.read()
-
-    def set(self, key, data, valid):
-        if not get_setting("enable_cache"):
-            return False
-        cache_path = self.get_cache_path(key)
-        if isfile(cache_path):
-            self.delete(key)
-        if not data:
-            return False
-        if not isdir(self.cache_dir):
-            os.makedirs(self.cache_dir)
-        tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
-        assert valid.endswith(tuple(tdmap))
-        expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1]))
-
-        if not self._lock_dbindex():
-            return False
-
-        if not isdir(dirname(cache_path)):
-            os.makedirs(dirname(cache_path))
-        try:
-            with codecs.open(cache_path, "wb", encoding="utf8") as fp:
-                fp.write(data)
-            with open(self._db_path, "a") as fp:
-                fp.write("%s=%s\n" % (str(expire_time), cache_path))
-        except UnicodeError:
-            if isfile(cache_path):
-                try:
-                    remove(cache_path)
-                except OSError:
-                    pass
-
-        return self._unlock_dbindex()
-
-    def delete(self, keys=None):
-        """ Keys=None, delete expired items """
-        if not isfile(self._db_path):
-            return None
-        if not keys:
-            keys = []
-        if not isinstance(keys, list):
-            keys = [keys]
-        paths_for_delete = [self.get_cache_path(k) for k in keys]
-        found = False
-        newlines = []
-        with open(self._db_path) as fp:
-            for line in fp.readlines():
-                line = line.strip()
-                if "=" not in line:
-                    continue
-                expire, path = line.split("=")
-                try:
-                    if (
-                        time() < int(expire)
-                        and isfile(path)
-                        and path not in paths_for_delete
-                    ):
-                        newlines.append(line)
-                        continue
-                except ValueError:
-                    pass
-                found = True
-                if isfile(path):
-                    try:
-                        remove(path)
-                        if not listdir(dirname(path)):
-                            fs.rmtree(dirname(path))
-                    except OSError:
-                        pass
-
-        if found and self._lock_dbindex():
-            with open(self._db_path, "w") as fp:
-                fp.write("\n".join(newlines) + "\n")
-            self._unlock_dbindex()
-
-        return True
-
-    def clean(self):
-        if not self.cache_dir or not isdir(self.cache_dir):
-            return
-        fs.rmtree(self.cache_dir)
-
-
-def clean_cache():
-    with ContentCache() as cc:
-        cc.clean()
-
-
 def sanitize_setting(name, value):
     if name not in DEFAULT_SETTINGS:
         raise exception.InvalidSettingName(name)
@@ -346,8 +193,8 @@ def delete_state_item(name):
 
 def get_setting(name):
     _env_name = "PLATFORMIO_SETTING_%s" % name.upper()
-    if _env_name in environ:
-        return sanitize_setting(name, getenv(_env_name))
+    if _env_name in os.environ:
+        return sanitize_setting(name, os.getenv(_env_name))
 
     with State() as state:
         if "settings" in state and name in state["settings"]:
@@ -383,8 +230,8 @@ def is_disabled_progressbar():
     return any(
         [
             get_session_var("force_option"),
-            is_ci(),
-            getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true",
+            proc.is_ci(),
+            os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true",
         ]
     )
 
@@ -394,33 +241,25 @@ def get_cid():
     if cid:
         return cid
     uid = None
-    if getenv("C9_UID"):
-        uid = getenv("C9_UID")
-    elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")):
-        try:
-            uid = (
-                requests.get(
-                    "{api}/user?token={token}".format(
-                        api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
-                        token=getenv("USER_TOKEN"),
-                    )
-                )
-                .json()
-                .get("id")
-            )
-        except:  # pylint: disable=bare-except
-            pass
+    if os.getenv("GITHUB_USER"):
+        uid = os.getenv("GITHUB_USER")
+    elif os.getenv("GITPOD_GIT_USER_NAME"):
+        uid = os.getenv("GITPOD_GIT_USER_NAME")
     if not uid:
         uid = uuid.getnode()
     cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest())
     cid = str(cid)
-    if WINDOWS or os.getuid() > 0:  # pylint: disable=no-member
+    if IS_WINDOWS or os.getuid() > 0:  # pylint: disable=no-member
         set_state_item("cid", cid)
     return cid
 
 
 def get_user_agent():
-    data = ["PlatformIO/%s" % __version__, "CI/%d" % int(is_ci())]
+    data = [
+        "PlatformIO/%s" % __version__,
+        "CI/%d" % int(proc.is_ci()),
+        "Container/%d" % int(proc.is_container()),
+    ]
     if get_session_var("caller_id"):
         data.append("Caller/%s" % get_session_var("caller_id"))
     if os.getenv("PLATFORMIO_IDE"):

platformio/builder/main.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+import os
 import sys
-from os import environ, makedirs
-from os.path import isdir, join
 from time import time
 
 import click
@@ -28,9 +28,8 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
-from platformio import fs
-from platformio.compat import dump_json_to_unicode
-from platformio.managers.platform import PlatformBase
+from platformio import compat, fs
+from platformio.platform.base import PlatformBase
 from platformio.proc import get_pythonexe_path
 from platformio.project.helpers import get_project_dir
 
@@ -45,46 +44,58 @@ clivars.AddVariables(
     ("PIOENV",),
     ("PIOTEST_RUNNING_NAME",),
     ("UPLOAD_PORT",),
+    ("PROGRAM_ARGS",),
 )
 
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
         "ar",
-        "as",
         "cc",
         "c++",
         "link",
+        "pioasm",
         "platformio",
-        "pioplatform",
         "pioproject",
+        "pioplatform",
+        "piotest",
+        "piotarget",
         "piomaxlen",
         "piolib",
         "pioupload",
-        "piomisc",
-        "pioide",
         "piosize",
+        "pioino",
+        "piomisc",
+        "piointegration",
     ],
-    toolpath=[join(fs.get_source_dir(), "builder", "tools")],
+    toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")],
     variables=clivars,
     # Propagating External Environment
-    ENV=environ,
+    ENV=os.environ,
     UNIX_TIME=int(time()),
-    BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"),
-    BUILD_SRC_DIR=join("$BUILD_DIR", "src"),
-    BUILD_TEST_DIR=join("$BUILD_DIR", "test"),
-    COMPILATIONDB_PATH=join("$BUILD_DIR", "compile_commands.json"),
+    BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
+    BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
+    BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
+    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
     LIBPATH=["$BUILD_DIR"],
     PROGNAME="program",
-    PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
+    PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
     PYTHONEXE=get_pythonexe_path(),
+    IDE_EXTRA_DATA={},
 )
 
+# Declare command verbose messages
+command_strings = dict(
+    ARCOM="Archiving",
+    LINKCOM="Linking",
+    RANLIBCOM="Indexing",
+    ASCOM="Compiling",
+    ASPPCOM="Compiling",
+    CCCOM="Compiling",
+    CXXCOM="Compiling",
+)
 if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
-    DEFAULT_ENV_OPTIONS["ARCOMSTR"] = "Archiving $TARGET"
-    DEFAULT_ENV_OPTIONS["LINKCOMSTR"] = "Linking $TARGET"
-    DEFAULT_ENV_OPTIONS["RANLIBCOMSTR"] = "Indexing $TARGET"
-    for k in ("ASCOMSTR", "ASPPCOMSTR", "CCCOMSTR", "CXXCOMSTR"):
-        DEFAULT_ENV_OPTIONS[k] = "Compiling $TARGET"
+    for name, value in command_strings.items():
+        DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value)
 
 env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
 
@@ -101,53 +112,70 @@ env.Replace(
 config = env.GetProjectConfig()
 env.Replace(
     PROJECT_DIR=get_project_dir(),
-    PROJECT_CORE_DIR=config.get_optional_dir("core"),
-    PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"),
-    PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"),
-    PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"),
-    PROJECT_INCLUDE_DIR=config.get_optional_dir("include"),
-    PROJECT_SRC_DIR=config.get_optional_dir("src"),
-    PROJECTSRC_DIR=config.get_optional_dir("src"),  # legacy for dev/platform
-    PROJECT_TEST_DIR=config.get_optional_dir("test"),
-    PROJECT_DATA_DIR=config.get_optional_dir("data"),
-    PROJECTDATA_DIR=config.get_optional_dir("data"),  # legacy for dev/platform
-    PROJECT_BUILD_DIR=config.get_optional_dir("build"),
-    BUILD_CACHE_DIR=config.get_optional_dir("build_cache"),
+    PROJECT_CORE_DIR=config.get("platformio", "core_dir"),
+    PROJECT_PACKAGES_DIR=config.get("platformio", "packages_dir"),
+    PROJECT_WORKSPACE_DIR=config.get("platformio", "workspace_dir"),
+    PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"),
+    PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"),
+    PROJECT_SRC_DIR=config.get("platformio", "src_dir"),
+    PROJECTSRC_DIR="$PROJECT_SRC_DIR",  # legacy for dev/platform
+    PROJECT_TEST_DIR=config.get("platformio", "test_dir"),
+    PROJECT_DATA_DIR=config.get("platformio", "data_dir"),
+    PROJECTDATA_DIR="$PROJECT_DATA_DIR",  # legacy for dev/platform
+    PROJECT_BUILD_DIR=config.get("platformio", "build_dir"),
+    BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"),
     LIBSOURCE_DIRS=[
-        config.get_optional_dir("lib"),
-        join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
-        config.get_optional_dir("globallib"),
+        config.get("platformio", "lib_dir"),
+        os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
+        config.get("platformio", "globallib_dir"),
     ],
 )
 
-if env.subst("$BUILD_CACHE_DIR"):
-    if not isdir(env.subst("$BUILD_CACHE_DIR")):
-        makedirs(env.subst("$BUILD_CACHE_DIR"))
-    env.CacheDir("$BUILD_CACHE_DIR")
-
 if int(ARGUMENTS.get("ISATTY", 0)):
     # pylint: disable=protected-access
     click._compat.isatty = lambda stream: True
 
-if env.GetOption("clean"):
-    env.PioClean(env.subst("$BUILD_DIR"))
+if compat.IS_WINDOWS and sys.version_info >= (3, 8) and os.getcwd().startswith("\\\\"):
+    click.secho("!!! WARNING !!!\t\t" * 3, fg="red")
+    click.secho(
+        "Your project is located on a mapped network drive but the "
+        "current command-line shell does not support the UNC paths.",
+        fg="yellow",
+    )
+    click.secho(
+        "Please move your project to a physical drive or check this workaround: "
+        "https://bit.ly/3kuU5mP\n",
+        fg="yellow",
+    )
+
+if env.subst("$BUILD_CACHE_DIR"):
+    if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")):
+        os.makedirs(env.subst("$BUILD_CACHE_DIR"))
+    env.CacheDir("$BUILD_CACHE_DIR")
+
+is_clean_all = "cleanall" in COMMAND_LINE_TARGETS
+if env.GetOption("clean") or is_clean_all:
+    env.PioClean(is_clean_all)
     env.Exit(0)
-elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
+
+if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     click.echo("Verbose mode can be enabled via `-v, --verbose` option")
 
 # Dynamically load dependent tools
 if "compiledb" in COMMAND_LINE_TARGETS:
     env.Tool("compilation_db")
 
-if not isdir(env.subst("$BUILD_DIR")):
-    makedirs(env.subst("$BUILD_DIR"))
+if not os.path.isdir(env.subst("$BUILD_DIR")):
+    os.makedirs(env.subst("$BUILD_DIR"))
 
 env.LoadProjectOptions()
 env.LoadPioPlatform()
 
 env.SConscriptChdir(0)
 env.SConsignFile(
-    join("$BUILD_DIR", ".sconsign%d%d.db" % (sys.version_info[0], sys.version_info[1]))
+    os.path.join(
+        "$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1])
+    )
 )
 
 for item in env.GetExtraScripts("pre"):
@@ -188,7 +216,7 @@ env.AddPreAction(
     ),
 )
 
-AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS))
+AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS))
 AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS))
 
 ##############################################################################
@@ -197,17 +225,21 @@ if "envdump" in COMMAND_LINE_TARGETS:
     click.echo(env.Dump())
     env.Exit(0)
 
-if "idedata" in COMMAND_LINE_TARGETS:
+if env.IsIntegrationDump():
+    projenv = None
     try:
         Import("projenv")
     except:  # pylint: disable=bare-except
         projenv = env
-    click.echo(
-        "\n%s\n"
-        % dump_json_to_unicode(
-            projenv.DumpIDEData()  # pylint: disable=undefined-variable
-        )
-    )
+    data = projenv.DumpIntegrationData(env)
+    # dump to file for the further reading by project.helpers.load_build_metadata
+    with open(
+        projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")),
+        mode="w",
+        encoding="utf8",
+    ) as fp:
+        json.dump(data, fp)
+    click.echo("\n%s\n" % json.dumps(data))  # pylint: disable=undefined-variable
     env.Exit(0)
 
 if "sizedata" in COMMAND_LINE_TARGETS:

platformio/builder/tools/compilation_db.py

@@ -1,17 +1,24 @@
 # Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-# Copyright 2015 MongoDB Inc.
+# Copyright 2020 MongoDB Inc.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
 #
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 # pylint: disable=unused-argument, protected-access, unused-variable, import-error
 # Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py
@@ -34,7 +41,7 @@ from platformio.proc import where_is_program
 # should hold the compilation database, otherwise, the file defaults to compile_commands.json,
 # which is the name that most clang tools search for by default.
 
-# TODO: Is there a better way to do this than this global? Right now this exists so that the
+# Is there a better way to do this than this global? Right now this exists so that the
 # emitter we add can record all of the things it emits, so that the scanner for the top level
 # compilation database can access the complete list, and also so that the writer has easy
 # access to write all of the files. But it seems clunky. How can the emitter and the scanner
@@ -51,7 +58,7 @@ class __CompilationDbNode(SCons.Node.Python.Value):
 
 
 def changed_since_last_build_node(*args, **kwargs):
-    """ Dummy decider to force always building"""
+    """Dummy decider to force always building"""
     return True
 
 
@@ -97,7 +104,7 @@ def makeEmitCompilationDbEntry(comstr):
             __COMPILATIONDB_ENV=env,
         )
 
-        # TODO: Technically, these next two lines should not be required: it should be fine to
+        # Technically, these next two lines should not be required: it should be fine to
         # cache the entries. However, they don't seem to update properly. Since they are quick
         # to re-generate disable caching and sidestep this problem.
         env.AlwaysBuild(entry)
@@ -145,7 +152,7 @@ def WriteCompilationDb(target, source, env):
         item["file"] = os.path.abspath(item["file"])
         entries.append(item)
 
-    with open(str(target[0]), "w") as target_file:
+    with open(str(target[0]), mode="w", encoding="utf8") as target_file:
         json.dump(
             entries, target_file, sort_keys=True, indent=4, separators=(",", ": ")
         )

platformio/builder/tools/pioasm.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import SCons.Tool.asm  # pylint: disable=import-error
+
+#
+# Resolve https://github.com/platformio/platformio-core/issues/3917
+# Avoid forcing .S to bare assembly on Windows OS
+#
+
+if ".S" in SCons.Tool.asm.ASSuffixes:
+    SCons.Tool.asm.ASSuffixes.remove(".S")
+if ".S" not in SCons.Tool.asm.ASPPSuffixes:
+    SCons.Tool.asm.ASPPSuffixes.append(".S")
+
+
+generate = SCons.Tool.asm.generate
+exists = SCons.Tool.asm.exists

platformio/builder/tools/pioino.py

@@ -0,0 +1,259 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import atexit
+import glob
+import io
+import os
+import re
+import tempfile
+
+import click
+
+from platformio.compat import get_filesystem_encoding, get_locale_encoding
+
+
+class InoToCPPConverter(object):
+
+    PROTOTYPE_RE = re.compile(
+        r"""^(
+        (?:template\<.*\>\s*)?      # template
+        ([a-z_\d\&]+\*?\s+){1,2}    # return type
+        ([a-z_\d]+\s*)              # name of prototype
+        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
+        )\s*(\{|;)                  # must end with `{` or `;`
+        """,
+        re.X | re.M | re.I,
+    )
+    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
+    PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
+
+    def __init__(self, env):
+        self.env = env
+        self._main_ino = None
+        self._safe_encoding = None
+
+    def read_safe_contents(self, path):
+        error_reported = False
+        for encoding in (
+            "utf-8",
+            None,
+            get_filesystem_encoding(),
+            get_locale_encoding(),
+            "latin-1",
+        ):
+            try:
+                with io.open(path, encoding=encoding) as fp:
+                    contents = fp.read()
+                    self._safe_encoding = encoding
+                    return contents
+            except UnicodeDecodeError:
+                if not error_reported:
+                    error_reported = True
+                    click.secho(
+                        "Unicode decode error has occurred, please remove invalid "
+                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
+                        % path,
+                        fg="yellow",
+                        err=True,
+                    )
+        return ""
+
+    def write_safe_contents(self, path, contents):
+        with io.open(
+            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
+        ) as fp:
+            return fp.write(contents)
+
+    def is_main_node(self, contents):
+        return self.DETECTMAIN_RE.search(contents)
+
+    def convert(self, nodes):
+        contents = self.merge(nodes)
+        if not contents:
+            return None
+        return self.process(contents)
+
+    def merge(self, nodes):
+        assert nodes
+        lines = []
+        for node in nodes:
+            contents = self.read_safe_contents(node.get_path())
+            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
+            if self.is_main_node(contents):
+                lines = _lines + lines
+                self._main_ino = node.get_path()
+            else:
+                lines.extend(_lines)
+
+        if not self._main_ino:
+            self._main_ino = nodes[0].get_path()
+
+        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
+
+    def process(self, contents):
+        out_file = self._main_ino + ".cpp"
+        assert self._gcc_preprocess(contents, out_file)
+        contents = self.read_safe_contents(out_file)
+        contents = self._join_multiline_strings(contents)
+        self.write_safe_contents(out_file, self.append_prototypes(contents))
+        return out_file
+
+    def _gcc_preprocess(self, contents, out_file):
+        tmp_path = tempfile.mkstemp()[1]
+        self.write_safe_contents(tmp_path, contents)
+        self.env.Execute(
+            self.env.VerboseAction(
+                '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
+                    out_file, tmp_path
+                ),
+                "Converting " + os.path.basename(out_file[:-4]),
+            )
+        )
+        atexit.register(_delete_file, tmp_path)
+        return os.path.isfile(out_file)
+
+    def _join_multiline_strings(self, contents):
+        if "\\\n" not in contents:
+            return contents
+        newlines = []
+        linenum = 0
+        stropen = False
+        for line in contents.split("\n"):
+            _linenum = self._parse_preproc_line_num(line)
+            if _linenum is not None:
+                linenum = _linenum
+            else:
+                linenum += 1
+
+            if line.endswith("\\"):
+                if line.startswith('"'):
+                    stropen = True
+                    newlines.append(line[:-1])
+                    continue
+                if stropen:
+                    newlines[len(newlines) - 1] += line[:-1]
+                    continue
+            elif stropen and line.endswith(('",', '";')):
+                newlines[len(newlines) - 1] += line
+                stropen = False
+                newlines.append(
+                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
+                )
+                continue
+
+            newlines.append(line)
+
+        return "\n".join(newlines)
+
+    @staticmethod
+    def _parse_preproc_line_num(line):
+        if not line.startswith("#"):
+            return None
+        tokens = line.split(" ", 3)
+        if len(tokens) > 2 and tokens[1].isdigit():
+            return int(tokens[1])
+        return None
+
+    def _parse_prototypes(self, contents):
+        prototypes = []
+        reserved_keywords = set(["if", "else", "while"])
+        for match in self.PROTOTYPE_RE.finditer(contents):
+            if (
+                set([match.group(2).strip(), match.group(3).strip()])
+                & reserved_keywords
+            ):
+                continue
+            prototypes.append(match)
+        return prototypes
+
+    def _get_total_lines(self, contents):
+        total = 0
+        if contents.endswith("\n"):
+            contents = contents[:-1]
+        for line in contents.split("\n")[::-1]:
+            linenum = self._parse_preproc_line_num(line)
+            if linenum is not None:
+                return total + linenum
+            total += 1
+        return total
+
+    def append_prototypes(self, contents):
+        prototypes = self._parse_prototypes(contents) or []
+
+        # skip already declared prototypes
+        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
+        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
+
+        if not prototypes:
+            return contents
+
+        prototype_names = set(m.group(3).strip() for m in prototypes)
+        split_pos = prototypes[0].start()
+        match_ptrs = re.search(
+            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
+            contents[:split_pos],
+            re.M,
+        )
+        if match_ptrs:
+            split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
+
+        result = []
+        result.append(contents[:split_pos].strip())
+        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
+        result.append(
+            '#line %d "%s"'
+            % (
+                self._get_total_lines(contents[:split_pos]),
+                self._main_ino.replace("\\", "/"),
+            )
+        )
+        result.append(contents[split_pos:].strip())
+        return "\n".join(result)
+
+
+def FindInoNodes(env):
+    src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
+    return env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
+        os.path.join(src_dir, "*.pde")
+    )
+
+
+def ConvertInoToCpp(env):
+    ino_nodes = env.FindInoNodes()
+    if not ino_nodes:
+        return
+    c = InoToCPPConverter(env)
+    out_file = c.convert(ino_nodes)
+
+    atexit.register(_delete_file, out_file)
+
+
+def _delete_file(path):
+    try:
+        if os.path.isfile(path):
+            os.remove(path)
+    except:  # pylint: disable=bare-except
+        pass
+
+
+def generate(env):
+    env.AddMethod(FindInoNodes)
+    env.AddMethod(ConvertInoToCpp)
+
+
+def exists(_):
+    return True

platformio/builder/tools/piointegration.py

@@ -14,41 +14,45 @@
 
 from __future__ import absolute_import
 
+import glob
 import os
-from glob import glob
 
-from SCons.Defaults import processDefines  # pylint: disable=import-error
+import SCons.Defaults  # pylint: disable=import-error
+import SCons.Subst  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 
-from platformio.compat import glob_escape
-from platformio.managers.core import get_core_package_dir
 from platformio.proc import exec_command, where_is_program
 
 
-def _dump_includes(env):
-    includes = {}
+def IsIntegrationDump(_):
+    return set(["_idedata", "idedata"]) & set(COMMAND_LINE_TARGETS)
 
-    includes["build"] = [
-        env.subst("$PROJECT_INCLUDE_DIR"),
-        env.subst("$PROJECT_SRC_DIR"),
-    ]
-    includes["build"].extend(
-        [os.path.realpath(env.subst(item)) for item in env.get("CPPPATH", [])]
+
+def DumpIntegrationIncludes(env):
+    result = dict(build=[], compatlib=[], toolchain=[])
+
+    result["build"].extend(
+        [
+            env.subst("$PROJECT_INCLUDE_DIR"),
+            env.subst("$PROJECT_SRC_DIR"),
+        ]
+    )
+    result["build"].extend(
+        [os.path.abspath(env.subst(item)) for item in env.get("CPPPATH", [])]
     )
 
     # installed libs
-    includes["compatlib"] = []
     for lb in env.GetLibBuilders():
-        includes["compatlib"].extend(
-            [os.path.realpath(inc) for inc in lb.get_include_dirs()]
+        result["compatlib"].extend(
+            [os.path.abspath(inc) for inc in lb.get_include_dirs()]
         )
 
     # includes from toolchains
     p = env.PioPlatform()
-    includes["toolchain"] = []
-    for name in p.get_installed_packages():
-        if p.get_package_type(name) != "toolchain":
+    for pkg in p.get_installed_packages(with_optional=False):
+        if p.get_package_type(pkg.metadata.name) != "toolchain":
             continue
-        toolchain_dir = glob_escape(p.get_package_dir(name))
+        toolchain_dir = glob.escape(pkg.path)
         toolchain_incglobs = [
             os.path.join(toolchain_dir, "*", "include", "c++", "*"),
             os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"),
@@ -56,17 +60,12 @@ def _dump_includes(env):
             os.path.join(toolchain_dir, "*", "include*"),
         ]
         for g in toolchain_incglobs:
-            includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)])
+            result["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)])
 
-    includes["unity"] = []
-    unity_dir = get_core_package_dir("tool-unity")
-    if unity_dir:
-        includes["unity"].append(unity_dir)
-
-    return includes
+    return result
 
 
-def _get_gcc_defines(env):
+def get_gcc_defines(env):
     items = []
     try:
         sysenv = os.environ.copy()
@@ -89,11 +88,13 @@ def _get_gcc_defines(env):
     return items
 
 
-def _dump_defines(env):
+def dump_defines(env):
     defines = []
     # global symbols
-    for item in processDefines(env.get("CPPDEFINES", [])):
-        defines.append(env.subst(item).replace("\\", ""))
+    for item in SCons.Defaults.processDefines(env.get("CPPDEFINES", [])):
+        item = item.strip()
+        if item:
+            defines.append(env.subst(item).replace("\\", ""))
 
     # special symbol for Atmel AVR MCU
     if env["PIOPLATFORM"] == "atmelavr":
@@ -112,15 +113,15 @@ def _dump_defines(env):
 
     # built-in GCC marcos
     # if env.GetCompilerType() == "gcc":
-    #     defines.extend(_get_gcc_defines(env))
+    #     defines.extend(get_gcc_defines(env))
 
     return defines
 
 
-def _get_svd_path(env):
+def dump_svd_path(env):
     svd_path = env.GetProjectOption("debug_svd_path")
     if svd_path:
-        return os.path.realpath(svd_path)
+        return os.path.abspath(svd_path)
 
     if "BOARD" not in env:
         return None
@@ -135,46 +136,45 @@ def _get_svd_path(env):
     # default file from ./platform/misc/svd folder
     p = env.PioPlatform()
     if os.path.isfile(os.path.join(p.get_dir(), "misc", "svd", svd_path)):
-        return os.path.realpath(os.path.join(p.get_dir(), "misc", "svd", svd_path))
+        return os.path.abspath(os.path.join(p.get_dir(), "misc", "svd", svd_path))
     return None
 
 
-def _escape_build_flag(flags):
-    return [flag if " " not in flag else '"%s"' % flag for flag in flags]
+def _subst_cmd(env, cmd):
+    args = env.subst_list(cmd, SCons.Subst.SUBST_CMD)[0]
+    return " ".join([SCons.Subst.quote_spaces(arg) for arg in args])
 
 
-def DumpIDEData(env):
-
-    env["__escape_build_flag"] = _escape_build_flag
-
-    LINTCCOM = (
-        "${__escape_build_flag(CFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS"
-    )
-    LINTCXXCOM = (
-        "${__escape_build_flag(CXXFLAGS)} ${__escape_build_flag(CCFLAGS)} $CPPFLAGS"
-    )
+def DumpIntegrationData(env, globalenv):
+    """env here is `projenv`"""
 
     data = {
         "env_name": env["PIOENV"],
-        "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()],
-        "defines": _dump_defines(env),
-        "includes": _dump_includes(env),
+        "libsource_dirs": [env.subst(item) for item in env.GetLibSourceDirs()],
+        "defines": dump_defines(env),
+        "includes": env.DumpIntegrationIncludes(),
         "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
         "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
         "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
         "prog_path": env.subst("$PROG_PATH"),
-        "flash_extra_images": [
-            {"offset": item[0], "path": env.subst(item[1])}
-            for item in env.get("FLASH_EXTRA_IMAGES", [])
-        ],
-        "svd_path": _get_svd_path(env),
+        "svd_path": dump_svd_path(env),
         "compiler_type": env.GetCompilerType(),
+        "targets": globalenv.DumpTargets(),
+        "extra": dict(
+            flash_images=[
+                {"offset": item[0], "path": env.subst(item[1])}
+                for item in env.get("FLASH_EXTRA_IMAGES", [])
+            ]
+        ),
     }
+    data["extra"].update(
+        env.get("INTEGRATION_EXTRA_DATA", env.get("IDE_EXTRA_DATA", {}))
+    )
 
     env_ = env.Clone()
     # https://github.com/platformio/platformio-atom-ide/issues/34
     _new_defines = []
-    for item in processDefines(env_.get("CPPDEFINES", [])):
+    for item in SCons.Defaults.processDefines(env_.get("CPPDEFINES", [])):
         item = item.replace('\\"', '"')
         if " " in item:
             _new_defines.append(item.replace(" ", "\\\\ "))
@@ -182,7 +182,13 @@ def DumpIDEData(env):
             _new_defines.append(item)
     env_.Replace(CPPDEFINES=_new_defines)
 
-    data.update({"cc_flags": env_.subst(LINTCCOM), "cxx_flags": env_.subst(LINTCXXCOM)})
+    # export C/C++ build flags
+    data.update(
+        {
+            "cc_flags": _subst_cmd(env_, "$CFLAGS $CCFLAGS $CPPFLAGS"),
+            "cxx_flags": _subst_cmd(env_, "$CXXFLAGS $CCFLAGS $CPPFLAGS"),
+        }
+    )
 
     return data
 
@@ -192,5 +198,7 @@ def exists(_):
 
 
 def generate(env):
-    env.AddMethod(DumpIDEData)
+    env.AddMethod(IsIntegrationDump)
+    env.AddMethod(DumpIntegrationIncludes)
+    env.AddMethod(DumpIntegrationData)
     return env

platformio/builder/tools/piolib.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=no-member, no-self-use, unused-argument, too-many-lines
+# pylint: disable=no-self-use, unused-argument, too-many-lines
 # pylint: disable=too-many-instance-attributes, too-many-public-methods
 # pylint: disable=assignment-from-no-return
 
@@ -23,22 +23,26 @@ import io
 import os
 import re
 import sys
-from os.path import basename, commonprefix, isdir, isfile, join, realpath, sep
 
 import click
 import SCons.Scanner  # pylint: disable=import-error
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
-from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import exception, fs, util
 from platformio.builder.tools import platformio as piotool
-from platformio.compat import WINDOWS, hashlib_encode_data, string_types
-from platformio.managers.lib import LibraryManager
+from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
+from platformio.http import HTTPClientError, InternetIsOffline
+from platformio.package.exception import (
+    MissingPackageManifestError,
+    UnknownPackageError,
+)
+from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manifest.parser import (
     ManifestParserError,
     ManifestParserFactory,
 )
+from platformio.package.meta import PackageItem
 from platformio.project.options import ProjectOptions
 
 
@@ -46,29 +50,39 @@ class LibBuilderFactory(object):
     @staticmethod
     def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))):
         clsname = "UnknownLibBuilder"
-        if isfile(join(path, "library.json")):
+        if os.path.isfile(os.path.join(path, "library.json")):
             clsname = "PlatformIOLibBuilder"
         else:
             used_frameworks = LibBuilderFactory.get_used_frameworks(env, path)
             common_frameworks = set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks)
             if common_frameworks:
-                clsname = "%sLibBuilder" % list(common_frameworks)[0].title()
+                clsname = "%sLibBuilder" % list(common_frameworks)[0].capitalize()
             elif used_frameworks:
-                clsname = "%sLibBuilder" % used_frameworks[0].title()
+                clsname = "%sLibBuilder" % used_frameworks[0].capitalize()
+
+        obj = globals()[clsname](env, path, verbose=verbose)
+
+        # Handle PlatformIOLibBuilder.manifest.build.builder
+        # pylint: disable=protected-access
+        if isinstance(obj, PlatformIOLibBuilder) and obj._manifest.get("build", {}).get(
+            "builder"
+        ):
+            obj = globals()[obj._manifest.get("build", {}).get("builder")](
+                env, path, verbose=verbose
+            )
 
-        obj = getattr(sys.modules[__name__], clsname)(env, path, verbose=verbose)
         assert isinstance(obj, LibBuilderBase)
         return obj
 
     @staticmethod
     def get_used_frameworks(env, path):
         if any(
-            isfile(join(path, fname))
+            os.path.isfile(os.path.join(path, fname))
             for fname in ("library.properties", "keywords.txt")
         ):
             return ["arduino"]
 
-        if isfile(join(path, "module.json")):
+        if os.path.isfile(os.path.join(path, "module.json")):
             return ["mbed"]
 
         include_re = re.compile(
@@ -84,7 +98,9 @@ class LibBuilderFactory(object):
                     fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT
                 ):
                     continue
-                with io.open(join(root, fname), errors="ignore") as fp:
+                with io.open(
+                    os.path.join(root, fname), encoding="utf8", errors="ignore"
+                ) as fp:
                     content = fp.read()
                 if not content:
                     continue
@@ -95,7 +111,7 @@ class LibBuilderFactory(object):
         return []
 
 
-class LibBuilderBase(object):
+class LibBuilderBase:
 
     CLASSIC_SCANNER = SCons.Scanner.C.CScanner()
     CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner()
@@ -111,7 +127,7 @@ class LibBuilderBase(object):
     def __init__(self, env, path, manifest=None, verbose=False):
         self.env = env.Clone()
         self.envorigin = env.Clone()
-        self.path = realpath(env.subst(path))
+        self.path = os.path.abspath(env.subst(path))
         self.verbose = verbose
 
         try:
@@ -122,11 +138,13 @@ class LibBuilderBase(object):
             )
             self._manifest = {}
 
-        self._is_dependent = False
-        self._is_built = False
-        self._depbuilders = list()
-        self._circular_deps = list()
-        self._processed_files = list()
+        self.is_dependent = False
+        self.is_built = False
+        self.depbuilders = []
+
+        self._deps_are_processed = False
+        self._circular_deps = []
+        self._processed_files = []
 
         # reset source filter, could be overridden with extra script
         self.env["SRC_FILTER"] = ""
@@ -140,21 +158,31 @@ class LibBuilderBase(object):
     def __contains__(self, path):
         p1 = self.path
         p2 = path
-        if WINDOWS:
+        if IS_WINDOWS:
             p1 = p1.lower()
             p2 = p2.lower()
         if p1 == p2:
             return True
-        return commonprefix((p1 + sep, p2)) == p1 + sep
+        if os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep:
+            return True
+        # try to resolve paths
+        p1 = os.path.os.path.realpath(p1)
+        p2 = os.path.os.path.realpath(p2)
+        return os.path.commonprefix([p1 + os.path.sep, p2]) == p1 + os.path.sep
 
     @property
     def name(self):
-        return self._manifest.get("name", basename(self.path))
+        return self._manifest.get("name", os.path.basename(self.path))
 
     @property
     def version(self):
         return self._manifest.get("version")
 
+    @property
+    def dependent(self):
+        """Backward compatibility with ESP-IDF"""
+        return self.is_dependent
+
     @property
     def dependencies(self):
         return self._manifest.get("dependencies")
@@ -170,13 +198,19 @@ class LibBuilderBase(object):
 
     @property
     def include_dir(self):
-        if not all(isdir(join(self.path, d)) for d in ("include", "src")):
-            return None
-        return join(self.path, "include")
+        for name in ("include", "Include"):
+            d = os.path.join(self.path, name)
+            if os.path.isdir(d):
+                return d
+        return None
 
     @property
     def src_dir(self):
-        return join(self.path, "src") if isdir(join(self.path, "src")) else self.path
+        for name in ("src", "Src"):
+            d = os.path.join(self.path, name)
+            if os.path.isdir(d):
+                return d
+        return self.path
 
     def get_include_dirs(self):
         items = []
@@ -189,7 +223,9 @@ class LibBuilderBase(object):
     @property
     def build_dir(self):
         lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3]
-        return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path))
+        return os.path.join(
+            "$BUILD_DIR", "lib%s" % lib_hash, os.path.basename(self.path)
+        )
 
     @property
     def build_flags(self):
@@ -203,18 +239,6 @@ class LibBuilderBase(object):
     def extra_script(self):
         return None
 
-    @property
-    def depbuilders(self):
-        return self._depbuilders
-
-    @property
-    def dependent(self):
-        return self._is_dependent
-
-    @property
-    def is_built(self):
-        return self._is_built
-
     @property
     def lib_archive(self):
         return self.env.GetProjectOption("lib_archive")
@@ -268,14 +292,15 @@ class LibBuilderBase(object):
             if self.extra_script:
                 self.env.SConscriptChdir(1)
                 self.env.SConscript(
-                    realpath(self.extra_script),
+                    os.path.abspath(self.extra_script),
                     exports={"env": self.env, "pio_lib_builder": self},
                 )
             self.env.ProcessUnFlags(self.build_unflags)
 
     def process_dependencies(self):
-        if not self.dependencies:
+        if not self.dependencies or self._deps_are_processed:
             return
+        self._deps_are_processed = True
         for item in self.dependencies:
             found = False
             for lb in self.env.GetLibBuilders():
@@ -283,7 +308,7 @@ class LibBuilderBase(object):
                     continue
                 found = True
                 if lb not in self.depbuilders:
-                    self.depend_recursive(lb)
+                    self.depend_on(lb)
                 break
 
             if not found and self.verbose:
@@ -294,14 +319,14 @@ class LibBuilderBase(object):
 
     def get_search_files(self):
         items = [
-            join(self.src_dir, item)
+            os.path.join(self.src_dir, item)
             for item in self.env.MatchSourceFiles(self.src_dir, self.src_filter)
         ]
         include_dir = self.include_dir
         if include_dir:
             items.extend(
                 [
-                    join(include_dir, item)
+                    os.path.join(include_dir, item)
                     for item in self.env.MatchSourceFiles(include_dir)
                 ]
             )
@@ -370,7 +395,7 @@ class LibBuilderBase(object):
                     continue
                 _f_part = _h_path[: _h_path.rindex(".")]
                 for ext in piotool.SRC_C_EXT + piotool.SRC_CXX_EXT:
-                    if not isfile("%s.%s" % (_f_part, ext)):
+                    if not os.path.isfile("%s.%s" % (_f_part, ext)):
                         continue
                     _c_path = self.env.File("%s.%s" % (_f_part, ext))
                     if _c_path not in result:
@@ -378,7 +403,29 @@ class LibBuilderBase(object):
 
         return result
 
-    def depend_recursive(self, lb, search_files=None):
+    def search_deps_recursive(self, search_files=None):
+        self.process_dependencies()
+
+        # when LDF is disabled
+        if self.lib_ldf_mode == "off":
+            return
+
+        if self.lib_ldf_mode.startswith("deep"):
+            search_files = self.get_search_files()
+
+        lib_inc_map = {}
+        for inc in self._get_found_includes(search_files):
+            for lb in self.env.GetLibBuilders():
+                if inc.get_abspath() in lb:
+                    if lb not in lib_inc_map:
+                        lib_inc_map[lb] = []
+                    lib_inc_map[lb].append(inc.get_abspath())
+                    break
+
+        for lb, lb_search_files in lib_inc_map.items():
+            self.depend_on(lb, search_files=lb_search_files)
+
+    def depend_on(self, lb, search_files=None, recursive=True):
         def _already_depends(_lb):
             if self in _lb.depbuilders:
                 return True
@@ -396,49 +443,34 @@ class LibBuilderBase(object):
                         "between `%s` and `%s`\n" % (self.path, lb.path)
                     )
                 self._circular_deps.append(lb)
-            elif lb not in self._depbuilders:
-                self._depbuilders.append(lb)
+            elif lb not in self.depbuilders:
+                self.depbuilders.append(lb)
+                lb.is_dependent = True
                 LibBuilderBase._INCLUDE_DIRS_CACHE = None
-        lb.search_deps_recursive(search_files)
 
-    def search_deps_recursive(self, search_files=None):
-        if not self._is_dependent:
-            self._is_dependent = True
-            self.process_dependencies()
-
-            if self.lib_ldf_mode.startswith("deep"):
-                search_files = self.get_search_files()
-
-        # when LDF is disabled
-        if self.lib_ldf_mode == "off":
-            return
-
-        lib_inc_map = {}
-        for inc in self._get_found_includes(search_files):
-            for lb in self.env.GetLibBuilders():
-                if inc.get_abspath() in lb:
-                    if lb not in lib_inc_map:
-                        lib_inc_map[lb] = []
-                    lib_inc_map[lb].append(inc.get_abspath())
-                    break
-
-        for lb, lb_search_files in lib_inc_map.items():
-            self.depend_recursive(lb, lb_search_files)
+        if recursive:
+            lb.search_deps_recursive(search_files)
 
     def build(self):
         libs = []
-        for lb in self._depbuilders:
+        shared_scopes = ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS")
+        for lb in self.depbuilders:
             libs.extend(lb.build())
             # copy shared information to self env
-            for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
-                self.env.PrependUnique(**{key: lb.env.get(key)})
+            self.env.PrependUnique(
+                **{
+                    scope: lb.env.get(scope)
+                    for scope in shared_scopes
+                    if lb.env.get(scope)
+                }
+            )
 
         for lb in self._circular_deps:
             self.env.PrependUnique(CPPPATH=lb.get_include_dirs())
 
-        if self._is_built:
+        if self.is_built:
             return libs
-        self._is_built = True
+        self.is_built = True
 
         self.env.PrependUnique(CPPPATH=self.get_include_dirs())
 
@@ -446,15 +478,30 @@ class LibBuilderBase(object):
             for lb in self.env.GetLibBuilders():
                 if self == lb or not lb.is_built:
                     continue
-                for key in ("CPPPATH", "LIBPATH", "LIBS", "LINKFLAGS"):
-                    self.env.PrependUnique(**{key: lb.env.get(key)})
+                self.env.PrependUnique(
+                    **{
+                        scope: lb.env.get(scope)
+                        for scope in shared_scopes
+                        if lb.env.get(scope)
+                    }
+                )
 
-        if self.lib_archive:
-            libs.append(
-                self.env.BuildLibrary(self.build_dir, self.src_dir, self.src_filter)
+        do_not_archive = not self.lib_archive
+        if not do_not_archive:
+            nodes = self.env.CollectBuildFiles(
+                self.build_dir, self.src_dir, self.src_filter
             )
-        else:
+            if nodes:
+                libs.append(
+                    self.env.BuildLibrary(
+                        self.build_dir, self.src_dir, self.src_filter, nodes
+                    )
+                )
+            else:
+                do_not_archive = True
+        if do_not_archive:
             self.env.BuildSources(self.build_dir, self.src_dir, self.src_filter)
+
         return libs
 
 
@@ -464,23 +511,32 @@ class UnknownLibBuilder(LibBuilderBase):
 
 class ArduinoLibBuilder(LibBuilderBase):
     def load_manifest(self):
-        manifest_path = join(self.path, "library.properties")
-        if not isfile(manifest_path):
+        manifest_path = os.path.join(self.path, "library.properties")
+        if not os.path.isfile(manifest_path):
             return {}
         return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
+    @property
+    def include_dir(self):
+        if not all(
+            os.path.isdir(os.path.join(self.path, d)) for d in ("include", "src")
+        ):
+            return None
+        return os.path.join(self.path, "include")
+
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
-        if isdir(join(self.path, "src")):
+        include_dirs = super().get_include_dirs()
+        if os.path.isdir(os.path.join(self.path, "src")):
             return include_dirs
-        if isdir(join(self.path, "utility")):
-            include_dirs.append(join(self.path, "utility"))
+        if os.path.isdir(os.path.join(self.path, "utility")):
+            include_dirs.append(os.path.join(self.path, "utility"))
         return include_dirs
 
     @property
     def src_filter(self):
-        src_dir = join(self.path, "src")
-        if isdir(src_dir):
+        src_dir = os.path.join(self.path, "src")
+        if os.path.isdir(src_dir):
+            # pylint: disable=no-member
             src_filter = LibBuilderBase.src_filter.fget(self)
             for root, _, files in os.walk(src_dir, followlinks=True):
                 found = False
@@ -491,65 +547,95 @@ class ArduinoLibBuilder(LibBuilderBase):
                 if not found:
                     continue
                 rel_path = root.replace(src_dir, "")
-                if rel_path.startswith(sep):
-                    rel_path = rel_path[1:] + sep
+                if rel_path.startswith(os.path.sep):
+                    rel_path = rel_path[1:] + os.path.sep
                 src_filter.append("-<%s*.[aA][sS][mM]>" % rel_path)
             return src_filter
 
         src_filter = []
-        is_utility = isdir(join(self.path, "utility"))
+        is_utility = os.path.isdir(os.path.join(self.path, "utility"))
         for ext in piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT:
             # arduino ide ignores files with .asm or .ASM extensions
             if ext.lower() == "asm":
                 continue
             src_filter.append("+<*.%s>" % ext)
             if is_utility:
-                src_filter.append("+<utility%s*.%s>" % (sep, ext))
+                src_filter.append("+<utility%s*.%s>" % (os.path.sep, ext))
         return src_filter
 
+    @property
+    def dependencies(self):
+        # do not include automatically all libraries for build
+        # chain+ will decide later
+        return None
+
+    @property
+    def lib_ldf_mode(self):
+        # pylint: disable=no-member
+        if not self._manifest.get("dependencies"):
+            return LibBuilderBase.lib_ldf_mode.fget(self)
+        missing = object()
+        global_value = self.env.GetProjectConfig().getraw(
+            "env:" + self.env["PIOENV"], "lib_ldf_mode", missing
+        )
+        if global_value != missing:
+            return LibBuilderBase.lib_ldf_mode.fget(self)
+        # automatically enable C++ Preprocessing in runtime
+        # (Arduino IDE has this behavior)
+        return "chain+"
+
     def is_frameworks_compatible(self, frameworks):
         return util.items_in_list(frameworks, ["arduino", "energia"])
 
     def is_platforms_compatible(self, platforms):
-        items = self._manifest.get("platforms", [])
-        if not items:
-            return LibBuilderBase.is_platforms_compatible(self, platforms)
-        return util.items_in_list(platforms, items)
+        return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"])
+
+    @property
+    def build_flags(self):
+        ldflags = [
+            LibBuilderBase.build_flags.fget(self),  # pylint: disable=no-member
+            self._manifest.get("ldflags"),
+        ]
+        if self._manifest.get("precompiled") in ("true", "full"):
+            # add to LDPATH {build.mcu} folder
+            board_config = self.env.BoardConfig()
+            for key in ("build.mcu", "build.cpu"):
+                libpath = os.path.join(self.src_dir, board_config.get(key, ""))
+                if not os.path.isdir(libpath):
+                    continue
+                self.env.PrependUnique(LIBPATH=libpath)
+                break
+        ldflags = [flag for flag in ldflags if flag]  # remove empty
+        return " ".join(ldflags) if ldflags else None
 
 
 class MbedLibBuilder(LibBuilderBase):
     def load_manifest(self):
-        manifest_path = join(self.path, "module.json")
-        if not isfile(manifest_path):
+        manifest_path = os.path.join(self.path, "module.json")
+        if not os.path.isfile(manifest_path):
             return {}
         return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
-    @property
-    def include_dir(self):
-        if isdir(join(self.path, "include")):
-            return join(self.path, "include")
-        return None
-
     @property
     def src_dir(self):
-        if isdir(join(self.path, "source")):
-            return join(self.path, "source")
-        return LibBuilderBase.src_dir.fget(self)
+        if os.path.isdir(os.path.join(self.path, "source")):
+            return os.path.join(self.path, "source")
+        return LibBuilderBase.src_dir.fget(self)  # pylint: disable=no-member
 
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
+        include_dirs = super().get_include_dirs()
         if self.path not in include_dirs:
             include_dirs.append(self.path)
 
         # library with module.json
         for p in self._manifest.get("extraIncludes", []):
-            include_dirs.append(join(self.path, p))
+            include_dirs.append(os.path.join(self.path, p))
 
         # old mbed library without manifest, add to CPPPATH all folders
         if not self._manifest:
             for root, _, __ in os.walk(self.path):
                 part = root.replace(self.path, "").lower()
-                if any(s in part for s in ("%s." % sep, "test", "example")):
+                if any(s in part for s in ("%s." % os.path.sep, "test", "example")):
                     continue
                 if root not in include_dirs:
                     include_dirs.append(root)
@@ -561,11 +647,11 @@ class MbedLibBuilder(LibBuilderBase):
 
     def process_extra_options(self):
         self._process_mbed_lib_confs()
-        return super(MbedLibBuilder, self).process_extra_options()
+        return super().process_extra_options()
 
     def _process_mbed_lib_confs(self):
         mbed_lib_paths = [
-            join(root, "mbed_lib.json")
+            os.path.join(root, "mbed_lib.json")
             for root, _, files in os.walk(self.path)
             if "mbed_lib.json" in files
         ]
@@ -574,8 +660,8 @@ class MbedLibBuilder(LibBuilderBase):
 
         mbed_config_path = None
         for p in self.env.get("CPPPATH"):
-            mbed_config_path = join(self.env.subst(p), "mbed_config.h")
-            if isfile(mbed_config_path):
+            mbed_config_path = os.path.join(self.env.subst(p), "mbed_config.h")
+            if os.path.isfile(mbed_config_path):
                 break
             mbed_config_path = None
         if not mbed_config_path:
@@ -642,7 +728,7 @@ class MbedLibBuilder(LibBuilderBase):
 
     def _mbed_conf_append_macros(self, mbed_config_path, macros):
         lines = []
-        with open(mbed_config_path) as fp:
+        with open(mbed_config_path, encoding="utf8") as fp:
             for line in fp.readlines():
                 line = line.strip()
                 if line == "#endif":
@@ -661,36 +747,37 @@ class MbedLibBuilder(LibBuilderBase):
                 if len(tokens) < 2 or tokens[1] not in macros:
                     lines.append(line)
         lines.append("")
-        with open(mbed_config_path, "w") as fp:
+        with open(mbed_config_path, mode="w", encoding="utf8") as fp:
             fp.write("\n".join(lines))
 
 
 class PlatformIOLibBuilder(LibBuilderBase):
     def load_manifest(self):
-        manifest_path = join(self.path, "library.json")
-        if not isfile(manifest_path):
+        manifest_path = os.path.join(self.path, "library.json")
+        if not os.path.isfile(manifest_path):
             return {}
         return ManifestParserFactory.new_from_file(manifest_path).as_dict()
 
     def _has_arduino_manifest(self):
-        return isfile(join(self.path, "library.properties"))
+        return os.path.isfile(os.path.join(self.path, "library.properties"))
 
     @property
     def include_dir(self):
         if "includeDir" in self._manifest.get("build", {}):
             with fs.cd(self.path):
-                return realpath(self._manifest.get("build").get("includeDir"))
-        return LibBuilderBase.include_dir.fget(self)
+                return os.path.abspath(self._manifest.get("build").get("includeDir"))
+        return LibBuilderBase.include_dir.fget(self)  # pylint: disable=no-member
 
     @property
     def src_dir(self):
         if "srcDir" in self._manifest.get("build", {}):
             with fs.cd(self.path):
-                return realpath(self._manifest.get("build").get("srcDir"))
-        return LibBuilderBase.src_dir.fget(self)
+                return os.path.abspath(self._manifest.get("build").get("srcDir"))
+        return LibBuilderBase.src_dir.fget(self)  # pylint: disable=no-member
 
     @property
     def src_filter(self):
+        # pylint: disable=no-member
         if "srcFilter" in self._manifest.get("build", {}):
             return self._manifest.get("build").get("srcFilter")
         if self.env["SRC_FILTER"]:
@@ -703,19 +790,19 @@ class PlatformIOLibBuilder(LibBuilderBase):
     def build_flags(self):
         if "flags" in self._manifest.get("build", {}):
             return self._manifest.get("build").get("flags")
-        return LibBuilderBase.build_flags.fget(self)
+        return LibBuilderBase.build_flags.fget(self)  # pylint: disable=no-member
 
     @property
     def build_unflags(self):
         if "unflags" in self._manifest.get("build", {}):
             return self._manifest.get("build").get("unflags")
-        return LibBuilderBase.build_unflags.fget(self)
+        return LibBuilderBase.build_unflags.fget(self)  # pylint: disable=no-member
 
     @property
     def extra_script(self):
         if "extraScript" in self._manifest.get("build", {}):
             return self._manifest.get("build").get("extraScript")
-        return LibBuilderBase.extra_script.fget(self)
+        return LibBuilderBase.extra_script.fget(self)  # pylint: disable=no-member
 
     @property
     def lib_archive(self):
@@ -727,12 +814,14 @@ class PlatformIOLibBuilder(LibBuilderBase):
             return self.env.GetProjectConfig().get(
                 "env:" + self.env["PIOENV"], "lib_archive"
             )
+        # pylint: disable=no-member
         return self._manifest.get("build", {}).get(
             "libArchive", LibBuilderBase.lib_archive.fget(self)
         )
 
     @property
     def lib_ldf_mode(self):
+        # pylint: disable=no-member
         return self.validate_ldf_mode(
             self._manifest.get("build", {}).get(
                 "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self)
@@ -741,6 +830,7 @@ class PlatformIOLibBuilder(LibBuilderBase):
 
     @property
     def lib_compat_mode(self):
+        # pylint: disable=no-member
         return self.validate_compat_mode(
             self._manifest.get("build", {}).get(
                 "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self)
@@ -748,28 +838,22 @@ class PlatformIOLibBuilder(LibBuilderBase):
         )
 
     def is_platforms_compatible(self, platforms):
-        items = self._manifest.get("platforms")
-        if not items:
-            return LibBuilderBase.is_platforms_compatible(self, platforms)
-        return util.items_in_list(platforms, items)
+        return util.items_in_list(platforms, self._manifest.get("platforms") or ["*"])
 
     def is_frameworks_compatible(self, frameworks):
-        items = self._manifest.get("frameworks")
-        if not items:
-            return LibBuilderBase.is_frameworks_compatible(self, frameworks)
-        return util.items_in_list(frameworks, items)
+        return util.items_in_list(frameworks, self._manifest.get("frameworks") or ["*"])
 
     def get_include_dirs(self):
-        include_dirs = LibBuilderBase.get_include_dirs(self)
+        include_dirs = super().get_include_dirs()
 
         # backwards compatibility with PlatformIO 2.0
         if (
             "build" not in self._manifest
             and self._has_arduino_manifest()
-            and not isdir(join(self.path, "src"))
-            and isdir(join(self.path, "utility"))
+            and not os.path.isdir(os.path.join(self.path, "src"))
+            and os.path.isdir(os.path.join(self.path, "utility"))
         ):
-            include_dirs.append(join(self.path, "utility"))
+            include_dirs.append(os.path.join(self.path, "utility"))
 
         for path in self.env.get("CPPPATH", []):
             if path not in self.envorigin.get("CPPPATH", []):
@@ -782,13 +866,13 @@ class ProjectAsLibBuilder(LibBuilderBase):
     def __init__(self, env, *args, **kwargs):
         # backup original value, will be reset in base.__init__
         project_src_filter = env.get("SRC_FILTER")
-        super(ProjectAsLibBuilder, self).__init__(env, *args, **kwargs)
+        super().__init__(env, *args, **kwargs)
         self.env["SRC_FILTER"] = project_src_filter
 
     @property
     def include_dir(self):
         include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
-        return include_dir if isdir(include_dir) else None
+        return include_dir if os.path.isdir(include_dir) else None
 
     @property
     def src_dir(self):
@@ -797,21 +881,21 @@ class ProjectAsLibBuilder(LibBuilderBase):
     def get_include_dirs(self):
         include_dirs = []
         project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
-        if isdir(project_include_dir):
+        if os.path.isdir(project_include_dir):
             include_dirs.append(project_include_dir)
-        for include_dir in LibBuilderBase.get_include_dirs(self):
+        for include_dir in super().get_include_dirs():
             if include_dir not in include_dirs:
                 include_dirs.append(include_dir)
         return include_dirs
 
     def get_search_files(self):
         # project files
-        items = LibBuilderBase.get_search_files(self)
+        items = super().get_search_files()
         # test files
-        if "__test" in COMMAND_LINE_TARGETS:
+        if "test" in self.env.GetBuildType():
             items.extend(
                 [
-                    join("$PROJECT_TEST_DIR", item)
+                    os.path.join("$PROJECT_TEST_DIR", item)
                     for item in self.env.MatchSourceFiles(
                         "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
                     )
@@ -821,7 +905,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
 
     @property
     def lib_ldf_mode(self):
-        mode = LibBuilderBase.lib_ldf_mode.fget(self)
+        mode = LibBuilderBase.lib_ldf_mode.fget(self)  # pylint: disable=no-member
         if not mode.startswith("chain"):
             return mode
         # parse all project files
@@ -829,45 +913,54 @@ class ProjectAsLibBuilder(LibBuilderBase):
 
     @property
     def src_filter(self):
+        # pylint: disable=no-member
         return self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self)
 
+    @property
+    def build_flags(self):
+        # pylint: disable=no-member
+        return self.env.get("SRC_BUILD_FLAGS") or LibBuilderBase.build_flags.fget(self)
+
     @property
     def dependencies(self):
         return self.env.GetProjectOption("lib_deps", [])
 
     def process_extra_options(self):
-        # skip for project, options are already processed
-        pass
+        with fs.cd(self.path):
+            self.env.ProcessFlags(self.build_flags)
+            self.env.ProcessUnFlags(self.build_unflags)
 
     def install_dependencies(self):
-        def _is_builtin(uri):
+        def _is_builtin(spec):
             for lb in self.env.GetLibBuilders():
-                if lb.name == uri:
+                if lb.name == spec:
                     return True
             return False
 
-        not_found_uri = []
-        for uri in self.dependencies:
+        not_found_specs = []
+        for spec in self.dependencies:
             # check if built-in library
-            if _is_builtin(uri):
+            if _is_builtin(spec):
                 continue
 
             found = False
             for storage_dir in self.env.GetLibSourceDirs():
-                lm = LibraryManager(storage_dir)
-                if lm.get_package_dir(*lm.parse_pkg_uri(uri)):
+                lm = LibraryPackageManager(storage_dir)
+                if lm.get_package(spec):
                     found = True
                     break
             if not found:
-                not_found_uri.append(uri)
+                not_found_specs.append(spec)
 
         did_install = False
-        lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV")))
-        for uri in not_found_uri:
+        lm = LibraryPackageManager(
+            self.env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))
+        )
+        for spec in not_found_specs:
             try:
-                lm.install(uri)
+                lm.install(spec)
                 did_install = True
-            except (exception.LibNotFound, exception.InternetIsOffline) as e:
+            except (HTTPClientError, UnknownPackageError, InternetIsOffline) as e:
                 click.secho("Warning! %s" % e, fg="yellow")
 
         # reset cache
@@ -875,20 +968,22 @@ class ProjectAsLibBuilder(LibBuilderBase):
             DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None)
 
     def process_dependencies(self):  # pylint: disable=too-many-branches
-        for uri in self.dependencies:
+        found_lbs = []
+        for spec in self.dependencies:
             found = False
             for storage_dir in self.env.GetLibSourceDirs():
                 if found:
                     break
-                lm = LibraryManager(storage_dir)
-                lib_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri))
-                if not lib_dir:
+                lm = LibraryPackageManager(storage_dir)
+                pkg = lm.get_package(spec)
+                if not pkg:
                     continue
                 for lb in self.env.GetLibBuilders():
-                    if lib_dir != lb.path:
+                    if pkg.path != lb.path:
                         continue
                     if lb not in self.depbuilders:
-                        self.depend_recursive(lb)
+                        self.depend_on(lb, recursive=False)
+                        found_lbs.append(lb)
                     found = True
                     break
             if found:
@@ -897,16 +992,20 @@ class ProjectAsLibBuilder(LibBuilderBase):
             # look for built-in libraries by a name
             # which don't have package manifest
             for lb in self.env.GetLibBuilders():
-                if lb.name != uri:
+                if lb.name != spec:
                     continue
                 if lb not in self.depbuilders:
-                    self.depend_recursive(lb)
+                    self.depend_on(lb)
                 found = True
                 break
 
+        # process library dependencies
+        for lb in found_lbs:
+            lb.search_deps_recursive()
+
     def build(self):
-        self._is_built = True  # do not build Project now
-        result = LibBuilderBase.build(self)
+        self.is_built = True  # do not build Project now
+        result = super().build()
         self.env.PrependUnique(CPPPATH=self.get_include_dirs())
         return result
 
@@ -939,25 +1038,30 @@ def IsCompatibleLibBuilder(env, lb, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)))
     return True
 
 
-def GetLibBuilders(env):  # pylint: disable=too-many-branches
-    if DefaultEnvironment().get("__PIO_LIB_BUILDERS", None) is not None:
+def GetLibBuilders(_):  # pylint: disable=too-many-branches
+    env = DefaultEnvironment()
+    if env.get("__PIO_LIB_BUILDERS", None) is not None:
         return sorted(
-            DefaultEnvironment()["__PIO_LIB_BUILDERS"],
-            key=lambda lb: 0 if lb.dependent else 1,
+            env["__PIO_LIB_BUILDERS"],
+            key=lambda lb: 0 if lb.is_dependent else 1,
         )
 
-    DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=[])
+    env.Replace(__PIO_LIB_BUILDERS=[])
 
     verbose = int(ARGUMENTS.get("PIOVERBOSE", 0))
     found_incompat = False
 
     for storage_dir in env.GetLibSourceDirs():
-        storage_dir = realpath(storage_dir)
-        if not isdir(storage_dir):
+        storage_dir = os.path.abspath(storage_dir)
+        if not os.path.isdir(storage_dir):
             continue
         for item in sorted(os.listdir(storage_dir)):
-            lib_dir = join(storage_dir, item)
-            if item == "__cores__" or not isdir(lib_dir):
+            lib_dir = os.path.join(storage_dir, item)
+            if item == "__cores__":
+                continue
+            if LibraryPackageManager.is_symlink(lib_dir):
+                lib_dir, _ = LibraryPackageManager.resolve_symlink(lib_dir)
+            if not lib_dir or not os.path.isdir(lib_dir):
                 continue
             try:
                 lb = LibBuilderFactory.new(env, lib_dir)
@@ -968,13 +1072,13 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
                     )
                 continue
             if env.IsCompatibleLibBuilder(lb):
-                DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
+                env.Append(__PIO_LIB_BUILDERS=[lb])
             else:
                 found_incompat = True
 
     for lb in env.get("EXTRA_LIB_BUILDERS", []):
         if env.IsCompatibleLibBuilder(lb):
-            DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb])
+            env.Append(__PIO_LIB_BUILDERS=[lb])
         else:
             found_incompat = True
 
@@ -985,17 +1089,25 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
             "ldf-compat-mode\n"
         )
 
-    return DefaultEnvironment()["__PIO_LIB_BUILDERS"]
+    return env["__PIO_LIB_BUILDERS"]
 
 
 def ConfigureProjectLibBuilder(env):
-    def _get_vcs_info(lb):
-        path = LibraryManager.get_src_manifest_path(lb.path)
-        return fs.load_json(path) if path else None
+    _pm_storage = {}
+
+    def _get_lib_license(pkg):
+        storage_dir = os.path.dirname(os.path.dirname(pkg.path))
+        if storage_dir not in _pm_storage:
+            _pm_storage[storage_dir] = LibraryPackageManager(storage_dir)
+        try:
+            return (_pm_storage[storage_dir].load_manifest(pkg) or {}).get("license")
+        except MissingPackageManifestError:
+            pass
+        return None
 
     def _correct_found_libs(lib_builders):
         # build full dependency graph
-        found_lbs = [lb for lb in lib_builders if lb.dependent]
+        found_lbs = [lb for lb in lib_builders if lb.is_dependent]
         for lb in lib_builders:
             if lb in found_lbs:
                 lb.search_deps_recursive(lb.get_search_files())
@@ -1007,27 +1119,29 @@ def ConfigureProjectLibBuilder(env):
     def _print_deps_tree(root, level=0):
         margin = "|   " * (level)
         for lb in root.depbuilders:
-            title = "<%s>" % lb.name
-            vcs_info = _get_vcs_info(lb)
-            if lb.version:
-                title += " %s" % lb.version
-            if vcs_info and vcs_info.get("version"):
-                title += " #%s" % vcs_info.get("version")
+            title = lb.name
+            pkg = PackageItem(lb.path)
+            if pkg.metadata:
+                title += " @ %s" % pkg.metadata.version
+            elif lb.version:
+                title += " @ %s" % lb.version
             click.echo("%s|-- %s" % (margin, title), nl=False)
             if int(ARGUMENTS.get("PIOVERBOSE", 0)):
-                if vcs_info:
-                    click.echo(" [%s]" % vcs_info.get("url"), nl=False)
-                click.echo(" (", nl=False)
-                click.echo(lb.path, nl=False)
+                click.echo(
+                    " (License: %s, " % (_get_lib_license(pkg) or "Unknown"), nl=False
+                )
+                if pkg.metadata and pkg.metadata.spec.external:
+                    click.echo("URI: %s, " % pkg.metadata.spec.uri, nl=False)
+                click.echo("Path: %s" % lb.path, nl=False)
                 click.echo(")", nl=False)
             click.echo("")
             if lb.depbuilders:
                 _print_deps_tree(lb, level + 1)
 
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
-    ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project)
+    ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project)  # pylint: disable=no-member
 
-    click.echo("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf")
+    click.echo("LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf")
     click.echo(
         "LDF Modes: Finder ~ %s, Compatibility ~ %s"
         % (ldf_mode, project.lib_compat_mode)

platformio/builder/tools/piomaxlen.py

@@ -14,15 +14,30 @@
 
 from __future__ import absolute_import
 
-from hashlib import md5
-from os import makedirs
-from os.path import isdir, isfile, join
+import hashlib
+import os
+import re
 
-from platformio.compat import WINDOWS, hashlib_encode_data
+from SCons.Platform import TempFileMunge  # pylint: disable=import-error
+from SCons.Subst import quote_spaces  # pylint: disable=import-error
 
-# Windows CLI has limit with command length to 8192
-# Leave 2000 chars for flags and other options
-MAX_LINE_LENGTH = 6000 if WINDOWS else 128072
+from platformio.compat import IS_WINDOWS, hashlib_encode_data
+
+# There are the next limits depending on a platform:
+# - Windows = 8192
+# - Unix    = 131072
+# We need ~512 characters for compiler and temporary file paths
+MAX_LINE_LENGTH = (8192 if IS_WINDOWS else 131072) - 512
+
+WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)")
+
+
+def tempfile_arg_esc_func(arg):
+    arg = quote_spaces(arg)
+    if not IS_WINDOWS:
+        return arg
+    # GCC requires double Windows slashes, let's use UNIX separator
+    return WINPATHSEP_RE.sub(r"/\1", arg)
 
 
 def long_sources_hook(env, sources):
@@ -41,32 +56,16 @@ def long_sources_hook(env, sources):
     return '@"%s"' % _file_long_data(env, " ".join(data))
 
 
-def long_incflags_hook(env, incflags):
-    _incflags = env.subst(incflags).replace("\\", "/")
-    if len(_incflags) < MAX_LINE_LENGTH:
-        return incflags
-
-    # fix space in paths
-    data = []
-    for line in _incflags.split(" -I"):
-        line = line.strip()
-        if not line.startswith("-I"):
-            line = "-I" + line
-        data.append('-I"%s"' % line[2:])
-
-    return '@"%s"' % _file_long_data(env, " ".join(data))
-
-
 def _file_long_data(env, data):
     build_dir = env.subst("$BUILD_DIR")
-    if not isdir(build_dir):
-        makedirs(build_dir)
-    tmp_file = join(
-        build_dir, "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest()
+    if not os.path.isdir(build_dir):
+        os.makedirs(build_dir)
+    tmp_file = os.path.join(
+        build_dir, "longcmd-%s" % hashlib.md5(hashlib_encode_data(data)).hexdigest()
     )
-    if isfile(tmp_file):
+    if os.path.isfile(tmp_file):
         return tmp_file
-    with open(tmp_file, "w") as fp:
+    with open(tmp_file, mode="w", encoding="utf8") as fp:
         fp.write(data)
     return tmp_file
 
@@ -76,17 +75,21 @@ def exists(_):
 
 
 def generate(env):
-    env.Replace(_long_sources_hook=long_sources_hook)
-    env.Replace(_long_incflags_hook=long_incflags_hook)
-    coms = {}
-    for key in ("ARCOM", "LINKCOM"):
-        coms[key] = env.get(key, "").replace(
-            "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}"
-        )
-    for key in ("_CCCOMCOM", "ASPPCOM"):
-        coms[key] = env.get(key, "").replace(
-            "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}"
-        )
-    env.Replace(**coms)
+    kwargs = dict(
+        _long_sources_hook=long_sources_hook,
+        TEMPFILE=TempFileMunge,
+        MAXLINELENGTH=MAX_LINE_LENGTH,
+        TEMPFILEARGESCFUNC=tempfile_arg_esc_func,
+        TEMPFILESUFFIX=".tmp",
+        TEMPFILEDIR="$BUILD_DIR",
+    )
+
+    for name in ("LINKCOM", "ASCOM", "ASPPCOM", "CCCOM", "CXXCOM"):
+        kwargs[name] = "${TEMPFILE('%s','$%sSTR')}" % (env.get(name), name)
+
+    kwargs["ARCOM"] = env.get("ARCOM", "").replace(
+        "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}"
+    )
+    env.Replace(**kwargs)
 
     return env

platformio/builder/tools/piomisc.py

@@ -14,248 +14,19 @@
 
 from __future__ import absolute_import
 
-import atexit
-import io
-import re
+import os
 import sys
-from os import environ, remove, walk
-from os.path import basename, isdir, isfile, join, realpath, relpath, sep
-from tempfile import mkstemp
-
-import click
-from SCons.Action import Action  # pylint: disable=import-error
-from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 
 from platformio import fs, util
-from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape
-from platformio.managers.core import get_core_package_dir
 from platformio.proc import exec_command
 
 
-class InoToCPPConverter(object):
-
-    PROTOTYPE_RE = re.compile(
-        r"""^(
-        (?:template\<.*\>\s*)?      # template
-        ([a-z_\d\&]+\*?\s+){1,2}    # return type
-        ([a-z_\d]+\s*)              # name of prototype
-        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
-        )\s*(\{|;)                  # must end with `{` or `;`
-        """,
-        re.X | re.M | re.I,
-    )
-    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
-    PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
-
-    def __init__(self, env):
-        self.env = env
-        self._main_ino = None
-        self._safe_encoding = None
-
-    def read_safe_contents(self, path):
-        error_reported = False
-        for encoding in (
-            "utf-8",
-            None,
-            get_filesystem_encoding(),
-            get_locale_encoding(),
-            "latin-1",
-        ):
-            try:
-                with io.open(path, encoding=encoding) as fp:
-                    contents = fp.read()
-                    self._safe_encoding = encoding
-                    return contents
-            except UnicodeDecodeError:
-                if not error_reported:
-                    error_reported = True
-                    click.secho(
-                        "Unicode decode error has occurred, please remove invalid "
-                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
-                        % path,
-                        fg="yellow",
-                        err=True,
-                    )
-        return ""
-
-    def write_safe_contents(self, path, contents):
-        with io.open(
-            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
-        ) as fp:
-            return fp.write(contents)
-
-    def is_main_node(self, contents):
-        return self.DETECTMAIN_RE.search(contents)
-
-    def convert(self, nodes):
-        contents = self.merge(nodes)
-        if not contents:
-            return None
-        return self.process(contents)
-
-    def merge(self, nodes):
-        assert nodes
-        lines = []
-        for node in nodes:
-            contents = self.read_safe_contents(node.get_path())
-            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
-            if self.is_main_node(contents):
-                lines = _lines + lines
-                self._main_ino = node.get_path()
-            else:
-                lines.extend(_lines)
-
-        if not self._main_ino:
-            self._main_ino = nodes[0].get_path()
-
-        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
-
-    def process(self, contents):
-        out_file = self._main_ino + ".cpp"
-        assert self._gcc_preprocess(contents, out_file)
-        contents = self.read_safe_contents(out_file)
-        contents = self._join_multiline_strings(contents)
-        self.write_safe_contents(out_file, self.append_prototypes(contents))
-        return out_file
-
-    def _gcc_preprocess(self, contents, out_file):
-        tmp_path = mkstemp()[1]
-        self.write_safe_contents(tmp_path, contents)
-        self.env.Execute(
-            self.env.VerboseAction(
-                '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
-                    out_file, tmp_path
-                ),
-                "Converting " + basename(out_file[:-4]),
-            )
-        )
-        atexit.register(_delete_file, tmp_path)
-        return isfile(out_file)
-
-    def _join_multiline_strings(self, contents):
-        if "\\\n" not in contents:
-            return contents
-        newlines = []
-        linenum = 0
-        stropen = False
-        for line in contents.split("\n"):
-            _linenum = self._parse_preproc_line_num(line)
-            if _linenum is not None:
-                linenum = _linenum
-            else:
-                linenum += 1
-
-            if line.endswith("\\"):
-                if line.startswith('"'):
-                    stropen = True
-                    newlines.append(line[:-1])
-                    continue
-                if stropen:
-                    newlines[len(newlines) - 1] += line[:-1]
-                    continue
-            elif stropen and line.endswith(('",', '";')):
-                newlines[len(newlines) - 1] += line
-                stropen = False
-                newlines.append(
-                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
-                )
-                continue
-
-            newlines.append(line)
-
-        return "\n".join(newlines)
-
-    @staticmethod
-    def _parse_preproc_line_num(line):
-        if not line.startswith("#"):
-            return None
-        tokens = line.split(" ", 3)
-        if len(tokens) > 2 and tokens[1].isdigit():
-            return int(tokens[1])
-        return None
-
-    def _parse_prototypes(self, contents):
-        prototypes = []
-        reserved_keywords = set(["if", "else", "while"])
-        for match in self.PROTOTYPE_RE.finditer(contents):
-            if (
-                set([match.group(2).strip(), match.group(3).strip()])
-                & reserved_keywords
-            ):
-                continue
-            prototypes.append(match)
-        return prototypes
-
-    def _get_total_lines(self, contents):
-        total = 0
-        if contents.endswith("\n"):
-            contents = contents[:-1]
-        for line in contents.split("\n")[::-1]:
-            linenum = self._parse_preproc_line_num(line)
-            if linenum is not None:
-                return total + linenum
-            total += 1
-        return total
-
-    def append_prototypes(self, contents):
-        prototypes = self._parse_prototypes(contents) or []
-
-        # skip already declared prototypes
-        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
-        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
-
-        if not prototypes:
-            return contents
-
-        prototype_names = set(m.group(3).strip() for m in prototypes)
-        split_pos = prototypes[0].start()
-        match_ptrs = re.search(
-            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
-            contents[:split_pos],
-            re.M,
-        )
-        if match_ptrs:
-            split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
-
-        result = []
-        result.append(contents[:split_pos].strip())
-        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
-        result.append(
-            '#line %d "%s"'
-            % (
-                self._get_total_lines(contents[:split_pos]),
-                self._main_ino.replace("\\", "/"),
-            )
-        )
-        result.append(contents[split_pos:].strip())
-        return "\n".join(result)
-
-
-def ConvertInoToCpp(env):
-    src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR"))
-    ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde"))
-    if not ino_nodes:
-        return
-    c = InoToCPPConverter(env)
-    out_file = c.convert(ino_nodes)
-
-    atexit.register(_delete_file, out_file)
-
-
-def _delete_file(path):
-    try:
-        if isfile(path):
-            remove(path)
-    except:  # pylint: disable=bare-except
-        pass
-
-
 @util.memoized()
-def _get_compiler_type(env):
+def GetCompilerType(env):
     if env.subst("$CC").endswith("-gcc"):
         return "gcc"
     try:
-        sysenv = environ.copy()
+        sysenv = os.environ.copy()
         sysenv["PATH"] = str(env["ENV"]["PATH"])
         result = exec_command([env.subst("$CC"), "-v"], env=sysenv)
     except OSError:
@@ -270,15 +41,11 @@ def _get_compiler_type(env):
     return None
 
 
-def GetCompilerType(env):
-    return _get_compiler_type(env)
-
-
 def GetActualLDScript(env):
     def _lookup_in_ldpath(script):
         for d in env.get("LIBPATH", []):
-            path = join(env.subst(d), script)
-            if isfile(path):
+            path = os.path.join(env.subst(d), script)
+            if os.path.isfile(path):
                 return path
         return None
 
@@ -297,7 +64,7 @@ def GetActualLDScript(env):
         else:
             continue
         script = env.subst(raw_script.replace('"', "").strip())
-        if isfile(script):
+        if os.path.isfile(script):
             return script
         path = _lookup_in_ldpath(script)
         if path:
@@ -319,30 +86,7 @@ def GetActualLDScript(env):
     env.Exit(1)
 
 
-def VerboseAction(_, act, actstr):
-    if int(ARGUMENTS.get("PIOVERBOSE", 0)):
-        return act
-    return Action(act, actstr)
-
-
-def PioClean(env, clean_dir):
-    if not isdir(clean_dir):
-        print("Build environment is clean")
-        env.Exit(0)
-    clean_rel_path = relpath(clean_dir)
-    for root, _, files in walk(clean_dir):
-        for f in files:
-            dst = join(root, f)
-            remove(dst)
-            print(
-                "Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst))
-            )
-    print("Done cleaning")
-    fs.rmtree(clean_dir)
-    env.Exit(0)
-
-
-def ConfigureDebugFlags(env):
+def ConfigureDebugTarget(env):
     def _cleanup_debug_flags(scope):
         if scope not in env:
             return
@@ -357,30 +101,28 @@ def ConfigureDebugFlags(env):
     for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"):
         _cleanup_debug_flags(scope)
 
-    debug_flags = env.ParseFlags(env.GetProjectOption("debug_build_flags"))
+    debug_flags = env.ParseFlags(
+        env.get("PIODEBUGFLAGS")
+        if env.get("PIODEBUGFLAGS")
+        and not env.GetProjectOptions(as_dict=True).get("debug_build_flags")
+        else env.GetProjectOption("debug_build_flags")
+    )
+
     env.MergeFlags(debug_flags)
     optimization_flags = [
         f for f in debug_flags.get("CCFLAGS", []) if f.startswith(("-O", "-g"))
     ]
 
     if optimization_flags:
-        env.AppendUnique(ASFLAGS=optimization_flags, LINKFLAGS=optimization_flags)
-
-
-def ConfigureTestTarget(env):
-    env.Append(
-        CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
-        CPPPATH=[join("$BUILD_DIR", "UnityTestLib")],
-    )
-    unitylib = env.BuildLibrary(
-        join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity")
-    )
-    env.Prepend(LIBS=[unitylib])
-
-    src_filter = ["+<*.cpp>", "+<*.c>"]
-    if "PIOTEST_RUNNING_NAME" in env:
-        src_filter.append("+<%s%s>" % (env["PIOTEST_RUNNING_NAME"], sep))
-    env.Replace(PIOTEST_SRC_FILTER=src_filter)
+        env.AppendUnique(
+            ASFLAGS=[
+                # skip -O flags for assembler
+                f
+                for f in optimization_flags
+                if f.startswith("-g")
+            ],
+            LINKFLAGS=optimization_flags,
+        )
 
 
 def GetExtraScripts(env, scope):
@@ -393,20 +135,17 @@ def GetExtraScripts(env, scope):
     if not items:
         return items
     with fs.cd(env.subst("$PROJECT_DIR")):
-        return [realpath(item) for item in items]
+        return [os.path.abspath(env.subst(item)) for item in items]
+
+
+def generate(env):
+    env.AddMethod(GetCompilerType)
+    env.AddMethod(GetActualLDScript)
+    env.AddMethod(ConfigureDebugTarget)
+    env.AddMethod(GetExtraScripts)
+    # bakward-compatibility with Zephyr build script
+    env.AddMethod(ConfigureDebugTarget, "ConfigureDebugFlags")
 
 
 def exists(_):
     return True
-
-
-def generate(env):
-    env.AddMethod(ConvertInoToCpp)
-    env.AddMethod(GetCompilerType)
-    env.AddMethod(GetActualLDScript)
-    env.AddMethod(VerboseAction)
-    env.AddMethod(PioClean)
-    env.AddMethod(ConfigureDebugFlags)
-    env.AddMethod(ConfigureTestTarget)
-    env.AddMethod(GetExtraScripts)
-    return env

platformio/builder/tools/pioplatform.py

@@ -14,31 +14,36 @@
 
 from __future__ import absolute_import
 
+import os
 import sys
-from os.path import isdir, isfile, join
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
+from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
-from platformio import exception, fs, util
-from platformio.compat import WINDOWS
-from platformio.managers.platform import PlatformFactory
+from platformio import fs, util
+from platformio.compat import IS_MACOS, IS_WINDOWS
+from platformio.package.meta import PackageItem
+from platformio.package.version import get_original_version
+from platformio.platform.exception import UnknownBoard
+from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectOptions
 
 # pylint: disable=too-many-branches, too-many-locals
 
 
 @util.memoized()
-def PioPlatform(env):
-    variables = env.GetProjectOptions(as_dict=True)
-    if "framework" in variables:
-        # support PIO Core 3.0 dev/platforms
-        variables["pioframework"] = variables["framework"]
-    p = PlatformFactory.newPlatform(env["PLATFORM_MANIFEST"])
-    p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
+def _PioPlatform():
+    env = DefaultEnvironment()
+    p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"]))
+    p.configure_project_packages(env["PIOENV"], COMMAND_LINE_TARGETS)
     return p
 
 
+def PioPlatform(_):
+    return _PioPlatform()
+
+
 def BoardConfig(env, board=None):
     with fs.cd(env.subst("$PROJECT_DIR")):
         try:
@@ -46,46 +51,51 @@ def BoardConfig(env, board=None):
             board = board or env.get("BOARD")
             assert board, "BoardConfig: Board is not defined"
             return p.board_config(board)
-        except (AssertionError, exception.UnknownBoard) as e:
+        except (AssertionError, UnknownBoard) as e:
             sys.stderr.write("Error: %s\n" % str(e))
             env.Exit(1)
+    return None
 
 
 def GetFrameworkScript(env, framework):
     p = env.PioPlatform()
     assert p.frameworks and framework in p.frameworks
     script_path = env.subst(p.frameworks[framework]["script"])
-    if not isfile(script_path):
-        script_path = join(p.get_dir(), script_path)
+    if not os.path.isfile(script_path):
+        script_path = os.path.join(p.get_dir(), script_path)
     return script_path
 
 
 def LoadPioPlatform(env):
     p = env.PioPlatform()
-    installed_packages = p.get_installed_packages()
 
     # Ensure real platform name
     env["PIOPLATFORM"] = p.name
 
     # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH
-    systype = util.get_systype()
-    for name in installed_packages:
-        type_ = p.get_package_type(name)
+    for pkg in p.get_installed_packages():
+        type_ = p.get_package_type(pkg.metadata.name)
         if type_ not in ("toolchain", "uploader", "debugger"):
             continue
-        pkg_dir = p.get_package_dir(name)
         env.PrependENVPath(
-            "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir
+            "PATH",
+            os.path.join(pkg.path, "bin")
+            if os.path.isdir(os.path.join(pkg.path, "bin"))
+            else pkg.path,
         )
-        if not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain":
+        if (
+            not IS_WINDOWS
+            and os.path.isdir(os.path.join(pkg.path, "lib"))
+            and type_ != "toolchain"
+        ):
             env.PrependENVPath(
-                "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH",
-                join(pkg_dir, "lib"),
+                "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH",
+                os.path.join(pkg.path, "lib"),
             )
 
     # Platform specific LD Scripts
-    if isdir(join(p.get_dir(), "ldscripts")):
-        env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")])
+    if os.path.isdir(os.path.join(p.get_dir(), "ldscripts")):
+        env.Prepend(LIBPATH=[os.path.join(p.get_dir(), "ldscripts")])
 
     if "BOARD" not in env:
         return
@@ -125,6 +135,7 @@ def LoadPioPlatform(env):
 
 def PrintConfiguration(env):  # pylint: disable=too-many-statements
     platform = env.PioPlatform()
+    pkg_metadata = PackageItem(platform.get_dir()).metadata
     board_config = env.BoardConfig() if "BOARD" in env else None
 
     def _get_configuration_data():
@@ -139,11 +150,19 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
         )
 
     def _get_plaform_data():
-        data = ["PLATFORM: %s %s" % (platform.title, platform.version)]
-        if platform.src_version:
-            data.append("#" + platform.src_version)
-        if int(ARGUMENTS.get("PIOVERBOSE", 0)) and platform.src_url:
-            data.append("(%s)" % platform.src_url)
+        data = [
+            "PLATFORM: %s (%s)"
+            % (
+                platform.title,
+                pkg_metadata.version if pkg_metadata else platform.version,
+            )
+        ]
+        if (
+            int(ARGUMENTS.get("PIOVERBOSE", 0))
+            and pkg_metadata
+            and pkg_metadata.spec.external
+        ):
+            data.append("(%s)" % pkg_metadata.spec.uri)
         if board_config:
             data.extend([">", board_config.get("name")])
         return data
@@ -162,7 +181,8 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
         ram = board_config.get("upload", {}).get("maximum_ram_size")
         flash = board_config.get("upload", {}).get("maximum_size")
         data.append(
-            "%s RAM, %s Flash" % (fs.format_filesize(ram), fs.format_filesize(flash))
+            "%s RAM, %s Flash"
+            % (fs.humanize_file_size(ram), fs.humanize_file_size(flash))
         )
         return data
 
@@ -194,8 +214,8 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
     def _get_packages_data():
         data = []
         for item in platform.dump_used_packages():
-            original_version = util.get_original_version(item["version"])
-            info = "%s %s" % (item["name"], item["version"])
+            original_version = get_original_version(item["version"])
+            info = "%s @ %s" % (item["name"], item["version"])
             extra = []
             if original_version:
                 extra.append(original_version)

platformio/builder/tools/pioproject.py

@@ -14,7 +14,8 @@
 
 from __future__ import absolute_import
 
-from platformio.project.config import MISSING, ProjectConfig, ProjectOptions
+from platformio.compat import MISSING
+from platformio.project.config import ProjectConfig
 
 
 def GetProjectConfig(env):
@@ -30,15 +31,17 @@ def GetProjectOption(env, option, default=MISSING):
 
 
 def LoadProjectOptions(env):
-    for option, value in env.GetProjectOptions():
-        option_meta = ProjectOptions.get("env." + option)
+    config = env.GetProjectConfig()
+    section = "env:" + env["PIOENV"]
+    for option in config.options(section):
+        option_meta = config.find_option_meta(section, option)
         if (
             not option_meta
             or not option_meta.buildenvvar
             or option_meta.buildenvvar in env
         ):
             continue
-        env[option_meta.buildenvvar] = value
+        env[option_meta.buildenvvar] = config.get(section, option)
 
 
 def exists(_):

platformio/builder/tools/piosize.py

@@ -16,6 +16,7 @@
 
 from __future__ import absolute_import
 
+import json
 import sys
 from os import environ, makedirs, remove
 from os.path import isdir, join, splitdrive
@@ -23,9 +24,8 @@ from os.path import isdir, join, splitdrive
 from elftools.elf.descriptions import describe_sh_flags
 from elftools.elf.elffile import ELFFile
 
-from platformio.compat import dump_json_to_unicode
+from platformio.compat import IS_WINDOWS
 from platformio.proc import exec_command
-from platformio.util import get_systype
 
 
 def _run_tool(cmd, env, tool_args):
@@ -37,7 +37,7 @@ def _run_tool(cmd, env, tool_args):
         makedirs(build_dir)
     tmp_file = join(build_dir, "size-data-longcmd.txt")
 
-    with open(tmp_file, "w") as fp:
+    with open(tmp_file, mode="w", encoding="utf8") as fp:
         fp.write("\n".join(tool_args))
 
     cmd.append("@" + tmp_file)
@@ -164,7 +164,7 @@ def _collect_symbols_info(env, elffile, elf_path, sections):
         location = symbol_locations.get(hex(symbol["addr"]))
         if not location or "?" in location:
             continue
-        if "windows" in get_systype():
+        if IS_WINDOWS:
             drive, tail = splitdrive(location)
             location = join(drive.upper(), tail)
         symbol["file"] = location
@@ -220,7 +220,7 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
             "sections": sections,
         }
 
-        files = dict()
+        files = {}
         for symbol in _collect_symbols_info(env, elffile, elf_path, sections):
             file_path = symbol.get("file") or "unknown"
             if not files.get(file_path, {}):
@@ -235,14 +235,16 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
 
             files[file_path]["symbols"].append(symbol)
 
-        data["memory"]["files"] = list()
+        data["memory"]["files"] = []
         for k, v in files.items():
             file_data = {"path": k}
             file_data.update(v)
             data["memory"]["files"].append(file_data)
 
-    with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp:
-        fp.write(dump_json_to_unicode(data))
+    with open(
+        join(env.subst("$BUILD_DIR"), "sizedata.json"), mode="w", encoding="utf8"
+    ) as fp:
+        fp.write(json.dumps(data))
 
 
 def exists(_):

platformio/builder/tools/piotarget.py

@@ -0,0 +1,134 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import os
+
+from SCons.Action import Action  # pylint: disable=import-error
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
+from SCons.Script import AlwaysBuild  # pylint: disable=import-error
+
+from platformio import compat, fs
+
+
+def VerboseAction(_, act, actstr):
+    if int(ARGUMENTS.get("PIOVERBOSE", 0)):
+        return act
+    return Action(act, actstr)
+
+
+def PioClean(env, clean_all=False):
+    def _relpath(path):
+        if compat.IS_WINDOWS:
+            prefix = os.getcwd()[:2].lower()
+            if (
+                ":" not in prefix
+                or not path.lower().startswith(prefix)
+                or os.path.relpath(path).startswith("..")
+            ):
+                return path
+        return os.path.relpath(path)
+
+    def _clean_dir(path):
+        clean_rel_path = _relpath(path)
+        for root, _, files in os.walk(path):
+            for f in files:
+                dst = os.path.join(root, f)
+                os.remove(dst)
+                print(
+                    "Removed %s"
+                    % (dst if not clean_rel_path.startswith(".") else _relpath(dst))
+                )
+
+    build_dir = env.subst("$BUILD_DIR")
+    libdeps_dir = env.subst("$PROJECT_LIBDEPS_DIR")
+    if os.path.isdir(build_dir):
+        _clean_dir(build_dir)
+        fs.rmtree(build_dir)
+    else:
+        print("Build environment is clean")
+
+    if clean_all and os.path.isdir(libdeps_dir):
+        _clean_dir(libdeps_dir)
+        fs.rmtree(libdeps_dir)
+
+    print("Done cleaning")
+
+
+def AddTarget(  # pylint: disable=too-many-arguments
+    env,
+    name,
+    dependencies,
+    actions,
+    title=None,
+    description=None,
+    group="General",
+    always_build=True,
+):
+    if "__PIO_TARGETS" not in env:
+        env["__PIO_TARGETS"] = {}
+    assert name not in env["__PIO_TARGETS"]
+    env["__PIO_TARGETS"][name] = dict(
+        name=name, title=title, description=description, group=group
+    )
+    target = env.Alias(name, dependencies, actions)
+    if always_build:
+        AlwaysBuild(target)
+    return target
+
+
+def AddPlatformTarget(env, *args, **kwargs):
+    return env.AddTarget(group="Platform", *args, **kwargs)
+
+
+def AddCustomTarget(env, *args, **kwargs):
+    return env.AddTarget(group="Custom", *args, **kwargs)
+
+
+def DumpTargets(env):
+    targets = env.get("__PIO_TARGETS") or {}
+    # pre-fill default targets if embedded dev-platform
+    if env.PioPlatform().is_embedded() and not any(
+        t["group"] == "Platform" for t in targets.values()
+    ):
+        targets["upload"] = dict(name="upload", group="Platform", title="Upload")
+    targets["compiledb"] = dict(
+        name="compiledb",
+        title="Compilation Database",
+        description="Generate compilation database `compile_commands.json`",
+        group="Advanced",
+    )
+    targets["clean"] = dict(name="clean", title="Clean", group="General")
+    targets["cleanall"] = dict(
+        name="cleanall",
+        title="Clean All",
+        group="General",
+        description="Clean a build environment and installed library dependencies",
+    )
+    return list(targets.values())
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    env.AddMethod(VerboseAction)
+    env.AddMethod(PioClean)
+    env.AddMethod(AddTarget)
+    env.AddMethod(AddPlatformTarget)
+    env.AddMethod(AddCustomTarget)
+    env.AddMethod(DumpTargets)
+    return env

platformio/builder/tools/piotest.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import os
+
+from platformio.builder.tools import platformio as piotool
+from platformio.test.result import TestSuite
+from platformio.test.runners.factory import TestRunnerFactory
+
+
+def ConfigureTestTarget(env):
+    env.Append(
+        CPPDEFINES=["UNIT_TEST", "PIO_UNIT_TESTING"],
+        PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piotool.SRC_BUILD_EXT],
+    )
+    env.Prepend(CPPPATH=["$PROJECT_TEST_DIR"])
+
+    if "PIOTEST_RUNNING_NAME" in env:
+        test_name = env["PIOTEST_RUNNING_NAME"]
+        while True:
+            test_name = os.path.dirname(test_name)  # parent dir
+            # skip nested tests (user's side issue?)
+            if not test_name or os.path.basename(test_name).startswith("test_"):
+                break
+            env.Prepend(
+                PIOTEST_SRC_FILTER=[
+                    f"+<{test_name}{os.path.sep}*.{ext}>"
+                    for ext in piotool.SRC_BUILD_EXT
+                ],
+                CPPPATH=[os.path.join("$PROJECT_TEST_DIR", test_name)],
+            )
+
+        env.Prepend(
+            PIOTEST_SRC_FILTER=[f"+<$PIOTEST_RUNNING_NAME{os.path.sep}>"],
+            CPPPATH=[os.path.join("$PROJECT_TEST_DIR", "$PIOTEST_RUNNING_NAME")],
+        )
+
+    test_runner = TestRunnerFactory.new(
+        TestSuite(env["PIOENV"], env.get("PIOTEST_RUNNING_NAME", "*")),
+        env.GetProjectConfig(),
+    )
+    test_runner.configure_build_env(env)
+
+
+def generate(env):
+    env.AddMethod(ConfigureTestTarget)
+
+
+def exists(_):
+    return True

platformio/builder/tools/pioupload.py

@@ -12,25 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=unused-argument
+
 from __future__ import absolute_import
 
+import os
 import re
 import sys
-from fnmatch import fnmatch
-from os import environ
-from os.path import isfile, join
 from shutil import copyfile
 from time import sleep
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from serial import Serial, SerialException
 
-from platformio import exception, fs, util
-from platformio.compat import WINDOWS
+from platformio import exception, fs
+from platformio.device.finder import find_mbed_disk, find_serial_port, is_pattern_port
+from platformio.device.list import list_serial_ports
 from platformio.proc import exec_command
 
-# pylint: disable=unused-argument
-
 
 def FlushSerialBuffer(env, port):
     s = Serial(env.subst(port))
@@ -62,7 +61,7 @@ def WaitForNewSerialPort(env, before):
     elapsed = 0
     before = [p["port"] for p in before]
     while elapsed < 5 and new_port is None:
-        now = [p["port"] for p in util.get_serial_ports()]
+        now = [p["port"] for p in list_serial_ports()]
         for p in now:
             if p not in before:
                 new_port = p
@@ -97,67 +96,28 @@ def WaitForNewSerialPort(env, before):
 
 def AutodetectUploadPort(*args, **kwargs):
     env = args[0]
-
-    def _get_pattern():
-        if "UPLOAD_PORT" not in env:
-            return None
-        if set(["*", "?", "[", "]"]) & set(env["UPLOAD_PORT"]):
-            return env["UPLOAD_PORT"]
-        return None
-
-    def _is_match_pattern(port):
-        pattern = _get_pattern()
-        if not pattern:
-            return True
-        return fnmatch(port, pattern)
-
-    def _look_for_mbed_disk():
-        msdlabels = ("mbed", "nucleo", "frdm", "microbit")
-        for item in util.get_logical_devices():
-            if item["path"].startswith("/net") or not _is_match_pattern(item["path"]):
-                continue
-            mbed_pages = [join(item["path"], n) for n in ("mbed.htm", "mbed.html")]
-            if any(isfile(p) for p in mbed_pages):
-                return item["path"]
-            if item["name"] and any(l in item["name"].lower() for l in msdlabels):
-                return item["path"]
-        return None
-
-    def _look_for_serial_port():
-        port = None
-        board_hwids = []
-        upload_protocol = env.subst("$UPLOAD_PROTOCOL")
-        if "BOARD" in env and "build.hwids" in env.BoardConfig():
-            board_hwids = env.BoardConfig().get("build.hwids")
-        for item in util.get_serial_ports(filter_hwid=True):
-            if not _is_match_pattern(item["port"]):
-                continue
-            port = item["port"]
-            if upload_protocol.startswith("blackmagic"):
-                if WINDOWS and port.startswith("COM") and len(port) > 4:
-                    port = "\\\\.\\%s" % port
-                if "GDB" in item["description"]:
-                    return port
-            for hwid in board_hwids:
-                hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                if hwid_str in item["hwid"]:
-                    return port
-        return port
-
-    if "UPLOAD_PORT" in env and not _get_pattern():
-        print(env.subst("Use manually specified: $UPLOAD_PORT"))
+    initial_port = env.subst("$UPLOAD_PORT")
+    upload_protocol = env.subst("$UPLOAD_PROTOCOL")
+    if initial_port and not is_pattern_port(initial_port):
+        print(env.subst("Using manually specified: $UPLOAD_PORT"))
         return
 
-    if env.subst("$UPLOAD_PROTOCOL") == "mbed" or (
-        "mbed" in env.subst("$PIOFRAMEWORK") and not env.subst("$UPLOAD_PROTOCOL")
+    if upload_protocol == "mbed" or (
+        "mbed" in env.subst("$PIOFRAMEWORK") and not upload_protocol
     ):
-        env.Replace(UPLOAD_PORT=_look_for_mbed_disk())
+        env.Replace(UPLOAD_PORT=find_mbed_disk(initial_port))
     else:
         try:
             fs.ensure_udev_rules()
         except exception.InvalidUdevRules as e:
             sys.stderr.write("\n%s\n\n" % e)
-        env.Replace(UPLOAD_PORT=_look_for_serial_port())
+        env.Replace(
+            UPLOAD_PORT=find_serial_port(
+                initial_port=initial_port,
+                board_config=env.BoardConfig() if "BOARD" in env else None,
+                upload_protocol=upload_protocol,
+            )
+        )
 
     if env.subst("$UPLOAD_PORT"):
         print(env.subst("Auto-detected: $UPLOAD_PORT"))
@@ -175,10 +135,12 @@ def UploadToDisk(_, target, source, env):
     assert "UPLOAD_PORT" in env
     progname = env.subst("$PROGNAME")
     for ext in ("bin", "hex"):
-        fpath = join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
-        if not isfile(fpath):
+        fpath = os.path.join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
+        if not os.path.isfile(fpath):
             continue
-        copyfile(fpath, join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)))
+        copyfile(
+            fpath, os.path.join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext))
+        )
     print(
         "Firmware has been successfully uploaded.\n"
         "(Some boards may require manual hard reset)"
@@ -211,7 +173,7 @@ def CheckUploadSize(_, target, source, env):
         if not isinstance(cmd, list):
             cmd = cmd.split()
         cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
-        sysenv = environ.copy()
+        sysenv = os.environ.copy()
         sysenv["PATH"] = str(env["ENV"]["PATH"])
         result = exec_command(env.subst(cmd), env=sysenv)
         if result["returncode"] != 0:
@@ -236,9 +198,9 @@ def CheckUploadSize(_, target, source, env):
     def _format_availale_bytes(value, total):
         percent_raw = float(value) / float(total)
         blocks_per_progress = 10
-        used_blocks = int(round(blocks_per_progress * percent_raw))
-        if used_blocks > blocks_per_progress:
-            used_blocks = blocks_per_progress
+        used_blocks = min(
+            int(round(blocks_per_progress * percent_raw)), blocks_per_progress
+        )
         return "[{:{}}] {: 6.1%} (used {:d} bytes from {:d} bytes)".format(
             "=" * used_blocks, blocks_per_progress, percent_raw, value, total
         )

platformio/builder/tools/platformio.py

@@ -26,9 +26,9 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from SCons.Script import Export  # pylint: disable=import-error
 from SCons.Script import SConscript  # pylint: disable=import-error
 
-from platformio import fs
-from platformio.compat import string_types
-from platformio.util import pioversion_to_intstr
+from platformio import __version__, fs
+from platformio.compat import IS_MACOS, string_types
+from platformio.package.version import pepver_to_semver
 
 SRC_HEADER_EXT = ["h", "hpp"]
 SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"]
@@ -47,14 +47,16 @@ def scons_patched_match_splitext(path, suffixes=None):
 
 
 def GetBuildType(env):
-    return (
-        "debug"
-        if (
-            set(["debug", "sizedata"]) & set(COMMAND_LINE_TARGETS)
-            or env.GetProjectOption("build_type") == "debug"
-        )
-        else "release"
-    )
+    modes = []
+    if (
+        set(["__debug", "sizedata"])  # sizedata = for memory inspection
+        & set(COMMAND_LINE_TARGETS)
+        or env.GetProjectOption("build_type") == "debug"
+    ):
+        modes.append("debug")
+    if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test":
+        modes.append("test")
+    return "+".join(modes or ["release"])
 
 
 def BuildProgram(env):
@@ -66,12 +68,17 @@ def BuildProgram(env):
         env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")])
 
     # enable "cyclic reference" for linker
-    if env.get("LIBS") and env.GetCompilerType() == "gcc":
+    if (
+        env.get("LIBS")
+        and env.GetCompilerType() == "gcc"
+        and (env.PioPlatform().is_embedded() or not IS_MACOS)
+    ):
         env.Prepend(_LIBFLAGS="-Wl,--start-group ")
         env.Append(_LIBFLAGS=" -Wl,--end-group")
 
     program = env.Program(
-        os.path.join("$BUILD_DIR", env.subst("$PROGNAME")), env["PIOBUILDFILES"]
+        os.path.join("$BUILD_DIR", env.subst("$PROGNAME$PROGSUFFIX")),
+        env["PIOBUILDFILES"],
     )
     env.Replace(PIOMAINPROG=program)
 
@@ -90,11 +97,16 @@ def BuildProgram(env):
 
 def ProcessProgramDeps(env):
     def _append_pio_macros():
+        core_version = pepver_to_semver(__version__)
         env.AppendUnique(
             CPPDEFINES=[
                 (
                     "PLATFORMIO",
-                    int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())),
+                    int(
+                        "{0:02d}{1:02d}{2:02d}".format(
+                            core_version.major, core_version.minor, core_version.patch
+                        )
+                    ),
                 )
             ]
         )
@@ -103,10 +115,6 @@ def ProcessProgramDeps(env):
 
     env.PrintConfiguration()
 
-    # fix ASM handling under non case-sensitive OS
-    if not Util.case_sensitive_suffixes(".s", ".S"):
-        env.Replace(AS="$CC", ASCOM="$ASPPCOM")
-
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
         env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
@@ -117,18 +125,25 @@ def ProcessProgramDeps(env):
     # process framework scripts
     env.BuildFrameworks(env.get("PIOFRAMEWORK"))
 
-    if env.GetBuildType() == "debug":
-        env.ConfigureDebugFlags()
+    if "debug" in env.GetBuildType():
+        env.ConfigureDebugTarget()
+    if "test" in env.GetBuildType():
+        env.ConfigureTestTarget()
 
     # remove specified flags
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
-    if "__test" in COMMAND_LINE_TARGETS:
-        env.ConfigureTestTarget()
+    if "compiledb" in COMMAND_LINE_TARGETS and env.get(
+        "COMPILATIONDB_INCLUDE_TOOLCHAIN"
+    ):
+        for scope, includes in env.DumpIntegrationIncludes().items():
+            if scope in ("toolchain",):
+                env.Append(CPPPATH=includes)
 
 
 def ProcessProjectDeps(env):
     project_lib_builder = env.ConfigureProjectLibBuilder()
+    projenv = project_lib_builder.env
 
     # prepend project libs to the beginning of list
     env.Prepend(LIBS=project_lib_builder.build())
@@ -141,19 +156,19 @@ def ProcessProjectDeps(env):
         }
     )
 
-    projenv = env.Clone()
-
-    # CPPPATH from dependencies
-    projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH"))
-    # extra build flags from `platformio.ini`
-    projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
-
-    is_test = "__test" in COMMAND_LINE_TARGETS
-    if is_test:
+    if "test" in env.GetBuildType():
+        build_files_before_nums = len(env.get("PIOBUILDFILES", []))
         projenv.BuildSources(
             "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
         )
-    if not is_test or env.GetProjectOption("test_build_project_src"):
+        if len(env.get("PIOBUILDFILES", [])) - build_files_before_nums < 1:
+            sys.stderr.write(
+                "Error: Nothing to build. Please put your test suites "
+                "to the '%s' folder\n" % env.subst("$PROJECT_TEST_DIR")
+            )
+            env.Exit(1)
+
+    if "test" not in env.GetBuildType() or env.GetProjectOption("test_build_src"):
         projenv.BuildSources(
             "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER")
         )
@@ -161,7 +176,7 @@ def ProcessProjectDeps(env):
     if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
         sys.stderr.write(
             "Error: Nothing to build. Please put your source code files "
-            "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
+            "to the '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
         )
         env.Exit(1)
 
@@ -197,12 +212,12 @@ def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
     for k in ("CPPPATH", "LIBPATH"):
         for i, p in enumerate(result.get(k, [])):
             if os.path.isdir(p):
-                result[k][i] = os.path.realpath(p)
+                result[k][i] = os.path.abspath(p)
 
     # fix relative path for "-include"
     for i, f in enumerate(result.get("CCFLAGS", [])):
         if isinstance(f, tuple) and f[0] == "-include":
-            result["CCFLAGS"][i] = (f[0], env.File(os.path.realpath(f[1].get_path())))
+            result["CCFLAGS"][i] = (f[0], env.File(os.path.abspath(f[1].get_path())))
 
     return result
 
@@ -282,18 +297,21 @@ def CollectBuildFiles(
         if fs.path_endswith_ext(item, SRC_BUILD_EXT):
             sources.append(env.File(os.path.join(_var_dir, os.path.basename(item))))
 
-    for callback, pattern in env.get("__PIO_BUILD_MIDDLEWARES", []):
-        tmp = []
-        for node in sources:
-            if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern):
-                tmp.append(node)
-                continue
-            n = callback(node)
-            if n:
-                tmp.append(n)
-        sources = tmp
+    middlewares = env.get("__PIO_BUILD_MIDDLEWARES")
+    if not middlewares:
+        return sources
 
-    return sources
+    new_sources = []
+    for node in sources:
+        new_node = node
+        for callback, pattern in middlewares:
+            if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern):
+                continue
+            new_node = callback(new_node)
+        if new_node:
+            new_sources.append(new_node)
+
+    return new_sources
 
 
 def AddBuildMiddleware(env, callback, pattern=None):
@@ -311,33 +329,25 @@ def BuildFrameworks(env, frameworks):
         )
         env.Exit(1)
 
-    board_frameworks = env.BoardConfig().get("frameworks", [])
-    if frameworks == ["platformio"]:
-        if board_frameworks:
-            frameworks.insert(0, board_frameworks[0])
-        else:
-            sys.stderr.write("Error: Please specify `board` in `platformio.ini`\n")
-            env.Exit(1)
-
-    for f in frameworks:
-        if f == "arduino":
-            # Arduino IDE appends .o the end of filename
+    supported_frameworks = env.BoardConfig().get("frameworks", [])
+    for name in frameworks:
+        if name == "arduino":
+            # Arduino IDE appends .o to the end of filename
             Builder.match_splitext = scons_patched_match_splitext
             if "nobuild" not in COMMAND_LINE_TARGETS:
                 env.ConvertInoToCpp()
 
-        if f in board_frameworks:
-            SConscript(env.GetFrameworkScript(f), exports="env")
+        if name in supported_frameworks:
+            SConscript(env.GetFrameworkScript(name), exports="env")
         else:
-            sys.stderr.write("Error: This board doesn't support %s framework!\n" % f)
+            sys.stderr.write("Error: This board doesn't support %s framework!\n" % name)
             env.Exit(1)
 
 
-def BuildLibrary(env, variant_dir, src_dir, src_filter=None):
+def BuildLibrary(env, variant_dir, src_dir, src_filter=None, nodes=None):
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
-    return env.StaticLibrary(
-        env.subst(variant_dir), env.CollectBuildFiles(variant_dir, src_dir, src_filter)
-    )
+    nodes = nodes or env.CollectBuildFiles(variant_dir, src_dir, src_filter)
+    return env.StaticLibrary(env.subst(variant_dir), nodes)
 
 
 def BuildSources(env, variant_dir, src_dir, src_filter=None):

platformio/cache.py

@@ -0,0 +1,165 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import codecs
+import hashlib
+import os
+from time import time
+
+from platformio import app, fs
+from platformio.compat import hashlib_encode_data
+from platformio.package.lockfile import LockFile
+from platformio.project.helpers import get_project_cache_dir
+
+
+class ContentCache(object):
+    def __init__(self, namespace=None):
+        self.cache_dir = os.path.join(get_project_cache_dir(), namespace or "content")
+        self._db_path = os.path.join(self.cache_dir, "db.data")
+        self._lockfile = None
+        if not os.path.isdir(self.cache_dir):
+            os.makedirs(self.cache_dir)
+
+    def __enter__(self):
+        # cleanup obsolete items
+        self.delete()
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        pass
+
+    @staticmethod
+    def key_from_args(*args):
+        h = hashlib.sha1()
+        for arg in args:
+            if arg:
+                h.update(hashlib_encode_data(arg))
+        return h.hexdigest()
+
+    def get_cache_path(self, key):
+        assert "/" not in key and "\\" not in key
+        key = str(key)
+        assert len(key) > 3
+        return os.path.join(self.cache_dir, key)
+
+    def get(self, key):
+        cache_path = self.get_cache_path(key)
+        if not os.path.isfile(cache_path):
+            return None
+        with codecs.open(cache_path, "rb", encoding="utf8") as fp:
+            return fp.read()
+
+    def set(self, key, data, valid):
+        if not app.get_setting("enable_cache"):
+            return False
+        cache_path = self.get_cache_path(key)
+        if os.path.isfile(cache_path):
+            self.delete(key)
+        if not data:
+            return False
+        tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
+        assert valid.endswith(tuple(tdmap))
+        expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1]))
+
+        if not self._lock_dbindex():
+            return False
+
+        if not os.path.isdir(os.path.dirname(cache_path)):
+            os.makedirs(os.path.dirname(cache_path))
+        try:
+            with codecs.open(cache_path, mode="wb", encoding="utf8") as fp:
+                fp.write(data)
+            with open(self._db_path, mode="a", encoding="utf8") as fp:
+                fp.write("%s=%s\n" % (str(expire_time), os.path.basename(cache_path)))
+        except UnicodeError:
+            if os.path.isfile(cache_path):
+                try:
+                    os.remove(cache_path)
+                except OSError:
+                    pass
+
+        return self._unlock_dbindex()
+
+    def delete(self, keys=None):
+        """Keys=None, delete expired items"""
+        if not os.path.isfile(self._db_path):
+            return None
+        if not keys:
+            keys = []
+        if not isinstance(keys, list):
+            keys = [keys]
+        paths_for_delete = [self.get_cache_path(k) for k in keys]
+        found = False
+        newlines = []
+        with open(self._db_path, encoding="utf8") as fp:
+            for line in fp.readlines():
+                line = line.strip()
+                if "=" not in line:
+                    continue
+                expire, fname = line.split("=")
+                path = os.path.join(self.cache_dir, fname)
+                try:
+                    if (
+                        time() < int(expire)
+                        and os.path.isfile(path)
+                        and path not in paths_for_delete
+                    ):
+                        newlines.append(line)
+                        continue
+                except ValueError:
+                    pass
+                found = True
+                if os.path.isfile(path):
+                    try:
+                        os.remove(path)
+                        if not os.listdir(os.path.dirname(path)):
+                            fs.rmtree(os.path.dirname(path))
+                    except OSError:
+                        pass
+
+        if found and self._lock_dbindex():
+            with open(self._db_path, mode="w", encoding="utf8") as fp:
+                fp.write("\n".join(newlines) + "\n")
+            self._unlock_dbindex()
+
+        return True
+
+    def clean(self):
+        if not os.path.isdir(self.cache_dir):
+            return
+        fs.rmtree(self.cache_dir)
+
+    def _lock_dbindex(self):
+        self._lockfile = LockFile(self.cache_dir)
+        try:
+            self._lockfile.acquire()
+        except:  # pylint: disable=bare-except
+            return False
+
+        return True
+
+    def _unlock_dbindex(self):
+        if self._lockfile:
+            self._lockfile.release()
+        return True
+
+
+#
+# Helpers
+#
+
+
+def cleanup_content_cache(namespace=None):
+    with ContentCache(namespace) as cc:
+        cc.clean()

platformio/check/cli.py

@@ -15,7 +15,9 @@
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
 # pylint: disable=redefined-builtin,too-many-statements
 
+import json
 import os
+import shutil
 from collections import Counter
 from os.path import dirname, isfile
 from time import time
@@ -24,14 +26,13 @@ import click
 from tabulate import tabulate
 
 from platformio import app, exception, fs, util
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools import CheckToolFactory
-from platformio.compat import dump_json_to_unicode
+from platformio.check.defect import DefectItem
+from platformio.check.tools import CheckToolFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import find_project_dir_above, get_project_dir
 
 
-@click.command("check", short_help="Run a static analysis tool on code")
+@click.command("check", short_help="Static Code Analysis")
 @click.option("-e", "--environment", multiple=True)
 @click.option(
     "-d",
@@ -105,8 +106,8 @@ def cli(
                 )
 
             default_patterns = [
-                config.get_optional_dir("src"),
-                config.get_optional_dir("include"),
+                config.get("platformio", "src_dir"),
+                config.get("platformio", "include_dir"),
             ]
             tool_options = dict(
                 verbose=verbose,
@@ -117,6 +118,7 @@ def cli(
                 if silent
                 else severity or config.get("env:" + envname, "check_severity"),
                 skip_packages=skip_packages or env_options.get("check_skip_packages"),
+                platform_packages=env_options.get("platform_packages"),
             )
 
             for tool in config.get("env:" + envname, "check_tool"):
@@ -163,9 +165,12 @@ def cli(
                     print_processing_footer(result)
 
         if json_output:
-            click.echo(dump_json_to_unicode(results_to_json(results)))
+            click.echo(json.dumps(results_to_json(results)))
         elif not silent:
-            print_check_summary(results)
+            print_check_summary(results, verbose=verbose)
+
+    # Reset custom project config
+    app.set_session_var("custom_project_conf", None)
 
     command_failed = any(r.get("succeeded") is False for r in results)
     if command_failed:
@@ -193,7 +198,7 @@ def print_processing_header(tool, envname, envdump):
         "Checking %s > %s (%s)"
         % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump))
     )
-    terminal_width, _ = click.get_terminal_size()
+    terminal_width, _ = shutil.get_terminal_size()
     click.secho("-" * terminal_width, bold=True)
 
 
@@ -214,7 +219,7 @@ def print_processing_footer(result):
 
 
 def collect_component_stats(result):
-    components = dict()
+    components = {}
 
     def _append_defect(component, defect):
         if not components.get(component):
@@ -249,7 +254,7 @@ def print_defects_stats(results):
 
     severity_labels = list(DefectItem.SEVERITY_LABELS.values())
     severity_labels.reverse()
-    tabular_data = list()
+    tabular_data = []
     for k, v in component_stats.items():
         tool_defect = [v.get(s, 0) for s in severity_labels]
         tabular_data.append([k] + tool_defect)
@@ -266,7 +271,7 @@ def print_defects_stats(results):
     click.echo()
 
 
-def print_check_summary(results):
+def print_check_summary(results, verbose=False):
     click.echo()
 
     tabular_data = []
@@ -283,6 +288,8 @@ def print_check_summary(results):
             status_str = click.style("FAILED", fg="red")
         elif result.get("succeeded") is None:
             status_str = "IGNORED"
+            if not verbose:
+                continue
         else:
             succeeded_nums += 1
             status_str = click.style("PASSED", fg="green")

platformio/check/defect.py

@@ -34,7 +34,7 @@ class DefectItem(object):
         severity,
         category,
         message,
-        file="unknown",
+        file=None,
         line=0,
         column=0,
         id=None,
@@ -50,7 +50,7 @@ class DefectItem(object):
         self.callstack = callstack
         self.cwe = cwe
         self.id = id
-        self.file = file
+        self.file = file or "unknown"
         if file.lower().startswith(get_project_dir().lower()):
             self.file = os.path.relpath(file, get_project_dir())
 
@@ -86,7 +86,7 @@ class DefectItem(object):
             "severity": self.SEVERITY_LABELS[self.severity],
             "category": self.category,
             "message": self.message,
-            "file": os.path.realpath(self.file),
+            "file": os.path.abspath(self.file),
             "line": self.line,
             "column": self.column,
             "callstack": self.callstack,

platformio/check/tools/__init__.py

@@ -13,9 +13,9 @@
 # limitations under the License.
 
 from platformio import exception
-from platformio.commands.check.tools.clangtidy import ClangtidyCheckTool
-from platformio.commands.check.tools.cppcheck import CppcheckCheckTool
-from platformio.commands.check.tools.pvsstudio import PvsStudioCheckTool
+from platformio.check.tools.clangtidy import ClangtidyCheckTool
+from platformio.check.tools.cppcheck import CppcheckCheckTool
+from platformio.check.tools.pvsstudio import PvsStudioCheckTool
 
 
 class CheckToolFactory(object):

platformio/check/tools/base.py

@@ -14,13 +14,15 @@
 
 import glob
 import os
-from tempfile import NamedTemporaryFile
+import tempfile
 
 import click
 
 from platformio import fs, proc
-from platformio.commands.check.defect import DefectItem
-from platformio.project.helpers import load_project_ide_data
+from platformio.check.defect import DefectItem
+from platformio.package.manager.core import get_core_package_dir
+from platformio.package.meta import PackageSpec
+from platformio.project.helpers import load_build_metadata
 
 
 class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
@@ -55,7 +57,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         ]
 
     def _load_cpp_data(self, project_dir):
-        data = load_project_ide_data(project_dir, self.envname)
+        data = load_build_metadata(project_dir, self.envname)
         if not data:
             return
         self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", ""))
@@ -66,6 +68,13 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         self.cxx_path = data.get("cxx_path")
         self.toolchain_defines = self._get_toolchain_defines()
 
+    def get_tool_dir(self, pkg_name):
+        for spec in self.options["platform_packages"] or []:
+            spec = PackageSpec(spec)
+            if spec.name == pkg_name:
+                return get_core_package_dir(pkg_name, spec=spec)
+        return get_core_package_dir(pkg_name)
+
     def get_flags(self, tool):
         result = []
         flags = self.options.get("flags") or []
@@ -84,7 +93,9 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
             cmd = "echo | %s -x %s %s %s -dM -E -" % (
                 self.cc_path,
                 language,
-                " ".join([f for f in build_flags if f.startswith(("-m", "-f"))]),
+                " ".join(
+                    [f for f in build_flags if f.startswith(("-m", "-f", "-std"))]
+                ),
                 includes_file,
             )
             result = proc.exec_command(cmd, shell=True)
@@ -103,7 +114,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         return {lang: _extract_defines(lang, incflags_file) for lang in ("c", "c++")}
 
     def _create_tmp_file(self, data):
-        with NamedTemporaryFile("w", delete=False) as fp:
+        with tempfile.NamedTemporaryFile("w", delete=False) as fp:
             fp.write(data)
             self._tmp_files.append(fp.name)
             return fp.name
@@ -166,6 +177,29 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
             if os.path.isfile(f):
                 os.remove(f)
 
+    @staticmethod
+    def is_check_successful(cmd_result):
+        return cmd_result["returncode"] == 0
+
+    def execute_check_cmd(self, cmd):
+        result = proc.exec_command(
+            cmd,
+            stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
+            stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
+        )
+
+        if not self.is_check_successful(result):
+            click.echo(
+                "\nError: Failed to execute check command! Exited with code %d."
+                % result["returncode"]
+            )
+            if self.options.get("verbose"):
+                click.echo(result["out"])
+                click.echo(result["err"])
+            self._bad_input = True
+
+        return result
+
     @staticmethod
     def get_project_target_files(patterns):
         c_extension = (".c",)
@@ -176,14 +210,14 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
 
         def _add_file(path):
             if path.endswith(header_extensions):
-                result["headers"].append(os.path.realpath(path))
+                result["headers"].append(os.path.abspath(path))
             elif path.endswith(c_extension):
-                result["c"].append(os.path.realpath(path))
+                result["c"].append(os.path.abspath(path))
             elif path.endswith(cpp_extensions):
-                result["c++"].append(os.path.realpath(path))
+                result["c++"].append(os.path.abspath(path))
 
         for pattern in patterns:
-            for item in glob.glob(pattern):
+            for item in glob.glob(pattern, recursive=True):
                 if not os.path.isdir(item):
                     _add_file(item)
                 for root, _, files in os.walk(item, followlinks=True):
@@ -199,11 +233,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
             if self.options.get("verbose"):
                 click.echo(" ".join(cmd))
 
-            proc.exec_command(
-                cmd,
-                stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
-                stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
-            )
+            self.execute_check_cmd(cmd)
 
         else:
             if self.options.get("verbose"):

platformio/check/tools/clangtidy.py

@@ -15,13 +15,12 @@
 import re
 from os.path import join
 
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
-from platformio.managers.core import get_core_package_dir
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 
 
 class ClangtidyCheckTool(CheckToolBase):
-    def tool_output_filter(self, line):
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
         if not self.options.get("verbose") and "[clang-diagnostic-error]" in line:
             return ""
 
@@ -34,7 +33,7 @@ class ClangtidyCheckTool(CheckToolBase):
 
         return ""
 
-    def parse_defect(self, raw_line):
+    def parse_defect(self, raw_line):  # pylint: disable=arguments-differ
         match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line)
         if not match:
             return raw_line
@@ -49,12 +48,20 @@ class ClangtidyCheckTool(CheckToolBase):
 
         return DefectItem(severity, category, message, file_, line, column, defect_id)
 
+    @staticmethod
+    def is_check_successful(cmd_result):
+        # Note: Clang-Tidy returns 1 for not critical compilation errors,
+        # so 0 and 1 are only acceptable values
+        return cmd_result["returncode"] < 2
+
     def configure_command(self):
-        tool_path = join(get_core_package_dir("tool-clangtidy"), "clang-tidy")
+        tool_path = join(self.get_tool_dir("tool-clangtidy"), "clang-tidy")
 
         cmd = [tool_path, "--quiet"]
         flags = self.get_flags("clangtidy")
-        if not self.is_flag_set("--checks", flags):
+        if not (
+            self.is_flag_set("--checks", flags) or self.is_flag_set("--config", flags)
+        ):
             cmd.append("--checks=*")
 
         project_files = self.get_project_target_files(self.options["patterns"])
@@ -63,10 +70,7 @@ class ClangtidyCheckTool(CheckToolBase):
         for scope in project_files:
             src_files.extend(project_files[scope])
 
-        cmd.extend(flags)
-        cmd.extend(src_files)
-        cmd.append("--")
-
+        cmd.extend(flags + src_files + ["--"])
         cmd.extend(
             ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines["c++"]]
         )
@@ -74,11 +78,11 @@ class ClangtidyCheckTool(CheckToolBase):
         includes = []
         for inc in self.cpp_includes:
             if self.options.get("skip_packages") and inc.lower().startswith(
-                self.config.get_optional_dir("packages").lower()
+                self.config.get("platformio", "packages_dir").lower()
             ):
                 continue
             includes.append(inc)
 
-        cmd.append("--extra-arg=" + self._long_includes_hook(includes))
+        cmd.extend(["-I%s" % inc for inc in includes])
 
         return cmd

platformio/check/tools/cppcheck.py

@@ -17,13 +17,15 @@ import os
 import click
 
 from platformio import proc
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
-from platformio.managers.core import get_core_package_dir
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
 
 
 class CppcheckCheckTool(CheckToolBase):
     def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._field_delimiter = "<&PIO&>"
+        self._buffer = ""
         self.defect_fields = [
             "severity",
             "message",
@@ -34,9 +36,8 @@ class CppcheckCheckTool(CheckToolBase):
             "cwe",
             "id",
         ]
-        super(CppcheckCheckTool, self).__init__(*args, **kwargs)
 
-    def tool_output_filter(self, line):
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
         if (
             not self.options.get("verbose")
             and "--suppress=unmatchedSuppression:" in line
@@ -48,20 +49,23 @@ class CppcheckCheckTool(CheckToolBase):
             for msg in (
                 "No C or C++ source files found",
                 "unrecognized command line option",
+                "there was an internal error",
             )
         ):
             self._bad_input = True
 
         return line
 
-    def parse_defect(self, raw_line):
-        if "<&PIO&>" not in raw_line or any(
-            f not in raw_line for f in self.defect_fields
-        ):
+    def parse_defect(self, raw_line):  # pylint: disable=arguments-differ
+        if self._field_delimiter not in raw_line:
             return None
 
-        args = dict()
-        for field in raw_line.split("<&PIO&>"):
+        self._buffer += raw_line
+        if any(f not in self._buffer for f in self.defect_fields):
+            return None
+
+        args = {}
+        for field in self._buffer.split(self._field_delimiter):
             field = field.strip().replace('"', "")
             name, value = field.split("=", 1)
             args[name] = value
@@ -80,7 +84,7 @@ class CppcheckCheckTool(CheckToolBase):
         if (
             args.get("file", "")
             .lower()
-            .startswith(self.config.get_optional_dir("packages").lower())
+            .startswith(self.config.get("platformio", "packages_dir").lower())
         ):
             if args["id"] in breaking_defect_ids:
                 if self.options.get("verbose"):
@@ -92,24 +96,27 @@ class CppcheckCheckTool(CheckToolBase):
                     )
                     click.echo()
                 self._bad_input = True
+                self._buffer = ""
             return None
 
+        self._buffer = ""
         return DefectItem(**args)
 
-    def configure_command(
-        self, language, src_files
-    ):  # pylint: disable=arguments-differ
-        tool_path = os.path.join(get_core_package_dir("tool-cppcheck"), "cppcheck")
+    def configure_command(self, language, src_file):  # pylint: disable=arguments-differ
+        tool_path = os.path.join(self.get_tool_dir("tool-cppcheck"), "cppcheck")
 
         cmd = [
             tool_path,
-            "--error-exitcode=1",
+            "--addon-python=%s" % proc.get_pythonexe_path(),
+            "--error-exitcode=3",
             "--verbose" if self.options.get("verbose") else "--quiet",
         ]
 
         cmd.append(
             '--template="%s"'
-            % "<&PIO&>".join(["{0}={{{0}}}".format(f) for f in self.defect_fields])
+            % self._field_delimiter.join(
+                ["{0}={{{0}}}".format(f) for f in self.defect_fields]
+            )
         )
 
         flags = self.get_flags("cppcheck")
@@ -134,10 +141,11 @@ class CppcheckCheckTool(CheckToolBase):
 
         build_flags = self.cxx_flags if language == "c++" else self.cc_flags
 
-        for flag in build_flags:
-            if "-std" in flag:
-                # Standards with GNU extensions are not allowed
-                cmd.append("-" + flag.replace("gnu", "c"))
+        if not self.is_flag_set("--std", flags):
+            # Try to guess the standard version from the build flags
+            for flag in build_flags:
+                if "-std" in flag:
+                    cmd.append("-" + self.convert_language_standard(flag))
 
         cmd.extend(
             ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines[language]]
@@ -149,8 +157,8 @@ class CppcheckCheckTool(CheckToolBase):
             "--include=" + inc
             for inc in self.get_forced_includes(build_flags, self.cpp_includes)
         )
-        cmd.append("--file-list=%s" % self._generate_src_file(src_files))
         cmd.append("--includes-file=%s" % self._generate_inc_file())
+        cmd.append('"%s"' % src_file)
 
         return cmd
 
@@ -193,14 +201,14 @@ class CppcheckCheckTool(CheckToolBase):
         result = []
         for inc in self.cpp_includes:
             if self.options.get("skip_packages") and inc.lower().startswith(
-                self.config.get_optional_dir("packages").lower()
+                self.config.get("platformio", "packages_dir").lower()
             ):
                 continue
             result.append(inc)
         return self._create_tmp_file("\n".join(result))
 
     def clean_up(self):
-        super(CppcheckCheckTool, self).clean_up()
+        super().clean_up()
 
         # delete temporary dump files generated by addons
         if not self.is_flag_set("--addon", self.get_flags("cppcheck")):
@@ -212,29 +220,47 @@ class CppcheckCheckTool(CheckToolBase):
                 if os.path.isfile(dump_file):
                     os.remove(dump_file)
 
+    @staticmethod
+    def is_check_successful(cmd_result):
+        # Cppcheck is configured to return '3' if a defect is found
+        return cmd_result["returncode"] in (0, 3)
+
+    @staticmethod
+    def convert_language_standard(flag):
+        cpp_standards_map = {
+            "0x": "11",
+            "1y": "14",
+            "1z": "17",
+            "2a": "20",
+        }
+
+        standard = flag[-2:]
+        # Note: GNU extensions are not supported and converted to regular standards
+        return flag.replace("gnu", "c").replace(
+            standard, cpp_standards_map.get(standard, standard)
+        )
+
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
-        project_files = self.get_project_target_files(self.options["patterns"])
 
-        languages = ("c", "c++")
-        if not any([project_files[t] for t in languages]):
+        project_files = self.get_project_target_files(self.options["patterns"])
+        src_files_scope = ("c", "c++")
+        if not any(project_files[t] for t in src_files_scope):
             click.echo("Error: Nothing to check.")
             return True
-        for language in languages:
-            if not project_files[language]:
-                continue
-            cmd = self.configure_command(language, project_files[language])
-            if not cmd:
-                self._bad_input = True
-                continue
-            if self.options.get("verbose"):
-                click.echo(" ".join(cmd))
 
-            proc.exec_command(
-                cmd,
-                stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
-                stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
-            )
+        for scope, files in project_files.items():
+            if scope not in src_files_scope:
+                continue
+            for src_file in files:
+                cmd = self.configure_command(scope, src_file)
+                if not cmd:
+                    self._bad_input = True
+                    continue
+                if self.options.get("verbose"):
+                    click.echo(" ".join(cmd))
+
+                self.execute_check_cmd(cmd)
 
         self.clean_up()
 

platformio/check/tools/pvsstudio.py

@@ -19,39 +19,50 @@ from xml.etree.ElementTree import fromstring
 
 import click
 
-from platformio import proc, util
-from platformio.commands.check.defect import DefectItem
-from platformio.commands.check.tools.base import CheckToolBase
-from platformio.managers.core import get_core_package_dir
+from platformio import proc
+from platformio.check.defect import DefectItem
+from platformio.check.tools.base import CheckToolBase
+from platformio.compat import IS_WINDOWS
 
 
 class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-attributes
     def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self._tmp_dir = tempfile.mkdtemp(prefix="piocheck")
         self._tmp_preprocessed_file = self._generate_tmp_file_path() + ".i"
         self._tmp_output_file = self._generate_tmp_file_path() + ".pvs"
         self._tmp_cfg_file = self._generate_tmp_file_path() + ".cfg"
         self._tmp_cmd_file = self._generate_tmp_file_path() + ".cmd"
         self.tool_path = os.path.join(
-            get_core_package_dir("tool-pvs-studio"),
-            "x64" if "windows" in util.get_systype() else "bin",
+            self.get_tool_dir("tool-pvs-studio"),
+            "x64" if IS_WINDOWS else "bin",
             "pvs-studio",
         )
-        super(PvsStudioCheckTool, self).__init__(*args, **kwargs)
 
-        with open(self._tmp_cfg_file, "w") as fp:
+        with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp:
             fp.write(
                 "exclude-path = "
-                + self.config.get_optional_dir("packages").replace("\\", "/")
+                + self.config.get("platformio", "packages_dir").replace("\\", "/")
             )
 
-        with open(self._tmp_cmd_file, "w") as fp:
+        with open(self._tmp_cmd_file, mode="w", encoding="utf8") as fp:
             fp.write(
                 " ".join(
                     ['-I"%s"' % inc.replace("\\", "/") for inc in self.cpp_includes]
                 )
             )
 
+    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
+        if any(
+            err_msg in line.lower()
+            for err_msg in (
+                "license was not entered",
+                "license information is incorrect",
+            )
+        ):
+            self._bad_input = True
+        return line
+
     def _process_defects(self, defects):
         for defect in defects:
             if not isinstance(defect, DefectItem):
@@ -64,10 +75,8 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
 
     def _demangle_report(self, output_file):
         converter_tool = os.path.join(
-            get_core_package_dir("tool-pvs-studio"),
-            "HtmlGenerator"
-            if "windows" in util.get_systype()
-            else os.path.join("bin", "plog-converter"),
+            self.get_tool_dir("tool-pvs-studio"),
+            "HtmlGenerator" if IS_WINDOWS else os.path.join("bin", "plog-converter"),
         )
 
         cmd = (
@@ -182,9 +191,15 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
             flags = self.cc_flags
             compiler = self.cc_path
 
-        cmd = [compiler, src_file, "-E", "-o", self._tmp_preprocessed_file]
+        cmd = [
+            compiler,
+            '"%s"' % src_file,
+            "-E",
+            "-o",
+            '"%s"' % self._tmp_preprocessed_file,
+        ]
         cmd.extend([f for f in flags if f])
-        cmd.extend(["-D%s" % d for d in self.cpp_defines])
+        cmd.extend(['"-D%s"' % d.replace('"', '\\"') for d in self.cpp_defines])
         cmd.append('@"%s"' % self._tmp_cmd_file)
 
         # Explicitly specify C++ as the language used in .ino files
@@ -199,10 +214,16 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
             self._bad_input = True
 
     def clean_up(self):
-        super(PvsStudioCheckTool, self).clean_up()
+        super().clean_up()
         if os.path.isdir(self._tmp_dir):
             shutil.rmtree(self._tmp_dir)
 
+    @staticmethod
+    def is_check_successful(cmd_result):
+        return (
+            "license" not in cmd_result["err"].lower() and cmd_result["returncode"] == 0
+        )
+
     def check(self, on_defect_callback=None):
         self._on_defect_callback = on_defect_callback
         for scope, files in self.get_project_target_files(
@@ -219,11 +240,8 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
                     self._bad_input = True
                     continue
 
-                result = proc.exec_command(cmd)
-                # pylint: disable=unsupported-membership-test
-                if result["returncode"] != 0 or "license" in result["err"].lower():
-                    self._bad_input = True
-                    click.echo(result["err"])
+                result = self.execute_check_cmd(cmd)
+                if result["returncode"] != 0:
                     continue
 
                 self._process_defects(self.parse_defects(self._tmp_output_file))

platformio/cli.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+from pathlib import Path
+
+import click
+
+
+class PlatformioCLI(click.MultiCommand):
+
+    leftover_args = []
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._pio_root_path = Path(__file__).parent
+        self._pio_cmd_aliases = dict(package="pkg")
+
+    def _find_pio_commands(self):
+        def _to_module_path(p):
+            return (
+                "platformio." + ".".join(p.relative_to(self._pio_root_path).parts)[:-3]
+            )
+
+        result = {}
+        for p in self._pio_root_path.rglob("cli.py"):
+            # skip this module
+            if p.parent == self._pio_root_path:
+                continue
+            cmd_name = p.parent.name
+            result[self._pio_cmd_aliases.get(cmd_name, cmd_name)] = _to_module_path(p)
+
+        # find legacy commands
+        for p in (self._pio_root_path / "commands").iterdir():
+            if p.name.startswith("_"):
+                continue
+            if (p / "command.py").is_file():
+                result[p.name] = _to_module_path(p / "command.py")
+            elif p.name.endswith(".py"):
+                result[p.name[:-3]] = _to_module_path(p)
+
+        return result
+
+    @staticmethod
+    def in_silence():
+        args = PlatformioCLI.leftover_args
+        return args and any(
+            [
+                args[0] == "debug" and "--interpreter" in " ".join(args),
+                args[0] == "upgrade",
+                "--json-output" in args,
+                "--version" in args,
+            ]
+        )
+
+    def invoke(self, ctx):
+        PlatformioCLI.leftover_args = ctx.args
+        if hasattr(ctx, "protected_args"):
+            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
+        return super().invoke(ctx)
+
+    def list_commands(self, ctx):
+        return sorted(list(self._find_pio_commands()))
+
+    def get_command(self, ctx, cmd_name):
+        commands = self._find_pio_commands()
+        if cmd_name not in commands:
+            return self._handle_obsolate_command(ctx, cmd_name)
+        module = importlib.import_module(commands[cmd_name])
+        return getattr(module, "cli")
+
+    @staticmethod
+    def _handle_obsolate_command(ctx, cmd_name):
+        # pylint: disable=import-outside-toplevel
+        if cmd_name == "init":
+            from platformio.project.commands.init import project_init_cmd
+
+            return project_init_cmd
+
+        if cmd_name == "package":
+            from platformio.package.cli import cli
+
+            return cli
+
+        raise click.UsageError('No such command "%s"' % cmd_name, ctx)

platformio/commands/__init__.py

@@ -11,70 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import os
-
-import click
-
-
-class PlatformioCLI(click.MultiCommand):
-
-    leftover_args = []
-
-    def __init__(self, *args, **kwargs):
-        super(PlatformioCLI, self).__init__(*args, **kwargs)
-        self._pio_cmds_dir = os.path.dirname(__file__)
-
-    @staticmethod
-    def in_silence():
-        args = PlatformioCLI.leftover_args
-        return args and any(
-            [
-                args[0] == "debug" and "--interpreter" in " ".join(args),
-                args[0] == "upgrade",
-                "--json-output" in args,
-                "--version" in args,
-            ]
-        )
-
-    def invoke(self, ctx):
-        PlatformioCLI.leftover_args = ctx.args
-        if hasattr(ctx, "protected_args"):
-            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
-        return super(PlatformioCLI, self).invoke(ctx)
-
-    def list_commands(self, ctx):
-        cmds = []
-        for cmd_name in os.listdir(self._pio_cmds_dir):
-            if cmd_name.startswith("__init__"):
-                continue
-            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
-                cmds.append(cmd_name)
-            elif cmd_name.endswith(".py"):
-                cmds.append(cmd_name[:-3])
-        cmds.sort()
-        return cmds
-
-    def get_command(self, ctx, cmd_name):
-        mod = None
-        try:
-            mod_path = "platformio.commands." + cmd_name
-            if os.path.isfile(os.path.join(self._pio_cmds_dir, cmd_name, "command.py")):
-                mod_path = "platformio.commands.%s.command" % cmd_name
-            mod = __import__(mod_path, None, None, ["cli"])
-        except ImportError:
-            try:
-                return self._handle_obsolate_command(cmd_name)
-            except AttributeError:
-                pass
-            raise click.UsageError('No such command "%s"' % cmd_name, ctx)
-        return mod.cli
-
-    @staticmethod
-    def _handle_obsolate_command(name):
-        # pylint: disable=import-outside-toplevel
-        if name == "init":
-            from platformio.commands.project import project_init
-
-            return project_init
-        raise AttributeError()

platformio/commands/account/client.py

@@ -1,235 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import os
-import time
-
-import requests.adapters
-from requests.packages.urllib3.util.retry import Retry  # pylint:disable=import-error
-
-from platformio import __pioaccount_api__, app
-from platformio.commands.account import exception
-
-
-class AccountClient(object):
-    def __init__(
-        self, api_base_url=__pioaccount_api__, retries=3,
-    ):
-        if api_base_url.endswith("/"):
-            api_base_url = api_base_url[:-1]
-        self.api_base_url = api_base_url
-        self._session = requests.Session()
-        self._session.headers.update({"User-Agent": app.get_user_agent()})
-        retry = Retry(
-            total=retries,
-            read=retries,
-            connect=retries,
-            backoff_factor=2,
-            method_whitelist=list(Retry.DEFAULT_METHOD_WHITELIST) + ["POST"],
-        )
-        adapter = requests.adapters.HTTPAdapter(max_retries=retry)
-        self._session.mount(api_base_url, adapter)
-
-    def login(self, username, password):
-        try:
-            self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            pass
-        else:
-            raise exception.AccountAlreadyAuthenticated(
-                app.get_state_item("account", {}).get("email", "")
-            )
-
-        response = self._session.post(
-            self.api_base_url + "/v1/login",
-            data={"username": username, "password": password},
-        )
-        result = self.raise_error_from_response(response)
-        app.set_state_item("account", result)
-        return result
-
-    def login_with_code(self, client_id, code, redirect_uri):
-        try:
-            self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            pass
-        else:
-            raise exception.AccountAlreadyAuthenticated(
-                app.get_state_item("account", {}).get("email", "")
-            )
-
-        response = self._session.post(
-            self.api_base_url + "/v1/login/code",
-            data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri},
-        )
-        result = self.raise_error_from_response(response)
-        app.set_state_item("account", result)
-        return result
-
-    def logout(self):
-        try:
-            refresh_token = self.get_refresh_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
-        response = requests.post(
-            self.api_base_url + "/v1/logout", data={"refresh_token": refresh_token},
-        )
-        try:
-            self.raise_error_from_response(response)
-        except exception.AccountError:
-            pass
-        app.delete_state_item("account")
-        return True
-
-    def change_password(self, old_password, new_password):
-        try:
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
-        response = self._session.post(
-            self.api_base_url + "/v1/password",
-            headers={"Authorization": "Bearer %s" % token},
-            data={"old_password": old_password, "new_password": new_password},
-        )
-        self.raise_error_from_response(response)
-        return True
-
-    def registration(
-        self, username, email, password, firstname, lastname
-    ):  # pylint:disable=too-many-arguments
-        try:
-            self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            pass
-        else:
-            raise exception.AccountAlreadyAuthenticated(
-                app.get_state_item("account", {}).get("email", "")
-            )
-
-        response = self._session.post(
-            self.api_base_url + "/v1/registration",
-            data={
-                "username": username,
-                "email": email,
-                "password": password,
-                "firstname": firstname,
-                "lastname": lastname,
-            },
-        )
-        return self.raise_error_from_response(response)
-
-    def auth_token(self, password, regenerate):
-        try:
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
-        response = self._session.post(
-            self.api_base_url + "/v1/token",
-            headers={"Authorization": "Bearer %s" % token},
-            data={"password": password, "regenerate": 1 if regenerate else 0},
-        )
-        return self.raise_error_from_response(response).get("auth_token")
-
-    def forgot_password(self, username):
-        response = self._session.post(
-            self.api_base_url + "/v1/forgot", data={"username": username},
-        )
-        return self.raise_error_from_response(response).get("auth_token")
-
-    def get_profile(self):
-        try:
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
-        response = self._session.get(
-            self.api_base_url + "/v1/profile",
-            headers={"Authorization": "Bearer %s" % token},
-        )
-        return self.raise_error_from_response(response)
-
-    def update_profile(self, profile, current_password):
-        try:
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
-        profile["current_password"] = current_password
-        response = self._session.put(
-            self.api_base_url + "/v1/profile",
-            headers={"Authorization": "Bearer %s" % token},
-            data=profile,
-        )
-        return self.raise_error_from_response(response)
-
-    def get_account_info(self, offline):
-        if offline:
-            account = app.get_state_item("account")
-            if not account:
-                raise exception.AccountNotAuthenticated()
-            return {
-                "profile": {
-                    "email": account.get("email"),
-                    "username": account.get("username"),
-                }
-            }
-        try:
-            token = self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
-        response = self._session.get(
-            self.api_base_url + "/v1/summary",
-            headers={"Authorization": "Bearer %s" % token},
-        )
-        return self.raise_error_from_response(response)
-
-    def fetch_authentication_token(self):
-        if "PLATFORMIO_AUTH_TOKEN" in os.environ:
-            return os.environ["PLATFORMIO_AUTH_TOKEN"]
-        auth = app.get_state_item("account", {}).get("auth", {})
-        if auth.get("access_token") and auth.get("access_token_expire"):
-            if auth.get("access_token_expire") > time.time():
-                return auth.get("access_token")
-            if auth.get("refresh_token"):
-                response = self._session.post(
-                    self.api_base_url + "/v1/login",
-                    headers={"Authorization": "Bearer %s" % auth.get("refresh_token")},
-                )
-                result = self.raise_error_from_response(response)
-                app.set_state_item("account", result)
-                return result.get("auth").get("access_token")
-        raise exception.AccountNotAuthenticated()
-
-    @staticmethod
-    def get_refresh_token():
-        try:
-            auth = app.get_state_item("account").get("auth").get("refresh_token")
-            return auth
-        except:  # pylint:disable=bare-except
-            raise exception.AccountNotAuthenticated()
-
-    @staticmethod
-    def raise_error_from_response(response, expected_codes=(200, 201, 202)):
-        if response.status_code in expected_codes:
-            try:
-                return response.json()
-            except ValueError:
-                pass
-        try:
-            message = response.json()["message"]
-        except (KeyError, ValueError):
-            message = response.text
-        if "Authorization session has been expired" in message:
-            app.delete_state_item("account")
-        raise exception.AccountError(message)

platformio/commands/account/command.py

@@ -1,278 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=unused-argument
-
-import datetime
-import json
-import re
-
-import click
-from tabulate import tabulate
-
-from platformio.commands.account import exception
-from platformio.commands.account.client import AccountClient
-
-
-@click.group("account", short_help="Manage PIO Account")
-def cli():
-    pass
-
-
-def validate_username(value):
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){3,38}$", value, flags=re.I):
-        raise click.BadParameter(
-            "Invalid username format. "
-            "Username must contain at least 4 characters including single hyphens,"
-            " and cannot begin or end with a hyphen"
-        )
-    return value
-
-
-def validate_email(value):
-    value = str(value).strip()
-    if not re.match(r"^[a-z\d_.+-]+@[a-z\d\-]+\.[a-z\d\-.]+$", value, flags=re.I):
-        raise click.BadParameter("Invalid email address")
-    return value
-
-
-def validate_password(value):
-    value = str(value).strip()
-    if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
-        raise click.BadParameter(
-            "Invalid password format. "
-            "Password must contain at least 8 characters"
-            " including a number and a lowercase letter"
-        )
-    return value
-
-
-@cli.command("register", short_help="Create new PIO Account")
-@click.option(
-    "-u",
-    "--username",
-    prompt=True,
-    callback=lambda _, __, value: validate_username(value),
-)
-@click.option(
-    "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value)
-)
-@click.option(
-    "-p",
-    "--password",
-    prompt=True,
-    hide_input=True,
-    confirmation_prompt=True,
-    callback=lambda _, __, value: validate_password(value),
-)
-@click.option("--firstname", prompt=True)
-@click.option("--lastname", prompt=True)
-def account_register(username, email, password, firstname, lastname):
-    client = AccountClient()
-    client.registration(username, email, password, firstname, lastname)
-    return click.secho(
-        "An account has been successfully created. "
-        "Please check your mail to activate your account and verify your email address.",
-        fg="green",
-    )
-
-
-@cli.command("login", short_help="Log in to PIO Account")
-@click.option("-u", "--username", prompt="Username or email")
-@click.option("-p", "--password", prompt=True, hide_input=True)
-def account_login(username, password):
-    client = AccountClient()
-    client.login(username, password)
-    return click.secho("Successfully logged in!", fg="green")
-
-
-@cli.command("logout", short_help="Log out of PIO Account")
-def account_logout():
-    client = AccountClient()
-    client.logout()
-    return click.secho("Successfully logged out!", fg="green")
-
-
-@cli.command("password", short_help="Change password")
-@click.option("--old-password", prompt=True, hide_input=True)
-@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
-def account_password(old_password, new_password):
-    client = AccountClient()
-    client.change_password(old_password, new_password)
-    return click.secho("Password successfully changed!", fg="green")
-
-
-@cli.command("token", short_help="Get or regenerate Authentication Token")
-@click.option("-p", "--password", prompt=True, hide_input=True)
-@click.option("--regenerate", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_token(password, regenerate, json_output):
-    client = AccountClient()
-    auth_token = client.auth_token(password, regenerate)
-    if json_output:
-        return click.echo(json.dumps({"status": "success", "result": auth_token}))
-    return click.secho("Personal Authentication Token: %s" % auth_token, fg="green")
-
-
-@cli.command("forgot", short_help="Forgot password")
-@click.option("--username", prompt="Username or email")
-def account_forgot(username):
-    client = AccountClient()
-    client.forgot_password(username)
-    return click.secho(
-        "If this account is registered, we will send the "
-        "further instructions to your email.",
-        fg="green",
-    )
-
-
-@cli.command("update", short_help="Update profile information")
-@click.option("--current-password", prompt=True, hide_input=True)
-@click.option("--username")
-@click.option("--email")
-@click.option("--firstname")
-@click.option("--lastname")
-def account_update(current_password, **kwargs):
-    client = AccountClient()
-    profile = client.get_profile()
-    new_profile = profile.copy()
-    if not any(kwargs.values()):
-        for field in profile:
-            new_profile[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=profile[field]
-            )
-            if field == "email":
-                validate_email(new_profile[field])
-            if field == "username":
-                validate_username(new_profile[field])
-    else:
-        new_profile.update({key: value for key, value in kwargs.items() if value})
-    client.update_profile(new_profile, current_password)
-    click.secho("Profile successfully updated!", fg="green")
-    username_changed = new_profile["username"] != profile["username"]
-    email_changed = new_profile["email"] != profile["email"]
-    if not username_changed and not email_changed:
-        return None
-    try:
-        client.logout()
-    except exception.AccountNotAuthenticated:
-        pass
-    if email_changed:
-        return click.secho(
-            "Please check your mail to verify your new email address and re-login. ",
-            fg="yellow",
-        )
-    return click.secho("Please re-login.", fg="yellow")
-
-
-@cli.command("show", short_help="PIO Account information")
-@click.option("--offline", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_show(offline, json_output):
-    client = AccountClient()
-    info = client.get_account_info(offline)
-    if json_output:
-        return click.echo(json.dumps(info))
-    click.echo()
-    if info.get("profile"):
-        print_profile(info["profile"])
-    if info.get("packages"):
-        print_packages(info["packages"])
-    if info.get("subscriptions"):
-        print_subscriptions(info["subscriptions"])
-    return click.echo()
-
-
-def print_profile(profile):
-    click.secho("Profile", fg="cyan", bold=True)
-    click.echo("=" * len("Profile"))
-    data = []
-    if profile.get("username"):
-        data.append(("Username:", profile["username"]))
-    if profile.get("email"):
-        data.append(("Email:", profile["email"]))
-    if profile.get("firstname"):
-        data.append(("First name:", profile["firstname"]))
-    if profile.get("lastname"):
-        data.append(("Last name:", profile["lastname"]))
-    click.echo(tabulate(data, tablefmt="plain"))
-
-
-def print_packages(packages):
-    click.echo()
-    click.secho("Packages", fg="cyan")
-    click.echo("=" * len("Packages"))
-    for package in packages:
-        click.echo()
-        click.secho(package.get("name"), bold=True)
-        click.echo("-" * len(package.get("name")))
-        if package.get("description"):
-            click.echo(package.get("description"))
-        data = []
-        expire = "-"
-        if "subscription" in package:
-            expire = datetime.datetime.strptime(
-                (
-                    package["subscription"].get("end_at")
-                    or package["subscription"].get("next_bill_at")
-                ),
-                "%Y-%m-%dT%H:%M:%SZ",
-            ).strftime("%Y-%m-%d")
-        data.append(("Expire:", expire))
-        services = []
-        for key in package:
-            if not key.startswith("service."):
-                continue
-            if isinstance(package[key], dict):
-                services.append(package[key].get("title"))
-            else:
-                services.append(package[key])
-        if services:
-            data.append(("Services:", ", ".join(services)))
-        click.echo(tabulate(data, tablefmt="plain"))
-
-
-def print_subscriptions(subscriptions):
-    click.echo()
-    click.secho("Subscriptions", fg="cyan")
-    click.echo("=" * len("Subscriptions"))
-    for subscription in subscriptions:
-        click.echo()
-        click.secho(subscription.get("product_name"), bold=True)
-        click.echo("-" * len(subscription.get("product_name")))
-        data = [("State:", subscription.get("status"))]
-        begin_at = datetime.datetime.strptime(
-            subscription.get("begin_at"), "%Y-%m-%dT%H:%M:%SZ"
-        ).strftime("%Y-%m-%d %H:%M:%S")
-        data.append(("Start date:", begin_at or "-"))
-        end_at = subscription.get("end_at")
-        if end_at:
-            end_at = datetime.datetime.strptime(
-                subscription.get("end_at"), "%Y-%m-%dT%H:%M:%SZ"
-            ).strftime("%Y-%m-%d %H:%M:%S")
-        data.append(("End date:", end_at or "-"))
-        next_bill_at = subscription.get("next_bill_at")
-        if next_bill_at:
-            next_bill_at = datetime.datetime.strptime(
-                subscription.get("next_bill_at"), "%Y-%m-%dT%H:%M:%SZ"
-            ).strftime("%Y-%m-%d %H:%M:%S")
-        data.append(("Next payment:", next_bill_at or "-"))
-        data.append(
-            ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")
-        )
-        data.append(
-            ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-")
-        )
-        click.echo(tabulate(data, tablefmt="plain"))

platformio/commands/boards.py

@@ -13,16 +13,16 @@
 # limitations under the License.
 
 import json
+import shutil
 
 import click
 from tabulate import tabulate
 
 from platformio import fs
-from platformio.compat import dump_json_to_unicode
-from platformio.managers.platform import PlatformManager
+from platformio.package.manager.platform import PlatformPackageManager
 
 
-@click.command("boards", short_help="Embedded Board Explorer")
+@click.command("boards", short_help="Board Explorer")
 @click.argument("query", required=False)
 @click.option("--installed", is_flag=True)
 @click.option("--json-output", is_flag=True)
@@ -41,7 +41,7 @@ def cli(query, installed, json_output):  # pylint: disable=R0912
             grpboards[board["platform"]] = []
         grpboards[board["platform"]].append(board)
 
-    terminal_width, _ = click.get_terminal_size()
+    terminal_width, _ = shutil.get_terminal_size()
     for (platform, boards) in sorted(grpboards.items()):
         click.echo("")
         click.echo("Platform: ", nl=False)
@@ -59,8 +59,8 @@ def print_boards(boards):
                     click.style(b["id"], fg="cyan"),
                     b["mcu"],
                     "%dMHz" % (b["fcpu"] / 1000000),
-                    fs.format_filesize(b["rom"]),
-                    fs.format_filesize(b["ram"]),
+                    fs.humanize_file_size(b["rom"]),
+                    fs.humanize_file_size(b["ram"]),
                     b["name"],
                 )
                 for b in boards
@@ -71,7 +71,7 @@ def print_boards(boards):
 
 
 def _get_boards(installed=False):
-    pm = PlatformManager()
+    pm = PlatformPackageManager()
     return pm.get_installed_boards() if installed else pm.get_all_boards()
 
 
@@ -83,4 +83,4 @@ def _print_boards_json(query, installed=False):
             if query.lower() not in search_data.lower():
                 continue
         result.append(board)
-    click.echo(dump_json_to_unicode(result))
+    click.echo(json.dumps(result))

platformio/commands/ci.py

@@ -12,21 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from glob import glob
-from os import getenv, makedirs, remove
-from os.path import basename, isdir, isfile, join, realpath
-from shutil import copyfile, copytree
-from tempfile import mkdtemp
+import glob
+import os
+import shutil
+import tempfile
 
 import click
 
 from platformio import app, fs
-from platformio.commands.project import project_init as cmd_project_init
-from platformio.commands.project import validate_boards
-from platformio.commands.run.command import cli as cmd_run
-from platformio.compat import glob_escape
 from platformio.exception import CIBuildEnvsEmpty
+from platformio.project.commands.init import project_init_cmd, validate_boards
 from platformio.project.config import ProjectConfig
+from platformio.run.cli import cli as cmd_run
 
 
 def validate_path(ctx, param, value):  # pylint: disable=unused-argument
@@ -35,8 +32,8 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
     for i, p in enumerate(value):
         if p.startswith("~"):
             value[i] = fs.expanduser(p)
-        value[i] = realpath(value[i])
-        if not glob(value[i]):
+        value[i] = os.path.abspath(value[i])
+        if not glob.glob(value[i], recursive=True):
             invalid_path = p
             break
     try:
@@ -53,7 +50,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
 @click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
 @click.option(
     "--build-dir",
-    default=mkdtemp,
+    default=tempfile.mkdtemp,
     type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
 )
 @click.option("--keep-build-dir", is_flag=True)
@@ -80,28 +77,28 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
     verbose,
 ):
 
-    if not src and getenv("PLATFORMIO_CI_SRC"):
-        src = validate_path(ctx, None, getenv("PLATFORMIO_CI_SRC").split(":"))
+    if not src and os.getenv("PLATFORMIO_CI_SRC"):
+        src = validate_path(ctx, None, os.getenv("PLATFORMIO_CI_SRC").split(":"))
     if not src:
         raise click.BadParameter("Missing argument 'src'")
 
     try:
         app.set_session_var("force_option", True)
 
-        if not keep_build_dir and isdir(build_dir):
+        if not keep_build_dir and os.path.isdir(build_dir):
             fs.rmtree(build_dir)
-        if not isdir(build_dir):
-            makedirs(build_dir)
+        if not os.path.isdir(build_dir):
+            os.makedirs(build_dir)
 
         for dir_name, patterns in dict(lib=lib, src=src).items():
             if not patterns:
                 continue
             contents = []
             for p in patterns:
-                contents += glob(p)
-            _copy_contents(join(build_dir, dir_name), contents)
+                contents += glob.glob(p, recursive=True)
+            _copy_contents(os.path.join(build_dir, dir_name), contents)
 
-        if project_conf and isfile(project_conf):
+        if project_conf and os.path.isfile(project_conf):
             _copy_project_conf(build_dir, project_conf)
         elif not board:
             raise CIBuildEnvsEmpty()
@@ -111,7 +108,7 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
 
         # initialise project
         ctx.invoke(
-            cmd_project_init,
+            project_init_cmd,
             project_dir=build_dir,
             board=board,
             project_option=project_option,
@@ -124,52 +121,55 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
             fs.rmtree(build_dir)
 
 
-def _copy_contents(dst_dir, contents):
+def _copy_contents(dst_dir, contents):  # pylint: disable=too-many-branches
     items = {"dirs": set(), "files": set()}
 
     for path in contents:
-        if isdir(path):
+        if os.path.isdir(path):
             items["dirs"].add(path)
-        elif isfile(path):
+        elif os.path.isfile(path):
             items["files"].add(path)
 
-    dst_dir_name = basename(dst_dir)
+    dst_dir_name = os.path.basename(dst_dir)
 
     if dst_dir_name == "src" and len(items["dirs"]) == 1:
-        copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
+        if not os.path.isdir(dst_dir):
+            shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
     else:
-        if not isdir(dst_dir):
-            makedirs(dst_dir)
+        if not os.path.isdir(dst_dir):
+            os.makedirs(dst_dir)
         for d in items["dirs"]:
-            copytree(d, join(dst_dir, basename(d)), symlinks=True)
+            src_dst_dir = os.path.join(dst_dir, os.path.basename(d))
+            if not os.path.isdir(src_dst_dir):
+                shutil.copytree(d, src_dst_dir, symlinks=True)
 
     if not items["files"]:
         return
 
     if dst_dir_name == "lib":
-        dst_dir = join(dst_dir, mkdtemp(dir=dst_dir))
+        dst_dir = os.path.join(dst_dir, tempfile.mkdtemp(dir=dst_dir))
 
     for f in items["files"]:
-        dst_file = join(dst_dir, basename(f))
+        dst_file = os.path.join(dst_dir, os.path.basename(f))
         if f == dst_file:
             continue
-        copyfile(f, dst_file)
+        shutil.copyfile(f, dst_file)
 
 
 def _exclude_contents(dst_dir, patterns):
     contents = []
     for p in patterns:
-        contents += glob(join(glob_escape(dst_dir), p))
+        contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True)
     for path in contents:
-        path = realpath(path)
-        if isdir(path):
+        path = os.path.abspath(path)
+        if os.path.isdir(path):
             fs.rmtree(path)
-        elif isfile(path):
-            remove(path)
+        elif os.path.isfile(path):
+            os.remove(path)
 
 
 def _copy_project_conf(build_dir, project_conf):
     config = ProjectConfig(project_conf, parse_extra=False)
     if config.has_section("platformio"):
         config.remove_section("platformio")
-    config.save(join(build_dir, "platformio.ini"))
+    config.save(os.path.join(build_dir, "platformio.ini"))

platformio/commands/debug/command.py

@@ -1,159 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=too-many-arguments, too-many-statements
-# pylint: disable=too-many-locals, too-many-branches
-
-import os
-import signal
-from os.path import isfile
-
-import click
-
-from platformio import app, exception, fs, proc, util
-from platformio.commands.debug import helpers
-from platformio.commands.debug.exception import DebugInvalidOptionsError
-from platformio.managers.core import inject_contrib_pysite
-from platformio.project.config import ProjectConfig
-from platformio.project.exception import ProjectEnvsNotAvailableError
-from platformio.project.helpers import is_platformio_project, load_project_ide_data
-
-
-@click.command(
-    "debug",
-    context_settings=dict(ignore_unknown_options=True),
-    short_help="PIO Unified Debugger",
-)
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
-)
-@click.option(
-    "-c",
-    "--project-conf",
-    type=click.Path(
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
-)
-@click.option("--environment", "-e", metavar="<environment>")
-@click.option("--verbose", "-v", is_flag=True)
-@click.option("--interface", type=click.Choice(["gdb"]))
-@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
-@click.pass_context
-def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unprocessed):
-    app.set_session_var("custom_project_conf", project_conf)
-
-    # use env variables from Eclipse or CLion
-    for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"):
-        if is_platformio_project(project_dir):
-            break
-        if os.getenv(sysenv):
-            project_dir = os.getenv(sysenv)
-
-    with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(project_conf)
-        config.validate(envs=[environment] if environment else None)
-
-        env_name = environment or helpers.get_default_debug_env(config)
-        env_options = config.items(env=env_name, as_dict=True)
-        if not set(env_options.keys()) >= set(["platform", "board"]):
-            raise ProjectEnvsNotAvailableError()
-        debug_options = helpers.validate_debug_options(ctx, env_options)
-        assert debug_options
-
-    if not interface:
-        return helpers.predebug_project(ctx, project_dir, env_name, False, verbose)
-
-    configuration = load_project_ide_data(project_dir, env_name)
-    if not configuration:
-        raise DebugInvalidOptionsError("Could not load debug configuration")
-
-    if "--version" in __unprocessed:
-        result = proc.exec_command([configuration["gdb_path"], "--version"])
-        if result["returncode"] == 0:
-            return click.echo(result["out"])
-        raise exception.PlatformioException("\n".join([result["out"], result["err"]]))
-
-    try:
-        fs.ensure_udev_rules()
-    except exception.InvalidUdevRules as e:
-        click.echo(
-            helpers.escape_gdbmi_stream("~", str(e) + "\n")
-            if helpers.is_gdbmi_mode()
-            else str(e) + "\n",
-            nl=False,
-        )
-
-    debug_options["load_cmds"] = helpers.configure_esp32_load_cmds(
-        debug_options, configuration
-    )
-
-    rebuild_prog = False
-    preload = debug_options["load_cmds"] == ["preload"]
-    load_mode = debug_options["load_mode"]
-    if load_mode == "always":
-        rebuild_prog = preload or not helpers.has_debug_symbols(
-            configuration["prog_path"]
-        )
-    elif load_mode == "modified":
-        rebuild_prog = helpers.is_prog_obsolete(
-            configuration["prog_path"]
-        ) or not helpers.has_debug_symbols(configuration["prog_path"])
-    else:
-        rebuild_prog = not isfile(configuration["prog_path"])
-
-    if preload or (not rebuild_prog and load_mode != "always"):
-        # don't load firmware through debug server
-        debug_options["load_cmds"] = []
-
-    if rebuild_prog:
-        if helpers.is_gdbmi_mode():
-            click.echo(
-                helpers.escape_gdbmi_stream(
-                    "~", "Preparing firmware for debugging...\n"
-                ),
-                nl=False,
-            )
-            stream = helpers.GDBMIConsoleStream()
-            with util.capture_std_streams(stream):
-                helpers.predebug_project(ctx, project_dir, env_name, preload, verbose)
-            stream.close()
-        else:
-            click.echo("Preparing firmware for debugging...")
-            helpers.predebug_project(ctx, project_dir, env_name, preload, verbose)
-
-        # save SHA sum of newly created prog
-        if load_mode == "modified":
-            helpers.is_prog_obsolete(configuration["prog_path"])
-
-    if not isfile(configuration["prog_path"]):
-        raise DebugInvalidOptionsError("Program/firmware is missed")
-
-    # run debugging client
-    inject_contrib_pysite()
-
-    # pylint: disable=import-outside-toplevel
-    from platformio.commands.debug.process.client import GDBClient, reactor
-
-    client = GDBClient(project_dir, __unprocessed, debug_options, env_options)
-    client.spawn(configuration["gdb_path"], configuration["prog_path"])
-
-    signal.signal(signal.SIGINT, lambda *args, **kwargs: None)
-    reactor.run()
-
-    return True

platformio/commands/debug/helpers.py

@@ -1,311 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-import sys
-import time
-from fnmatch import fnmatch
-from hashlib import sha1
-from io import BytesIO
-from os.path import isfile
-
-from platformio import exception, fs, util
-from platformio.commands import PlatformioCLI
-from platformio.commands.debug.exception import DebugInvalidOptionsError
-from platformio.commands.platform import platform_install as cmd_platform_install
-from platformio.commands.run.command import cli as cmd_run
-from platformio.compat import is_bytes
-from platformio.managers.platform import PlatformFactory
-from platformio.project.config import ProjectConfig
-from platformio.project.options import ProjectOptions
-
-
-class GDBMIConsoleStream(BytesIO):  # pylint: disable=too-few-public-methods
-
-    STDOUT = sys.stdout
-
-    def write(self, text):
-        self.STDOUT.write(escape_gdbmi_stream("~", text))
-        self.STDOUT.flush()
-
-
-def is_gdbmi_mode():
-    return "--interpreter" in " ".join(PlatformioCLI.leftover_args)
-
-
-def escape_gdbmi_stream(prefix, stream):
-    bytes_stream = False
-    if is_bytes(stream):
-        bytes_stream = True
-        stream = stream.decode()
-
-    if not stream:
-        return b"" if bytes_stream else ""
-
-    ends_nl = stream.endswith("\n")
-    stream = re.sub(r"\\+", "\\\\\\\\", stream)
-    stream = stream.replace('"', '\\"')
-    stream = stream.replace("\n", "\\n")
-    stream = '%s"%s"' % (prefix, stream)
-    if ends_nl:
-        stream += "\n"
-
-    return stream.encode() if bytes_stream else stream
-
-
-def get_default_debug_env(config):
-    default_envs = config.default_envs()
-    all_envs = config.envs()
-    for env in default_envs:
-        if config.get("env:" + env, "build_type") == "debug":
-            return env
-    for env in all_envs:
-        if config.get("env:" + env, "build_type") == "debug":
-            return env
-    return default_envs[0] if default_envs else all_envs[0]
-
-
-def predebug_project(ctx, project_dir, env_name, preload, verbose):
-    ctx.invoke(
-        cmd_run,
-        project_dir=project_dir,
-        environment=[env_name],
-        target=["debug"] + (["upload"] if preload else []),
-        verbose=verbose,
-    )
-    if preload:
-        time.sleep(5)
-
-
-def validate_debug_options(cmd_ctx, env_options):
-    def _cleanup_cmds(items):
-        items = ProjectConfig.parse_multi_values(items)
-        return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items]
-
-    try:
-        platform = PlatformFactory.newPlatform(env_options["platform"])
-    except exception.UnknownPlatform:
-        cmd_ctx.invoke(
-            cmd_platform_install,
-            platforms=[env_options["platform"]],
-            skip_default_package=True,
-        )
-        platform = PlatformFactory.newPlatform(env_options["platform"])
-
-    board_config = platform.board_config(env_options["board"])
-    tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool"))
-    tool_settings = board_config.get("debug", {}).get("tools", {}).get(tool_name, {})
-    server_options = None
-
-    # specific server per a system
-    if isinstance(tool_settings.get("server", {}), list):
-        for item in tool_settings["server"][:]:
-            tool_settings["server"] = item
-            if util.get_systype() in item.get("system", []):
-                break
-
-    # user overwrites debug server
-    if env_options.get("debug_server"):
-        server_options = {
-            "cwd": None,
-            "executable": None,
-            "arguments": env_options.get("debug_server"),
-        }
-        server_options["executable"] = server_options["arguments"][0]
-        server_options["arguments"] = server_options["arguments"][1:]
-    elif "server" in tool_settings:
-        server_options = tool_settings["server"]
-        server_package = server_options.get("package")
-        server_package_dir = (
-            platform.get_package_dir(server_package) if server_package else None
-        )
-        if server_package and not server_package_dir:
-            platform.install_packages(
-                with_packages=[server_package], skip_default_package=True, silent=True
-            )
-            server_package_dir = platform.get_package_dir(server_package)
-        server_options.update(
-            dict(
-                cwd=server_package_dir if server_package else None,
-                executable=server_options.get("executable"),
-                arguments=[
-                    a.replace("$PACKAGE_DIR", server_package_dir)
-                    if server_package_dir
-                    else a
-                    for a in server_options.get("arguments", [])
-                ],
-            )
-        )
-
-    extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds"))
-    extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds")))
-    result = dict(
-        tool=tool_name,
-        upload_protocol=env_options.get(
-            "upload_protocol", board_config.get("upload", {}).get("protocol")
-        ),
-        load_cmds=_cleanup_cmds(
-            env_options.get(
-                "debug_load_cmds",
-                tool_settings.get(
-                    "load_cmds",
-                    tool_settings.get(
-                        "load_cmd", ProjectOptions["env.debug_load_cmds"].default
-                    ),
-                ),
-            )
-        ),
-        load_mode=env_options.get(
-            "debug_load_mode",
-            tool_settings.get(
-                "load_mode", ProjectOptions["env.debug_load_mode"].default
-            ),
-        ),
-        init_break=env_options.get(
-            "debug_init_break",
-            tool_settings.get(
-                "init_break", ProjectOptions["env.debug_init_break"].default
-            ),
-        ),
-        init_cmds=_cleanup_cmds(
-            env_options.get("debug_init_cmds", tool_settings.get("init_cmds"))
-        ),
-        extra_cmds=extra_cmds,
-        require_debug_port=tool_settings.get("require_debug_port", False),
-        port=reveal_debug_port(
-            env_options.get("debug_port", tool_settings.get("port")),
-            tool_name,
-            tool_settings,
-        ),
-        server=server_options,
-    )
-    return result
-
-
-def configure_esp32_load_cmds(debug_options, configuration):
-    ignore_conds = [
-        debug_options["load_cmds"] != ["load"],
-        "xtensa-esp32" not in configuration.get("cc_path", ""),
-        not configuration.get("flash_extra_images"),
-        not all(
-            [isfile(item["path"]) for item in configuration.get("flash_extra_images")]
-        ),
-    ]
-    if any(ignore_conds):
-        return debug_options["load_cmds"]
-
-    mon_cmds = [
-        'monitor program_esp32 "{{{path}}}" {offset} verify'.format(
-            path=fs.to_unix_path(item["path"]), offset=item["offset"]
-        )
-        for item in configuration.get("flash_extra_images")
-    ]
-    mon_cmds.append(
-        'monitor program_esp32 "{%s.bin}" 0x10000 verify'
-        % fs.to_unix_path(configuration["prog_path"][:-4])
-    )
-    return mon_cmds
-
-
-def has_debug_symbols(prog_path):
-    if not isfile(prog_path):
-        return False
-    matched = {
-        b".debug_info": False,
-        b".debug_abbrev": False,
-        b" -Og": False,
-        b" -g": False,
-        b"__PLATFORMIO_BUILD_DEBUG__": False,
-    }
-    with open(prog_path, "rb") as fp:
-        last_data = b""
-        while True:
-            data = fp.read(1024)
-            if not data:
-                break
-            for pattern, found in matched.items():
-                if found:
-                    continue
-                if pattern in last_data + data:
-                    matched[pattern] = True
-            last_data = data
-    return all(matched.values())
-
-
-def is_prog_obsolete(prog_path):
-    prog_hash_path = prog_path + ".sha1"
-    if not isfile(prog_path):
-        return True
-    shasum = sha1()
-    with open(prog_path, "rb") as fp:
-        while True:
-            data = fp.read(1024)
-            if not data:
-                break
-            shasum.update(data)
-    new_digest = shasum.hexdigest()
-    old_digest = None
-    if isfile(prog_hash_path):
-        with open(prog_hash_path) as fp:
-            old_digest = fp.read()
-    if new_digest == old_digest:
-        return False
-    with open(prog_hash_path, "w") as fp:
-        fp.write(new_digest)
-    return True
-
-
-def reveal_debug_port(env_debug_port, tool_name, tool_settings):
-    def _get_pattern():
-        if not env_debug_port:
-            return None
-        if set(["*", "?", "[", "]"]) & set(env_debug_port):
-            return env_debug_port
-        return None
-
-    def _is_match_pattern(port):
-        pattern = _get_pattern()
-        if not pattern:
-            return True
-        return fnmatch(port, pattern)
-
-    def _look_for_serial_port(hwids):
-        for item in util.get_serialports(filter_hwid=True):
-            if not _is_match_pattern(item["port"]):
-                continue
-            port = item["port"]
-            if tool_name.startswith("blackmagic"):
-                if (
-                    "windows" in util.get_systype()
-                    and port.startswith("COM")
-                    and len(port) > 4
-                ):
-                    port = "\\\\.\\%s" % port
-                if "GDB" in item["description"]:
-                    return port
-            for hwid in hwids:
-                hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                if hwid_str in item["hwid"]:
-                    return port
-        return None
-
-    if env_debug_port and not _get_pattern():
-        return env_debug_port
-    if not tool_settings.get("require_debug_port"):
-        return None
-
-    debug_port = _look_for_serial_port(tool_settings.get("hwids", []))
-    if not debug_port:
-        raise DebugInvalidOptionsError("Please specify `debug_port` for environment")
-    return debug_port

platformio/commands/debug/initcfgs.py

@@ -1,161 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-GDB_DEFAULT_INIT_CONFIG = """
-define pio_reset_halt_target
-    monitor reset halt
-end
-
-define pio_reset_run_target
-    monitor reset
-end
-
-target extended-remote $DEBUG_PORT
-monitor init
-$LOAD_CMDS
-pio_reset_halt_target
-$INIT_BREAK
-"""
-
-GDB_STUTIL_INIT_CONFIG = """
-define pio_reset_halt_target
-    monitor reset
-    monitor halt
-end
-
-define pio_reset_run_target
-    monitor reset
-end
-
-target extended-remote $DEBUG_PORT
-$LOAD_CMDS
-pio_reset_halt_target
-$INIT_BREAK
-"""
-
-GDB_JLINK_INIT_CONFIG = """
-define pio_reset_halt_target
-    monitor reset
-    monitor halt
-end
-
-define pio_reset_run_target
-    monitor clrbp
-    monitor reset
-    monitor go
-end
-
-target extended-remote $DEBUG_PORT
-monitor clrbp
-monitor speed auto
-pio_reset_halt_target
-$LOAD_CMDS
-$INIT_BREAK
-"""
-
-GDB_BLACKMAGIC_INIT_CONFIG = """
-define pio_reset_halt_target
-    set language c
-    set *0xE000ED0C = 0x05FA0004
-    set $busy = (*0xE000ED0C & 0x4)
-    while ($busy)
-        set $busy = (*0xE000ED0C & 0x4)
-    end
-    set language auto
-end
-
-define pio_reset_run_target
-    pio_reset_halt_target
-end
-
-target extended-remote $DEBUG_PORT
-monitor swdp_scan
-attach 1
-set mem inaccessible-by-default off
-$LOAD_CMDS
-$INIT_BREAK
-
-set language c
-set *0xE000ED0C = 0x05FA0004
-set $busy = (*0xE000ED0C & 0x4)
-while ($busy)
-    set $busy = (*0xE000ED0C & 0x4)
-end
-set language auto
-"""
-
-GDB_MSPDEBUG_INIT_CONFIG = """
-define pio_reset_halt_target
-end
-
-define pio_reset_run_target
-end
-
-target extended-remote $DEBUG_PORT
-monitor erase
-$LOAD_CMDS
-pio_reset_halt_target
-$INIT_BREAK
-"""
-
-GDB_QEMU_INIT_CONFIG = """
-define pio_reset_halt_target
-    monitor system_reset
-end
-
-define pio_reset_run_target
-    monitor system_reset
-end
-
-target extended-remote $DEBUG_PORT
-$LOAD_CMDS
-pio_reset_halt_target
-$INIT_BREAK
-"""
-
-GDB_RENODE_INIT_CONFIG = """
-define pio_reset_halt_target
-    monitor machine Reset
-    $LOAD_CMDS
-    monitor start
-end
-
-define pio_reset_run_target
-    pio_reset_halt_target
-end
-
-target extended-remote $DEBUG_PORT
-$LOAD_CMDS
-$INIT_BREAK
-monitor start
-"""
-
-
-TOOL_TO_CONFIG = {
-    "jlink": GDB_JLINK_INIT_CONFIG,
-    "mspdebug": GDB_MSPDEBUG_INIT_CONFIG,
-    "qemu": GDB_QEMU_INIT_CONFIG,
-    "blackmagic": GDB_BLACKMAGIC_INIT_CONFIG,
-    "renode": GDB_RENODE_INIT_CONFIG,
-}
-
-
-def get_gdb_init_config(debug_options):
-    tool = debug_options.get("tool")
-    if tool and tool in TOOL_TO_CONFIG:
-        return TOOL_TO_CONFIG[tool]
-    server_exe = (debug_options.get("server") or {}).get("executable", "").lower()
-    if "st-util" in server_exe:
-        return GDB_STUTIL_INIT_CONFIG
-    return GDB_DEFAULT_INIT_CONFIG

platformio/commands/debug/process/base.py

@@ -1,93 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import signal
-import time
-
-import click
-from twisted.internet import protocol  # pylint: disable=import-error
-
-from platformio import fs
-from platformio.compat import string_types
-from platformio.proc import get_pythonexe_path
-from platformio.project.helpers import get_project_core_dir
-
-
-class BaseProcess(protocol.ProcessProtocol, object):
-
-    STDOUT_CHUNK_SIZE = 2048
-    LOG_FILE = None
-
-    COMMON_PATTERNS = {
-        "PLATFORMIO_HOME_DIR": get_project_core_dir(),
-        "PLATFORMIO_CORE_DIR": get_project_core_dir(),
-        "PYTHONEXE": get_pythonexe_path(),
-    }
-
-    def __init__(self):
-        self._last_activity = 0
-
-    def apply_patterns(self, source, patterns=None):
-        _patterns = self.COMMON_PATTERNS.copy()
-        _patterns.update(patterns or {})
-
-        for key, value in _patterns.items():
-            if key.endswith(("_DIR", "_PATH")):
-                _patterns[key] = fs.to_unix_path(value)
-
-        def _replace(text):
-            for key, value in _patterns.items():
-                pattern = "$%s" % key
-                text = text.replace(pattern, value or "")
-            return text
-
-        if isinstance(source, string_types):
-            source = _replace(source)
-        elif isinstance(source, (list, dict)):
-            items = enumerate(source) if isinstance(source, list) else source.items()
-            for key, value in items:
-                if isinstance(value, string_types):
-                    source[key] = _replace(value)
-                elif isinstance(value, (list, dict)):
-                    source[key] = self.apply_patterns(value, patterns)
-
-        return source
-
-    def onStdInData(self, data):
-        self._last_activity = time.time()
-        if self.LOG_FILE:
-            with open(self.LOG_FILE, "ab") as fp:
-                fp.write(data)
-
-    def outReceived(self, data):
-        self._last_activity = time.time()
-        if self.LOG_FILE:
-            with open(self.LOG_FILE, "ab") as fp:
-                fp.write(data)
-        while data:
-            chunk = data[: self.STDOUT_CHUNK_SIZE]
-            click.echo(chunk, nl=False)
-            data = data[self.STDOUT_CHUNK_SIZE :]
-
-    def errReceived(self, data):
-        self._last_activity = time.time()
-        if self.LOG_FILE:
-            with open(self.LOG_FILE, "ab") as fp:
-                fp.write(data)
-        click.echo(data, nl=False, err=True)
-
-    def processEnded(self, _):
-        self._last_activity = time.time()
-        # Allow terminating via SIGINT/CTRL+C
-        signal.signal(signal.SIGINT, signal.default_int_handler)

platformio/commands/debug/process/client.py

@@ -1,279 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import re
-import signal
-import time
-from hashlib import sha1
-from os.path import basename, dirname, isdir, join, realpath, splitext
-from tempfile import mkdtemp
-
-from twisted.internet import defer  # pylint: disable=import-error
-from twisted.internet import protocol  # pylint: disable=import-error
-from twisted.internet import reactor  # pylint: disable=import-error
-from twisted.internet import stdio  # pylint: disable=import-error
-from twisted.internet import task  # pylint: disable=import-error
-
-from platformio import app, fs, proc, telemetry, util
-from platformio.commands.debug import helpers
-from platformio.commands.debug.exception import DebugInvalidOptionsError
-from platformio.commands.debug.initcfgs import get_gdb_init_config
-from platformio.commands.debug.process.base import BaseProcess
-from platformio.commands.debug.process.server import DebugServer
-from platformio.compat import hashlib_encode_data, is_bytes
-from platformio.project.helpers import get_project_cache_dir
-
-
-class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
-
-    PIO_SRC_NAME = ".pioinit"
-    INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed"
-
-    def __init__(self, project_dir, args, debug_options, env_options):
-        super(GDBClient, self).__init__()
-        self.project_dir = project_dir
-        self.args = list(args)
-        self.debug_options = debug_options
-        self.env_options = env_options
-
-        self._debug_server = DebugServer(debug_options, env_options)
-        self._session_id = None
-
-        if not isdir(get_project_cache_dir()):
-            os.makedirs(get_project_cache_dir())
-        self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), prefix=".piodebug-")
-
-        self._target_is_run = False
-        self._auto_continue_timer = None
-        self._errors_buffer = b""
-
-    @defer.inlineCallbacks
-    def spawn(self, gdb_path, prog_path):
-        session_hash = gdb_path + prog_path
-        self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest()
-        self._kill_previous_session()
-
-        patterns = {
-            "PROJECT_DIR": self.project_dir,
-            "PROG_PATH": prog_path,
-            "PROG_DIR": dirname(prog_path),
-            "PROG_NAME": basename(splitext(prog_path)[0]),
-            "DEBUG_PORT": self.debug_options["port"],
-            "UPLOAD_PROTOCOL": self.debug_options["upload_protocol"],
-            "INIT_BREAK": self.debug_options["init_break"] or "",
-            "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []),
-        }
-
-        yield self._debug_server.spawn(patterns)
-        if not patterns["DEBUG_PORT"]:
-            patterns["DEBUG_PORT"] = self._debug_server.get_debug_port()
-
-        self.generate_pioinit(self._gdbsrc_dir, patterns)
-
-        # start GDB client
-        args = [
-            "piogdb",
-            "-q",
-            "--directory",
-            self._gdbsrc_dir,
-            "--directory",
-            self.project_dir,
-            "-l",
-            "10",
-        ]
-        args.extend(self.args)
-        if not gdb_path:
-            raise DebugInvalidOptionsError("GDB client is not configured")
-        gdb_data_dir = self._get_data_dir(gdb_path)
-        if gdb_data_dir:
-            args.extend(["--data-directory", gdb_data_dir])
-        args.append(patterns["PROG_PATH"])
-
-        transport = reactor.spawnProcess(
-            self, gdb_path, args, path=self.project_dir, env=os.environ
-        )
-        defer.returnValue(transport)
-
-    @staticmethod
-    def _get_data_dir(gdb_path):
-        if "msp430" in gdb_path:
-            return None
-        gdb_data_dir = realpath(join(dirname(gdb_path), "..", "share", "gdb"))
-        return gdb_data_dir if isdir(gdb_data_dir) else None
-
-    def generate_pioinit(self, dst_dir, patterns):
-        # default GDB init commands depending on debug tool
-        commands = get_gdb_init_config(self.debug_options).split("\n")
-
-        if self.debug_options["init_cmds"]:
-            commands = self.debug_options["init_cmds"]
-        commands.extend(self.debug_options["extra_cmds"])
-
-        if not any("define pio_reset_run_target" in cmd for cmd in commands):
-            commands = [
-                "define pio_reset_run_target",
-                "   echo Warning! Undefined pio_reset_run_target command\\n",
-                "   monitor reset",
-                "end",
-            ] + commands
-        if not any("define pio_reset_halt_target" in cmd for cmd in commands):
-            commands = [
-                "define pio_reset_halt_target",
-                "   echo Warning! Undefined pio_reset_halt_target command\\n",
-                "   monitor reset halt",
-                "end",
-            ] + commands
-        if not any("define pio_restart_target" in cmd for cmd in commands):
-            commands += [
-                "define pio_restart_target",
-                "   pio_reset_halt_target",
-                "   $INIT_BREAK",
-                "   %s" % ("continue" if patterns["INIT_BREAK"] else "next"),
-                "end",
-            ]
-
-        banner = [
-            "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n",
-            "echo PlatformIO: debug_tool = %s\\n" % self.debug_options["tool"],
-            "echo PlatformIO: Initializing remote target...\\n",
-        ]
-        footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER]
-        commands = banner + commands + footer
-
-        with open(join(dst_dir, self.PIO_SRC_NAME), "w") as fp:
-            fp.write("\n".join(self.apply_patterns(commands, patterns)))
-
-    def connectionMade(self):
-        self._lock_session(self.transport.pid)
-
-        p = protocol.Protocol()
-        p.dataReceived = self.onStdInData
-        stdio.StandardIO(p)
-
-    def onStdInData(self, data):
-        super(GDBClient, self).onStdInData(data)
-        if b"-exec-run" in data:
-            if self._target_is_run:
-                token, _ = data.split(b"-", 1)
-                self.outReceived(token + b"^running\n")
-                return
-            data = data.replace(b"-exec-run", b"-exec-continue")
-
-        if b"-exec-continue" in data:
-            self._target_is_run = True
-        if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"):
-            # Allow terminating via SIGINT/CTRL+C
-            signal.signal(signal.SIGINT, signal.default_int_handler)
-            self.transport.write(b"pio_reset_run_target\n")
-        self.transport.write(data)
-
-    def processEnded(self, reason):  # pylint: disable=unused-argument
-        self._unlock_session()
-        if self._gdbsrc_dir and isdir(self._gdbsrc_dir):
-            fs.rmtree(self._gdbsrc_dir)
-        if self._debug_server:
-            self._debug_server.terminate()
-
-        reactor.stop()
-
-    def outReceived(self, data):
-        super(GDBClient, self).outReceived(data)
-        self._handle_error(data)
-        # go to init break automatically
-        if self.INIT_COMPLETED_BANNER.encode() in data:
-            telemetry.send_event(
-                "Debug", "Started", telemetry.dump_run_environment(self.env_options)
-            )
-            self._auto_continue_timer = task.LoopingCall(self._auto_exec_continue)
-            self._auto_continue_timer.start(0.1)
-
-    def errReceived(self, data):
-        super(GDBClient, self).errReceived(data)
-        self._handle_error(data)
-
-    def console_log(self, msg):
-        if helpers.is_gdbmi_mode():
-            msg = helpers.escape_gdbmi_stream("~", msg)
-        self.outReceived(msg if is_bytes(msg) else msg.encode())
-
-    def _auto_exec_continue(self):
-        auto_exec_delay = 0.5  # in seconds
-        if self._last_activity > (time.time() - auto_exec_delay):
-            return
-        if self._auto_continue_timer:
-            self._auto_continue_timer.stop()
-        self._auto_continue_timer = None
-
-        if not self.debug_options["init_break"] or self._target_is_run:
-            return
-        self.console_log(
-            "PlatformIO: Resume the execution to `debug_init_break = %s`\n"
-            % self.debug_options["init_break"]
-        )
-        self.console_log(
-            "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n"
-        )
-        self.transport.write(
-            b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n"
-        )
-        self._target_is_run = True
-
-    def _handle_error(self, data):
-        self._errors_buffer = (self._errors_buffer + data)[-8192:]  # keep last 8 KBytes
-        if not (
-            self.PIO_SRC_NAME.encode() in self._errors_buffer
-            and b"Error in sourced" in self._errors_buffer
-        ):
-            return
-
-        last_erros = self._errors_buffer.decode()
-        last_erros = " ".join(reversed(last_erros.split("\n")))
-        last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M)
-
-        err = "%s -> %s" % (
-            telemetry.dump_run_environment(self.env_options),
-            last_erros,
-        )
-        telemetry.send_exception("DebugInitError: %s" % err)
-        self.transport.loseConnection()
-
-    def _kill_previous_session(self):
-        assert self._session_id
-        pid = None
-        with app.ContentCache() as cc:
-            pid = cc.get(self._session_id)
-            cc.delete(self._session_id)
-        if not pid:
-            return
-        if "windows" in util.get_systype():
-            kill = ["Taskkill", "/PID", pid, "/F"]
-        else:
-            kill = ["kill", pid]
-        try:
-            proc.exec_command(kill)
-        except:  # pylint: disable=bare-except
-            pass
-
-    def _lock_session(self, pid):
-        if not self._session_id:
-            return
-        with app.ContentCache() as cc:
-            cc.set(self._session_id, str(pid), "1h")
-
-    def _unlock_session(self):
-        if not self._session_id:
-            return
-        with app.ContentCache() as cc:
-            cc.delete(self._session_id)

platformio/commands/debug/process/server.py

@@ -1,166 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import time
-from os.path import isdir, isfile, join
-
-from twisted.internet import defer  # pylint: disable=import-error
-from twisted.internet import reactor  # pylint: disable=import-error
-
-from platformio import fs, util
-from platformio.commands.debug.exception import DebugInvalidOptionsError
-from platformio.commands.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode
-from platformio.commands.debug.process.base import BaseProcess
-from platformio.proc import where_is_program
-
-
-class DebugServer(BaseProcess):
-    def __init__(self, debug_options, env_options):
-        super(DebugServer, self).__init__()
-        self.debug_options = debug_options
-        self.env_options = env_options
-
-        self._debug_port = ":3333"
-        self._transport = None
-        self._process_ended = False
-        self._ready = False
-
-    @defer.inlineCallbacks
-    def spawn(self, patterns):  # pylint: disable=too-many-branches
-        systype = util.get_systype()
-        server = self.debug_options.get("server")
-        if not server:
-            defer.returnValue(None)
-        server = self.apply_patterns(server, patterns)
-        server_executable = server["executable"]
-        if not server_executable:
-            defer.returnValue(None)
-        if server["cwd"]:
-            server_executable = join(server["cwd"], server_executable)
-        if (
-            "windows" in systype
-            and not server_executable.endswith(".exe")
-            and isfile(server_executable + ".exe")
-        ):
-            server_executable = server_executable + ".exe"
-
-        if not isfile(server_executable):
-            server_executable = where_is_program(server_executable)
-        if not isfile(server_executable):
-            raise DebugInvalidOptionsError(
-                "\nCould not launch Debug Server '%s'. Please check that it "
-                "is installed and is included in a system PATH\n\n"
-                "See documentation or contact contact@platformio.org:\n"
-                "https://docs.platformio.org/page/plus/debugging.html\n"
-                % server_executable
-            )
-
-        openocd_pipe_allowed = all(
-            [not self.debug_options["port"], "openocd" in server_executable]
-        )
-        if openocd_pipe_allowed:
-            args = []
-            if server["cwd"]:
-                args.extend(["-s", server["cwd"]])
-            args.extend(
-                ["-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"]
-            )
-            args.extend(server["arguments"])
-            str_args = " ".join(
-                [arg if arg.startswith("-") else '"%s"' % arg for arg in args]
-            )
-            self._debug_port = '| "%s" %s' % (server_executable, str_args)
-            self._debug_port = fs.to_unix_path(self._debug_port)
-            defer.returnValue(self._debug_port)
-
-        env = os.environ.copy()
-        # prepend server "lib" folder to LD path
-        if (
-            "windows" not in systype
-            and server["cwd"]
-            and isdir(join(server["cwd"], "lib"))
-        ):
-            ld_key = "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH"
-            env[ld_key] = join(server["cwd"], "lib")
-            if os.environ.get(ld_key):
-                env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key))
-        # prepend BIN to PATH
-        if server["cwd"] and isdir(join(server["cwd"], "bin")):
-            env["PATH"] = "%s%s%s" % (
-                join(server["cwd"], "bin"),
-                os.pathsep,
-                os.environ.get("PATH", os.environ.get("Path", "")),
-            )
-
-        self._transport = reactor.spawnProcess(
-            self,
-            server_executable,
-            [server_executable] + server["arguments"],
-            path=server["cwd"],
-            env=env,
-        )
-        if "mspdebug" in server_executable.lower():
-            self._debug_port = ":2000"
-        elif "jlink" in server_executable.lower():
-            self._debug_port = ":2331"
-        elif "qemu" in server_executable.lower():
-            self._debug_port = ":1234"
-
-        yield self._wait_until_ready()
-
-        defer.returnValue(self._debug_port)
-
-    @defer.inlineCallbacks
-    def _wait_until_ready(self):
-        timeout = 10
-        elapsed = 0
-        delay = 0.5
-        auto_ready_delay = 0.5
-        while not self._ready and not self._process_ended and elapsed < timeout:
-            yield self.async_sleep(delay)
-            if not self.debug_options.get("server", {}).get("ready_pattern"):
-                self._ready = self._last_activity < (time.time() - auto_ready_delay)
-            elapsed += delay
-
-    @staticmethod
-    def async_sleep(secs):
-        d = defer.Deferred()
-        reactor.callLater(secs, d.callback, None)
-        return d
-
-    def get_debug_port(self):
-        return self._debug_port
-
-    def outReceived(self, data):
-        super(DebugServer, self).outReceived(
-            escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data
-        )
-        if self._ready:
-            return
-        ready_pattern = self.debug_options.get("server", {}).get("ready_pattern")
-        if ready_pattern:
-            self._ready = ready_pattern.encode() in data
-
-    def processEnded(self, reason):
-        self._process_ended = True
-        super(DebugServer, self).processEnded(reason)
-
-    def terminate(self):
-        if self._process_ended or not self._transport:
-            return
-        try:
-            self._transport.signalProcess("KILL")
-        except:  # pylint: disable=bare-except
-            pass

platformio/commands/device/__init__.py

@@ -12,4 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio.commands.device.filters.base import DeviceMonitorFilter
+# pylint: disable=unused-import
+from platformio.device.filters.base import (
+    DeviceMonitorFilterBase as DeviceMonitorFilter,
+)

platformio/commands/device/command.py

@@ -1,243 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import sys
-from fnmatch import fnmatch
-
-import click
-from serial.tools import miniterm
-
-from platformio import exception, fs, util
-from platformio.commands.device import helpers as device_helpers
-from platformio.compat import dump_json_to_unicode
-from platformio.managers.platform import PlatformFactory
-from platformio.project.exception import NotPlatformIOProjectError
-
-
-@click.group(short_help="Monitor device or list existing")
-def cli():
-    pass
-
-
-@cli.command("list", short_help="List devices")
-@click.option("--serial", is_flag=True, help="List serial ports, default")
-@click.option("--logical", is_flag=True, help="List logical devices")
-@click.option("--mdns", is_flag=True, help="List multicast DNS services")
-@click.option("--json-output", is_flag=True)
-def device_list(  # pylint: disable=too-many-branches
-    serial, logical, mdns, json_output
-):
-    if not logical and not mdns:
-        serial = True
-    data = {}
-    if serial:
-        data["serial"] = util.get_serial_ports()
-    if logical:
-        data["logical"] = util.get_logical_devices()
-    if mdns:
-        data["mdns"] = util.get_mdns_services()
-
-    single_key = list(data)[0] if len(list(data)) == 1 else None
-
-    if json_output:
-        return click.echo(
-            dump_json_to_unicode(data[single_key] if single_key else data)
-        )
-
-    titles = {
-        "serial": "Serial Ports",
-        "logical": "Logical Devices",
-        "mdns": "Multicast DNS Services",
-    }
-
-    for key, value in data.items():
-        if not single_key:
-            click.secho(titles[key], bold=True)
-            click.echo("=" * len(titles[key]))
-
-        if key == "serial":
-            for item in value:
-                click.secho(item["port"], fg="cyan")
-                click.echo("-" * len(item["port"]))
-                click.echo("Hardware ID: %s" % item["hwid"])
-                click.echo("Description: %s" % item["description"])
-                click.echo("")
-
-        if key == "logical":
-            for item in value:
-                click.secho(item["path"], fg="cyan")
-                click.echo("-" * len(item["path"]))
-                click.echo("Name: %s" % item["name"])
-                click.echo("")
-
-        if key == "mdns":
-            for item in value:
-                click.secho(item["name"], fg="cyan")
-                click.echo("-" * len(item["name"]))
-                click.echo("Type: %s" % item["type"])
-                click.echo("IP: %s" % item["ip"])
-                click.echo("Port: %s" % item["port"])
-                if item["properties"]:
-                    click.echo(
-                        "Properties: %s"
-                        % (
-                            "; ".join(
-                                [
-                                    "%s=%s" % (k, v)
-                                    for k, v in item["properties"].items()
-                                ]
-                            )
-                        )
-                    )
-                click.echo("")
-
-        if single_key:
-            click.echo("")
-
-    return True
-
-
-@cli.command("monitor", short_help="Monitor device (Serial)")
-@click.option("--port", "-p", help="Port, a number or a device name")
-@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
-@click.option(
-    "--parity",
-    default="N",
-    type=click.Choice(["N", "E", "O", "S", "M"]),
-    help="Set parity, default=N",
-)
-@click.option("--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off")
-@click.option(
-    "--xonxoff", is_flag=True, help="Enable software flow control, default=Off"
-)
-@click.option(
-    "--rts", default=None, type=click.IntRange(0, 1), help="Set initial RTS line state"
-)
-@click.option(
-    "--dtr", default=None, type=click.IntRange(0, 1), help="Set initial DTR line state"
-)
-@click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
-@click.option(
-    "--encoding",
-    default="UTF-8",
-    help="Set the encoding for the serial port (e.g. hexlify, "
-    "Latin1, UTF-8), default: UTF-8",
-)
-@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations")
-@click.option(
-    "--eol",
-    default="CRLF",
-    type=click.Choice(["CR", "LF", "CRLF"]),
-    help="End of line mode, default=CRLF",
-)
-@click.option("--raw", is_flag=True, help="Do not apply any encodings/transformations")
-@click.option(
-    "--exit-char",
-    type=int,
-    default=3,
-    help="ASCII code of special character that is used to exit "
-    "the application, default=3 (Ctrl+C)",
-)
-@click.option(
-    "--menu-char",
-    type=int,
-    default=20,
-    help="ASCII code of special character that is used to "
-    "control miniterm (menu), default=20 (DEC)",
-)
-@click.option(
-    "--quiet",
-    is_flag=True,
-    help="Diagnostics: suppress non-error messages, default=Off",
-)
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
-)
-@click.option(
-    "-e",
-    "--environment",
-    help="Load configuration from `platformio.ini` and specified environment",
-)
-def device_monitor(**kwargs):  # pylint: disable=too-many-branches
-    # load default monitor filters
-    filters_dir = os.path.join(fs.get_source_dir(), "commands", "device", "filters")
-    for name in os.listdir(filters_dir):
-        if not name.endswith(".py"):
-            continue
-        device_helpers.load_monitor_filter(os.path.join(filters_dir, name))
-
-    project_options = {}
-    try:
-        with fs.cd(kwargs["project_dir"]):
-            project_options = device_helpers.get_project_options(kwargs["environment"])
-        kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options)
-    except NotPlatformIOProjectError:
-        pass
-
-    platform = None
-    if "platform" in project_options:
-        with fs.cd(kwargs["project_dir"]):
-            platform = PlatformFactory.newPlatform(project_options["platform"])
-            device_helpers.register_platform_filters(
-                platform, kwargs["project_dir"], kwargs["environment"]
-            )
-
-    if not kwargs["port"]:
-        ports = util.get_serial_ports(filter_hwid=True)
-        if len(ports) == 1:
-            kwargs["port"] = ports[0]["port"]
-        elif "platform" in project_options and "board" in project_options:
-            board_hwids = device_helpers.get_board_hwids(
-                kwargs["project_dir"], platform, project_options["board"],
-            )
-            for item in ports:
-                for hwid in board_hwids:
-                    hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
-                    if hwid_str in item["hwid"]:
-                        kwargs["port"] = item["port"]
-                        break
-                if kwargs["port"]:
-                    break
-    elif kwargs["port"] and (set(["*", "?", "[", "]"]) & set(kwargs["port"])):
-        for item in util.get_serial_ports():
-            if fnmatch(item["port"], kwargs["port"]):
-                kwargs["port"] = item["port"]
-                break
-
-    # override system argv with patched options
-    sys.argv = ["monitor"] + device_helpers.options_to_argv(
-        kwargs,
-        project_options,
-        ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"),
-    )
-
-    if not kwargs["quiet"]:
-        click.echo(
-            "--- Available filters and text transformations: %s"
-            % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys()))
-        )
-        click.echo("--- More details at http://bit.ly/pio-monitor-filters")
-    try:
-        miniterm.main(
-            default_port=kwargs["port"],
-            default_baudrate=kwargs["baud"] or 9600,
-            default_rts=kwargs["rts"],
-            default_dtr=kwargs["dtr"],
-        )
-    except Exception as e:
-        raise exception.MinitermException(e)

platformio/commands/device/filters/base.py

@@ -1,42 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from serial.tools import miniterm
-
-from platformio.project.config import ProjectConfig
-
-
-class DeviceMonitorFilter(miniterm.Transform):
-    def __init__(self, project_dir=None, environment=None):
-        """ Called by PlatformIO to pass context """
-        miniterm.Transform.__init__(self)
-
-        self.project_dir = project_dir
-        self.environment = environment
-
-        self.config = ProjectConfig.get_instance()
-        if not self.environment:
-            default_envs = self.config.default_envs()
-            if default_envs:
-                self.environment = default_envs[0]
-            elif self.config.envs():
-                self.environment = self.config.envs()[0]
-
-    def __call__(self):
-        """ Called by the miniterm library when the filter is actually used """
-        return self
-
-    @property
-    def NAME(self):
-        raise NotImplementedError("Please declare NAME attribute for the filter class")

platformio/commands/device/helpers.py

@@ -1,106 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import inspect
-import os
-
-from serial.tools import miniterm
-
-from platformio import fs
-from platformio.commands.device import DeviceMonitorFilter
-from platformio.compat import get_object_members, load_python_module
-from platformio.project.config import ProjectConfig
-
-
-def apply_project_monitor_options(cli_options, project_options):
-    for k in ("port", "speed", "rts", "dtr"):
-        k2 = "monitor_%s" % k
-        if k == "speed":
-            k = "baud"
-        if cli_options[k] is None and k2 in project_options:
-            cli_options[k] = project_options[k2]
-            if k != "port":
-                cli_options[k] = int(cli_options[k])
-    return cli_options
-
-
-def options_to_argv(cli_options, project_options, ignore=None):
-    confmon_flags = project_options.get("monitor_flags", [])
-    result = confmon_flags[::]
-
-    for f in project_options.get("monitor_filters", []):
-        result.extend(["--filter", f])
-
-    for k, v in cli_options.items():
-        if v is None or (ignore and k in ignore):
-            continue
-        k = "--" + k.replace("_", "-")
-        if k in confmon_flags:
-            continue
-        if isinstance(v, bool):
-            if v:
-                result.append(k)
-        elif isinstance(v, tuple):
-            for i in v:
-                result.extend([k, i])
-        else:
-            result.extend([k, str(v)])
-    return result
-
-
-def get_project_options(environment=None):
-    config = ProjectConfig.get_instance()
-    config.validate(envs=[environment] if environment else None)
-    if not environment:
-        default_envs = config.default_envs()
-        if default_envs:
-            environment = default_envs[0]
-        else:
-            environment = config.envs()[0]
-    return config.items(env=environment, as_dict=True)
-
-
-def get_board_hwids(project_dir, platform, board):
-    with fs.cd(project_dir):
-        return platform.board_config(board).get("build.hwids", [])
-
-
-def load_monitor_filter(path, project_dir=None, environment=None):
-    name = os.path.basename(path)
-    name = name[: name.find(".")]
-    module = load_python_module("platformio.commands.device.filters.%s" % name, path)
-    for cls in get_object_members(module).values():
-        if (
-            not inspect.isclass(cls)
-            or not issubclass(cls, DeviceMonitorFilter)
-            or cls == DeviceMonitorFilter
-        ):
-            continue
-        obj = cls(project_dir, environment)
-        miniterm.TRANSFORMATIONS[obj.NAME] = obj
-    return True
-
-
-def register_platform_filters(platform, project_dir, environment):
-    monitor_dir = os.path.join(platform.get_dir(), "monitor")
-    if not os.path.isdir(monitor_dir):
-        return
-
-    for name in os.listdir(monitor_dir):
-        if not name.startswith("filter_") or not name.endswith(".py"):
-            continue
-        path = os.path.join(monitor_dir, name)
-        if not os.path.isfile(path):
-            continue
-        load_monitor_filter(path, project_dir, environment)

platformio/commands/home/command.py

@@ -1,161 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=too-many-locals,too-many-statements
-
-import mimetypes
-import socket
-from os.path import isdir
-
-import click
-
-from platformio import exception
-from platformio.compat import WINDOWS
-from platformio.managers.core import (
-    build_contrib_pysite_deps,
-    get_core_package_dir,
-    inject_contrib_pysite,
-)
-
-
-@click.command("home", short_help="PIO Home")
-@click.option("--port", type=int, default=8008, help="HTTP port, default=8008")
-@click.option(
-    "--host",
-    default="127.0.0.1",
-    help=(
-        "HTTP host, default=127.0.0.1. You can open PIO Home for inbound "
-        "connections with --host=0.0.0.0"
-    ),
-)
-@click.option("--no-open", is_flag=True)
-@click.option(
-    "--shutdown-timeout",
-    default=0,
-    type=int,
-    help=(
-        "Automatically shutdown server on timeout (in seconds) when no clients "
-        "are connected. Default is 0 which means never auto shutdown"
-    ),
-)
-def cli(port, host, no_open, shutdown_timeout):
-    # pylint: disable=import-error, import-outside-toplevel
-
-    # import contrib modules
-    inject_contrib_pysite()
-
-    try:
-        from autobahn.twisted.resource import WebSocketResource
-    except:  # pylint: disable=bare-except
-        build_contrib_pysite_deps(get_core_package_dir("contrib-pysite"))
-        from autobahn.twisted.resource import WebSocketResource
-
-    from twisted.internet import reactor
-    from twisted.web import server
-    from twisted.internet.error import CannotListenError
-
-    from platformio.commands.home.rpc.handlers.app import AppRPC
-    from platformio.commands.home.rpc.handlers.ide import IDERPC
-    from platformio.commands.home.rpc.handlers.misc import MiscRPC
-    from platformio.commands.home.rpc.handlers.os import OSRPC
-    from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
-    from platformio.commands.home.rpc.handlers.project import ProjectRPC
-    from platformio.commands.home.rpc.handlers.account import AccountRPC
-    from platformio.commands.home.rpc.server import JSONRPCServerFactory
-    from platformio.commands.home.web import WebRoot
-
-    factory = JSONRPCServerFactory(shutdown_timeout)
-    factory.addHandler(AppRPC(), namespace="app")
-    factory.addHandler(IDERPC(), namespace="ide")
-    factory.addHandler(MiscRPC(), namespace="misc")
-    factory.addHandler(OSRPC(), namespace="os")
-    factory.addHandler(PIOCoreRPC(), namespace="core")
-    factory.addHandler(ProjectRPC(), namespace="project")
-    factory.addHandler(AccountRPC(), namespace="account")
-
-    contrib_dir = get_core_package_dir("contrib-piohome")
-    if not isdir(contrib_dir):
-        raise exception.PlatformioException("Invalid path to PIO Home Contrib")
-
-    # Ensure PIO Home mimetypes are known
-    mimetypes.add_type("text/html", ".html")
-    mimetypes.add_type("text/css", ".css")
-    mimetypes.add_type("application/javascript", ".js")
-
-    root = WebRoot(contrib_dir)
-    root.putChild(b"wsrpc", WebSocketResource(factory))
-    site = server.Site(root)
-
-    # hook for `platformio-node-helpers`
-    if host == "__do_not_start__":
-        return
-
-    already_started = is_port_used(host, port)
-    home_url = "http://%s:%d" % (host, port)
-    if not no_open:
-        if already_started:
-            click.launch(home_url)
-        else:
-            reactor.callLater(1, lambda: click.launch(home_url))
-
-    click.echo(
-        "\n".join(
-            [
-                "",
-                "  ___I_",
-                " /\\-_--\\   PlatformIO Home",
-                "/  \\_-__\\",
-                "|[]| [] |  %s" % home_url,
-                "|__|____|______________%s" % ("_" * len(host)),
-            ]
-        )
-    )
-    click.echo("")
-    click.echo("Open PlatformIO Home in your browser by this URL => %s" % home_url)
-
-    try:
-        reactor.listenTCP(port, site, interface=host)
-    except CannotListenError as e:
-        click.secho(str(e), fg="red", err=True)
-        already_started = True
-
-    if already_started:
-        click.secho(
-            "PlatformIO Home server is already started in another process.", fg="yellow"
-        )
-        return
-
-    click.echo("PIO Home has been started. Press Ctrl+C to shutdown.")
-
-    reactor.run()
-
-
-def is_port_used(host, port):
-    socket.setdefaulttimeout(1)
-    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    if WINDOWS:
-        try:
-            s.bind((host, port))
-            s.close()
-            return False
-        except (OSError, socket.error):
-            pass
-    else:
-        try:
-            s.connect((host, port))
-            s.close()
-        except socket.error:
-            return False
-
-    return True

platformio/commands/home/helpers.py

@@ -1,69 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=keyword-arg-before-vararg,arguments-differ,signature-differs
-
-import os
-import socket
-
-import requests
-from twisted.internet import defer  # pylint: disable=import-error
-from twisted.internet import reactor  # pylint: disable=import-error
-from twisted.internet import threads  # pylint: disable=import-error
-
-from platformio import util
-from platformio.proc import where_is_program
-
-
-class AsyncSession(requests.Session):
-    def __init__(self, n=None, *args, **kwargs):
-        if n:
-            pool = reactor.getThreadPool()
-            pool.adjustPoolsize(0, n)
-
-        super(AsyncSession, self).__init__(*args, **kwargs)
-
-    def request(self, *args, **kwargs):
-        func = super(AsyncSession, self).request
-        return threads.deferToThread(func, *args, **kwargs)
-
-    def wrap(self, *args, **kwargs):  # pylint: disable=no-self-use
-        return defer.ensureDeferred(*args, **kwargs)
-
-
-@util.memoized(expire="60s")
-def requests_session():
-    return AsyncSession(n=5)
-
-
-@util.memoized(expire="60s")
-def get_core_fullpath():
-    return where_is_program(
-        "platformio" + (".exe" if "windows" in util.get_systype() else "")
-    )
-
-
-@util.memoized(expire="10s")
-def is_twitter_blocked():
-    ip = "104.244.42.1"
-    timeout = 2
-    try:
-        if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")):
-            requests.get("http://%s" % ip, allow_redirects=False, timeout=timeout)
-        else:
-            socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80))
-        return False
-    except:  # pylint: disable=bare-except
-        pass
-    return True

platformio/commands/home/rpc/handlers/ide.py

@@ -1,47 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import time
-
-import jsonrpc  # pylint: disable=import-error
-from twisted.internet import defer  # pylint: disable=import-error
-
-
-class IDERPC(object):
-    def __init__(self):
-        self._queue = {}
-
-    def send_command(self, sid, command, params):
-        if not self._queue.get(sid):
-            raise jsonrpc.exceptions.JSONRPCDispatchException(
-                code=4005, message="PIO Home IDE agent is not started"
-            )
-        while self._queue[sid]:
-            self._queue[sid].pop().callback(
-                {"id": time.time(), "method": command, "params": params}
-            )
-
-    def listen_commands(self, sid=0):
-        if sid not in self._queue:
-            self._queue[sid] = []
-        self._queue[sid].append(defer.Deferred())
-        return self._queue[sid][-1]
-
-    def open_project(self, sid, project_dir):
-        return self.send_command(sid, "open_project", project_dir)
-
-    def open_text_document(self, sid, path, line=None, column=None):
-        return self.send_command(
-            sid, "open_text_document", dict(path=path, line=line, column=column)
-        )

platformio/commands/home/rpc/server.py

@@ -1,101 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=import-error
-
-import click
-import jsonrpc
-from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol
-from jsonrpc.exceptions import JSONRPCDispatchException
-from twisted.internet import defer, reactor
-
-from platformio.compat import PY2, dump_json_to_unicode, is_bytes
-
-
-class JSONRPCServerProtocol(WebSocketServerProtocol):
-    def onOpen(self):
-        self.factory.connection_nums += 1
-        if self.factory.shutdown_timer:
-            self.factory.shutdown_timer.cancel()
-            self.factory.shutdown_timer = None
-
-    def onClose(self, wasClean, code, reason):  # pylint: disable=unused-argument
-        self.factory.connection_nums -= 1
-        if self.factory.connection_nums == 0:
-            self.factory.shutdownByTimeout()
-
-    def onMessage(self, payload, isBinary):  # pylint: disable=unused-argument
-        # click.echo("> %s" % payload)
-        response = jsonrpc.JSONRPCResponseManager.handle(
-            payload, self.factory.dispatcher
-        ).data
-        # if error
-        if "result" not in response:
-            self.sendJSONResponse(response)
-            return None
-
-        d = defer.maybeDeferred(lambda: response["result"])
-        d.addCallback(self._callback, response)
-        d.addErrback(self._errback, response)
-
-        return None
-
-    def _callback(self, result, response):
-        response["result"] = result
-        self.sendJSONResponse(response)
-
-    def _errback(self, failure, response):
-        if isinstance(failure.value, JSONRPCDispatchException):
-            e = failure.value
-        else:
-            e = JSONRPCDispatchException(code=4999, message=failure.getErrorMessage())
-        del response["result"]
-        response["error"] = e.error._data  # pylint: disable=protected-access
-        self.sendJSONResponse(response)
-
-    def sendJSONResponse(self, response):
-        # click.echo("< %s" % response)
-        if "error" in response:
-            click.secho("Error: %s" % response["error"], fg="red", err=True)
-        response = dump_json_to_unicode(response)
-        if not PY2 and not is_bytes(response):
-            response = response.encode("utf-8")
-        self.sendMessage(response)
-
-
-class JSONRPCServerFactory(WebSocketServerFactory):
-
-    protocol = JSONRPCServerProtocol
-    connection_nums = 0
-    shutdown_timer = 0
-
-    def __init__(self, shutdown_timeout=0):
-        super(JSONRPCServerFactory, self).__init__()
-        self.shutdown_timeout = shutdown_timeout
-        self.dispatcher = jsonrpc.Dispatcher()
-
-    def shutdownByTimeout(self):
-        if self.shutdown_timeout < 1:
-            return
-
-        def _auto_shutdown_server():
-            click.echo("Automatically shutdown server on timeout")
-            reactor.stop()
-
-        self.shutdown_timer = reactor.callLater(
-            self.shutdown_timeout, _auto_shutdown_server
-        )
-
-    def addHandler(self, handler, namespace):
-        self.dispatcher.build_method_map(handler, prefix="%s." % namespace)

platformio/commands/lib/command.py

@@ -14,29 +14,25 @@
 
 # pylint: disable=too-many-branches, too-many-locals
 
+import json
+import logging
 import os
 import time
+from urllib.parse import quote
 
 import click
-import semantic_version
 from tabulate import tabulate
 
 from platformio import exception, fs, util
-from platformio.commands import PlatformioCLI
-from platformio.compat import dump_json_to_unicode
-from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
-from platformio.package.manifest.parser import ManifestParserFactory
-from platformio.package.manifest.schema import ManifestSchema
+from platformio.cli import PlatformioCLI
+from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps
+from platformio.package.exception import NotGlobalLibDir, UnknownPackageError
+from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.meta import PackageItem, PackageSpec
 from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
-from platformio.project.exception import InvalidProjectConfError
 from platformio.project.helpers import get_project_dir, is_platformio_project
 
-try:
-    from urllib.parse import quote
-except ImportError:
-    from urllib import quote
-
 CTX_META_INPUT_DIRS_KEY = __name__ + ".input_dirs"
 CTX_META_PROJECT_ENVIRONMENTS_KEY = __name__ + ".project_environments"
 CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs"
@@ -44,10 +40,10 @@ CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps"
 
 
 def get_project_global_lib_dir():
-    return ProjectConfig.get_instance().get_optional_dir("globallib")
+    return ProjectConfig.get_instance().get("platformio", "globallib_dir")
 
 
-@click.group(short_help="Library Manager")
+@click.group(short_help="Library manager", hidden=True)
 @click.option(
     "-d",
     "--storage-dir",
@@ -72,6 +68,7 @@ def get_project_global_lib_dir():
 )
 @click.pass_context
 def cli(ctx, **options):
+    in_silence = PlatformioCLI.in_silence()
     storage_cmds = ("install", "uninstall", "update", "list")
     # skip commands that don't need storage folder
     if ctx.invoked_subcommand not in storage_cmds or (
@@ -94,11 +91,10 @@ def cli(ctx, **options):
             )
 
     if not storage_dirs:
-        raise exception.NotGlobalLibDir(
+        raise NotGlobalLibDir(
             get_project_dir(), get_project_global_lib_dir(), ctx.invoked_subcommand
         )
 
-    in_silence = PlatformioCLI.in_silence()
     ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options["environment"]
     ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs
     ctx.meta[CTX_META_STORAGE_DIRS_KEY] = []
@@ -112,7 +108,7 @@ def cli(ctx, **options):
                 os.path.join(storage_dir, "platformio.ini")
             )
             config.validate(options["environment"], silent=in_silence)
-            libdeps_dir = config.get_optional_dir("libdeps")
+            libdeps_dir = config.get("platformio", "libdeps_dir")
             for env in config.envs():
                 if options["environment"] and env not in options["environment"]:
                     continue
@@ -126,89 +122,108 @@ def cli(ctx, **options):
 @cli.command("install", short_help="Install library")
 @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]")
 @click.option(
-    "--save",
+    "--save/--no-save",
     is_flag=True,
-    help="Save installed libraries into the `platformio.ini` dependency list",
+    default=True,
+    help="Save installed libraries into the `platformio.ini` dependency list"
+    " (enabled by default)",
 )
 @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option(
-    "--interactive", is_flag=True, help="Allow to make a choice for all prompts"
+    "--interactive",
+    is_flag=True,
+    help="Deprecated! Please use a strict dependency specification (owner/libname)",
 )
 @click.option(
     "-f", "--force", is_flag=True, help="Reinstall/redownload library if exists"
 )
 @click.pass_context
-def lib_install(  # pylint: disable=too-many-arguments
+def lib_install(  # pylint: disable=too-many-arguments,unused-argument
     ctx, libraries, save, silent, interactive, force
 ):
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg install` instead.\n",
+        fg="yellow",
+    )
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, [])
 
-    installed_manifests = {}
+    installed_pkgs = {}
     for storage_dir in storage_dirs:
         if not silent and (libraries or storage_dir in storage_libdeps):
             print_storage_header(storage_dirs, storage_dir)
-        lm = LibraryManager(storage_dir)
-        if libraries:
-            for library in libraries:
-                pkg_dir = lm.install(
-                    library, silent=silent, interactive=interactive, force=force
-                )
-                installed_manifests[library] = lm.load_manifest(pkg_dir)
-        elif storage_dir in storage_libdeps:
-            builtin_lib_storages = None
-            for library in storage_libdeps[storage_dir]:
-                try:
-                    pkg_dir = lm.install(
-                        library, silent=silent, interactive=interactive, force=force
-                    )
-                    installed_manifests[library] = lm.load_manifest(pkg_dir)
-                except exception.LibNotFound as e:
-                    if builtin_lib_storages is None:
-                        builtin_lib_storages = get_builtin_libs()
-                    if not silent or not is_builtin_lib(builtin_lib_storages, library):
-                        click.secho("Warning! %s" % e, fg="yellow")
+        lm = LibraryPackageManager(storage_dir)
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
 
-    if not save or not libraries:
-        return
+        if libraries:
+            installed_pkgs = {
+                library: lm.install(library, force=force) for library in libraries
+            }
+
+        elif storage_dir in storage_libdeps:
+            for library in storage_libdeps[storage_dir]:
+                lm.install(library, force=force)
+
+    if save and installed_pkgs:
+        _save_deps(ctx, installed_pkgs)
+
+
+def _save_deps(ctx, pkgs, action="add"):
+    specs = []
+    for library, pkg in pkgs.items():
+        spec = PackageSpec(library)
+        if spec.external:
+            specs.append(spec)
+        else:
+            specs.append(
+                PackageSpec(
+                    owner=pkg.metadata.spec.owner,
+                    name=pkg.metadata.spec.name,
+                    requirements=spec.requirements
+                    or (
+                        ("^%s" % pkg.metadata.version)
+                        if not pkg.metadata.version.build
+                        else pkg.metadata.version
+                    ),
+                )
+            )
 
     input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
     project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
     for input_dir in input_dirs:
-        config = ProjectConfig.get_instance(os.path.join(input_dir, "platformio.ini"))
-        config.validate(project_environments)
-        for env in config.envs():
-            if project_environments and env not in project_environments:
-                continue
-            config.expand_interpolations = False
-            try:
-                lib_deps = config.get("env:" + env, "lib_deps")
-            except InvalidProjectConfError:
-                lib_deps = []
-            for library in libraries:
-                if library in lib_deps:
-                    continue
-                manifest = installed_manifests[library]
-                try:
-                    assert library.lower() == manifest["name"].lower()
-                    assert semantic_version.Version(manifest["version"])
-                    lib_deps.append("{name}@^{version}".format(**manifest))
-                except (AssertionError, ValueError):
-                    lib_deps.append(library)
-            config.set("env:" + env, "lib_deps", lib_deps)
-            config.save()
+        if not is_platformio_project(input_dir):
+            continue
+        save_project_libdeps(input_dir, specs, project_environments, action=action)
 
 
-@cli.command("uninstall", short_help="Uninstall libraries")
+@cli.command("uninstall", short_help="Remove libraries")
 @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]")
+@click.option(
+    "--save/--no-save",
+    is_flag=True,
+    default=True,
+    help="Remove libraries from the `platformio.ini` dependency list and save changes"
+    " (enabled by default)",
+)
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.pass_context
-def lib_uninstall(ctx, libraries):
+def lib_uninstall(ctx, libraries, save, silent):
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg uninstall` instead.\n",
+        fg="yellow",
+    )
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
+    uninstalled_pkgs = {}
     for storage_dir in storage_dirs:
         print_storage_header(storage_dirs, storage_dir)
-        lm = LibraryManager(storage_dir)
-        for library in libraries:
-            lm.uninstall(library)
+        lm = LibraryPackageManager(storage_dir)
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
+        uninstalled_pkgs = {library: lm.uninstall(library) for library in libraries}
+
+    if save and uninstalled_pkgs:
+        _save_deps(ctx, uninstalled_pkgs, action="remove")
 
 
 @cli.command("update", short_help="Update installed libraries")
@@ -222,46 +237,73 @@ def lib_uninstall(ctx, libraries):
 @click.option(
     "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
 )
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option("--json-output", is_flag=True)
 @click.pass_context
-def lib_update(ctx, libraries, only_check, dry_run, json_output):
-    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
+def lib_update(  # pylint: disable=too-many-arguments
+    ctx, libraries, only_check, dry_run, silent, json_output
+):
     only_check = dry_run or only_check
+    if only_check and not json_output:
+        raise exception.UserSideException(
+            "This command is deprecated, please use `pio pkg outdated` instead"
+        )
+
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg update` instead.\n",
+            fg="yellow",
+        )
+
+    storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     json_result = {}
     for storage_dir in storage_dirs:
         if not json_output:
             print_storage_header(storage_dirs, storage_dir)
-        lm = LibraryManager(storage_dir)
-
-        _libraries = libraries
-        if not _libraries:
-            _libraries = [manifest["__pkg_dir"] for manifest in lm.get_installed()]
+        lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, [])
+        lm = LibraryPackageManager(storage_dir)
+        lm.set_log_level(logging.WARN if silent else logging.DEBUG)
+        _libraries = libraries or lib_deps or lm.get_installed()
 
         if only_check and json_output:
             result = []
             for library in _libraries:
-                pkg_dir = library if os.path.isdir(library) else None
-                requirements = None
-                url = None
-                if not pkg_dir:
-                    name, requirements, url = lm.parse_pkg_uri(library)
-                    pkg_dir = lm.get_package_dir(name, requirements, url)
-                if not pkg_dir:
+                spec = None
+                pkg = None
+                if isinstance(library, PackageItem):
+                    pkg = library
+                else:
+                    spec = PackageSpec(library)
+                    pkg = lm.get_package(spec)
+                if not pkg:
                     continue
-                latest = lm.outdated(pkg_dir, requirements)
-                if not latest:
+                outdated = lm.outdated(pkg, spec)
+                if not outdated.is_outdated(allow_incompatible=True):
                     continue
-                manifest = lm.load_manifest(pkg_dir)
-                manifest["versionLatest"] = latest
+                manifest = lm.legacy_load_manifest(pkg)
+                manifest["versionWanted"] = (
+                    str(outdated.wanted) if outdated.wanted else None
+                )
+                manifest["versionLatest"] = (
+                    str(outdated.latest) if outdated.latest else None
+                )
                 result.append(manifest)
             json_result[storage_dir] = result
         else:
             for library in _libraries:
-                lm.update(library, only_check=only_check)
+                to_spec = (
+                    None if isinstance(library, PackageItem) else PackageSpec(library)
+                )
+                try:
+                    lm.update(library, to_spec=to_spec)
+                except UnknownPackageError as e:
+                    if library not in lib_deps:
+                        raise e
 
     if json_output:
         return click.echo(
-            dump_json_to_unicode(
+            json.dumps(
                 json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
             )
         )
@@ -273,13 +315,19 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output):
 @click.option("--json-output", is_flag=True)
 @click.pass_context
 def lib_list(ctx, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg list` instead.\n",
+            fg="yellow",
+        )
     storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
     json_result = {}
     for storage_dir in storage_dirs:
         if not json_output:
             print_storage_header(storage_dirs, storage_dir)
-        lm = LibraryManager(storage_dir)
-        items = lm.get_installed()
+        lm = LibraryPackageManager(storage_dir)
+        items = lm.legacy_get_installed()
         if json_output:
             json_result[storage_dir] = items
         elif items:
@@ -290,7 +338,7 @@ def lib_list(ctx, json_output):
 
     if json_output:
         return click.echo(
-            dump_json_to_unicode(
+            json.dumps(
                 json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
             )
         )
@@ -303,6 +351,7 @@ def lib_list(ctx, json_output):
 @click.option("--json-output", is_flag=True)
 @click.option("--page", type=click.INT, default=1)
 @click.option("--id", multiple=True)
+@click.option("-o", "--owner", multiple=True)
 @click.option("-n", "--name", multiple=True)
 @click.option("-a", "--author", multiple=True)
 @click.option("-k", "--keyword", multiple=True)
@@ -315,6 +364,13 @@ def lib_list(ctx, json_output):
     help="Do not prompt, automatically paginate with delay",
 )
 def lib_search(query, json_output, page, noninteractive, **filters):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg search` instead.\n",
+            fg="yellow",
+        )
+    regclient = LibraryPackageManager().get_registry_client_instance()
     if not query:
         query = []
     if not isinstance(query, list):
@@ -324,12 +380,15 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         for value in values:
             query.append('%s:"%s"' % (key, value))
 
-    result = util.get_api_result(
-        "/v2/lib/search", dict(query=" ".join(query), page=page), cache_valid="1d"
+    result = regclient.fetch_json_data(
+        "get",
+        "/v2/lib/search",
+        params=dict(query=" ".join(query), page=page),
+        x_cache_valid="1d",
     )
 
     if json_output:
-        click.echo(dump_json_to_unicode(result))
+        click.echo(json.dumps(result))
         return
 
     if result["total"] == 0:
@@ -374,10 +433,11 @@ def lib_search(query, json_output, page, noninteractive, **filters):
             time.sleep(5)
         elif not click.confirm("Show next libraries?"):
             break
-        result = util.get_api_result(
+        result = regclient.fetch_json_data(
+            "get",
             "/v2/lib/search",
-            {"query": " ".join(query), "page": int(result["page"]) + 1},
-            cache_valid="1d",
+            params=dict(query=" ".join(query), page=int(result["page"]) + 1),
+            x_cache_valid="1d",
         )
 
 
@@ -387,7 +447,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
 def lib_builtin(storage, json_output):
     items = get_builtin_libs(storage)
     if json_output:
-        return click.echo(dump_json_to_unicode(items))
+        return click.echo(json.dumps(items))
 
     for storage_ in items:
         if not storage_["items"]:
@@ -406,28 +466,34 @@ def lib_builtin(storage, json_output):
 @click.argument("library", metavar="[LIBRARY]")
 @click.option("--json-output", is_flag=True)
 def lib_show(library, json_output):
-    lm = LibraryManager()
-    name, requirements, _ = lm.parse_pkg_uri(library)
-    lib_id = lm.search_lib_id(
-        {"name": name, "requirements": requirements},
-        silent=json_output,
-        interactive=not json_output,
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg show` instead.\n",
+            fg="yellow",
+        )
+    lm = LibraryPackageManager()
+    lm.set_log_level(logging.ERROR if json_output else logging.DEBUG)
+    lib_id = lm.reveal_registry_package_id(library)
+    regclient = lm.get_registry_client_instance()
+    lib = regclient.fetch_json_data(
+        "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h"
     )
-    lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d")
     if json_output:
-        return click.echo(dump_json_to_unicode(lib))
+        return click.echo(json.dumps(lib))
 
-    click.secho(lib["name"], fg="cyan")
-    click.echo("=" * len(lib["name"]))
-    click.secho("#ID: %d" % lib["id"], bold=True)
+    title = "{ownername}/{name}".format(**lib)
+    click.secho(title, fg="cyan")
+    click.echo("=" * len(title))
     click.echo(lib["description"])
     click.echo()
 
+    click.secho("ID: %d" % lib["id"])
     click.echo(
         "Version: %s, released %s"
         % (
             lib["version"]["name"],
-            time.strftime("%c", util.parse_date(lib["version"]["released"])),
+            util.parse_datetime(lib["version"]["released"]).strftime("%c"),
         )
     )
     click.echo("Manifest: %s" % lib["confurl"])
@@ -435,9 +501,9 @@ def lib_show(library, json_output):
         if key not in lib or not lib[key]:
             continue
         if isinstance(lib[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(lib[key])))
+            click.echo("%s: %s" % (key.capitalize(), ", ".join(lib[key])))
         else:
-            click.echo("%s: %s" % (key.title(), lib[key]))
+            click.echo("%s: %s" % (key.capitalize(), lib[key]))
 
     blocks = []
 
@@ -445,7 +511,7 @@ def lib_show(library, json_output):
     for author in lib.get("authors", []):
         _data = []
         for key in ("name", "email", "url", "maintainer"):
-            if not author[key]:
+            if not author.get(key):
                 continue
             if key == "email":
                 _data.append("<%s>" % author[key])
@@ -469,7 +535,7 @@ def lib_show(library, json_output):
             "Versions",
             [
                 "%s, released %s"
-                % (v["name"], time.strftime("%c", util.parse_date(v["released"])))
+                % (v["name"], util.parse_datetime(v["released"]).strftime("%c"))
                 for v in lib["versions"]
             ],
         )
@@ -495,38 +561,28 @@ def lib_show(library, json_output):
     return True
 
 
-@cli.command("register", short_help="Register a new library")
+@cli.command("register", short_help="Deprecated")
 @click.argument("config_url")
-def lib_register(config_url):
-    if not config_url.startswith("http://") and not config_url.startswith("https://"):
-        raise exception.InvalidLibConfURL(config_url)
-
-    # Validate manifest
-    ManifestSchema().load_manifest(
-        ManifestParserFactory.new_from_url(config_url).as_dict()
+def lib_register(config_url):  # pylint: disable=unused-argument
+    raise exception.UserSideException(
+        "This command is deprecated. Please use `pio pkg publish` command."
     )
 
-    result = util.get_api_result("/lib/register", data=dict(config_url=config_url))
-    if "message" in result and result["message"]:
-        click.secho(
-            result["message"],
-            fg="green" if "successed" in result and result["successed"] else "red",
-        )
-
 
 @cli.command("stats", short_help="Library Registry Statistics")
 @click.option("--json-output", is_flag=True)
 def lib_stats(json_output):
-    result = util.get_api_result("/lib/stats", cache_valid="1h")
+    regclient = LibraryPackageManager().get_registry_client_instance()
+    result = regclient.fetch_json_data("get", "/v2/lib/stats", x_cache_valid="1h")
 
     if json_output:
-        return click.echo(dump_json_to_unicode(result))
+        return click.echo(json.dumps(result))
 
     for key in ("updated", "added"):
         tabular_data = [
             (
                 click.style(item["name"], fg="cyan"),
-                time.strftime("%c", util.parse_date(item["date"])),
+                util.parse_datetime(item["date"]).strftime("%c"),
                 "https://platformio.org/lib/show/%s/%s"
                 % (item["id"], quote(item["name"])),
             )
@@ -601,9 +657,9 @@ def print_lib_item(item):
         if key not in item or not item[key]:
             continue
         if isinstance(item[key], list):
-            click.echo("%s: %s" % (key.title(), ", ".join(item[key])))
+            click.echo("%s: %s" % (key.capitalize(), ", ".join(item[key])))
         else:
-            click.echo("%s: %s" % (key.title(), item[key]))
+            click.echo("%s: %s" % (key.capitalize(), item[key]))
 
     for key in ("frameworks", "platforms"):
         if key not in item:

platformio/commands/lib/helpers.py

@@ -0,0 +1,104 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from platformio import util
+from platformio.compat import ci_strings_are_equal
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.package.meta import PackageSpec
+from platformio.platform.factory import PlatformFactory
+from platformio.project.config import ProjectConfig
+from platformio.project.exception import InvalidProjectConfError
+
+
+@util.memoized(expire="60s")
+def get_builtin_libs(storage_names=None):
+    # pylint: disable=import-outside-toplevel
+    from platformio.package.manager.library import LibraryPackageManager
+
+    items = []
+    storage_names = storage_names or []
+    pm = PlatformPackageManager()
+    for pkg in pm.get_installed():
+        p = PlatformFactory.new(pkg)
+        for storage in p.get_lib_storages():
+            if storage_names and storage["name"] not in storage_names:
+                continue
+            lm = LibraryPackageManager(storage["path"])
+            items.append(
+                {
+                    "name": storage["name"],
+                    "path": storage["path"],
+                    "items": lm.legacy_get_installed(),
+                }
+            )
+    return items
+
+
+def is_builtin_lib(name):
+    for storage in get_builtin_libs():
+        for lib in storage["items"]:
+            if lib.get("name") == name:
+                return True
+    return False
+
+
+def ignore_deps_by_specs(deps, specs):
+    result = []
+    for dep in deps:
+        depspec = PackageSpec(dep)
+        if depspec.external:
+            result.append(dep)
+            continue
+        ignore_conditions = []
+        for spec in specs:
+            if depspec.owner:
+                ignore_conditions.append(
+                    ci_strings_are_equal(depspec.owner, spec.owner)
+                    and ci_strings_are_equal(depspec.name, spec.name)
+                )
+            else:
+                ignore_conditions.append(ci_strings_are_equal(depspec.name, spec.name))
+        if not any(ignore_conditions):
+            result.append(dep)
+    return result
+
+
+def save_project_libdeps(project_dir, specs, environments=None, action="add"):
+    config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini"))
+    config.validate(environments)
+    for env in config.envs():
+        if environments and env not in environments:
+            continue
+        config.expand_interpolations = False
+        candidates = []
+        try:
+            candidates = ignore_deps_by_specs(
+                config.get("env:" + env, "lib_deps"), specs
+            )
+        except InvalidProjectConfError:
+            pass
+        if action == "add":
+            candidates.extend(spec.as_dependency() for spec in specs)
+        if candidates:
+            result = []
+            for item in candidates:
+                item = item.strip()
+                if item and item not in result:
+                    result.append(item)
+            config.set("env:" + env, "lib_deps", result)
+        elif config.has_option("env:" + env, "lib_deps"):
+            config.remove_option("env:" + env, "lib_deps")
+    config.save()

platformio/commands/platform.py

@@ -12,163 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from os.path import dirname, isdir
+import json
+import logging
+import os
 
 import click
 
-from platformio import app, exception, util
 from platformio.commands.boards import print_boards
-from platformio.compat import dump_json_to_unicode
-from platformio.managers.platform import PlatformFactory, PlatformManager
-from platformio.package.pack import PackagePacker
+from platformio.exception import UserSideException
+from platformio.package.exception import UnknownPackageError
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.package.meta import PackageItem, PackageSpec
+from platformio.package.version import get_original_version
+from platformio.platform.exception import UnknownPlatform
+from platformio.platform.factory import PlatformFactory
 
 
-@click.group(short_help="Platform Manager")
+@click.group(short_help="Platform manager", hidden=True)
 def cli():
     pass
 
 
-def _print_platforms(platforms):
-    for platform in platforms:
-        click.echo(
-            "{name} ~ {title}".format(
-                name=click.style(platform["name"], fg="cyan"), title=platform["title"]
-            )
-        )
-        click.echo("=" * (3 + len(platform["name"] + platform["title"])))
-        click.echo(platform["description"])
-        click.echo()
-        if "homepage" in platform:
-            click.echo("Home: %s" % platform["homepage"])
-        if "frameworks" in platform and platform["frameworks"]:
-            click.echo("Frameworks: %s" % ", ".join(platform["frameworks"]))
-        if "packages" in platform:
-            click.echo("Packages: %s" % ", ".join(platform["packages"]))
-        if "version" in platform:
-            if "__src_url" in platform:
-                click.echo(
-                    "Version: #%s (%s)" % (platform["version"], platform["__src_url"])
-                )
-            else:
-                click.echo("Version: " + platform["version"])
-        click.echo()
-
-
-def _get_registry_platforms():
-    platforms = util.get_api_result("/platforms", cache_valid="7d")
-    pm = PlatformManager()
-    for platform in platforms or []:
-        platform["versions"] = pm.get_all_repo_versions(platform["name"])
-    return platforms
-
-
-def _get_platform_data(*args, **kwargs):
-    try:
-        return _get_installed_platform_data(*args, **kwargs)
-    except exception.UnknownPlatform:
-        return _get_registry_platform_data(*args, **kwargs)
-
-
-def _get_installed_platform_data(platform, with_boards=True, expose_packages=True):
-    p = PlatformFactory.newPlatform(platform)
-    data = dict(
-        name=p.name,
-        title=p.title,
-        description=p.description,
-        version=p.version,
-        homepage=p.homepage,
-        repository=p.repository_url,
-        url=p.vendor_url,
-        docs=p.docs_url,
-        license=p.license,
-        forDesktop=not p.is_embedded(),
-        frameworks=sorted(list(p.frameworks) if p.frameworks else []),
-        packages=list(p.packages) if p.packages else [],
-    )
-
-    # if dump to API
-    # del data['version']
-    # return data
-
-    # overwrite VCS version and add extra fields
-    manifest = PlatformManager().load_manifest(dirname(p.manifest_path))
-    assert manifest
-    for key in manifest:
-        if key == "version" or key.startswith("__"):
-            data[key] = manifest[key]
-
-    if with_boards:
-        data["boards"] = [c.get_brief_data() for c in p.get_boards().values()]
-
-    if not data["packages"] or not expose_packages:
-        return data
-
-    data["packages"] = []
-    installed_pkgs = p.get_installed_packages()
-    for name, opts in p.packages.items():
-        item = dict(
-            name=name,
-            type=p.get_package_type(name),
-            requirements=opts.get("version"),
-            optional=opts.get("optional") is True,
-        )
-        if name in installed_pkgs:
-            for key, value in installed_pkgs[name].items():
-                if key not in ("url", "version", "description"):
-                    continue
-                item[key] = value
-                if key == "version":
-                    item["originalVersion"] = util.get_original_version(value)
-        data["packages"].append(item)
-
-    return data
-
-
-def _get_registry_platform_data(  # pylint: disable=unused-argument
-    platform, with_boards=True, expose_packages=True
-):
-    _data = None
-    for p in _get_registry_platforms():
-        if p["name"] == platform:
-            _data = p
-            break
-
-    if not _data:
-        return None
-
-    data = dict(
-        name=_data["name"],
-        title=_data["title"],
-        description=_data["description"],
-        homepage=_data["homepage"],
-        repository=_data["repository"],
-        url=_data["url"],
-        license=_data["license"],
-        forDesktop=_data["forDesktop"],
-        frameworks=_data["frameworks"],
-        packages=_data["packages"],
-        versions=_data["versions"],
-    )
-
-    if with_boards:
-        data["boards"] = [
-            board
-            for board in PlatformManager().get_registered_boards()
-            if board["platform"] == _data["name"]
-        ]
-
-    return data
-
-
 @cli.command("search", short_help="Search for development platform")
 @click.argument("query", required=False)
 @click.option("--json-output", is_flag=True)
 def platform_search(query, json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg search` instead.\n",
+            fg="yellow",
+        )
     platforms = []
     for platform in _get_registry_platforms():
         if query == "all":
             query = ""
-        search_data = dump_json_to_unicode(platform)
+        search_data = json.dumps(platform)
         if query and query.lower() not in search_data.lower():
             continue
         platforms.append(
@@ -178,7 +57,7 @@ def platform_search(query, json_output):
         )
 
     if json_output:
-        click.echo(dump_json_to_unicode(platforms))
+        click.echo(json.dumps(platforms))
     else:
         _print_platforms(platforms)
 
@@ -187,11 +66,14 @@ def platform_search(query, json_output):
 @click.argument("query", required=False)
 @click.option("--json-output", is_flag=True)
 def platform_frameworks(query, json_output):
+    regclient = PlatformPackageManager().get_registry_client_instance()
     frameworks = []
-    for framework in util.get_api_result("/frameworks", cache_valid="7d"):
+    for framework in regclient.fetch_json_data(
+        "get", "/v2/frameworks", x_cache_valid="1d"
+    ):
         if query == "all":
             query = ""
-        search_data = dump_json_to_unicode(framework)
+        search_data = json.dumps(framework)
         if query and query.lower() not in search_data.lower():
             continue
         framework["homepage"] = "https://platformio.org/frameworks/" + framework["name"]
@@ -204,7 +86,7 @@ def platform_frameworks(query, json_output):
 
     frameworks = sorted(frameworks, key=lambda manifest: manifest["name"])
     if json_output:
-        click.echo(dump_json_to_unicode(frameworks))
+        click.echo(json.dumps(frameworks))
     else:
         _print_platforms(frameworks)
 
@@ -212,18 +94,22 @@ def platform_frameworks(query, json_output):
 @cli.command("list", short_help="List installed development platforms")
 @click.option("--json-output", is_flag=True)
 def platform_list(json_output):
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg list` instead.\n",
+            fg="yellow",
+        )
     platforms = []
-    pm = PlatformManager()
-    for manifest in pm.get_installed():
+    pm = PlatformPackageManager()
+    for pkg in pm.get_installed():
         platforms.append(
-            _get_installed_platform_data(
-                manifest["__pkg_dir"], with_boards=False, expose_packages=False
-            )
+            _get_installed_platform_data(pkg, with_boards=False, expose_packages=False)
         )
 
     platforms = sorted(platforms, key=lambda manifest: manifest["name"])
     if json_output:
-        click.echo(dump_json_to_unicode(platforms))
+        click.echo(json.dumps(platforms))
     else:
         _print_platforms(platforms)
 
@@ -232,18 +118,23 @@ def platform_list(json_output):
 @click.argument("platform")
 @click.option("--json-output", is_flag=True)
 def platform_show(platform, json_output):  # pylint: disable=too-many-branches
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg show` instead.\n",
+            fg="yellow",
+        )
     data = _get_platform_data(platform)
     if not data:
-        raise exception.UnknownPlatform(platform)
+        raise UnknownPlatform(platform)
     if json_output:
-        return click.echo(dump_json_to_unicode(data))
+        return click.echo(json.dumps(data))
 
+    dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"]
     click.echo(
-        "{name} ~ {title}".format(
-            name=click.style(data["name"], fg="cyan"), title=data["title"]
-        )
+        "{dep} ~ {title}".format(dep=click.style(dep, fg="cyan"), title=data["title"])
     )
-    click.echo("=" * (3 + len(data["name"] + data["title"])))
+    click.echo("=" * (3 + len(dep + data["title"])))
     click.echo(data["description"])
     click.echo()
     if "version" in data:
@@ -300,33 +191,76 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
 @click.option("--without-package", multiple=True)
 @click.option("--skip-default-package", is_flag=True)
 @click.option("--with-all-packages", is_flag=True)
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option(
     "-f",
     "--force",
     is_flag=True,
     help="Reinstall/redownload dev/platform and its packages if exist",
 )
-def platform_install(  # pylint: disable=too-many-arguments
+def platform_install(  # pylint: disable=too-many-arguments,too-many-locals
     platforms,
     with_package,
     without_package,
     skip_default_package,
     with_all_packages,
+    silent,
     force,
 ):
-    pm = PlatformManager()
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg install` instead.\n",
+        fg="yellow",
+    )
+
+    def _find_pkg_names(p, candidates):
+        result = []
+        for candidate in candidates:
+            found = False
+            # lookup by package types
+            for _name, _opts in p.packages.items():
+                if _opts.get("type") == candidate:
+                    result.append(_name)
+                    found = True
+            if (
+                p.frameworks
+                and candidate.startswith("framework-")
+                and candidate[10:] in p.frameworks
+            ):
+                result.append(p.frameworks[candidate[10:]]["package"])
+                found = True
+            if not found:
+                result.append(candidate)
+        return result
+
+    pm = PlatformPackageManager()
+    pm.set_log_level(logging.WARN if silent else logging.DEBUG)
     for platform in platforms:
-        if pm.install(
-            name=platform,
-            with_packages=with_package,
-            without_packages=without_package,
-            skip_default_package=skip_default_package,
-            with_all_packages=with_all_packages,
-            force=force,
-        ):
+        if with_package or without_package or with_all_packages:
+            pkg = pm.install(platform, skip_dependencies=True)
+            p = PlatformFactory.new(pkg)
+            if with_all_packages:
+                with_package = list(p.packages)
+            with_package = set(_find_pkg_names(p, with_package or []))
+            without_package = set(_find_pkg_names(p, without_package or []))
+            upkgs = with_package | without_package
+            ppkgs = set(p.packages)
+            if not upkgs.issubset(ppkgs):
+                raise UnknownPackageError(", ".join(upkgs - ppkgs))
+            for name, options in p.packages.items():
+                if name in without_package:
+                    continue
+                if name in with_package or not (
+                    skip_default_package or options.get("optional", False)
+                ):
+                    p.pm.install(p.get_package_spec(name), force=force)
+        else:
+            pkg = pm.install(platform, skip_dependencies=skip_default_package)
+
+        if pkg and not silent:
             click.secho(
                 "The platform '%s' has been successfully installed!\n"
-                "The rest of packages will be installed automatically "
+                "The rest of the packages will be installed later "
                 "depending on your build environment." % platform,
                 fg="green",
             )
@@ -335,11 +269,17 @@ def platform_install(  # pylint: disable=too-many-arguments
 @cli.command("uninstall", short_help="Uninstall development platform")
 @click.argument("platforms", nargs=-1, required=True, metavar="[PLATFORM...]")
 def platform_uninstall(platforms):
-    pm = PlatformManager()
+    click.secho(
+        "\nWARNING: This command is deprecated and will be removed in "
+        "the next releases. \nPlease use `pio pkg uninstall` instead.\n",
+        fg="yellow",
+    )
+    pm = PlatformPackageManager()
+    pm.set_log_level(logging.DEBUG)
     for platform in platforms:
         if pm.uninstall(platform):
             click.secho(
-                "The platform '%s' has been successfully uninstalled!" % platform,
+                "The platform '%s' has been successfully removed!" % platform,
                 fg="green",
             )
 
@@ -358,66 +298,206 @@ def platform_uninstall(platforms):
 @click.option(
     "--dry-run", is_flag=True, help="Do not update, only check for the new versions"
 )
+@click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting")
 @click.option("--json-output", is_flag=True)
-def platform_update(  # pylint: disable=too-many-locals
-    platforms, only_packages, only_check, dry_run, json_output
+def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
+    platforms, only_check, dry_run, silent, json_output, **_
 ):
-    pm = PlatformManager()
-    pkg_dir_to_name = {}
-    if not platforms:
-        platforms = []
-        for manifest in pm.get_installed():
-            platforms.append(manifest["__pkg_dir"])
-            pkg_dir_to_name[manifest["__pkg_dir"]] = manifest.get(
-                "title", manifest["name"]
-            )
+    if only_check and not json_output:
+        raise UserSideException(
+            "This command is deprecated, please use `pio pkg outdated` instead"
+        )
 
+    if not json_output:
+        click.secho(
+            "\nWARNING: This command is deprecated and will be removed in "
+            "the next releases. \nPlease use `pio pkg update` instead.\n",
+            fg="yellow",
+        )
+
+    pm = PlatformPackageManager()
+    pm.set_log_level(logging.WARN if silent else logging.DEBUG)
+    platforms = platforms or pm.get_installed()
     only_check = dry_run or only_check
 
     if only_check and json_output:
         result = []
         for platform in platforms:
-            pkg_dir = platform if isdir(platform) else None
-            requirements = None
-            url = None
-            if not pkg_dir:
-                name, requirements, url = pm.parse_pkg_uri(platform)
-                pkg_dir = pm.get_package_dir(name, requirements, url)
-            if not pkg_dir:
+            spec = None
+            pkg = None
+            if isinstance(platform, PackageItem):
+                pkg = platform
+            else:
+                spec = PackageSpec(platform)
+                pkg = pm.get_package(spec)
+            if not pkg:
                 continue
-            latest = pm.outdated(pkg_dir, requirements)
+            outdated = pm.outdated(pkg, spec)
             if (
-                not latest
-                and not PlatformFactory.newPlatform(pkg_dir).are_outdated_packages()
+                not outdated.is_outdated(allow_incompatible=True)
+                and not PlatformFactory.new(pkg).are_outdated_packages()
             ):
                 continue
             data = _get_installed_platform_data(
-                pkg_dir, with_boards=False, expose_packages=False
+                pkg, with_boards=False, expose_packages=False
             )
-            if latest:
-                data["versionLatest"] = latest
+            if outdated.is_outdated(allow_incompatible=True):
+                data["versionLatest"] = (
+                    str(outdated.latest) if outdated.latest else None
+                )
             result.append(data)
-        return click.echo(dump_json_to_unicode(result))
+        return click.echo(json.dumps(result))
 
-    # cleanup cached board and platform lists
-    app.clean_cache()
     for platform in platforms:
         click.echo(
             "Platform %s"
-            % click.style(pkg_dir_to_name.get(platform, platform), fg="cyan")
+            % click.style(
+                platform.metadata.name
+                if isinstance(platform, PackageItem)
+                else platform,
+                fg="cyan",
+            )
         )
         click.echo("--------")
-        pm.update(platform, only_packages=only_packages, only_check=only_check)
+        pm.update(platform)
         click.echo()
 
     return True
 
 
-@cli.command(
-    "pack", short_help="Create a tarball from development platform/tool package"
-)
-@click.argument("package", required=True, metavar="[source directory, tar.gz or zip]")
-def platform_pack(package):
-    p = PackagePacker(package)
-    tarball_path = p.pack()
-    click.secho('Wrote a tarball to "%s"' % tarball_path, fg="green")
+#
+# Helpers
+#
+
+
+def _print_platforms(platforms):
+    for platform in platforms:
+        click.echo(
+            "{name} ~ {title}".format(
+                name=click.style(platform["name"], fg="cyan"), title=platform["title"]
+            )
+        )
+        click.echo("=" * (3 + len(platform["name"] + platform["title"])))
+        click.echo(platform["description"])
+        click.echo()
+        if "homepage" in platform:
+            click.echo("Home: %s" % platform["homepage"])
+        if "frameworks" in platform and platform["frameworks"]:
+            click.echo("Frameworks: %s" % ", ".join(platform["frameworks"]))
+        if "packages" in platform:
+            click.echo("Packages: %s" % ", ".join(platform["packages"]))
+        if "version" in platform:
+            if "__src_url" in platform:
+                click.echo(
+                    "Version: %s (%s)" % (platform["version"], platform["__src_url"])
+                )
+            else:
+                click.echo("Version: " + platform["version"])
+        click.echo()
+
+
+def _get_registry_platforms():
+    regclient = PlatformPackageManager().get_registry_client_instance()
+    return regclient.fetch_json_data("get", "/v2/platforms", x_cache_valid="1d")
+
+
+def _get_platform_data(*args, **kwargs):
+    try:
+        return _get_installed_platform_data(*args, **kwargs)
+    except UnknownPlatform:
+        return _get_registry_platform_data(*args, **kwargs)
+
+
+def _get_installed_platform_data(platform, with_boards=True, expose_packages=True):
+    p = PlatformFactory.new(platform)
+    data = dict(
+        name=p.name,
+        title=p.title,
+        description=p.description,
+        version=p.version,
+        homepage=p.homepage,
+        url=p.homepage,
+        repository=p.repository_url,
+        license=p.license,
+        forDesktop=not p.is_embedded(),
+        frameworks=sorted(list(p.frameworks) if p.frameworks else []),
+        packages=list(p.packages) if p.packages else [],
+    )
+
+    # if dump to API
+    # del data['version']
+    # return data
+
+    # overwrite VCS version and add extra fields
+    manifest = PlatformPackageManager().legacy_load_manifest(
+        os.path.dirname(p.manifest_path)
+    )
+    assert manifest
+    for key in manifest:
+        if key == "version" or key.startswith("__"):
+            data[key] = manifest[key]
+
+    if with_boards:
+        data["boards"] = [c.get_brief_data() for c in p.get_boards().values()]
+
+    if not data["packages"] or not expose_packages:
+        return data
+
+    data["packages"] = []
+    installed_pkgs = {
+        pkg.metadata.name: p.pm.load_manifest(pkg) for pkg in p.get_installed_packages()
+    }
+    for name, options in p.packages.items():
+        item = dict(
+            name=name,
+            type=p.get_package_type(name),
+            requirements=options.get("version"),
+            optional=options.get("optional") is True,
+        )
+        if name in installed_pkgs:
+            for key, value in installed_pkgs[name].items():
+                if key not in ("url", "version", "description"):
+                    continue
+                item[key] = value
+                if key == "version":
+                    item["originalVersion"] = get_original_version(value)
+        data["packages"].append(item)
+
+    return data
+
+
+def _get_registry_platform_data(  # pylint: disable=unused-argument
+    platform, with_boards=True, expose_packages=True
+):
+    _data = None
+    for p in _get_registry_platforms():
+        if p["name"] == platform:
+            _data = p
+            break
+
+    if not _data:
+        return None
+
+    data = dict(
+        ownername=_data.get("ownername"),
+        name=_data["name"],
+        title=_data["title"],
+        description=_data["description"],
+        homepage=_data["homepage"],
+        repository=_data["repository"],
+        url=_data["url"],
+        license=_data["license"],
+        forDesktop=_data["forDesktop"],
+        frameworks=_data["frameworks"],
+        packages=_data["packages"],
+        versions=_data.get("versions"),
+    )
+
+    if with_boards:
+        data["boards"] = [
+            board
+            for board in PlatformPackageManager().get_registered_boards()
+            if board["platform"] == _data["name"]
+        ]
+
+    return data

platformio/commands/settings.py

@@ -27,7 +27,7 @@ def format_value(raw):
     return str(raw)
 
 
-@click.group(short_help="Manage PlatformIO settings")
+@click.group(short_help="Manage system settings")
 def cli():
     pass
 

platformio/commands/test/command.py

@@ -1,273 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
-
-from fnmatch import fnmatch
-from os import getcwd, listdir
-from os.path import isdir, join
-from time import time
-
-import click
-from tabulate import tabulate
-
-from platformio import app, exception, fs, util
-from platformio.commands.test.embedded import EmbeddedTestProcessor
-from platformio.commands.test.native import NativeTestProcessor
-from platformio.project.config import ProjectConfig
-
-
-@click.command("test", short_help="Unit Testing")
-@click.option("--environment", "-e", multiple=True, metavar="<environment>")
-@click.option(
-    "--filter",
-    "-f",
-    multiple=True,
-    metavar="<pattern>",
-    help="Filter tests by a pattern",
-)
-@click.option(
-    "--ignore",
-    "-i",
-    multiple=True,
-    metavar="<pattern>",
-    help="Ignore tests by a pattern",
-)
-@click.option("--upload-port")
-@click.option("--test-port")
-@click.option(
-    "-d",
-    "--project-dir",
-    default=getcwd,
-    type=click.Path(
-        exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True
-    ),
-)
-@click.option(
-    "-c",
-    "--project-conf",
-    type=click.Path(
-        exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True
-    ),
-)
-@click.option("--without-building", is_flag=True)
-@click.option("--without-uploading", is_flag=True)
-@click.option("--without-testing", is_flag=True)
-@click.option("--no-reset", is_flag=True)
-@click.option(
-    "--monitor-rts",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial RTS line state for Serial Monitor",
-)
-@click.option(
-    "--monitor-dtr",
-    default=None,
-    type=click.IntRange(0, 1),
-    help="Set initial DTR line state for Serial Monitor",
-)
-@click.option("--verbose", "-v", is_flag=True)
-@click.pass_context
-def cli(  # pylint: disable=redefined-builtin
-    ctx,
-    environment,
-    ignore,
-    filter,
-    upload_port,
-    test_port,
-    project_dir,
-    project_conf,
-    without_building,
-    without_uploading,
-    without_testing,
-    no_reset,
-    monitor_rts,
-    monitor_dtr,
-    verbose,
-):
-    app.set_session_var("custom_project_conf", project_conf)
-
-    with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(project_conf)
-        config.validate(envs=environment)
-
-        test_dir = config.get_optional_dir("test")
-        if not isdir(test_dir):
-            raise exception.TestDirNotExists(test_dir)
-        test_names = get_test_names(test_dir)
-
-        if not verbose:
-            click.echo("Verbose mode can be enabled via `-v, --verbose` option")
-        click.secho("Collected %d items" % len(test_names), bold=True)
-
-        results = []
-        default_envs = config.default_envs()
-        for testname in test_names:
-
-            for envname in config.envs():
-                section = "env:%s" % envname
-
-                # filter and ignore patterns
-                patterns = dict(filter=list(filter), ignore=list(ignore))
-                for key in patterns:
-                    patterns[key].extend(config.get(section, "test_%s" % key, []))
-
-                skip_conditions = [
-                    environment and envname not in environment,
-                    not environment and default_envs and envname not in default_envs,
-                    testname != "*"
-                    and patterns["filter"]
-                    and not any([fnmatch(testname, p) for p in patterns["filter"]]),
-                    testname != "*"
-                    and any([fnmatch(testname, p) for p in patterns["ignore"]]),
-                ]
-                if any(skip_conditions):
-                    results.append({"env": envname, "test": testname})
-                    continue
-
-                click.echo()
-                print_processing_header(testname, envname)
-
-                cls = (
-                    NativeTestProcessor
-                    if config.get(section, "platform") == "native"
-                    else EmbeddedTestProcessor
-                )
-                tp = cls(
-                    ctx,
-                    testname,
-                    envname,
-                    dict(
-                        project_config=config,
-                        project_dir=project_dir,
-                        upload_port=upload_port,
-                        test_port=test_port,
-                        without_building=without_building,
-                        without_uploading=without_uploading,
-                        without_testing=without_testing,
-                        no_reset=no_reset,
-                        monitor_rts=monitor_rts,
-                        monitor_dtr=monitor_dtr,
-                        verbose=verbose,
-                        silent=not verbose,
-                    ),
-                )
-                result = {
-                    "env": envname,
-                    "test": testname,
-                    "duration": time(),
-                    "succeeded": tp.process(),
-                }
-                result["duration"] = time() - result["duration"]
-                results.append(result)
-
-                print_processing_footer(result)
-
-    if without_testing:
-        return
-
-    print_testing_summary(results)
-
-    command_failed = any(r.get("succeeded") is False for r in results)
-    if command_failed:
-        raise exception.ReturnErrorCode(1)
-
-
-def get_test_names(test_dir):
-    names = []
-    for item in sorted(listdir(test_dir)):
-        if isdir(join(test_dir, item)):
-            names.append(item)
-    if not names:
-        names = ["*"]
-    return names
-
-
-def print_processing_header(test, env):
-    click.echo(
-        "Processing %s in %s environment"
-        % (
-            click.style(test, fg="yellow", bold=True),
-            click.style(env, fg="cyan", bold=True),
-        )
-    )
-    terminal_width, _ = click.get_terminal_size()
-    click.secho("-" * terminal_width, bold=True)
-
-
-def print_processing_footer(result):
-    is_failed = not result.get("succeeded")
-    util.print_labeled_bar(
-        "[%s] Took %.2f seconds"
-        % (
-            (
-                click.style("FAILED", fg="red", bold=True)
-                if is_failed
-                else click.style("PASSED", fg="green", bold=True)
-            ),
-            result["duration"],
-        ),
-        is_error=is_failed,
-    )
-
-
-def print_testing_summary(results):
-    click.echo()
-
-    tabular_data = []
-    succeeded_nums = 0
-    failed_nums = 0
-    duration = 0
-
-    for result in results:
-        duration += result.get("duration", 0)
-        if result.get("succeeded") is False:
-            failed_nums += 1
-            status_str = click.style("FAILED", fg="red")
-        elif result.get("succeeded") is None:
-            status_str = "IGNORED"
-        else:
-            succeeded_nums += 1
-            status_str = click.style("PASSED", fg="green")
-
-        tabular_data.append(
-            (
-                result["test"],
-                click.style(result["env"], fg="cyan"),
-                status_str,
-                util.humanize_duration_time(result.get("duration")),
-            )
-        )
-
-    click.echo(
-        tabulate(
-            tabular_data,
-            headers=[
-                click.style(s, bold=True)
-                for s in ("Test", "Environment", "Status", "Duration")
-            ],
-        ),
-        err=failed_nums,
-    )
-
-    util.print_labeled_bar(
-        "%s%d succeeded in %s"
-        % (
-            "%d failed, " % failed_nums if failed_nums else "",
-            succeeded_nums,
-            util.humanize_duration_time(duration),
-        ),
-        is_error=failed_nums,
-        fg="red" if failed_nums else "green",
-    )