Fixed an issue with preprocessing of `*.ino` when macros were not handled // Resolve #2972

.appveyor.yml

@@ -0,0 +1,28 @@
+build: off
+
+platform:
+  - x64
+
+environment:
+  matrix:
+    - TOXENV: "py27"
+      PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P2_{build}
+
+    - TOXENV: "py36"
+      PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P3_{build}
+
+install:
+  - cmd: git submodule update --init --recursive
+  - cmd: SET PATH=C:\MinGW\bin;%PATH%
+  - cmd: pip install --force-reinstall tox
+
+test_script:
+  - cmd: tox
+
+notifications:
+  - provider: Slack
+    incoming_webhook:
+      secure: E9H0SU0Ju7WLDvgxsV8cs3J62T3nTTX7QkEjsczN0Sto/c9hWkVfhc5gGWUkxhlD975cokHByKGJIdwYwCewqOI+7BrcT8U+nlga4Uau7J8=
+    on_build_success: false
+    on_build_failure: true
+    on_build_status_changed: true

.coveragerc

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=unused-import
-from platformio.device.monitor.filters.base import (
-    DeviceMonitorFilterBase as DeviceMonitorFilter,
-)
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    raise AssertionError
+    raise NotImplementedError

.github/ISSUE_TEMPLATE.md

@@ -6,8 +6,9 @@ What kind of issue is this?
       use [Community Forums](https://community.platformio.org) or [Premium Support](https://platformio.org/support)
 
 - [ ] **PlatformIO IDE**.
-      All issues related to PlatformIO IDE should be reported to the
-      [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues) repository
+      All issues related to PlatformIO IDE should be reported to appropriate repository:
+      [PlatformIO IDE for Atom](https://github.com/platformio/platformio-atom-ide/issues) or
+      [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues)
 
 - [ ] **Development Platform or Board**.
       All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms

.github/workflows/core.yml

@@ -1,47 +0,0 @@
-name: Core
-
-on: [push, pull_request]
-
-jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-20.04, windows-latest, macos-latest]
-        python-version: ["3.6", "3.9", "3.11"]
-
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: "recursive"
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
-
-      - name: Python Lint
-        if: ${{ matrix.python-version != '3.6' }}
-        run: |
-          tox -e lint
-
-      - name: Integration Tests
-        run: |
-          tox -e testcore
-
-      - name: Slack Notification
-        uses: homoluctus/slatify@master
-        if: failure()
-        with:
-          type: ${{ job.status }}
-          job_name: '*Core*'
-          commit: true
-          url: ${{ secrets.SLACK_BUILD_WEBHOOK }}
-          token: ${{ secrets.SLACK_GITHUB_TOKEN }}

.github/workflows/deployment.yml

@@ -1,45 +0,0 @@
-name: Deployment
-
-on:
-  push:
-    branches:
-      - "master"
-      - "release/**"
-
-jobs:
-  deployment:
-    runs-on: ubuntu-latest
-    environment: production
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: "recursive"
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
-
-      - name: Deployment Tests
-        env:
-          TEST_EMAIL_LOGIN: ${{ secrets.TEST_EMAIL_LOGIN }}
-          TEST_EMAIL_PASSWORD: ${{ secrets.TEST_EMAIL_PASSWORD }}
-          TEST_EMAIL_IMAP_SERVER: ${{ secrets.TEST_EMAIL_IMAP_SERVER }}
-        run: |
-          tox -e testcore
-
-      - name: Build Python source tarball
-        run:  python setup.py sdist bdist_wheel
-
-      - name: Publish package to PyPI
-        if: ${{ github.ref == 'refs/heads/master' }}
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/docs.yml

@@ -1,109 +0,0 @@
-name: Docs
-
-on: [push, pull_request]
-
-jobs:
-  build:
-    name: Build Docs
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: "recursive"
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
-
-      - name: Build docs
-        run: |
-          tox -e docs
-
-      - name: Slack Notification
-        uses: homoluctus/slatify@master
-        if: failure()
-        with:
-          type: ${{ job.status }}
-          job_name: '*Docs*'
-          commit: true
-          url: ${{ secrets.SLACK_BUILD_WEBHOOK }}
-          token: ${{ secrets.SLACK_GITHUB_TOKEN }}
-
-      - name: Preserve Docs
-        if: ${{ github.event_name == 'push' }}
-        run: |
-          tar -czvf docs.tar.gz -C docs/_build html rtdpage
-
-      - name: Save artifact
-        if: ${{ github.event_name == 'push' }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: docs
-          path: ./docs.tar.gz
-
-  deploy:
-    name: Deploy Docs
-    needs: build
-    runs-on: ubuntu-latest
-    env:
-      DOCS_REPO: platformio/platformio-docs
-      DOCS_DIR: platformio-docs
-      LATEST_DOCS_DIR: latest-docs
-      RELEASE_BUILD: ${{ startsWith(github.ref, 'refs/tags/v') }}
-    if: ${{ github.event_name == 'push' }}
-    steps:
-      - name: Download artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: docs
-      - name: Unpack artifact
-        run: |
-          mkdir ./${{ env.LATEST_DOCS_DIR }}
-          tar -xzf ./docs.tar.gz -C ./${{ env.LATEST_DOCS_DIR }}
-      - name: Delete Artifact
-        uses: geekyeggo/delete-artifact@v2
-        with:
-          name: docs
-      - name: Select Docs type
-        id: get-destination-dir
-        run: |
-          if [[ ${{ env.RELEASE_BUILD }} == true ]]; then
-            echo "::set-output name=dst_dir::stable"
-          else
-            echo "::set-output name=dst_dir::latest"
-          fi
-      - name: Checkout latest Docs
-        continue-on-error: true
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ env.DOCS_REPO }}
-          path: ${{ env.DOCS_DIR }}
-          ref: gh-pages
-      - name: Synchronize Docs
-        run: |
-          rm -rf ${{ env.DOCS_DIR }}/.git
-          rm -rf ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }}
-          mkdir -p ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }}
-          cp -rf ${{ env.LATEST_DOCS_DIR }}/html/* ${{ env.DOCS_DIR }}/en/${{ steps.get-destination-dir.outputs.dst_dir }}
-          if [[ ${{ env.RELEASE_BUILD }} == false ]]; then
-            rm -rf ${{ env.DOCS_DIR }}/page
-            mkdir -p ${{ env.DOCS_DIR }}/page
-            cp -rf ${{ env.LATEST_DOCS_DIR }}/rtdpage/* ${{ env.DOCS_DIR }}/page
-          fi
-      - name: Validate Docs
-        run: |
-          if [ -z "$(ls -A ${{ env.DOCS_DIR }})" ]; then
-             echo "Docs folder is empty. Aborting!"
-             exit 1
-          fi
-      - name: Deploy to Github Pages
-        uses: peaceiris/actions-gh-pages@v3
-        with:
-          personal_token: ${{ secrets.DEPLOY_GH_DOCS_TOKEN }}
-          external_repository: ${{ env.DOCS_REPO }}
-          publish_dir: ./${{ env.DOCS_DIR }}
-          commit_message: Sync Docs

.github/workflows/examples.yml

@@ -1,63 +0,0 @@
-name: Examples
-
-on: [push, pull_request]
-
-
-jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-    env:
-      PIO_INSTALL_DEVPLATFORM_OWNERNAMES: "platformio"
-      PIO_INSTALL_DEVPLATFORM_NAMES: "aceinna_imu,atmelavr,atmelmegaavr,atmelsam,espressif32,espressif8266,nordicnrf52,raspberrypi,ststm32,teensy"
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: "recursive"
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
-
-      - name: Run on Linux
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          # Free space
-          sudo apt clean
-          docker rmi $(docker image ls -aq)
-          df -h
-          tox -e testexamples
-
-      - name: Run on macOS
-        if: startsWith(matrix.os, 'macos')
-        run: |
-          df -h
-          tox -e testexamples
-
-      - name: Run on Windows
-        if: startsWith(matrix.os, 'windows')
-        env:
-          PLATFORMIO_CORE_DIR: C:/pio
-          PLATFORMIO_WORKSPACE_DIR: C:/pio-workspace/$PROJECT_HASH
-        run: |
-          tox -e testexamples
-
-      - name: Slack Notification
-        uses: homoluctus/slatify@master
-        if: failure()
-        with:
-          type: ${{ job.status }}
-          job_name: '*Examples*'
-          commit: true
-          url: ${{ secrets.SLACK_BUILD_WEBHOOK }}
-          token: ${{ secrets.SLACK_GITHUB_TOKEN }}

.github/workflows/projects.yml

@@ -1,69 +0,0 @@
-name: Projects
-
-on: [push, pull_request]
-
-jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        project:
-          - marlin:
-            repository: "MarlinFirmware/Marlin"
-            folder: "Marlin"
-            config_dir: "Marlin"
-            env_name: "mega2560"
-          - esphome:
-            repository: "esphome/esphome"
-            folder: "esphome"
-            config_dir: "esphome"
-            env_name: "esp32-arduino"
-          - smartknob:
-            repository: "scottbez1/smartknob"
-            folder: "smartknob"
-            config_dir: "smartknob"
-            env_name: "view"
-          - espurna:
-            repository: "xoseperez/espurna"
-            folder: "espurna"
-            config_dir: "espurna/code"
-            env_name: "nodemcu-lolin"
-          - OpenMQTTGateway:
-            repository: "1technophile/OpenMQTTGateway"
-            folder: "OpenMQTTGateway"
-            config_dir: "OpenMQTTGateway"
-            env_name: "esp32-m5atom-lite"
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        exclude:
-        - os: windows-latest
-          project: {"esphome": "", "repository": "esphome/esphome", "folder": "esphome", "config_dir": "esphome", "env_name": "esp32-arduino"}
-
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: "recursive"
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-
-      - name: Install PlatformIO
-        run: pip install -U .
-
-      - name: Check out ${{ matrix.project.repository }}
-        uses: actions/checkout@v3
-        with:
-          submodules: "recursive"
-          repository: ${{ matrix.project.repository }}
-          path: ${{ matrix.project.folder }}
-
-      - name: Install ESPHome dependencies
-        # Requires esptool package as it's used in a custom prescript
-        if: ${{ contains(matrix.project.repository, 'esphome') }}
-        run: pip install esptool==3.*
-
-      - name: Compile ${{ matrix.project.repository }}
-        run: pio run -d ${{ matrix.project.config_dir }} -e ${{ matrix.project.env_name }}
-

.gitignore

@@ -1,6 +1,6 @@
 *.egg-info
 *.pyc
-__pycache__
+.pioenvs
 .tox
 docs/_build
 dist

.isort.cfg

@@ -0,0 +1,3 @@
+[settings]
+line_length=79
+known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,jsonrpc,tabulate

.pylintrc

@@ -1,12 +1,12 @@
-[REPORTS]
-output-format=colorized
-
 [MESSAGES CONTROL]
 disable=
     missing-docstring,
-    duplicate-code,
+    ungrouped-imports,
     invalid-name,
-    too-few-public-methods,
-    consider-using-f-string,
     cyclic-import,
-    use-dict-literal
+    duplicate-code,
+    superfluous-parens,
+    too-few-public-methods,
+    useless-object-inheritance,
+    useless-import-alias,
+    fixme

.style.yapf

@@ -0,0 +1,3 @@
+[style]
+blank_line_before_nested_class_or_def = true
+allow_multiline_lambdas = true

.travis.yml

@@ -0,0 +1,39 @@
+language: python
+
+matrix:
+  include:
+    - os: linux
+      sudo: false
+      python: 2.7
+      env: TOX_ENV=docs
+    - os: linux
+      sudo: required
+      python: 2.7
+      env: TOX_ENV=py27 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d)
+    - os: linux
+      sudo: required
+      python: 3.6
+      env: TOX_ENV=py36 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d)
+    - os: osx
+      language: generic
+      env: TOX_ENV=skipexamples
+
+install:
+  - git submodule update --init --recursive
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl -fsSL https://bootstrap.pypa.io/get-pip.py | sudo python; fi
+  - pip install -U tox
+
+  # ChipKIT issue: install 32-bit support for GCC PIC32
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi
+
+script:
+  - tox -e $TOX_ENV
+
+notifications:
+  email: false
+
+  slack:
+    rooms:
+      secure: JD6VGfN4+SLU2CwDdiIOr1VgwD+zbYUCE/srwyGuHavnjIkPItkl6T6Bn8Y4VrU6ysbuKotfdV2TAJJ82ivFbY8BvZBc7FBcYp/AGQ4FaCCV5ySv8RDAcQgdE12oaGzMdODiLqsB85f65zOlAFa+htaXyEiRTcotn6Y2hupatrI=
+    on_failure: always
+    on_success: change

CODE_OF_CONDUCT.md

@@ -1,3 +0,0 @@
-# Code of Conduct
-
-See https://piolabs.com/legal/code-of-conduct.html

CONTRIBUTING.md

@@ -1,22 +1,21 @@
 Contributing
 ------------
 
-To get started, <a href="https://cla-assistant.io/platformio/platformio-core">sign the Contributor License Agreement</a>.
+To get started, <a href="https://www.clahub.com/agreements/platformio/platformio-core">sign the Contributor License Agreement</a>.
 
-1. Fork the repository on GitHub
+1. Fork the repository on GitHub.
 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git`
 3. Run `pip install tox`
-4. Go to the root of the PlatformIO Core project where `tox.ini` is located (``cd platformio-core``) and run `tox -e py39`.
-   You can replace `py39` with your own Python version. For example, `py311` means Python 3.11.
+4. Go to the root of project where is located `tox.ini` and run `tox -e py27`
 5. Activate current development environment:
 
-   * Windows: `.tox\py39\Scripts\activate`
-   * Bash/ZSH: `source .tox/py39/bin/activate`
-   * Fish: `source .tox/py39/bin/activate.fish`
+   * Windows: `.tox\py27\Scripts\activate`
+   * Bash/ZSH: `source .tox/py27/bin/activate`
+   * Fish: `source .tox/py27/bin/activate.fish`
 
 6. Make changes to code, documentation, etc.
-7. Lint source code `make before-commit`
+7. Lint source code `make lint`
 8. Run the tests `make test`
 9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html)
 10. Commit changes to your forked repository
-11. Submit a Pull Request on GitHub
+11. Submit a Pull Request on GitHub.

Makefile

@@ -1,19 +1,17 @@
 lint:
-	pylint --rcfile=./.pylintrc ./tests
 	pylint --rcfile=./.pylintrc ./platformio
 
 isort:
-	isort ./platformio
-	isort ./tests
+	isort -rc ./platformio
+	isort -rc ./tests
 
-format:
-	black ./platformio
-	black ./tests
+yapf:
+	yapf --recursive --in-place platformio/
 
 test:
-	py.test --verbose --exitfirst -n 6 --dist=loadscope tests --ignore tests/test_examples.py
+	py.test --verbose --capture=no --exitfirst -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py
 
-before-commit: isort format lint
+before-commit: isort yapf lint test
 
 clean-docs:
 	rm -rf docs/_build
@@ -28,11 +26,8 @@ clean: clean-docs
 
 profile:
 	# Usage $ > make PIOARGS="boards" profile
-	python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS}
+	python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS}
 	snakeviz .tox/.tmp/cprofile.prof
 
-pack:
-	python setup.py sdist
-
 publish:
 	python setup.py sdist upload

README.rst

@@ -1,79 +1,127 @@
-PlatformIO Core
-===============
+PlatformIO
+==========
 
-.. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg
-    :target: https://docs.platformio.org/en/latest/core/index.html
-    :alt:  CI Build for PlatformIO Core
-.. image:: https://github.com/platformio/platformio-core/workflows/Docs/badge.svg
-    :target: https://docs.platformio.org?utm_source=github&utm_medium=core
-    :alt:  CI Build for Docs
-.. image:: https://github.com/platformio/platformio-core/workflows/Examples/badge.svg
-    :target: https://github.com/platformio/platformio-examples
-    :alt:  CI Build for dev-platform examples
-.. image:: https://github.com/platformio/platformio-core/workflows/Projects/badge.svg
-    :target: https://docs.platformio.org/en/latest/tutorials/index.html#projects
-    :alt:  CI Build for the Community Projects
+.. image:: https://travis-ci.org/platformio/platformio-core.svg?branch=develop
+    :target: https://travis-ci.org/platformio/platformio-core
+    :alt: Travis.CI Build Status
+.. image:: https://ci.appveyor.com/api/projects/status/unnpw0n3c5k14btn/branch/develop?svg=true
+    :target: https://ci.appveyor.com/project/ivankravets/platformio-core
+    :alt: AppVeyor.CI Build Status
 .. image:: https://img.shields.io/pypi/v/platformio.svg
     :target: https://pypi.python.org/pypi/platformio/
     :alt: Latest Version
-.. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg
-   :alt: PlatformIO Labs
-   :target: https://piolabs.com/?utm_source=github&utm_medium=core
+.. image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg
+    :target: https://pypi.python.org/pypi/platformio/
+    :alt:  License
+.. image:: https://img.shields.io/badge/PlatformIO-Community-orange.svg
+   :alt: Community Forums
+   :target: https://community.platformio.org?utm_source=github&utm_medium=core
 
-**Quick Links:** `Homepage <https://platformio.org?utm_source=github&utm_medium=core>`_ |
+**Quick Links:** `Web <https://platformio.org?utm_source=github&utm_medium=core>`_ |
 `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_ |
-`Registry <https://registry.platformio.org?utm_source=github&utm_medium=core>`_ |
 `Project Examples <https://github.com/platformio/platformio-examples/>`__ |
 `Docs <https://docs.platformio.org?utm_source=github&utm_medium=core>`_ |
 `Donate <https://platformio.org/donate?utm_source=github&utm_medium=core>`_ |
-`Contact Us <https://piolabs.com/?utm_source=github&utm_medium=core>`_
+`Contact Us <https://platformio.org/contact?utm_source=github&utm_medium=core>`_
 
-**Social:** `LinkedIn <https://www.linkedin.com/company/platformio/>`_ |
-`Twitter <https://twitter.com/PlatformIO_Org>`_ |
+**Social:** `Twitter <https://twitter.com/PlatformIO_Org>`_ |
+`LinkedIn <https://www.linkedin.com/company/platformio/>`_ |
 `Facebook <https://www.facebook.com/platformio>`_ |
-`Community Forums <https://community.platformio.org?utm_source=github&utm_medium=core>`_
+`Hackaday <https://hackaday.io/project/7980-platformio>`_ |
+`Bintray <https://bintray.com/platformio>`_ |
+`Community <https://community.platformio.org?utm_source=github&utm_medium=core>`_
 
 .. image:: https://raw.githubusercontent.com/platformio/platformio-web/develop/app/images/platformio-ide-laptop.png
     :target: https://platformio.org?utm_source=github&utm_medium=core
 
-`PlatformIO <https://platformio.org>`_ is a professional collaborative platform for embedded development.
-
-**A place where Developers and Teams have true Freedom! No more vendor lock-in!**
-
-* Open source, maximum permissive Apache 2.0 license
-* Cross-platform IDE and Unified Debugger
-* Static Code Analyzer and Remote Unit Testing
-* Multi-platform and Multi-architecture Build System
-* Firmware File Explorer and Memory Inspection
+`PlatformIO <https://platformio.org?utm_source=github&utm_medium=core>`_ is an open source ecosystem for IoT
+development. Cross-platform IDE and unified debugger. Remote unit testing and
+firmware updates.
 
 Get Started
 -----------
 
 * `What is PlatformIO? <https://docs.platformio.org/en/latest/what-is-platformio.html?utm_source=github&utm_medium=core>`_
+
+Open Source
+-----------
+
 * `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_
 * `PlatformIO Core (CLI) <https://docs.platformio.org/en/latest/core.html?utm_source=github&utm_medium=core>`_
+* `Library Management <https://docs.platformio.org/page/librarymanager/index.html?utm_source=github&utm_medium=core>`_
 * `Project Examples <https://github.com/platformio/platformio-examples?utm_source=github&utm_medium=core>`__
+* `Desktop IDEs Integration <https://docs.platformio.org/page/ide.html?utm_source=github&utm_medium=core>`_
+* `Continuous Integration <https://docs.platformio.org/page/ci/index.html?utm_source=github&utm_medium=core>`_
+* `Advanced Scripting API <https://docs.platformio.org/page/projectconf/advanced_scripting.html?utm_source=github&utm_medium=core>`_
 
-Solutions
----------
+PIO Plus
+--------
 
-* `Library Management <https://docs.platformio.org/en/latest/librarymanager/index.html?utm_source=github&utm_medium=core>`_
-* `Desktop IDEs Integration <https://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core>`_
-* `Continuous Integration <https://docs.platformio.org/en/latest/ci/index.html?utm_source=github&utm_medium=core>`_
-
-**Advanced**
-
-* `Debugging <https://docs.platformio.org/en/latest/plus/debugging.html?utm_source=github&utm_medium=core>`_
-* `Unit Testing <https://docs.platformio.org/en/latest/advanced/unit-testing/index.html?utm_source=github&utm_medium=core>`_
-* `Static Code Analysis <https://docs.platformio.org/en/latest/plus/pio-check.html?utm_source=github&utm_medium=core>`_
-* `Remote Development <https://docs.platformio.org/en/latest/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
+* `PIO Remote <https://docs.platformio.org/page/plus/pio-remote.html?utm_source=github&utm_medium=core>`_
+* `PIO Unified Debugger <https://docs.platformio.org/page/plus/debugging.html?utm_source=github&utm_medium=core>`_
+* `PIO Unit Testing <https://docs.platformio.org/en/latest/plus/unit-testing.html?utm_source=github&utm_medium=core>`_
+* `Cloud IDEs Integration <https://docs.platformio.org/en/latest/ide.html?utm_source=github&utm_medium=core#solution-pio-delivery>`_
+* `Integration Services <https://platformio.org/pricing?utm_source=github&utm_medium=core#enterprise-features>`_
 
 Registry
 --------
 
-* `Libraries <https://registry.platformio.org/search?t=library&utm_source=github&utm_medium=core>`_
-* `Development Platforms <https://registry.platformio.org/search?t=platform&utm_source=github&utm_medium=core>`_
-* `Development Tools <https://registry.platformio.org/search?t=tool&utm_source=github&utm_medium=core>`_
+* `Libraries <https://platformio.org/lib?utm_source=github&utm_medium=core>`_
+* `Development Platforms <https://platformio.org/platforms?utm_source=github&utm_medium=core>`_
+* `Frameworks <https://platformio.org/frameworks?utm_source=github&utm_medium=core>`_
+* `Embedded Boards <https://platformio.org/boards?utm_source=github&utm_medium=core>`_
+
+Development Platforms
+---------------------
+
+* `Aceinna IMU <https://platformio.org/platforms/aceinna_imu?utm_source=github&utm_medium=core>`_
+* `Atmel AVR <https://platformio.org/platforms/atmelavr?utm_source=github&utm_medium=core>`_
+* `Atmel SAM <https://platformio.org/platforms/atmelsam?utm_source=github&utm_medium=core>`_
+* `Espressif 32 <https://platformio.org/platforms/espressif32?utm_source=github&utm_medium=core>`_
+* `Espressif 8266 <https://platformio.org/platforms/espressif8266?utm_source=github&utm_medium=core>`_
+* `Freescale Kinetis <https://platformio.org/platforms/freescalekinetis?utm_source=github&utm_medium=core>`_
+* `Infineon XMC <https://platformio.org/platforms/infineonxmc?utm_source=github&utm_medium=core>`_
+* `Intel ARC32 <https://platformio.org/platforms/intel_arc32?utm_source=github&utm_medium=core>`_
+* `Intel MCS-51 (8051) <https://platformio.org/platforms/intel_mcs51?utm_source=github&utm_medium=core>`_
+* `Kendryte K210 <https://platformio.org/platforms/kendryte210?utm_source=github&utm_medium=core>`_
+* `Lattice iCE40 <https://platformio.org/platforms/lattice_ice40?utm_source=github&utm_medium=core>`_
+* `Maxim 32 <https://platformio.org/platforms/maxim32?utm_source=github&utm_medium=core>`_
+* `Microchip PIC32 <https://platformio.org/platforms/microchippic32?utm_source=github&utm_medium=core>`_
+* `Nordic nRF51 <https://platformio.org/platforms/nordicnrf51?utm_source=github&utm_medium=core>`_
+* `Nordic nRF52 <https://platformio.org/platforms/nordicnrf52?utm_source=github&utm_medium=core>`_
+* `NXP LPC <https://platformio.org/platforms/nxplpc?utm_source=github&utm_medium=core>`_
+* `RISC-V <https://platformio.org/platforms/riscv?utm_source=github&utm_medium=core>`_
+* `RISC-V GAP <https://platformio.org/platforms/riscv_gap?utm_source=github&utm_medium=core>`_
+* `Samsung ARTIK <https://platformio.org/platforms/samsung_artik?utm_source=github&utm_medium=core>`_
+* `Silicon Labs EFM32 <https://platformio.org/platforms/siliconlabsefm32?utm_source=github&utm_medium=core>`_
+* `ST STM32 <https://platformio.org/platforms/ststm32?utm_source=github&utm_medium=core>`_
+* `ST STM8 <https://platformio.org/platforms/ststm8?utm_source=github&utm_medium=core>`_
+* `Teensy <https://platformio.org/platforms/teensy?utm_source=github&utm_medium=core>`_
+* `TI MSP430 <https://platformio.org/platforms/timsp430?utm_source=github&utm_medium=core>`_
+* `TI Tiva <https://platformio.org/platforms/titiva?utm_source=github&utm_medium=core>`_
+* `WIZNet W7500 <https://platformio.org/platforms/wiznet7500?utm_source=github&utm_medium=core>`_
+
+Frameworks
+----------
+
+* `Arduino <https://platformio.org/frameworks/arduino?utm_source=github&utm_medium=core>`_
+* `ARTIK SDK <https://platformio.org/frameworks/artik-sdk?utm_source=github&utm_medium=core>`_
+* `CMSIS <https://platformio.org/frameworks/cmsis?utm_source=github&utm_medium=core>`_
+* `Energia <https://platformio.org/frameworks/energia?utm_source=github&utm_medium=core>`_
+* `ESP-IDF <https://platformio.org/frameworks/espidf?utm_source=github&utm_medium=core>`_
+* `ESP8266 Non-OS SDK <https://platformio.org/frameworks/esp8266-nonos-sdk?utm_source=github&utm_medium=core>`_
+* `ESP8266 RTOS SDK <https://platformio.org/frameworks/esp8266-rtos-sdk?utm_source=github&utm_medium=core>`_
+* `Freedom E SDK <https://platformio.org/frameworks/freedom-e-sdk?utm_source=github&utm_medium=core>`_
+* `Kendryte Standalone SDK <https://platformio.org/frameworks/kendryte-standalone-sdk?utm_source=github&utm_medium=core>`_
+* `libOpenCM3 <https://platformio.org/frameworks/libopencm3?utm_source=github&utm_medium=core>`_
+* `mbed <https://platformio.org/frameworks/mbed?utm_source=github&utm_medium=core>`_
+* `PULP OS <https://platformio.org/frameworks/pulp-os?utm_source=github&utm_medium=core>`_
+* `Pumbaa <https://platformio.org/frameworks/pumbaa?utm_source=github&utm_medium=core>`_
+* `Simba <https://platformio.org/frameworks/simba?utm_source=github&utm_medium=core>`_
+* `SPL <https://platformio.org/frameworks/spl?utm_source=github&utm_medium=core>`_
+* `STM32Cube <https://platformio.org/frameworks/stm32cube?utm_source=github&utm_medium=core>`_
+* `Tizen RT <https://platformio.org/frameworks/tizenrt?utm_source=github&utm_medium=core>`_
+* `WiringPi <https://platformio.org/frameworks/wiringpi?utm_source=github&utm_medium=core>`_
 
 Contributing
 ------------
@@ -87,6 +135,7 @@ Share minimal diagnostics and usage information to help us make PlatformIO bette
 It is enabled by default. For more information see:
 
 * `Telemetry Setting <https://docs.platformio.org/en/latest/userguide/cmd_settings.html?utm_source=github&utm_medium=core#enable-telemetry>`_
+* `SSL Setting <https://docs.platformio.org/en/latest/userguide/cmd_settings.html?utm_source=github&utm_medium=core#strict-ssl>`_
 
 License
 -------
@@ -95,7 +144,3 @@ Copyright (c) 2014-present PlatformIO <contact@platformio.org>
 
 The PlatformIO is licensed under the permissive Apache 2.0 license,
 so you can use it in both commercial and personal projects with confidence.
-
-.. image:: https://raw.githubusercontent.com/vshymanskyy/StandWithUkraine/main/banner-direct.svg
-    :target: https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md
-    :alt:  SWUbanner

SECURITY.md

@@ -1,34 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-We are committed to ensuring the security and protection of PlatformIO Core. 
-To this end, we support only the following versions:
-
-| Version | Supported          |
-| ------- | ------------------ |
-| 6.1.x   | :white_check_mark: |
-| < 6.1   | :x:                |
-
-Unsupported versions of the PlatformIO Core may have known vulnerabilities or security issues that could compromise the security of our organization's systems and data. 
-Therefore, it is important that all developers use only supported versions of the PlatformIO Core.
-
-## Reporting a Vulnerability
-
-We take the security of our systems and data very seriously. We encourage responsible disclosure of any vulnerabilities or security issues that you may find in our systems or applications. If you believe you have discovered a vulnerability, please report it to us immediately.
-
-To report a vulnerability, please send an email to our security team at contact@piolabs.com. Please include as much information as possible, including:
-
-- A description of the vulnerability and how it can be exploited
-- Steps to reproduce the vulnerability
-- Any additional information that can help us understand and reproduce the vulnerability
-
-Once we receive your report, our security team will acknowledge receipt within 24 hours and will work to validate the reported vulnerability. We will provide periodic updates on the progress of the vulnerability assessment, and will notify you once a fix has been deployed.
-
-If the vulnerability is accepted, we will work to remediate the issue as quickly as possible. We may also provide credit or recognition to the individual who reported the vulnerability, at our discretion.
-
-If the vulnerability is declined, we will provide a justification for our decision and may offer guidance on how to improve the report or how to test the system more effectively.
-
-Please note that we will not take any legal action against individuals who report vulnerabilities in good faith and in accordance with this policy.
-
-Thank you for helping us keep our systems and data secure.

platformio/__init__.py

@@ -12,47 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (6, 1, 8)
+VERSION = (4, 1, "0a1")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
 __description__ = (
-    "A professional collaborative platform for embedded development. "
-    "Cross-platform IDE and Unified Debugger. "
-    "Static Code Analyzer and Remote Unit Testing. "
-    "Multi-platform and Multi-architecture Build System. "
-    "Firmware File Explorer and Memory Inspection. "
-    "IoT, Arduino, CMSIS, ESP-IDF, FreeRTOS, libOpenCM3, mbedOS, Pulp OS, SPL, "
-    "STM32Cube, Zephyr RTOS, ARM, AVR, Espressif (ESP8266/ESP32), FPGA, "
-    "MCS-51 (8051), MSP430, Nordic (nRF51/nRF52), NXP i.MX RT, PIC32, RISC-V, "
-    "STMicroelectronics (STM8/STM32), Teensy"
-)
+    "An open source ecosystem for IoT development. "
+    "Cross-platform IDE and unified debugger. "
+    "Remote unit testing and firmware updates. "
+    "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, "
+    "FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3")
 __url__ = "https://platformio.org"
 
-__author__ = "PlatformIO Labs"
-__email__ = "contact@piolabs.com"
+__author__ = "PlatformIO"
+__email__ = "contact@platformio.org"
 
 __license__ = "Apache Software License"
-__copyright__ = "Copyright 2014-present PlatformIO Labs"
+__copyright__ = "Copyright 2014-present PlatformIO"
 
-__accounts_api__ = "https://api.accounts.platformio.org"
-__registry_mirror_hosts__ = [
-    "registry.platformio.org",
-    "registry.nm1.platformio.org",
-]
-__pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
-
-__core_packages__ = {
-    "contrib-piohome": "~3.4.2",
-    "contrib-pioremote": "~1.0.0",
-    "tool-scons": "~4.40502.0",
-    "tool-cppcheck": "~1.270.0",
-    "tool-clangtidy": "~1.150005.0",
-    "tool-pvs-studio": "~7.18.0",
-}
-
-__check_internet_hosts__ = [
-    "185.199.110.153",  # Github.com
-    "88.198.170.159",  # platformio.org
-    "github.com",
-] + __registry_mirror_hosts__
+__apiurl__ = "https://api.platformio.org"

platformio/__main__.py

@@ -14,67 +14,57 @@
 
 import os
 import sys
-import traceback
+from traceback import format_exc
 
 import click
 
-from platformio import __version__, exception, maintenance
-from platformio.cli import PlatformioCLI
-from platformio.compat import IS_CYGWIN, ensure_python3
+from platformio import __version__, exception, maintenance, util
+from platformio.commands import PlatformioCLI
+from platformio.compat import CYGWIN
 
 
-@click.command(
-    cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])
-)
-@click.version_option(__version__, prog_name="PlatformIO Core")
-@click.option("--force", "-f", is_flag=True, help="DEPRECATED", hidden=True)
-@click.option("--caller", "-c", help="Caller ID (service)")
-@click.option("--no-ansi", is_flag=True, help="Do not print ANSI control characters")
+@click.command(cls=PlatformioCLI,
+               context_settings=dict(help_option_names=["-h", "--help"]))
+@click.version_option(__version__, prog_name="PlatformIO")
+@click.option("--force",
+              "-f",
+              is_flag=True,
+              help="Force to accept any confirmation prompts.")
+@click.option("--caller", "-c", help="Caller ID (service).")
 @click.pass_context
-def cli(ctx, force, caller, no_ansi):  # pylint: disable=unused-argument
-    try:
-        if (
-            no_ansi
-            or str(
-                os.getenv("PLATFORMIO_NO_ANSI", os.getenv("PLATFORMIO_DISABLE_COLOR"))
-            ).lower()
-            == "true"
-        ):
-            # pylint: disable=protected-access
-            click._compat.isatty = lambda stream: False
-        elif (
-            str(
-                os.getenv("PLATFORMIO_FORCE_ANSI", os.getenv("PLATFORMIO_FORCE_COLOR"))
-            ).lower()
-            == "true"
-        ):
-            # pylint: disable=protected-access
-            click._compat.isatty = lambda stream: True
-    except:  # pylint: disable=bare-except
-        pass
-
-    maintenance.on_cmd_start(ctx, caller)
+def cli(ctx, force, caller):
+    maintenance.on_platformio_start(ctx, force, caller)
 
 
-@cli.result_callback()
+@cli.resultcallback()
 @click.pass_context
-def process_result(*_, **__):
-    maintenance.on_cmd_end()
+def process_result(ctx, result, force, caller):  # pylint: disable=W0613
+    maintenance.on_platformio_end(ctx, result)
 
 
+@util.memoized()
 def configure():
-    if IS_CYGWIN:
+    if CYGWIN:
         raise exception.CygwinEnvDetected()
 
     # https://urllib3.readthedocs.org
     # /en/latest/security.html#insecureplatformwarning
     try:
-        import urllib3  # pylint: disable=import-outside-toplevel
-
+        import urllib3
         urllib3.disable_warnings()
     except (AttributeError, ImportError):
         pass
 
+    try:
+        if str(os.getenv("PLATFORMIO_DISABLE_COLOR", "")).lower() == "true":
+            # pylint: disable=protected-access
+            click._compat.isatty = lambda stream: False
+        elif str(os.getenv("PLATFORMIO_FORCE_COLOR", "")).lower() == "true":
+            # pylint: disable=protected-access
+            click._compat.isatty = lambda stream: True
+    except:  # pylint: disable=bare-except
+        pass
+
     # Handle IOError issue with VSCode's Terminal (Windows)
     click_echo_origin = [click.echo, click.secho]
 
@@ -83,8 +73,7 @@ def configure():
             click_echo_origin[origin](*args, **kwargs)
         except IOError:
             (sys.stderr.write if kwargs.get("err") else sys.stdout.write)(
-                "%s\n" % (args[0] if args else "")
-            )
+                "%s\n" % (args[0] if args else ""))
 
     click.echo = lambda *args, **kwargs: _safe_echo(0, *args, **kwargs)
     click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs)
@@ -96,32 +85,29 @@ def main(argv=None):
     if argv:
         assert isinstance(argv, list)
         sys.argv = argv
-
     try:
-        ensure_python3(raise_exception=True)
         configure()
-        cli()  # pylint: disable=no-value-for-parameter
-    except SystemExit as exc:
-        if exc.code and str(exc.code).isdigit():
-            exit_code = int(exc.code)
-    except Exception as exc:  # pylint: disable=broad-except
-        if not isinstance(exc, exception.ReturnErrorCode):
-            maintenance.on_platformio_exception(exc)
-            error_str = f"{exc.__class__.__name__}: "
-            if isinstance(exc, exception.PlatformioException):
-                error_str += str(exc)
+        cli(None, None, None)
+    except SystemExit:
+        pass
+    except Exception as e:  # pylint: disable=broad-except
+        if not isinstance(e, exception.ReturnErrorCode):
+            maintenance.on_platformio_exception(e)
+            error_str = "Error: "
+            if isinstance(e, exception.PlatformioException):
+                error_str += str(e)
             else:
-                error_str += traceback.format_exc()
+                error_str += format_exc()
                 error_str += """
 ============================================================
 
 An unexpected error occurred. Further steps:
 
 * Verify that you have the latest version of PlatformIO using
-  `python -m pip install -U platformio` command
+  `pip install -U platformio` command
 
 * Try to find answer in FAQ Troubleshooting section
-  https://docs.platformio.org/page/faq/index.html
+  https://docs.platformio.org/page/faq.html
 
 * Report this problem to the developers
   https://github.com/platformio/platformio-core/issues
@@ -129,9 +115,7 @@ An unexpected error occurred. Further steps:
 ============================================================
 """
             click.secho(error_str, fg="red", err=True)
-        exit_code = int(str(exc)) if str(exc).isdigit() else 1
-
-    maintenance.on_platformio_exit()
+        exit_code = int(str(e)) if str(e).isdigit() else 1
     sys.argv = prev_sys_argv
     return exit_code
 

platformio/account/cli.py

@@ -1,44 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.commands.destroy import account_destroy_cmd
-from platformio.account.commands.forgot import account_forgot_cmd
-from platformio.account.commands.login import account_login_cmd
-from platformio.account.commands.logout import account_logout_cmd
-from platformio.account.commands.password import account_password_cmd
-from platformio.account.commands.register import account_register_cmd
-from platformio.account.commands.show import account_show_cmd
-from platformio.account.commands.token import account_token_cmd
-from platformio.account.commands.update import account_update_cmd
-
-
-@click.group(
-    "account",
-    commands=[
-        account_destroy_cmd,
-        account_forgot_cmd,
-        account_login_cmd,
-        account_logout_cmd,
-        account_password_cmd,
-        account_register_cmd,
-        account_show_cmd,
-        account_token_cmd,
-        account_update_cmd,
-    ],
-    short_help="Manage PlatformIO account",
-)
-def cli():
-    pass

platformio/account/client.py

@@ -1,352 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import time
-
-from platformio import __accounts_api__, app
-from platformio.exception import PlatformioException, UserSideException
-from platformio.http import HTTPClient, HTTPClientError
-
-
-class AccountError(PlatformioException):
-    MESSAGE = "{0}"
-
-
-class AccountNotAuthorized(AccountError, UserSideException):
-    MESSAGE = "You are not authorized! Please log in to PlatformIO Account."
-
-
-class AccountAlreadyAuthorized(AccountError, UserSideException):
-    MESSAGE = "You are already authorized with {0} account."
-
-
-class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
-    SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7
-
-    def __init__(self):
-        super().__init__(__accounts_api__)
-
-    @staticmethod
-    def get_refresh_token():
-        try:
-            return app.get_state_item("account").get("auth").get("refresh_token")
-        except Exception as exc:
-            raise AccountNotAuthorized() from exc
-
-    @staticmethod
-    def delete_local_session():
-        app.delete_state_item("account")
-
-    @staticmethod
-    def delete_local_state(key):
-        account = app.get_state_item("account")
-        if not account or key not in account:
-            return
-        del account[key]
-        app.set_state_item("account", account)
-
-    def fetch_json_data(self, *args, **kwargs):
-        try:
-            return super().fetch_json_data(*args, **kwargs)
-        except HTTPClientError as exc:
-            raise AccountError(exc) from exc
-
-    def fetch_authentication_token(self):
-        if os.environ.get("PLATFORMIO_AUTH_TOKEN"):
-            return os.environ.get("PLATFORMIO_AUTH_TOKEN")
-        auth = app.get_state_item("account", {}).get("auth", {})
-        if auth.get("access_token") and auth.get("access_token_expire"):
-            if auth.get("access_token_expire") > time.time():
-                return auth.get("access_token")
-            if auth.get("refresh_token"):
-                try:
-                    data = self.fetch_json_data(
-                        "post",
-                        "/v1/login",
-                        headers={
-                            "Authorization": "Bearer %s" % auth.get("refresh_token")
-                        },
-                    )
-                    app.set_state_item("account", data)
-                    return data.get("auth").get("access_token")
-                except AccountError:
-                    self.delete_local_session()
-        raise AccountNotAuthorized()
-
-    def login(self, username, password):
-        try:
-            self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            pass
-        else:
-            raise AccountAlreadyAuthorized(
-                app.get_state_item("account", {}).get("email", "")
-            )
-
-        data = self.fetch_json_data(
-            "post",
-            "/v1/login",
-            data={"username": username, "password": password},
-        )
-        app.set_state_item("account", data)
-        return data
-
-    def login_with_code(self, client_id, code, redirect_uri):
-        try:
-            self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            pass
-        else:
-            raise AccountAlreadyAuthorized(
-                app.get_state_item("account", {}).get("email", "")
-            )
-
-        result = self.fetch_json_data(
-            "post",
-            "/v1/login/code",
-            data={"client_id": client_id, "code": code, "redirect_uri": redirect_uri},
-        )
-        app.set_state_item("account", result)
-        return result
-
-    def logout(self):
-        refresh_token = self.get_refresh_token()
-        self.delete_local_session()
-        try:
-            self.fetch_json_data(
-                "post",
-                "/v1/logout",
-                data={"refresh_token": refresh_token},
-            )
-        except AccountError:
-            pass
-        return True
-
-    def change_password(self, old_password, new_password):
-        return self.fetch_json_data(
-            "post",
-            "/v1/password",
-            data={"old_password": old_password, "new_password": new_password},
-            x_with_authorization=True,
-        )
-
-    def registration(
-        self, username, email, password, firstname, lastname
-    ):  # pylint:disable=too-many-arguments
-        try:
-            self.fetch_authentication_token()
-        except:  # pylint:disable=bare-except
-            pass
-        else:
-            raise AccountAlreadyAuthorized(
-                app.get_state_item("account", {}).get("email", "")
-            )
-
-        return self.fetch_json_data(
-            "post",
-            "/v1/registration",
-            data={
-                "username": username,
-                "email": email,
-                "password": password,
-                "firstname": firstname,
-                "lastname": lastname,
-            },
-        )
-
-    def auth_token(self, password, regenerate):
-        return self.fetch_json_data(
-            "post",
-            "/v1/token",
-            data={"password": password, "regenerate": 1 if regenerate else 0},
-            x_with_authorization=True,
-        ).get("auth_token")
-
-    def forgot_password(self, username):
-        return self.fetch_json_data(
-            "post",
-            "/v1/forgot",
-            data={"username": username},
-        )
-
-    def get_profile(self):
-        return self.fetch_json_data(
-            "get",
-            "/v1/profile",
-            x_with_authorization=True,
-        )
-
-    def update_profile(self, profile, current_password):
-        profile["current_password"] = current_password
-        self.delete_local_state("summary")
-        response = self.fetch_json_data(
-            "put",
-            "/v1/profile",
-            data=profile,
-            x_with_authorization=True,
-        )
-        return response
-
-    def get_account_info(self, offline=False):
-        account = app.get_state_item("account") or {}
-        if (
-            account.get("summary")
-            and account["summary"].get("expire_at", 0) > time.time()
-        ):
-            return account["summary"]
-        if offline and account.get("email"):
-            return {
-                "profile": {
-                    "email": account.get("email"),
-                    "username": account.get("username"),
-                }
-            }
-        result = self.fetch_json_data(
-            "get",
-            "/v1/summary",
-            x_with_authorization=True,
-        )
-        account["summary"] = dict(
-            profile=result.get("profile"),
-            packages=result.get("packages"),
-            subscriptions=result.get("subscriptions"),
-            user_id=result.get("user_id"),
-            expire_at=int(time.time()) + self.SUMMARY_CACHE_TTL,
-        )
-        app.set_state_item("account", account)
-        return result
-
-    def get_logged_username(self):
-        return self.get_account_info(offline=True).get("profile").get("username")
-
-    def destroy_account(self):
-        return self.fetch_json_data(
-            "delete",
-            "/v1/account",
-            x_with_authorization=True,
-        )
-
-    def create_org(self, orgname, email, displayname):
-        return self.fetch_json_data(
-            "post",
-            "/v1/orgs",
-            data={"orgname": orgname, "email": email, "displayname": displayname},
-            x_with_authorization=True,
-        )
-
-    def get_org(self, orgname):
-        return self.fetch_json_data(
-            "get",
-            "/v1/orgs/%s" % orgname,
-            x_with_authorization=True,
-        )
-
-    def list_orgs(self):
-        return self.fetch_json_data(
-            "get",
-            "/v1/orgs",
-            x_with_authorization=True,
-        )
-
-    def update_org(self, orgname, data):
-        return self.fetch_json_data(
-            "put",
-            "/v1/orgs/%s" % orgname,
-            data={k: v for k, v in data.items() if v},
-            x_with_authorization=True,
-        )
-
-    def destroy_org(self, orgname):
-        return self.fetch_json_data(
-            "delete",
-            "/v1/orgs/%s" % orgname,
-            x_with_authorization=True,
-        )
-
-    def add_org_owner(self, orgname, username):
-        return self.fetch_json_data(
-            "post",
-            "/v1/orgs/%s/owners" % orgname,
-            data={"username": username},
-            x_with_authorization=True,
-        )
-
-    def list_org_owners(self, orgname):
-        return self.fetch_json_data(
-            "get",
-            "/v1/orgs/%s/owners" % orgname,
-            x_with_authorization=True,
-        )
-
-    def remove_org_owner(self, orgname, username):
-        return self.fetch_json_data(
-            "delete",
-            "/v1/orgs/%s/owners" % orgname,
-            params={"username": username},
-            x_with_authorization=True,
-        )
-
-    def create_team(self, orgname, teamname, description):
-        return self.fetch_json_data(
-            "post",
-            "/v1/orgs/%s/teams" % orgname,
-            data={"name": teamname, "description": description},
-            x_with_authorization=True,
-        )
-
-    def destroy_team(self, orgname, teamname):
-        return self.fetch_json_data(
-            "delete",
-            "/v1/orgs/%s/teams/%s" % (orgname, teamname),
-            x_with_authorization=True,
-        )
-
-    def get_team(self, orgname, teamname):
-        return self.fetch_json_data(
-            "get",
-            "/v1/orgs/%s/teams/%s" % (orgname, teamname),
-            x_with_authorization=True,
-        )
-
-    def list_teams(self, orgname):
-        return self.fetch_json_data(
-            "get",
-            "/v1/orgs/%s/teams" % orgname,
-            x_with_authorization=True,
-        )
-
-    def update_team(self, orgname, teamname, data):
-        return self.fetch_json_data(
-            "put",
-            "/v1/orgs/%s/teams/%s" % (orgname, teamname),
-            data={k: v for k, v in data.items() if v},
-            x_with_authorization=True,
-        )
-
-    def add_team_member(self, orgname, teamname, username):
-        return self.fetch_json_data(
-            "post",
-            "/v1/orgs/%s/teams/%s/members" % (orgname, teamname),
-            data={"username": username},
-            x_with_authorization=True,
-        )
-
-    def remove_team_member(self, orgname, teamname, username):
-        return self.fetch_json_data(
-            "delete",
-            "/v1/orgs/%s/teams/%s/members" % (orgname, teamname),
-            params={"username": username},
-            x_with_authorization=True,
-        )

platformio/account/commands/destroy.py

@@ -1,37 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient, AccountNotAuthorized
-
-
-@click.command("destroy", short_help="Destroy account")
-def account_destroy_cmd():
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the %s user account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % client.get_logged_username(),
-        abort=True,
-    )
-    client.destroy_account()
-    try:
-        client.logout()
-    except AccountNotAuthorized:
-        pass
-    click.secho(
-        "User account has been destroyed.",
-        fg="green",
-    )

platformio/account/commands/login.py

@@ -1,26 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-
-
-@click.command("login", short_help="Log in to PlatformIO Account")
-@click.option("-u", "--username", prompt="Username or email")
-@click.option("-p", "--password", prompt=True, hide_input=True)
-def account_login_cmd(username, password):
-    client = AccountClient()
-    client.login(username, password)
-    click.secho("Successfully logged in!", fg="green")

platformio/account/commands/logout.py

@@ -1,24 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-
-
-@click.command("logout", short_help="Log out of PlatformIO Account")
-def account_logout_cmd():
-    client = AccountClient()
-    client.logout()
-    click.secho("Successfully logged out!", fg="green")

platformio/account/commands/password.py

@@ -1,26 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-
-
-@click.command("password", short_help="Change password")
-@click.option("--old-password", prompt=True, hide_input=True)
-@click.option("--new-password", prompt=True, hide_input=True, confirmation_prompt=True)
-def account_password_cmd(old_password, new_password):
-    client = AccountClient()
-    client.change_password(old_password, new_password)
-    click.secho("Password successfully changed!", fg="green")

platformio/account/commands/register.py

@@ -1,52 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-from platformio.account.validate import (
-    validate_email,
-    validate_password,
-    validate_username,
-)
-
-
-@click.command("register", short_help="Create new PlatformIO Account")
-@click.option(
-    "-u",
-    "--username",
-    prompt=True,
-    callback=lambda _, __, value: validate_username(value),
-)
-@click.option(
-    "-e", "--email", prompt=True, callback=lambda _, __, value: validate_email(value)
-)
-@click.option(
-    "-p",
-    "--password",
-    prompt=True,
-    hide_input=True,
-    confirmation_prompt=True,
-    callback=lambda _, __, value: validate_password(value),
-)
-@click.option("--firstname", prompt=True)
-@click.option("--lastname", prompt=True)
-def account_register_cmd(username, email, password, firstname, lastname):
-    client = AccountClient()
-    client.registration(username, email, password, firstname, lastname)
-    click.secho(
-        "An account has been successfully created. "
-        "Please check your mail to activate your account and verify your email address.",
-        fg="green",
-    )

platformio/account/commands/show.py

@@ -1,116 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-
-import click
-from tabulate import tabulate
-
-from platformio import util
-from platformio.account.client import AccountClient
-
-
-@click.command("show", short_help="PlatformIO Account information")
-@click.option("--offline", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_show_cmd(offline, json_output):
-    client = AccountClient()
-    info = client.get_account_info(offline)
-    if json_output:
-        click.echo(json.dumps(info))
-        return
-    click.echo()
-    if info.get("profile"):
-        print_profile(info["profile"])
-    if info.get("packages"):
-        print_packages(info["packages"])
-    if info.get("subscriptions"):
-        print_subscriptions(info["subscriptions"])
-    click.echo()
-
-
-def print_profile(profile):
-    click.secho("Profile", fg="cyan", bold=True)
-    click.echo("=" * len("Profile"))
-    data = []
-    if profile.get("username"):
-        data.append(("Username:", profile["username"]))
-    if profile.get("email"):
-        data.append(("Email:", profile["email"]))
-    if profile.get("firstname"):
-        data.append(("First name:", profile["firstname"]))
-    if profile.get("lastname"):
-        data.append(("Last name:", profile["lastname"]))
-    click.echo(tabulate(data, tablefmt="plain"))
-
-
-def print_packages(packages):
-    click.echo()
-    click.secho("Packages", fg="cyan")
-    click.echo("=" * len("Packages"))
-    for package in packages:
-        click.echo()
-        click.secho(package.get("name"), bold=True)
-        click.echo("-" * len(package.get("name")))
-        if package.get("description"):
-            click.echo(package.get("description"))
-        data = []
-        expire = "-"
-        if "subscription" in package:
-            expire = util.parse_datetime(
-                package["subscription"].get("end_at")
-                or package["subscription"].get("next_bill_at")
-            ).strftime("%Y-%m-%d")
-        data.append(("Expire:", expire))
-        services = []
-        for key in package:
-            if not key.startswith("service."):
-                continue
-            if isinstance(package[key], dict):
-                services.append(package[key].get("title"))
-            else:
-                services.append(package[key])
-        if services:
-            data.append(("Services:", ", ".join(services)))
-        click.echo(tabulate(data, tablefmt="plain"))
-
-
-def print_subscriptions(subscriptions):
-    click.echo()
-    click.secho("Subscriptions", fg="cyan")
-    click.echo("=" * len("Subscriptions"))
-    for subscription in subscriptions:
-        click.echo()
-        click.secho(subscription.get("product_name"), bold=True)
-        click.echo("-" * len(subscription.get("product_name")))
-        data = [("State:", subscription.get("status"))]
-        begin_at = util.parse_datetime(subscription.get("begin_at")).strftime("%c")
-        data.append(("Start date:", begin_at or "-"))
-        end_at = subscription.get("end_at")
-        if end_at:
-            end_at = util.parse_datetime(subscription.get("end_at")).strftime("%c")
-        data.append(("End date:", end_at or "-"))
-        next_bill_at = subscription.get("next_bill_at")
-        if next_bill_at:
-            next_bill_at = util.parse_datetime(
-                subscription.get("next_bill_at")
-            ).strftime("%c")
-        data.append(("Next payment:", next_bill_at or "-"))
-        data.append(
-            ("Edit:", click.style(subscription.get("update_url"), fg="blue") or "-")
-        )
-        data.append(
-            ("Cancel:", click.style(subscription.get("cancel_url"), fg="blue") or "-")
-        )
-        click.echo(tabulate(data, tablefmt="plain"))

platformio/account/commands/token.py

@@ -1,32 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-
-import click
-
-from platformio.account.client import AccountClient
-
-
-@click.command("token", short_help="Get or regenerate Authentication Token")
-@click.option("-p", "--password", prompt=True, hide_input=True)
-@click.option("--regenerate", is_flag=True)
-@click.option("--json-output", is_flag=True)
-def account_token_cmd(password, regenerate, json_output):
-    client = AccountClient()
-    auth_token = client.auth_token(password, regenerate)
-    if json_output:
-        click.echo(json.dumps({"status": "success", "result": auth_token}))
-        return
-    click.secho("Personal Authentication Token: %s" % auth_token, fg="green")

platformio/account/commands/update.py

@@ -1,59 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient, AccountNotAuthorized
-from platformio.account.validate import validate_email, validate_username
-
-
-@click.command("update", short_help="Update profile information")
-@click.option("--current-password", prompt=True, hide_input=True)
-@click.option("--username")
-@click.option("--email")
-@click.option("--firstname")
-@click.option("--lastname")
-def account_update_cmd(current_password, **kwargs):
-    client = AccountClient()
-    profile = client.get_profile()
-    new_profile = profile.copy()
-    if not any(kwargs.values()):
-        for field in profile:
-            new_profile[field] = click.prompt(
-                field.replace("_", " ").capitalize(), default=profile[field]
-            )
-            if field == "email":
-                validate_email(new_profile[field])
-            if field == "username":
-                validate_username(new_profile[field])
-    else:
-        new_profile.update({key: value for key, value in kwargs.items() if value})
-    client.update_profile(new_profile, current_password)
-    click.secho("Profile successfully updated!", fg="green")
-    username_changed = new_profile["username"] != profile["username"]
-    email_changed = new_profile["email"] != profile["email"]
-    if not username_changed and not email_changed:
-        return None
-    try:
-        client.logout()
-    except AccountNotAuthorized:
-        pass
-    if email_changed:
-        click.secho(
-            "Please check your mail to verify your new email address and re-login. ",
-            fg="yellow",
-        )
-        return None
-    click.secho("Please re-login.", fg="yellow")
-    return None

platformio/account/org/cli.py

@@ -1,38 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.org.commands.add import org_add_cmd
-from platformio.account.org.commands.create import org_create_cmd
-from platformio.account.org.commands.destroy import org_destroy_cmd
-from platformio.account.org.commands.list import org_list_cmd
-from platformio.account.org.commands.remove import org_remove_cmd
-from platformio.account.org.commands.update import org_update_cmd
-
-
-@click.group(
-    "account",
-    commands=[
-        org_add_cmd,
-        org_create_cmd,
-        org_destroy_cmd,
-        org_list_cmd,
-        org_remove_cmd,
-        org_update_cmd,
-    ],
-    short_help="Manage organizations",
-)
-def cli():
-    pass

platformio/account/org/commands/add.py

@@ -1,34 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-
-
-@click.command("add", short_help="Add a new owner to organization")
-@click.argument(
-    "orgname",
-)
-@click.argument(
-    "username",
-)
-def org_add_cmd(orgname, username):
-    client = AccountClient()
-    client.add_org_owner(orgname, username)
-    return click.secho(
-        "The new owner `%s` has been successfully added to the `%s` organization."
-        % (username, orgname),
-        fg="green",
-    )

platformio/account/org/commands/create.py

@@ -1,38 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-from platformio.account.validate import validate_email, validate_orgname
-
-
-@click.command("create", short_help="Create a new organization")
-@click.argument(
-    "orgname",
-    callback=lambda _, __, value: validate_orgname(value),
-)
-@click.option(
-    "--email", callback=lambda _, __, value: validate_email(value) if value else value
-)
-@click.option(
-    "--displayname",
-)
-def org_create_cmd(orgname, email, displayname):
-    client = AccountClient()
-    client.create_org(orgname, email, displayname)
-    return click.secho(
-        "The organization `%s` has been successfully created." % orgname,
-        fg="green",
-    )

platformio/account/org/commands/destroy.py

@@ -1,34 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-
-
-@click.command("destroy", short_help="Destroy organization")
-@click.argument("orgname")
-def org_destroy_cmd(orgname):
-    client = AccountClient()
-    click.confirm(
-        "Are you sure you want to delete the `%s` organization account?\n"
-        "Warning! All linked data will be permanently removed and can not be restored."
-        % orgname,
-        abort=True,
-    )
-    client.destroy_org(orgname)
-    return click.secho(
-        "Organization `%s` has been destroyed." % orgname,
-        fg="green",
-    )

platformio/account/org/commands/list.py

@@ -1,48 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-
-import click
-from tabulate import tabulate
-
-from platformio.account.client import AccountClient
-
-
-@click.command("list", short_help="List organizations and their members")
-@click.option("--json-output", is_flag=True)
-def org_list_cmd(json_output):
-    client = AccountClient()
-    orgs = client.list_orgs()
-    if json_output:
-        return click.echo(json.dumps(orgs))
-    if not orgs:
-        return click.echo("You do not have any organization")
-    for org in orgs:
-        click.echo()
-        click.secho(org.get("orgname"), fg="cyan")
-        click.echo("-" * len(org.get("orgname")))
-        data = []
-        if org.get("displayname"):
-            data.append(("Display Name:", org.get("displayname")))
-        if org.get("email"):
-            data.append(("Email:", org.get("email")))
-        data.append(
-            (
-                "Owners:",
-                ", ".join((owner.get("username") for owner in org.get("owners"))),
-            )
-        )
-        click.echo(tabulate(data, tablefmt="plain"))
-    return click.echo()

platformio/account/org/commands/remove.py

@@ -1,34 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-
-
-@click.command("remove", short_help="Remove an owner from organization")
-@click.argument(
-    "orgname",
-)
-@click.argument(
-    "username",
-)
-def org_remove_cmd(orgname, username):
-    client = AccountClient()
-    client.remove_org_owner(orgname, username)
-    return click.secho(
-        "The `%s` owner has been successfully removed from the `%s` organization."
-        % (username, orgname),
-        fg="green",
-    )

platformio/account/org/commands/update.py

@@ -1,50 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-from platformio.account.validate import validate_email, validate_orgname
-
-
-@click.command("update", short_help="Update organization")
-@click.argument("cur_orgname")
-@click.option(
-    "--orgname",
-    callback=lambda _, __, value: validate_orgname(value) if value else value,
-    help="A new orgname",
-)
-@click.option(
-    "--email",
-    callback=lambda _, __, value: validate_email(value) if value else value,
-)
-@click.option("--displayname")
-def org_update_cmd(cur_orgname, **kwargs):
-    client = AccountClient()
-    org = client.get_org(cur_orgname)
-    new_org = {
-        key: value if value is not None else org[key] for key, value in kwargs.items()
-    }
-    if not any(kwargs.values()):
-        for key in kwargs:
-            new_org[key] = click.prompt(key.capitalize(), default=org[key])
-            if key == "email":
-                validate_email(new_org[key])
-            if key == "orgname":
-                validate_orgname(new_org[key])
-    client.update_org(cur_orgname, new_org)
-    return click.secho(
-        "The organization `%s` has been successfully updated." % cur_orgname,
-        fg="green",
-    )

platformio/account/team/cli.py

@@ -1,38 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.team.commands.add import team_add_cmd
-from platformio.account.team.commands.create import team_create_cmd
-from platformio.account.team.commands.destroy import team_destroy_cmd
-from platformio.account.team.commands.list import team_list_cmd
-from platformio.account.team.commands.remove import team_remove_cmd
-from platformio.account.team.commands.update import team_update_cmd
-
-
-@click.group(
-    "team",
-    commands=[
-        team_add_cmd,
-        team_create_cmd,
-        team_destroy_cmd,
-        team_list_cmd,
-        team_remove_cmd,
-        team_update_cmd,
-    ],
-    short_help="Manage organization teams",
-)
-def cli():
-    pass

platformio/account/team/commands/add.py

@@ -1,38 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-from platformio.account.validate import validate_orgname_teamname
-
-
-@click.command("add", short_help="Add a new member to team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.argument(
-    "username",
-)
-def team_add_cmd(orgname_teamname, username):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.add_team_member(orgname, teamname, username)
-    return click.secho(
-        "The new member %s has been successfully added to the %s team."
-        % (username, teamname),
-        fg="green",
-    )

platformio/account/team/commands/create.py

@@ -1,37 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-from platformio.account.validate import validate_orgname_teamname
-
-
-@click.command("create", short_help="Create a new team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.option(
-    "--description",
-)
-def team_create_cmd(orgname_teamname, description):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.create_team(orgname, teamname, description)
-    return click.secho(
-        "The team %s has been successfully created." % teamname,
-        fg="green",
-    )

platformio/account/team/commands/destroy.py

@@ -1,40 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-from platformio.account.validate import validate_orgname_teamname
-
-
-@click.command("destroy", short_help="Destroy a team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-def team_destroy_cmd(orgname_teamname):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    click.confirm(
-        click.style(
-            "Are you sure you want to destroy the %s team?" % teamname, fg="yellow"
-        ),
-        abort=True,
-    )
-    client = AccountClient()
-    client.destroy_team(orgname, teamname)
-    return click.secho(
-        "The team %s has been successfully destroyed." % teamname,
-        fg="green",
-    )

platformio/account/team/commands/list.py

@@ -1,59 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-
-import click
-from tabulate import tabulate
-
-from platformio.account.client import AccountClient
-
-
-@click.command("list", short_help="List teams")
-@click.argument("orgname", required=False)
-@click.option("--json-output", is_flag=True)
-def team_list_cmd(orgname, json_output):
-    client = AccountClient()
-    data = {}
-    if not orgname:
-        for item in client.list_orgs():
-            teams = client.list_teams(item.get("orgname"))
-            data[item.get("orgname")] = teams
-    else:
-        teams = client.list_teams(orgname)
-        data[orgname] = teams
-    if json_output:
-        return click.echo(json.dumps(data[orgname] if orgname else data))
-    if not any(data.values()):
-        return click.secho("You do not have any teams.", fg="yellow")
-    for org_name, teams in data.items():
-        for team in teams:
-            click.echo()
-            click.secho("%s:%s" % (org_name, team.get("name")), fg="cyan")
-            click.echo("-" * len("%s:%s" % (org_name, team.get("name"))))
-            table_data = []
-            if team.get("description"):
-                table_data.append(("Description:", team.get("description")))
-            table_data.append(
-                (
-                    "Members:",
-                    ", ".join(
-                        (member.get("username") for member in team.get("members"))
-                    )
-                    if team.get("members")
-                    else "-",
-                )
-            )
-            click.echo(tabulate(table_data, tablefmt="plain"))
-    return click.echo()

platformio/account/team/commands/remove.py

@@ -1,36 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-from platformio.account.validate import validate_orgname_teamname
-
-
-@click.command("remove", short_help="Remove a member from team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.argument("username")
-def team_remove_cmd(orgname_teamname, username):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    client.remove_team_member(orgname, teamname, username)
-    return click.secho(
-        "The %s member has been successfully removed from the %s team."
-        % (username, teamname),
-        fg="green",
-    )

platformio/account/team/commands/update.py

@@ -1,51 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import click
-
-from platformio.account.client import AccountClient
-from platformio.account.validate import validate_orgname_teamname, validate_teamname
-
-
-@click.command("update", short_help="Update team")
-@click.argument(
-    "orgname_teamname",
-    metavar="ORGNAME:TEAMNAME",
-    callback=lambda _, __, value: validate_orgname_teamname(value),
-)
-@click.option(
-    "--name",
-    callback=lambda _, __, value: validate_teamname(value) if value else value,
-    help="A new team name",
-)
-@click.option(
-    "--description",
-)
-def team_update_cmd(orgname_teamname, **kwargs):
-    orgname, teamname = orgname_teamname.split(":", 1)
-    client = AccountClient()
-    team = client.get_team(orgname, teamname)
-    new_team = {
-        key: value if value is not None else team[key] for key, value in kwargs.items()
-    }
-    if not any(kwargs.values()):
-        for key in kwargs:
-            new_team[key] = click.prompt(key.capitalize(), default=team[key])
-            if key == "name":
-                validate_teamname(new_team[key])
-    client.update_team(orgname, teamname, new_team)
-    return click.secho(
-        "The team %s has been successfully updated." % teamname,
-        fg="green",
-    )

platformio/account/validate.py

@@ -1,84 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-
-import click
-
-
-def validate_username(value, field="username"):
-    value = str(value).strip() if value else None
-    if not value or not re.match(
-        r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I
-    ):
-        raise click.BadParameter(
-            "Invalid %s format. "
-            "%s must contain only alphanumeric characters "
-            "or single hyphens, cannot begin or end with a hyphen, "
-            "and must not be longer than 38 characters."
-            % (field.lower(), field.capitalize())
-        )
-    return value
-
-
-def validate_orgname(value):
-    return validate_username(value, "Organization name")
-
-
-def validate_email(value):
-    value = str(value).strip() if value else None
-    if not value or not re.match(
-        r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I
-    ):
-        raise click.BadParameter("Invalid email address")
-    return value
-
-
-def validate_password(value):
-    value = str(value).strip() if value else None
-    if not value or not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value):
-        raise click.BadParameter(
-            "Invalid password format. "
-            "Password must contain at least 8 characters"
-            " including a number and a lowercase letter"
-        )
-    return value
-
-
-def validate_teamname(value):
-    value = str(value).strip() if value else None
-    if not value or not re.match(
-        r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I
-    ):
-        raise click.BadParameter(
-            "Invalid team name format. "
-            "Team name must only contain alphanumeric characters, "
-            "single hyphens, underscores, spaces. It can not "
-            "begin or end with a hyphen or a underscore and must"
-            " not be longer than 20 characters."
-        )
-    return value
-
-
-def validate_orgname_teamname(value):
-    value = str(value).strip() if value else None
-    if not value or ":" not in value:
-        raise click.BadParameter(
-            "Please specify organization and team name using the following"
-            " format - orgname:teamname. For example, mycompany:DreamTeam"
-        )
-    orgname, teamname = value.split(":", 1)
-    validate_orgname(orgname)
-    validate_teamname(teamname)
-    return value

platformio/app.py

@@ -12,80 +12,97 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import getpass
+import codecs
 import hashlib
-import json
 import os
-import platform
-import socket
-import time
 import uuid
+from os import environ, getenv, listdir, remove
+from os.path import abspath, dirname, expanduser, isdir, isfile, join
+from time import time
 
-from platformio import __version__, exception, fs, proc
-from platformio.compat import IS_WINDOWS, hashlib_encode_data
-from platformio.package.lockfile import LockFile
-from platformio.project.config import ProjectConfig
-from platformio.project.helpers import get_default_projects_dir
+import requests
+
+from platformio import exception, fs, lockfile
+from platformio.compat import (WINDOWS, dump_json_to_unicode,
+                               hashlib_encode_data)
+from platformio.proc import is_ci
+from platformio.project.helpers import (get_project_cache_dir,
+                                        get_project_core_dir)
+
+
+def get_default_projects_dir():
+    docs_dir = join(expanduser("~"), "Documents")
+    try:
+        assert WINDOWS
+        import ctypes.wintypes
+        buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH)
+        ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf)
+        docs_dir = buf.value
+    except:  # pylint: disable=bare-except
+        pass
+    return join(docs_dir, "PlatformIO", "Projects")
 
 
 def projects_dir_validate(projects_dir):
-    assert os.path.isdir(projects_dir)
-    return os.path.abspath(projects_dir)
+    assert isdir(projects_dir)
+    return abspath(projects_dir)
 
 
 DEFAULT_SETTINGS = {
-    "check_platformio_interval": {
-        "description": "Check for the new PlatformIO Core interval (days)",
-        "value": 7,
+    "auto_update_libraries": {
+        "description": "Automatically update libraries (Yes/No)",
+        "value": False
     },
-    "check_prune_system_threshold": {
-        "description": "Check for pruning unnecessary data threshold (megabytes)",
-        "value": 1024,
+    "auto_update_platforms": {
+        "description": "Automatically update platforms (Yes/No)",
+        "value": False
+    },
+    "check_libraries_interval": {
+        "description": "Check for the library updates interval (days)",
+        "value": 7
+    },
+    "check_platformio_interval": {
+        "description": "Check for the new PlatformIO interval (days)",
+        "value": 3
+    },
+    "check_platforms_interval": {
+        "description": "Check for the platform updates interval (days)",
+        "value": 7
     },
     "enable_cache": {
-        "description": "Enable caching for HTTP API requests",
-        "value": True,
+        "description": "Enable caching for API requests and Library Manager",
+        "value": True
+    },
+    "strict_ssl": {
+        "description": "Strict SSL for PlatformIO Services",
+        "value": False
     },
     "enable_telemetry": {
-        "description": ("Telemetry service <https://bit.ly/pio-telemetry> (Yes/No)"),
-        "value": True,
+        "description":
+        ("Telemetry service <http://bit.ly/pio-telemetry> (Yes/No)"),
+        "value": True
     },
     "force_verbose": {
         "description": "Force verbose output when processing environments",
-        "value": False,
+        "value": False
     },
     "projects_dir": {
-        "description": "Default location for PlatformIO projects (PlatformIO Home)",
+        "description": "Default location for PlatformIO projects (PIO Home)",
         "value": get_default_projects_dir(),
-        "validator": projects_dir_validate,
-    },
-    "enable_proxy_strict_ssl": {
-        "description": "Verify the proxy server certificate against the list of supplied CAs",
-        "value": True,
+        "validator": projects_dir_validate
     },
 }
 
-SESSION_VARS = {
-    "command_ctx": None,
-    "caller_id": None,
-    "custom_project_conf": None,
-    "pause_telemetry": False,
-}
+SESSION_VARS = {"command_ctx": None, "force_option": False, "caller_id": None}
 
 
-def resolve_state_path(conf_option_dir, file_name, ensure_dir_exists=True):
-    state_dir = ProjectConfig.get_instance().get("platformio", conf_option_dir)
-    if ensure_dir_exists and not os.path.isdir(state_dir):
-        os.makedirs(state_dir)
-    return os.path.join(state_dir, file_name)
+class State(object):
 
-
-class State:
     def __init__(self, path=None, lock=False):
         self.path = path
         self.lock = lock
         if not self.path:
-            self.path = resolve_state_path("core_dir", "appstate.json")
+            self.path = join(get_project_core_dir(), "appstate.json")
         self._storage = {}
         self._lockfile = None
         self.modified = False
@@ -93,37 +110,31 @@ class State:
     def __enter__(self):
         try:
             self._lock_state_file()
-            if os.path.isfile(self.path):
+            if isfile(self.path):
                 self._storage = fs.load_json(self.path)
             assert isinstance(self._storage, dict)
-        except (
-            AssertionError,
-            ValueError,
-            UnicodeDecodeError,
-            exception.InvalidJSONFile,
-        ):
+        except (AssertionError, ValueError, UnicodeDecodeError,
+                exception.InvalidJSONFile):
             self._storage = {}
         return self
 
     def __exit__(self, type_, value, traceback):
         if self.modified:
             try:
-                with open(self.path, mode="w", encoding="utf8") as fp:
-                    fp.write(json.dumps(self._storage))
-            except IOError as exc:
-                raise exception.HomeDirPermissionsError(
-                    os.path.dirname(self.path)
-                ) from exc
+                with open(self.path, "w") as fp:
+                    fp.write(dump_json_to_unicode(self._storage))
+            except IOError:
+                raise exception.HomeDirPermissionsError(get_project_core_dir())
         self._unlock_state_file()
 
     def _lock_state_file(self):
         if not self.lock:
             return
-        self._lockfile = LockFile(self.path)
+        self._lockfile = lockfile.LockFile(self.path)
         try:
             self._lockfile.acquire()
-        except IOError as exc:
-            raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) from exc
+        except IOError:
+            raise exception.HomeDirPermissionsError(dirname(self.path))
 
     def _unlock_state_file(self):
         if hasattr(self, "_lockfile") and self._lockfile:
@@ -137,9 +148,6 @@ class State:
     def as_dict(self):
         return self._storage
 
-    def keys(self):
-        return self._storage.keys()
-
     def get(self, key, default=True):
         return self._storage.get(key, default)
 
@@ -165,6 +173,143 @@ class State:
         return item in self._storage
 
 
+class ContentCache(object):
+
+    def __init__(self, cache_dir=None):
+        self.cache_dir = None
+        self._db_path = None
+        self._lockfile = None
+
+        self.cache_dir = cache_dir or get_project_cache_dir()
+        self._db_path = join(self.cache_dir, "db.data")
+
+    def __enter__(self):
+        self.delete()
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        pass
+
+    def _lock_dbindex(self):
+        if not self.cache_dir:
+            os.makedirs(self.cache_dir)
+        self._lockfile = lockfile.LockFile(self.cache_dir)
+        try:
+            self._lockfile.acquire()
+        except:  # pylint: disable=bare-except
+            return False
+
+        return True
+
+    def _unlock_dbindex(self):
+        if self._lockfile:
+            self._lockfile.release()
+        return True
+
+    def get_cache_path(self, key):
+        key = str(key)
+        assert len(key) > 3
+        return join(self.cache_dir, key[-2:], key)
+
+    @staticmethod
+    def key_from_args(*args):
+        h = hashlib.md5()
+        for arg in args:
+            if arg:
+                h.update(hashlib_encode_data(arg))
+        return h.hexdigest()
+
+    def get(self, key):
+        cache_path = self.get_cache_path(key)
+        if not isfile(cache_path):
+            return None
+        with codecs.open(cache_path, "rb", encoding="utf8") as fp:
+            return fp.read()
+
+    def set(self, key, data, valid):
+        if not get_setting("enable_cache"):
+            return False
+        cache_path = self.get_cache_path(key)
+        if isfile(cache_path):
+            self.delete(key)
+        if not data:
+            return False
+        if not isdir(self.cache_dir):
+            os.makedirs(self.cache_dir)
+        tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
+        assert valid.endswith(tuple(tdmap))
+        expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1]))
+
+        if not self._lock_dbindex():
+            return False
+
+        if not isdir(dirname(cache_path)):
+            os.makedirs(dirname(cache_path))
+        try:
+            with codecs.open(cache_path, "wb", encoding="utf8") as fp:
+                fp.write(data)
+            with open(self._db_path, "a") as fp:
+                fp.write("%s=%s\n" % (str(expire_time), cache_path))
+        except UnicodeError:
+            if isfile(cache_path):
+                try:
+                    remove(cache_path)
+                except OSError:
+                    pass
+
+        return self._unlock_dbindex()
+
+    def delete(self, keys=None):
+        """ Keys=None, delete expired items """
+        if not isfile(self._db_path):
+            return None
+        if not keys:
+            keys = []
+        if not isinstance(keys, list):
+            keys = [keys]
+        paths_for_delete = [self.get_cache_path(k) for k in keys]
+        found = False
+        newlines = []
+        with open(self._db_path) as fp:
+            for line in fp.readlines():
+                line = line.strip()
+                if "=" not in line:
+                    continue
+                expire, path = line.split("=")
+                try:
+                    if time() < int(expire) and isfile(path) and \
+                            path not in paths_for_delete:
+                        newlines.append(line)
+                        continue
+                except ValueError:
+                    pass
+                found = True
+                if isfile(path):
+                    try:
+                        remove(path)
+                        if not listdir(dirname(path)):
+                            fs.rmtree(dirname(path))
+                    except OSError:
+                        pass
+
+        if found and self._lock_dbindex():
+            with open(self._db_path, "w") as fp:
+                fp.write("\n".join(newlines) + "\n")
+            self._unlock_dbindex()
+
+        return True
+
+    def clean(self):
+        if not self.cache_dir or not isdir(self.cache_dir):
+            return
+        fs.rmtree(self.cache_dir)
+
+
+def clean_cache():
+    with ContentCache() as cc:
+        cc.clean()
+
+
 def sanitize_setting(name, value):
     if name not in DEFAULT_SETTINGS:
         raise exception.InvalidSettingName(name)
@@ -172,14 +317,14 @@ def sanitize_setting(name, value):
     defdata = DEFAULT_SETTINGS[name]
     try:
         if "validator" in defdata:
-            value = defdata["validator"](value)
-        elif isinstance(defdata["value"], bool):
+            value = defdata['validator'](value)
+        elif isinstance(defdata['value'], bool):
             if not isinstance(value, bool):
                 value = str(value).lower() in ("true", "yes", "y", "1")
-        elif isinstance(defdata["value"], int):
+        elif isinstance(defdata['value'], int):
             value = int(value)
-    except Exception as exc:
-        raise exception.InvalidSettingValue(value, name) from exc
+    except Exception:
+        raise exception.InvalidSettingValue(value, name)
     return value
 
 
@@ -202,28 +347,28 @@ def delete_state_item(name):
 
 def get_setting(name):
     _env_name = "PLATFORMIO_SETTING_%s" % name.upper()
-    if _env_name in os.environ:
-        return sanitize_setting(name, os.getenv(_env_name))
+    if _env_name in environ:
+        return sanitize_setting(name, getenv(_env_name))
 
     with State() as state:
-        if "settings" in state and name in state["settings"]:
-            return state["settings"][name]
+        if "settings" in state and name in state['settings']:
+            return state['settings'][name]
 
-    return DEFAULT_SETTINGS[name]["value"]
+    return DEFAULT_SETTINGS[name]['value']
 
 
 def set_setting(name, value):
     with State(lock=True) as state:
         if "settings" not in state:
-            state["settings"] = {}
-        state["settings"][name] = sanitize_setting(name, value)
+            state['settings'] = {}
+        state['settings'][name] = sanitize_setting(name, value)
         state.modified = True
 
 
 def reset_settings():
     with State(lock=True) as state:
         if "settings" in state:
-            del state["settings"]
+            del state['settings']
 
 
 def get_session_var(name, default=None):
@@ -236,7 +381,11 @@ def set_session_var(name, value):
 
 
 def is_disabled_progressbar():
-    return os.getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true"
+    return any([
+        get_session_var("force_option"),
+        is_ci(),
+        getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true"
+    ])
 
 
 def get_cid():
@@ -244,44 +393,19 @@ def get_cid():
     if cid:
         return cid
     uid = None
-    if os.getenv("GITHUB_USER"):
-        uid = os.getenv("GITHUB_USER")
-    elif os.getenv("GITPOD_GIT_USER_NAME"):
-        uid = os.getenv("GITPOD_GIT_USER_NAME")
+    if getenv("C9_UID"):
+        uid = getenv("C9_UID")
+    elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")):
+        try:
+            uid = requests.get("{api}/user?token={token}".format(
+                api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")),
+                token=getenv("USER_TOKEN"))).json().get("id")
+        except:  # pylint: disable=bare-except
+            pass
     if not uid:
         uid = uuid.getnode()
     cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest())
     cid = str(cid)
-    if IS_WINDOWS or os.getuid() > 0:  # pylint: disable=no-member
+    if WINDOWS or os.getuid() > 0:  # yapf: disable pylint: disable=no-member
         set_state_item("cid", cid)
-        set_state_item("created_at", int(time.time()))
     return cid
-
-
-def get_user_agent():
-    data = [
-        "PlatformIO/%s" % __version__,
-        "CI/%d" % int(proc.is_ci()),
-        "Container/%d" % int(proc.is_container()),
-    ]
-    if get_session_var("caller_id"):
-        data.append("Caller/%s" % get_session_var("caller_id"))
-    if os.getenv("PLATFORMIO_IDE"):
-        data.append("IDE/%s" % os.getenv("PLATFORMIO_IDE"))
-    data.append("Python/%s" % platform.python_version())
-    data.append("Platform/%s" % platform.platform())
-    return " ".join(data)
-
-
-def get_host_id():
-    h = hashlib.sha1(hashlib_encode_data(get_cid()))
-    try:
-        username = getpass.getuser()
-        h.update(hashlib_encode_data(username))
-    except:  # pylint: disable=bare-except
-        pass
-    return h.hexdigest()
-
-
-def get_host_name():
-    return str(socket.gethostname())[:255]

platformio/assets/schema/library.json

@@ -1,487 +0,0 @@
-{
-  "$id": "https://example.com/library.json",
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "title": "library.json schema",
-  "type": "object",
-  "properties": {
-    "name": {
-      "type": "string",
-      "maxLength": 50,
-      "description": "A name of a library.\nMust be unique in the PlatformIO Registry\nShould be slug style for simplicity, consistency, and compatibility. Example: HelloWorld\nCan contain a-z, digits, and dashes (but not start/end with them)\nConsecutive dashes and [:;/,@<>] chars are not allowed.",
-      "required": true
-    },
-    "version": {
-      "type": "string",
-      "maxLength": 20,
-      "description": "A version of a current library source code. Can contain a-z, digits, dots or dash and should be Semantic Versioning compatible.",
-      "required": true
-    },
-    "description": {
-      "type": "string",
-      "maxLength": 255,
-      "description": "The field helps users to identify and search for your library with a brief description. Describe the hardware devices (sensors, boards and etc.) which are suitable with it.",
-      "required": true
-    },
-    "keywords": {
-      "anyOf": [
-        {
-          "type": "string",
-          "maxLength": 255
-        },
-        {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "maxLength": 255
-          }
-        }
-      ],
-      "description": "Used for search by keyword. Helps to make your library easier to discover without people needing to know its name.\nThe keyword should be lowercased, can contain a-z, digits and dash (but not start/end with them). A list from the keywords can be specified with separator , or declared as Array.",
-      "required": true
-    },
-    "homepage": {
-      "type": "string",
-      "maxLength": 255,
-      "description": "Home page of a library (if is different from repository url).",
-      "required": false
-    },
-    "repository": {
-      "type": "object",
-      "properties": {
-        "type": {
-          "enum": [
-            "git",
-            "hg",
-            "svn"
-          ],
-          "description": "only “git”, “hg” or “svn” are supported"
-        },
-        "url": {
-          "type": "string"
-        },
-        "branch": {
-          "type": "string",
-          "description": "if is not specified, default branch will be used. This field will be ignored if tag/release exists with the value of version."
-        }
-      },
-      "description": "The repository in which the source code can be found.",
-      "required": false
-    },
-    "authors": {
-      "anyOf": [
-        {
-          "type": "object",
-          "properties": {
-            "name": {
-              "type": "string",
-              "required": true,
-              "description": "Full name"
-            },
-            "email": {
-              "type": "string"
-            },
-            "url": {
-              "type": "string",
-              "description": "An author’s contact page"
-            },
-            "maintainer": {
-              "type": "boolean",
-              "description": "Specify “maintainer” status"
-            }
-          }
-        },
-        {
-          "type": "array",
-          "items": {
-            "type": "object",
-            "properties": {
-              "name": {
-                "type": "string",
-                "required": true,
-                "description": "Full name"
-              },
-              "email": {
-                "type": "string"
-              },
-              "url": {
-                "type": "string",
-                "description": "An author’s contact page"
-              },
-              "maintainer": {
-                "type": "boolean",
-                "description": "Specify “maintainer” status"
-              }
-            }
-          }
-        }
-      ],
-      "description": "An author contact information\nIf authors field is not defined, PlatformIO will try to fetch data from VCS provider (Github, Gitlab, etc) if repository is declared.",
-      "required": false
-    },
-    "license": {
-      "type": "string",
-      "description": "A SPDX license ID or SPDX Expression. You can check the full list of SPDX license IDs (see “Identifier” column).",
-      "required": false
-    },
-    "frameworks": {
-      "anyOf": [
-        {
-          "type": "string",
-          "description": "espidf, freertos, *, etc'"
-        },
-        {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "description": "espidf, freertos, *, etc'"
-          }
-        }
-      ],
-      "description": "A list with compatible frameworks. The available framework names are defined in the Frameworks section.\nIf the library is compatible with the all frameworks, then do not declare this field or you use *",
-      "required": false
-    },
-    "platforms": {
-      "anyOf": [
-        {
-          "type": "string",
-          "description": "atmelavr, espressif8266, *, etc'"
-        },
-        {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "description": "atmelavr, espressif8266, *, etc'"
-          }
-        }
-      ],
-      "description": "A list with compatible development platforms. The available platform name are defined in Development Platforms section.\nIf the library is compatible with the all platforms, then do not declare this field or use *.\nPlatformIO does not check platforms for compatibility in default mode. See Compatibility Mode for details. If you need a strict checking for compatible platforms for a library, please set libCompatMode to strict.",
-      "required": false
-    },
-    "headers": {
-      "anyOf": [
-        {
-          "type": "string",
-          "description": "MyLibrary.h"
-        },
-        {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "description": "FooCore.h, FooFeature.h"
-          }
-        }
-      ],
-      "description": "A list of header files that can be included in a project source files using #include <...> directive.",
-      "required": false
-    },
-    "examples": {
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "name": {
-            "type": "string"
-          },
-          "base": {
-            "type": "string"
-          },
-          "files": {
-            "type": "array",
-            "items": {
-              "type": "string"
-            }
-          }
-        }
-      },
-      "description": "A list of example patterns.",
-      "required": "false"
-    },
-    "dependencies": {
-      "anyOf": [
-        {
-          "type": "object",
-          "properties": {
-            "owner": {
-              "type": "string",
-              "description": "an owner name (username) from the PlatformIO Registry"
-            },
-            "name": {
-              "type": "string",
-              "description": "library name"
-            },
-            "version": {
-              "type": "string",
-              "description": "Version Requirements or Package Specifications"
-            },
-            "frameworks": {
-              "anyOf": [
-                {
-                  "type": "string"
-                },
-                {
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                }
-              ],
-              "description": "project compatible Frameworks"
-            },
-            "platforms": {
-              "anyOf": [
-                {
-                  "type": "string"
-                },
-                {
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                }
-              ],
-              "description": " project compatible Development Platforms"
-            }
-          }
-        },
-        {
-          "type": "array",
-          "items": {
-            "type": "object",
-            "properties": {
-              "owner": {
-                "type": "string",
-                "description": "an owner name (username) from the PlatformIO Registry"
-              },
-              "name": {
-                "type": "string",
-                "description": "library name"
-              },
-              "version": {
-                "type": "string",
-                "description": "Version Requirements or Package Specifications"
-              },
-              "frameworks": {
-                "anyOf": [
-                  {
-                    "type": "string"
-                  },
-                  {
-                    "type": "array",
-                    "items": {
-                      "type": "string"
-                    }
-                  }
-                ],
-                "description": "project compatible Frameworks"
-              },
-              "platforms": {
-                "anyOf": [
-                  {
-                    "type": "string"
-                  },
-                  {
-                    "type": "array",
-                    "items": {
-                      "type": "string"
-                    }
-                  }
-                ],
-                "description": " project compatible Development Platforms"
-              }
-            }
-          }
-        }
-      ],
-      "description": "A list of dependent libraries that will be automatically installed.",
-      "required": false
-    },
-    "export": {
-      "type": "object",
-      "properties": {
-        "include": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          },
-          "description": "Export only files that matched declared patterns.\n* - matches everything\n? - matches any single character\n[seq] - matches any character in seq\n[!seq] - matches any character not in seq"
-        },
-        "exclude": {
-          "type": "array",
-          "items": {
-            "type": "string"
-          },
-          "description": "Exclude the directories and files which match with exclude patterns."
-        }
-      },
-      "description": "This option is useful if you need to exclude extra data (test code, docs, images, PDFs, etc). It allows one to reduce the size of the final archive.\nTo check which files will be included in the final packages, please use pio pkg pack command.",
-      "required": false
-    },
-    "scripts": {
-      "type": "object",
-      "properties": {
-        "postinstall": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            }
-          ],
-          "description": "runs a script AFTER the package has been installed.\nRun a custom Python script located in the package “scripts” folder AFTER the package is installed. Please note that you don’t need to specify a Python interpreter for Python scripts"
-        },
-        "preuninstall": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            }
-          ],
-          "description": "runs a script BEFORE the package is removed.\nRun a custom Bash script BEFORE the package is uninstalled. The script is declared as a list of command arguments and is located at the root of a package"
-        }
-      },
-      "description": "Execute custom scripts during the special Package Management CLI life cycle events",
-      "required": false
-    },
-    "build": {
-      "type": "object",
-      "properties": {
-        "flags": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            }
-          ],
-          "description": "Extra flags to control preprocessing, compilation, assembly, and linking processes. More details build_flags.\nKeep in mind when operating with the -I flag (directories to be searched for header files). The path should be relative to the root directory where the library.json manifest is located."
-        },
-        "unflags": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            }
-          ],
-          "description": "Remove base/initial flags which were set by development platform. More details build_unflags."
-        },
-        "includeDir": {
-          "type": "string",
-          "description": "Custom directory to be searched for header files. A default value is include and means that folder is located at the root of a library.\nThe Library Dependency Finder (LDF) will pick a library automatically only when a project or other dependent libraries include any header file located in includeDir or srcDir.",
-          "required": false
-        },
-        "srcDir": {
-          "type": "string",
-          "description": "Custom location of library source code. A default value is src and means that folder is located in the root of a library.",
-          "required": "false"
-        },
-        "srcFilter": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            }
-          ],
-          "description": "Specify which source files should be included/excluded from build process. The path in filter should be relative to the srcDir option of a library.\nSee syntax for build_src_filter.\nPlease note that you can generate source filter “on-the-fly” using extraScript",
-          "required": false
-        },
-        "extraScript": {
-          "type": "string",
-          "description": "Launch extra script before a build process.",
-          "required": "false"
-        },
-        "libArchive": {
-          "type": "boolean",
-          "description": "Create an archive (*.a, static library) from the object files and link it into a firmware (program). This is default behavior of PlatformIO Build System (\"libArchive\": true).\nSetting \"libArchive\": false will instruct PlatformIO Build System to link object files directly (in-line). This could be useful if you need to override weak symbols defined in framework or other libraries.\nYou can disable library archiving globally using lib_archive option in “platformio.ini” (Project Configuration File).",
-          "required": "false"
-        },
-        "libLDFMode": {
-          "anyOf": [
-            {
-              "enum": [
-                "off"
-              ],
-              "description": "“Manual mode”, does not process source files of a project and dependencies. Builds only the libraries that are specified in manifests (library.json, module.json) or using lib_deps option."
-            },
-            {
-              "enum": [
-                "chain"
-              ],
-              "description": "[DEFAULT] Parses ALL C/C++ source files of the project and follows only by nested includes (#include ..., chain...) from the libraries. It also parses C, CC, CPP files from libraries which have the same name as included header file. Does not evaluate C/C++ Preprocessor conditional syntax."
-            },
-            {
-              "enum": [
-                "deep"
-              ],
-              "description": "Parses ALL C/C++ source files of the project and parses ALL C/C++ source files of the each found dependency (recursively). Does not evaluate C/C++ Preprocessor conditional syntax."
-            },
-            {
-              "enum": [
-                "chain+"
-              ],
-              "description": "The same behavior as for the chain but evaluates C/C++ Preprocessor conditional syntax."
-            },
-            {
-              "enum": [
-                "deep+"
-              ],
-              "description": "The same behavior as for the deep but evaluates C/C++ Preprocessor conditional syntax."
-            }
-          ],
-          "description": "Specify Library Dependency Finder Mode. See Dependency Finder Mode for details.",
-          "required": false
-        },
-        "libCompatMode": {
-          "type": "string",
-          "description": "Specify Library Compatibility Mode. See Compatibility Mode for details.",
-          "required": false
-        },
-        "builder": {
-          "anyOf": [
-            {
-              "enum": [
-                "PlatformIOLibBuilder"
-              ],
-              "description": "Default Builder"
-            },
-            {
-              "enum": [
-                "ArduinoLibBuilder"
-              ]
-            },
-            {
-              "enum": [
-                "MbedLibBuilder"
-              ]
-            }
-          ],
-          "description": "Override default PlatformIOLibBuilder with another builder.",
-          "required": false
-        }
-      },
-      "required": false
-    }
-  }
-}

platformio/builder/main.py

@@ -12,9 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
-import os
-import sys
+from os import environ, makedirs
+from os.path import isdir, join
 from time import time
 
 import click
@@ -28,10 +27,11 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
-from platformio import app, fs
-from platformio.platform.base import PlatformBase
-from platformio.proc import get_pythonexe_path, where_is_program
-from platformio.project.helpers import get_project_dir
+from platformio import fs
+from platformio.compat import PY2, dump_json_to_unicode
+from platformio.managers.platform import PlatformBase
+from platformio.proc import get_pythonexe_path
+from platformio.project import helpers as project_helpers
 
 AllowSubstExceptions(NameError)
 
@@ -43,134 +43,85 @@ clivars.AddVariables(
     ("PROJECT_CONFIG",),
     ("PIOENV",),
     ("PIOTEST_RUNNING_NAME",),
-    ("UPLOAD_PORT",),
-    ("PROGRAM_ARGS",),
-)
+    ("UPLOAD_PORT",)
+)  # yapf: disable
 
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
-        "ar",
-        "cc",
-        "c++",
-        "link",
-        "piohooks",
-        "pioasm",
-        "piobuild",
-        "pioproject",
-        "pioplatform",
-        "piotest",
-        "piotarget",
-        "piolib",
-        "pioupload",
-        "piosize",
-        "pioino",
-        "piomisc",
-        "piointegration",
-        "piomaxlen",
+        "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform",
+        "pioproject", "piowinhooks", "piolib", "pioupload", "piomisc", "pioide"
     ],
-    toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")],
+    toolpath=[join(fs.get_source_dir(), "builder", "tools")],
     variables=clivars,
-    # Propagating External Environment
-    ENV=os.environ,
-    TIMESTAMP=int(time()),
-    UNIX_TIME="$TIMESTAMP",  # deprecated
-    BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
-    BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
-    BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
-    COMPILATIONDB_PATH=os.path.join("$PROJECT_DIR", "compile_commands.json"),
-    LIBPATH=["$BUILD_DIR"],
-    PROGNAME="program",
-    PROGPATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
-    PROG_PATH="$PROGPATH",  # deprecated
-    PYTHONEXE=get_pythonexe_path(),
-)
 
-# Declare command verbose messages
-command_strings = dict(
-    ARCOM="Archiving",
-    LINKCOM="Linking",
-    RANLIBCOM="Indexing",
-    ASCOM="Compiling",
-    ASPPCOM="Compiling",
-    CCCOM="Compiling",
-    CXXCOM="Compiling",
-)
+    # Propagating External Environment
+    ENV=environ,
+    UNIX_TIME=int(time()),
+    PROJECT_DIR=project_helpers.get_project_dir(),
+    PROJECTCORE_DIR=project_helpers.get_project_core_dir(),
+    PROJECTPACKAGES_DIR=project_helpers.get_project_packages_dir(),
+    PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(),
+    PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(),
+    PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(),
+    PROJECTSRC_DIR=project_helpers.get_project_src_dir(),
+    PROJECTTEST_DIR=project_helpers.get_project_test_dir(),
+    PROJECTDATA_DIR=project_helpers.get_project_data_dir(),
+    PROJECTBUILD_DIR=project_helpers.get_project_build_dir(),
+    BUILDCACHE_DIR=project_helpers.get_project_optional_dir("build_cache_dir"),
+    BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"),
+    BUILDSRC_DIR=join("$BUILD_DIR", "src"),
+    BUILDTEST_DIR=join("$BUILD_DIR", "test"),
+    LIBPATH=["$BUILD_DIR"],
+    LIBSOURCE_DIRS=[
+        project_helpers.get_project_lib_dir(),
+        join("$PROJECTLIBDEPS_DIR", "$PIOENV"),
+        project_helpers.get_project_global_lib_dir()
+    ],
+    PROGNAME="program",
+    PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
+    PYTHONEXE=get_pythonexe_path())
+
 if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
-    for name, value in command_strings.items():
-        DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value)
+    DEFAULT_ENV_OPTIONS['ARCOMSTR'] = "Archiving $TARGET"
+    DEFAULT_ENV_OPTIONS['LINKCOMSTR'] = "Linking $TARGET"
+    DEFAULT_ENV_OPTIONS['RANLIBCOMSTR'] = "Indexing $TARGET"
+    for k in ("ASCOMSTR", "ASPPCOMSTR", "CCCOMSTR", "CXXCOMSTR"):
+        DEFAULT_ENV_OPTIONS[k] = "Compiling $TARGET"
 
 env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
-env.SConscriptChdir(False)
 
 # Load variables from CLI
 env.Replace(
     **{
         key: PlatformBase.decode_scons_arg(env[key])
-        for key in list(clivars.keys())
-        if key in env
-    }
-)
+        for key in list(clivars.keys()) if key in env
+    })
 
-# Setup project optional directories
-config = env.GetProjectConfig()
-app.set_session_var("custom_project_conf", config.path)
-
-env.Replace(
-    PROJECT_DIR=get_project_dir(),
-    PROJECT_CORE_DIR=config.get("platformio", "core_dir"),
-    PROJECT_PACKAGES_DIR=config.get("platformio", "packages_dir"),
-    PROJECT_WORKSPACE_DIR=config.get("platformio", "workspace_dir"),
-    PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"),
-    PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"),
-    PROJECT_SRC_DIR=config.get("platformio", "src_dir"),
-    PROJECTSRC_DIR="$PROJECT_SRC_DIR",  # legacy for dev/platform
-    PROJECT_TEST_DIR=config.get("platformio", "test_dir"),
-    PROJECT_DATA_DIR=config.get("platformio", "data_dir"),
-    PROJECTDATA_DIR="$PROJECT_DATA_DIR",  # legacy for dev/platform
-    PROJECT_BUILD_DIR=config.get("platformio", "build_dir"),
-    BUILD_TYPE=env.GetBuildType(),
-    BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"),
-    LIBSOURCE_DIRS=[
-        config.get("platformio", "lib_dir"),
-        os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
-        config.get("platformio", "globallib_dir"),
-    ],
-)
+if env.subst("$BUILDCACHE_DIR"):
+    if not isdir(env.subst("$BUILDCACHE_DIR")):
+        makedirs(env.subst("$BUILDCACHE_DIR"))
+    env.CacheDir("$BUILDCACHE_DIR")
 
 if int(ARGUMENTS.get("ISATTY", 0)):
     # pylint: disable=protected-access
     click._compat.isatty = lambda stream: True
 
-if env.subst("$BUILD_CACHE_DIR"):
-    if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")):
-        os.makedirs(env.subst("$BUILD_CACHE_DIR"))
-    env.CacheDir("$BUILD_CACHE_DIR")
-
-if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
-    click.echo("Verbose mode can be enabled via `-v, --verbose` option")
-
-# Dynamically load dependent tools
-if "compiledb" in COMMAND_LINE_TARGETS:
-    env.Tool("compilation_db")
-
-if not os.path.isdir(env.subst("$BUILD_DIR")):
-    os.makedirs(env.subst("$BUILD_DIR"))
+if env.GetOption('clean'):
+    env.PioClean(env.subst("$BUILD_DIR"))
+    env.Exit(0)
+elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
+    print("Verbose mode can be enabled via `-v, --verbose` option")
 
 env.LoadProjectOptions()
 env.LoadPioPlatform()
 
+env.SConscriptChdir(0)
 env.SConsignFile(
-    os.path.join(
-        "$BUILD_CACHE_DIR" if env.subst("$BUILD_CACHE_DIR") else "$BUILD_DIR",
-        ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]),
-    )
-)
+    join("$PROJECTBUILD_DIR",
+         ".sconsign.dblite" if PY2 else ".sconsign3.dblite"))
 
-env.SConscript(env.GetExtraScripts("pre"), exports="env")
-
-if env.IsCleanTarget():
-    env.CleanProject(fullclean=int(ARGUMENTS.get("FULLCLEAN", 0)))
-    env.Exit(0)
+for item in env.GetExtraScripts("pre"):
+    env.SConscript(item, exports="env")
 
 env.SConscript("$BUILD_SCRIPT")
 
@@ -179,81 +130,38 @@ if "UPLOAD_FLAGS" in env:
 if env.GetProjectOption("upload_command"):
     env.Replace(UPLOADCMD=env.GetProjectOption("upload_command"))
 
-env.SConscript(env.GetExtraScripts("post"), exports="env")
+for item in env.GetExtraScripts("post"):
+    env.SConscript(item, exports="env")
 
 ##############################################################################
 
 # Checking program size
-if env.get("SIZETOOL") and not (
-    set(["nobuild", "sizedata"]) & set(COMMAND_LINE_TARGETS)
-):
-    env.Depends("upload", "checkprogsize")
+if env.get("SIZETOOL") and "nobuild" not in COMMAND_LINE_TARGETS:
+    env.Depends(["upload", "program"], "checkprogsize")
     # Replace platform's "size" target with our
     _new_targets = [t for t in DEFAULT_TARGETS if str(t) != "size"]
     Default(None)
     Default(_new_targets)
     Default("checkprogsize")
 
-if "compiledb" in COMMAND_LINE_TARGETS:
-    # Resolve absolute path of toolchain
-    for cmd in ("CC", "CXX", "AS"):
-        if cmd not in env:
-            continue
-        if os.path.isabs(env[cmd]):
-            continue
-        env[cmd] = where_is_program(env.subst("$%s" % cmd), env.subst("${ENV['PATH']}"))
-    env.Alias("compiledb", env.CompilationDatabase("$COMPILATIONDB_PATH"))
-
 # Print configured protocols
-env.AddPreAction(
-    "upload",
-    env.VerboseAction(
-        lambda source, target, env: env.PrintUploadInfo(),
-        "Configuring upload protocol...",
-    ),
-)
+env.AddPreAction(["upload", "program"],
+                 env.VerboseAction(
+                     lambda source, target, env: env.PrintUploadInfo(),
+                     "Configuring upload protocol..."))
 
-AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS))
+AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS))
 AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS))
 
-env.ProcessDelayedActions()
-
 ##############################################################################
 
 if "envdump" in COMMAND_LINE_TARGETS:
-    click.echo(env.Dump())
+    print(env.Dump())
     env.Exit(0)
 
-if env.IsIntegrationDump():
-    projenv = None
-    try:
-        Import("projenv")
-    except:  # pylint: disable=bare-except
-        projenv = env
-    data = projenv.DumpIntegrationData(env)
-    # dump to file for the further reading by project.helpers.load_build_metadata
-    with open(
-        projenv.subst(os.path.join("$BUILD_DIR", "idedata.json")),
-        mode="w",
-        encoding="utf8",
-    ) as fp:
-        json.dump(data, fp)
-    click.echo("\n%s\n" % json.dumps(data))  # pylint: disable=undefined-variable
+if "idedata" in COMMAND_LINE_TARGETS:
+    Import("projenv")
+    print("\n%s\n" % dump_json_to_unicode(
+        env.DumpIDEData(projenv)  # pylint: disable=undefined-variable
+    ))
     env.Exit(0)
-
-if "sizedata" in COMMAND_LINE_TARGETS:
-    AlwaysBuild(
-        env.Alias(
-            "sizedata",
-            DEFAULT_TARGETS,
-            env.VerboseAction(env.DumpSizeData, "Generating memory usage report..."),
-        )
-    )
-
-    Default("sizedata")
-
-# issue #4604: process targets sequentially
-for index, target in enumerate(
-    [t for t in COMMAND_LINE_TARGETS if not t.startswith("__")][1:]
-):
-    env.Depends(target, COMMAND_LINE_TARGETS[index])

platformio/builder/tools/pioasm.py

@@ -1,29 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import SCons.Tool.asm  # pylint: disable=import-error
-
-#
-# Resolve https://github.com/platformio/platformio-core/issues/3917
-# Avoid forcing .S to bare assembly on Windows OS
-#
-
-if ".S" in SCons.Tool.asm.ASSuffixes:
-    SCons.Tool.asm.ASSuffixes.remove(".S")
-if ".S" not in SCons.Tool.asm.ASPPSuffixes:
-    SCons.Tool.asm.ASPPSuffixes.append(".S")
-
-
-generate = SCons.Tool.asm.generate
-exists = SCons.Tool.asm.exists

platformio/builder/tools/piobuild.py

@@ -1,382 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import fnmatch
-import os
-import sys
-
-from SCons import Builder, Util  # pylint: disable=import-error
-from SCons.Node import FS  # pylint: disable=import-error
-from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
-from SCons.Script import AlwaysBuild  # pylint: disable=import-error
-from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
-from SCons.Script import SConscript  # pylint: disable=import-error
-
-from platformio import __version__, fs
-from platformio.compat import IS_MACOS, string_types
-from platformio.package.version import pepver_to_semver
-
-SRC_HEADER_EXT = ["h", "hpp"]
-SRC_ASM_EXT = ["S", "spp", "SPP", "sx", "s", "asm", "ASM"]
-SRC_C_EXT = ["c"]
-SRC_CXX_EXT = ["cc", "cpp", "cxx", "c++"]
-SRC_BUILD_EXT = SRC_C_EXT + SRC_CXX_EXT + SRC_ASM_EXT
-SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % os.sep, "-<.svn%s>" % os.sep]
-
-
-def scons_patched_match_splitext(path, suffixes=None):
-    """Patch SCons Builder, append $OBJSUFFIX to the end of each target"""
-    tokens = Util.splitext(path)
-    if suffixes and tokens[1] and tokens[1] in suffixes:
-        return (path, tokens[1])
-    return tokens
-
-
-def GetBuildType(env):
-    modes = []
-    if (
-        set(["__debug", "sizedata"])  # sizedata = for memory inspection
-        & set(COMMAND_LINE_TARGETS)
-        or env.GetProjectOption("build_type") == "debug"
-    ):
-        modes.append("debug")
-    if "__test" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "test":
-        modes.append("test")
-    return "+".join(modes or ["release"])
-
-
-def BuildProgram(env):
-    env.ProcessProgramDeps()
-    env.ProcessProjectDeps()
-
-    # append into the beginning a main LD script
-    if env.get("LDSCRIPT_PATH") and not any("-Wl,-T" in f for f in env["LINKFLAGS"]):
-        env.Prepend(LINKFLAGS=["-T", env.subst("$LDSCRIPT_PATH")])
-
-    # enable "cyclic reference" for linker
-    if (
-        env.get("LIBS")
-        and env.GetCompilerType() == "gcc"
-        and (env.PioPlatform().is_embedded() or not IS_MACOS)
-    ):
-        env.Prepend(_LIBFLAGS="-Wl,--start-group ")
-        env.Append(_LIBFLAGS=" -Wl,--end-group")
-
-    program = env.Program(env.subst("$PROGPATH"), env["PIOBUILDFILES"])
-    env.Replace(PIOMAINPROG=program)
-
-    AlwaysBuild(
-        env.Alias(
-            "checkprogsize",
-            program,
-            env.VerboseAction(env.CheckUploadSize, "Checking size $PIOMAINPROG"),
-        )
-    )
-
-    print("Building in %s mode" % env["BUILD_TYPE"])
-
-    return program
-
-
-def ProcessProgramDeps(env):
-    def _append_pio_macros():
-        core_version = pepver_to_semver(__version__)
-        env.AppendUnique(
-            CPPDEFINES=[
-                (
-                    "PLATFORMIO",
-                    int(
-                        "{0:02d}{1:02d}{2:02d}".format(
-                            core_version.major, core_version.minor, core_version.patch
-                        )
-                    ),
-                )
-            ]
-        )
-
-    _append_pio_macros()
-
-    env.PrintConfiguration()
-
-    # process extra flags from board
-    if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
-        env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
-
-    # apply user flags
-    env.ProcessFlags(env.get("BUILD_FLAGS"))
-
-    # process framework scripts
-    env.BuildFrameworks(env.get("PIOFRAMEWORK"))
-
-    if "debug" in env["BUILD_TYPE"]:
-        env.ConfigureDebugTarget()
-
-    # remove specified flags
-    env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
-
-    if "compiledb" in COMMAND_LINE_TARGETS and env.get(
-        "COMPILATIONDB_INCLUDE_TOOLCHAIN"
-    ):
-        for scope, includes in env.DumpIntegrationIncludes().items():
-            if scope in ("toolchain",):
-                env.Append(CPPPATH=includes)
-
-
-def ProcessProjectDeps(env):
-    plb = env.ConfigureProjectLibBuilder()
-
-    # prepend project libs to the beginning of list
-    env.Prepend(LIBS=plb.build())
-    # prepend extra linker related options from libs
-    env.PrependUnique(
-        **{
-            key: plb.env.get(key)
-            for key in ("LIBS", "LIBPATH", "LINKFLAGS")
-            if plb.env.get(key)
-        }
-    )
-
-    if "test" in env["BUILD_TYPE"]:
-        build_files_before_nums = len(env.get("PIOBUILDFILES", []))
-        plb.env.BuildSources(
-            "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
-        )
-        if len(env.get("PIOBUILDFILES", [])) - build_files_before_nums < 1:
-            sys.stderr.write(
-                "Error: Nothing to build. Please put your test suites "
-                "to the '%s' folder\n" % env.subst("$PROJECT_TEST_DIR")
-            )
-            env.Exit(1)
-
-    if "test" not in env["BUILD_TYPE"] or env.GetProjectOption("test_build_src"):
-        plb.env.BuildSources(
-            "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER")
-        )
-
-    if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
-        sys.stderr.write(
-            "Error: Nothing to build. Please put your source code files "
-            "to the '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
-        )
-        env.Exit(1)
-
-
-def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
-    if not isinstance(flags, list):
-        flags = [flags]
-    result = {}
-    for raw in flags:
-        for key, value in env.ParseFlags(str(raw)).items():
-            if key not in result:
-                result[key] = []
-            result[key].extend(value)
-
-    cppdefines = []
-    for item in result["CPPDEFINES"]:
-        if not Util.is_Sequence(item):
-            cppdefines.append(item)
-            continue
-        name, value = item[:2]
-        if '"' in value:
-            value = value.replace('"', '\\"')
-        elif value.isdigit():
-            value = int(value)
-        elif value.replace(".", "", 1).isdigit():
-            value = float(value)
-        cppdefines.append((name, value))
-    result["CPPDEFINES"] = cppdefines
-
-    # fix relative CPPPATH & LIBPATH
-    for k in ("CPPPATH", "LIBPATH"):
-        for i, p in enumerate(result.get(k, [])):
-            p = env.subst(p)
-            if os.path.isdir(p):
-                result[k][i] = os.path.abspath(p)
-
-    # fix relative path for "-include"
-    for i, f in enumerate(result.get("CCFLAGS", [])):
-        if isinstance(f, tuple) and f[0] == "-include":
-            p = env.subst(f[1].get_path())
-            if os.path.exists(p):
-                result["CCFLAGS"][i] = (f[0], os.path.abspath(p))
-
-    return result
-
-
-def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
-    if not flags:
-        return
-    env.Append(**env.ParseFlagsExtended(flags))
-
-    # Cancel any previous definition of name, either built in or
-    # provided with a -U option // Issue #191
-    undefines = [
-        u
-        for u in env.get("CCFLAGS", [])
-        if isinstance(u, string_types) and u.startswith("-U")
-    ]
-    if undefines:
-        for undef in undefines:
-            env["CCFLAGS"].remove(undef)
-            if undef[2:] in env["CPPDEFINES"]:
-                env["CPPDEFINES"].remove(undef[2:])
-        env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines))
-
-
-def ProcessUnFlags(env, flags):
-    if not flags:
-        return
-    parsed = env.ParseFlagsExtended(flags)
-    unflag_scopes = tuple(set(["ASPPFLAGS"] + list(parsed.keys())))
-    for scope in unflag_scopes:
-        for unflags in parsed.values():
-            for unflag in unflags:
-                for current in list(env.get(scope, [])):
-                    conditions = [
-                        unflag == current,
-                        not isinstance(unflag, (tuple, list))
-                        and isinstance(current, (tuple, list))
-                        and unflag == current[0],
-                    ]
-                    if any(conditions):
-                        env[scope].remove(current)
-
-
-def StringifyMacro(env, value):  # pylint: disable=unused-argument
-    return '\\"%s\\"' % value.replace('"', '\\\\\\"')
-
-
-def MatchSourceFiles(env, src_dir, src_filter=None, src_exts=None):
-    src_filter = env.subst(src_filter) if src_filter else None
-    src_filter = src_filter or SRC_FILTER_DEFAULT
-    src_exts = src_exts or (SRC_BUILD_EXT + SRC_HEADER_EXT)
-    return fs.match_src_files(env.subst(src_dir), src_filter, src_exts)
-
-
-def CollectBuildFiles(
-    env, variant_dir, src_dir, src_filter=None, duplicate=False
-):  # pylint: disable=too-many-locals
-    sources = []
-    variants = []
-
-    src_dir = env.subst(src_dir)
-    if src_dir.endswith(os.sep):
-        src_dir = src_dir[:-1]
-
-    for item in env.MatchSourceFiles(src_dir, src_filter, SRC_BUILD_EXT):
-        _reldir = os.path.dirname(item)
-        _src_dir = os.path.join(src_dir, _reldir) if _reldir else src_dir
-        _var_dir = os.path.join(variant_dir, _reldir) if _reldir else variant_dir
-
-        if _var_dir not in variants:
-            variants.append(_var_dir)
-            env.VariantDir(_var_dir, _src_dir, duplicate)
-
-        sources.append(env.File(os.path.join(_var_dir, os.path.basename(item))))
-
-    middlewares = env.get("__PIO_BUILD_MIDDLEWARES")
-    if not middlewares:
-        return sources
-
-    new_sources = []
-    for node in sources:
-        new_node = node
-        for callback, pattern in middlewares:
-            if pattern and not fnmatch.fnmatch(node.srcnode().get_path(), pattern):
-                continue
-            if callback.__code__.co_argcount == 2:
-                new_node = callback(env, new_node)
-            else:
-                new_node = callback(new_node)
-            if not new_node:
-                break
-        if new_node:
-            new_sources.append(new_node)
-
-    return new_sources
-
-
-def AddBuildMiddleware(env, callback, pattern=None):
-    env.Append(__PIO_BUILD_MIDDLEWARES=[(callback, pattern)])
-
-
-def BuildFrameworks(env, frameworks):
-    if not frameworks:
-        return
-
-    if "BOARD" not in env:
-        sys.stderr.write(
-            "Please specify `board` in `platformio.ini` to use "
-            "with '%s' framework\n" % ", ".join(frameworks)
-        )
-        env.Exit(1)
-
-    supported_frameworks = env.BoardConfig().get("frameworks", [])
-    for name in frameworks:
-        if name == "arduino":
-            # Arduino IDE appends .o to the end of filename
-            Builder.match_splitext = scons_patched_match_splitext
-            if "nobuild" not in COMMAND_LINE_TARGETS:
-                env.ConvertInoToCpp()
-
-        if name in supported_frameworks:
-            SConscript(env.GetFrameworkScript(name), exports="env")
-        else:
-            sys.stderr.write("Error: This board doesn't support %s framework!\n" % name)
-            env.Exit(1)
-
-
-def BuildLibrary(env, variant_dir, src_dir, src_filter=None, nodes=None):
-    env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
-    nodes = nodes or env.CollectBuildFiles(variant_dir, src_dir, src_filter)
-    return env.StaticLibrary(env.subst(variant_dir), nodes)
-
-
-def BuildSources(env, variant_dir, src_dir, src_filter=None):
-    if env.get("PIOMAINPROG"):
-        sys.stderr.write(
-            "Error: The main program is already constructed and the inline "
-            "source files are not allowed. Please use `env.BuildLibrary(...)` "
-            "or PRE-type script instead."
-        )
-        env.Exit(1)
-
-    nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter)
-    DefaultEnvironment().Append(
-        PIOBUILDFILES=[
-            env.Object(node) if isinstance(node, FS.File) else node for node in nodes
-        ]
-    )
-
-
-def exists(_):
-    return True
-
-
-def generate(env):
-    env.AddMethod(GetBuildType)
-    env.AddMethod(BuildProgram)
-    env.AddMethod(ProcessProgramDeps)
-    env.AddMethod(ProcessProjectDeps)
-    env.AddMethod(ParseFlagsExtended)
-    env.AddMethod(ProcessFlags)
-    env.AddMethod(ProcessUnFlags)
-    env.AddMethod(StringifyMacro)
-    env.AddMethod(MatchSourceFiles)
-    env.AddMethod(CollectBuildFiles)
-    env.AddMethod(AddBuildMiddleware)
-    env.AddMethod(BuildFrameworks)
-    env.AddMethod(BuildLibrary)
-    env.AddMethod(BuildSources)
-    return env

platformio/builder/tools/piohooks.py

@@ -1,50 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-def AddActionWrapper(handler):
-    def wraps(env, files, action):
-        if not isinstance(files, (list, tuple, set)):
-            files = [files]
-        known_nodes = []
-        unknown_files = []
-        for item in files:
-            nodes = env.arg2nodes(item, env.fs.Entry)
-            if nodes and nodes[0].exists():
-                known_nodes.extend(nodes)
-            else:
-                unknown_files.append(item)
-        if unknown_files:
-            env.Append(**{"_PIO_DELAYED_ACTIONS": [(handler, unknown_files, action)]})
-        if known_nodes:
-            return handler(known_nodes, action)
-        return []
-
-    return wraps
-
-
-def ProcessDelayedActions(env):
-    for func, nodes, action in env.get("_PIO_DELAYED_ACTIONS", []):
-        func(nodes, action)
-
-
-def generate(env):
-    env.Replace(**{"_PIO_DELAYED_ACTIONS": []})
-    env.AddMethod(AddActionWrapper(env.AddPreAction), "AddPreAction")
-    env.AddMethod(AddActionWrapper(env.AddPostAction), "AddPostAction")
-    env.AddMethod(ProcessDelayedActions)
-
-
-def exists(_):
-    return True

platformio/builder/tools/pioide.py

@@ -0,0 +1,197 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from glob import glob
+from os import environ
+from os.path import abspath, isfile, join
+
+from SCons.Defaults import processDefines  # pylint: disable=import-error
+
+from platformio.compat import glob_escape
+from platformio.managers.core import get_core_package_dir
+from platformio.proc import exec_command, where_is_program
+
+
+def _dump_includes(env, projenv):
+    includes = []
+
+    for item in projenv.get("CPPPATH", []):
+        includes.append(projenv.subst(item))
+
+    # installed libs
+    for lb in env.GetLibBuilders():
+        includes.extend(lb.get_include_dirs())
+
+    # includes from toolchains
+    p = env.PioPlatform()
+    for name in p.get_installed_packages():
+        if p.get_package_type(name) != "toolchain":
+            continue
+        toolchain_dir = glob_escape(p.get_package_dir(name))
+        toolchain_incglobs = [
+            join(toolchain_dir, "*", "include*"),
+            join(toolchain_dir, "*", "include", "c++", "*"),
+            join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"),
+            join(toolchain_dir, "lib", "gcc", "*", "*", "include*")
+        ]
+        for g in toolchain_incglobs:
+            includes.extend(glob(g))
+
+    unity_dir = get_core_package_dir("tool-unity")
+    if unity_dir:
+        includes.append(unity_dir)
+
+    includes.extend(
+        [env.subst("$PROJECTINCLUDE_DIR"),
+         env.subst("$PROJECTSRC_DIR")])
+
+    # remove duplicates
+    result = []
+    for item in includes:
+        if item not in result:
+            result.append(abspath(item))
+
+    return result
+
+
+def _get_gcc_defines(env):
+    items = []
+    try:
+        sysenv = environ.copy()
+        sysenv['PATH'] = str(env['ENV']['PATH'])
+        result = exec_command("echo | %s -dM -E -" % env.subst("$CC"),
+                              env=sysenv,
+                              shell=True)
+    except OSError:
+        return items
+    if result['returncode'] != 0:
+        return items
+    for line in result['out'].split("\n"):
+        tokens = line.strip().split(" ", 2)
+        if not tokens or tokens[0] != "#define":
+            continue
+        if len(tokens) > 2:
+            items.append("%s=%s" % (tokens[1], tokens[2]))
+        else:
+            items.append(tokens[1])
+    return items
+
+
+def _dump_defines(env):
+    defines = []
+    # global symbols
+    for item in processDefines(env.get("CPPDEFINES", [])):
+        defines.append(env.subst(item).replace('\\', ''))
+
+    # special symbol for Atmel AVR MCU
+    if env['PIOPLATFORM'] == "atmelavr":
+        board_mcu = env.get("BOARD_MCU")
+        if not board_mcu and "BOARD" in env:
+            board_mcu = env.BoardConfig().get("build.mcu")
+        if board_mcu:
+            defines.append(
+                str("__AVR_%s__" % board_mcu.upper().replace(
+                    "ATMEGA", "ATmega").replace("ATTINY", "ATtiny")))
+
+    # built-in GCC marcos
+    # if env.GetCompilerType() == "gcc":
+    #     defines.extend(_get_gcc_defines(env))
+
+    return defines
+
+
+def _get_svd_path(env):
+    svd_path = env.GetProjectOption("debug_svd_path")
+    if svd_path:
+        return abspath(svd_path)
+
+    if "BOARD" not in env:
+        return None
+    try:
+        svd_path = env.BoardConfig().get("debug.svd_path")
+        assert svd_path
+    except (AssertionError, KeyError):
+        return None
+    # custom path to SVD file
+    if isfile(svd_path):
+        return svd_path
+    # default file from ./platform/misc/svd folder
+    p = env.PioPlatform()
+    if isfile(join(p.get_dir(), "misc", "svd", svd_path)):
+        return abspath(join(p.get_dir(), "misc", "svd", svd_path))
+    return None
+
+
+def DumpIDEData(env, projenv):
+    LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS"
+    LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS"
+
+    data = {
+        "env_name":
+        env['PIOENV'],
+        "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()],
+        "defines":
+        _dump_defines(env),
+        "includes":
+        _dump_includes(env, projenv),
+        "cc_flags":
+        env.subst(LINTCCOM),
+        "cxx_flags":
+        env.subst(LINTCXXCOM),
+        "cc_path":
+        where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
+        "cxx_path":
+        where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
+        "gdb_path":
+        where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),
+        "prog_path":
+        env.subst("$PROG_PATH"),
+        "flash_extra_images": [{
+            "offset": item[0],
+            "path": env.subst(item[1])
+        } for item in env.get("FLASH_EXTRA_IMAGES", [])],
+        "svd_path":
+        _get_svd_path(env),
+        "compiler_type":
+        env.GetCompilerType()
+    }
+
+    env_ = env.Clone()
+    # https://github.com/platformio/platformio-atom-ide/issues/34
+    _new_defines = []
+    for item in processDefines(env_.get("CPPDEFINES", [])):
+        item = item.replace('\\"', '"')
+        if " " in item:
+            _new_defines.append(item.replace(" ", "\\\\ "))
+        else:
+            _new_defines.append(item)
+    env_.Replace(CPPDEFINES=_new_defines)
+
+    data.update({
+        "cc_flags": env_.subst(LINTCCOM),
+        "cxx_flags": env_.subst(LINTCXXCOM)
+    })
+
+    return data
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    env.AddMethod(DumpIDEData)
+    return env

platformio/builder/tools/pioino.py

@@ -1,256 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import atexit
-import glob
-import io
-import os
-import re
-import tempfile
-
-import click
-
-from platformio.compat import get_filesystem_encoding, get_locale_encoding
-
-
-class InoToCPPConverter:
-    PROTOTYPE_RE = re.compile(
-        r"""^(
-        (?:template\<.*\>\s*)?      # template
-        ([a-z_\d\&]+\*?\s+){1,2}    # return type
-        ([a-z_\d]+\s*)              # name of prototype
-        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
-        )\s*(\{|;)                  # must end with `{` or `;`
-        """,
-        re.X | re.M | re.I,
-    )
-    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
-    PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
-
-    def __init__(self, env):
-        self.env = env
-        self._main_ino = None
-        self._safe_encoding = None
-
-    def read_safe_contents(self, path):
-        error_reported = False
-        for encoding in (
-            "utf-8",
-            None,
-            get_filesystem_encoding(),
-            get_locale_encoding(),
-            "latin-1",
-        ):
-            try:
-                with io.open(path, encoding=encoding) as fp:
-                    contents = fp.read()
-                    self._safe_encoding = encoding
-                    return contents
-            except UnicodeDecodeError:
-                if not error_reported:
-                    error_reported = True
-                    click.secho(
-                        "Unicode decode error has occurred, please remove invalid "
-                        "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8"
-                        % path,
-                        fg="yellow",
-                        err=True,
-                    )
-        return ""
-
-    def write_safe_contents(self, path, contents):
-        with io.open(
-            path, "w", encoding=self._safe_encoding, errors="backslashreplace"
-        ) as fp:
-            return fp.write(contents)
-
-    def is_main_node(self, contents):
-        return self.DETECTMAIN_RE.search(contents)
-
-    def convert(self, nodes):
-        contents = self.merge(nodes)
-        if not contents:
-            return None
-        return self.process(contents)
-
-    def merge(self, nodes):
-        assert nodes
-        lines = []
-        for node in nodes:
-            contents = self.read_safe_contents(node.get_path())
-            _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents]
-            if self.is_main_node(contents):
-                lines = _lines + lines
-                self._main_ino = node.get_path()
-            else:
-                lines.extend(_lines)
-
-        if not self._main_ino:
-            self._main_ino = nodes[0].get_path()
-
-        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
-
-    def process(self, contents):
-        out_file = re.sub(r"[\"\'\;]+", "", self._main_ino) + ".cpp"
-        assert self._gcc_preprocess(contents, out_file)
-        contents = self.read_safe_contents(out_file)
-        contents = self._join_multiline_strings(contents)
-        self.write_safe_contents(out_file, self.append_prototypes(contents))
-        return out_file
-
-    def _gcc_preprocess(self, contents, out_file):
-        tmp_path = tempfile.mkstemp()[1]
-        self.write_safe_contents(tmp_path, contents)
-        self.env.Execute(
-            self.env.VerboseAction(
-                '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format(
-                    out_file, tmp_path
-                ),
-                "Converting " + os.path.basename(out_file[:-4]),
-            )
-        )
-        atexit.register(_delete_file, tmp_path)
-        return os.path.isfile(out_file)
-
-    def _join_multiline_strings(self, contents):
-        if "\\\n" not in contents:
-            return contents
-        newlines = []
-        linenum = 0
-        stropen = False
-        for line in contents.split("\n"):
-            _linenum = self._parse_preproc_line_num(line)
-            if _linenum is not None:
-                linenum = _linenum
-            else:
-                linenum += 1
-
-            if line.endswith("\\"):
-                if line.startswith('"'):
-                    stropen = True
-                    newlines.append(line[:-1])
-                    continue
-                if stropen:
-                    newlines[len(newlines) - 1] += line[:-1]
-                    continue
-            elif stropen and line.endswith(('",', '";')):
-                newlines[len(newlines) - 1] += line
-                stropen = False
-                newlines.append(
-                    '#line %d "%s"' % (linenum, self._main_ino.replace("\\", "/"))
-                )
-                continue
-
-            newlines.append(line)
-
-        return "\n".join(newlines)
-
-    @staticmethod
-    def _parse_preproc_line_num(line):
-        if not line.startswith("#"):
-            return None
-        tokens = line.split(" ", 3)
-        if len(tokens) > 2 and tokens[1].isdigit():
-            return int(tokens[1])
-        return None
-
-    def _parse_prototypes(self, contents):
-        prototypes = []
-        reserved_keywords = set(["if", "else", "while"])
-        for match in self.PROTOTYPE_RE.finditer(contents):
-            if (
-                set([match.group(2).strip(), match.group(3).strip()])
-                & reserved_keywords
-            ):
-                continue
-            prototypes.append(match)
-        return prototypes
-
-    def _get_total_lines(self, contents):
-        total = 0
-        if contents.endswith("\n"):
-            contents = contents[:-1]
-        for line in contents.split("\n")[::-1]:
-            linenum = self._parse_preproc_line_num(line)
-            if linenum is not None:
-                return total + linenum
-            total += 1
-        return total
-
-    def append_prototypes(self, contents):
-        prototypes = self._parse_prototypes(contents) or []
-
-        # skip already declared prototypes
-        declared = set(m.group(1).strip() for m in prototypes if m.group(4) == ";")
-        prototypes = [m for m in prototypes if m.group(1).strip() not in declared]
-
-        if not prototypes:
-            return contents
-
-        prototype_names = set(m.group(3).strip() for m in prototypes)
-        split_pos = prototypes[0].start()
-        match_ptrs = re.search(
-            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
-            contents[:split_pos],
-            re.M,
-        )
-        if match_ptrs:
-            split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
-
-        result = []
-        result.append(contents[:split_pos].strip())
-        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
-        result.append(
-            '#line %d "%s"'
-            % (
-                self._get_total_lines(contents[:split_pos]),
-                self._main_ino.replace("\\", "/"),
-            )
-        )
-        result.append(contents[split_pos:].strip())
-        return "\n".join(result)
-
-
-def FindInoNodes(env):
-    src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
-    return env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
-        os.path.join(src_dir, "*.pde")
-    )
-
-
-def ConvertInoToCpp(env):
-    ino_nodes = env.FindInoNodes()
-    if not ino_nodes:
-        return
-    c = InoToCPPConverter(env)
-    out_file = c.convert(ino_nodes)
-
-    atexit.register(_delete_file, out_file)
-
-
-def _delete_file(path):
-    try:
-        if os.path.isfile(path):
-            os.remove(path)
-    except:  # pylint: disable=bare-except
-        pass
-
-
-def generate(env):
-    env.AddMethod(FindInoNodes)
-    env.AddMethod(ConvertInoToCpp)
-
-
-def exists(_):
-    return True

platformio/builder/tools/piointegration.py

@@ -1,199 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import glob
-import os
-
-import click
-import SCons.Defaults  # pylint: disable=import-error
-import SCons.Subst  # pylint: disable=import-error
-from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
-
-from platformio.proc import exec_command, where_is_program
-
-
-def IsIntegrationDump(_):
-    return set(["__idedata", "idedata"]) & set(COMMAND_LINE_TARGETS)
-
-
-def DumpIntegrationIncludes(env):
-    result = dict(build=[], compatlib=[], toolchain=[])
-
-    result["build"].extend(
-        [
-            env.subst("$PROJECT_INCLUDE_DIR"),
-            env.subst("$PROJECT_SRC_DIR"),
-        ]
-    )
-    result["build"].extend(
-        [os.path.abspath(env.subst(item)) for item in env.get("CPPPATH", [])]
-    )
-
-    # installed libs
-    for lb in env.GetLibBuilders():
-        result["compatlib"].extend(
-            [os.path.abspath(inc) for inc in lb.get_include_dirs()]
-        )
-
-    # includes from toolchains
-    p = env.PioPlatform()
-    for pkg in p.get_installed_packages(with_optional=False):
-        if p.get_package_type(pkg.metadata.name) != "toolchain":
-            continue
-        toolchain_dir = glob.escape(pkg.path)
-        toolchain_incglobs = [
-            os.path.join(toolchain_dir, "*", "include", "c++", "*"),
-            os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"),
-            os.path.join(toolchain_dir, "lib", "gcc", "*", "*", "include*"),
-            os.path.join(toolchain_dir, "*", "include*"),
-        ]
-        for g in toolchain_incglobs:
-            result["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)])
-
-    return result
-
-
-def get_gcc_defines(env):
-    items = []
-    try:
-        sysenv = os.environ.copy()
-        sysenv["PATH"] = str(env["ENV"]["PATH"])
-        result = exec_command(
-            "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True
-        )
-    except OSError:
-        return items
-    if result["returncode"] != 0:
-        return items
-    for line in result["out"].split("\n"):
-        tokens = line.strip().split(" ", 2)
-        if not tokens or tokens[0] != "#define":
-            continue
-        if len(tokens) > 2:
-            items.append("%s=%s" % (tokens[1], tokens[2]))
-        else:
-            items.append(tokens[1])
-    return items
-
-
-def dump_defines(env):
-    defines = []
-    # global symbols
-    for item in SCons.Defaults.processDefines(env.get("CPPDEFINES", [])):
-        item = item.strip()
-        if item:
-            defines.append(env.subst(item).replace('\\"', '"'))
-
-    # special symbol for Atmel AVR MCU
-    if env["PIOPLATFORM"] == "atmelavr":
-        board_mcu = env.get("BOARD_MCU")
-        if not board_mcu and "BOARD" in env:
-            board_mcu = env.BoardConfig().get("build.mcu")
-        if board_mcu:
-            defines.append(
-                str(
-                    "__AVR_%s__"
-                    % board_mcu.upper()
-                    .replace("ATMEGA", "ATmega")
-                    .replace("ATTINY", "ATtiny")
-                )
-            )
-
-    # built-in GCC marcos
-    # if env.GetCompilerType() == "gcc":
-    #     defines.extend(get_gcc_defines(env))
-
-    return defines
-
-
-def dump_svd_path(env):
-    svd_path = env.GetProjectOption("debug_svd_path")
-    if svd_path:
-        return os.path.abspath(svd_path)
-
-    if "BOARD" not in env:
-        return None
-    try:
-        svd_path = env.BoardConfig().get("debug.svd_path")
-        assert svd_path
-    except (AssertionError, KeyError):
-        return None
-    # custom path to SVD file
-    if os.path.isfile(svd_path):
-        return svd_path
-    # default file from ./platform/misc/svd folder
-    p = env.PioPlatform()
-    if os.path.isfile(os.path.join(p.get_dir(), "misc", "svd", svd_path)):
-        return os.path.abspath(os.path.join(p.get_dir(), "misc", "svd", svd_path))
-    return None
-
-
-def _subst_cmd(env, cmd):
-    args = env.subst_list(cmd, SCons.Subst.SUBST_CMD)[0]
-    return " ".join([SCons.Subst.quote_spaces(arg) for arg in args])
-
-
-def DumpIntegrationData(*args):
-    projenv, globalenv = args[0:2]  # pylint: disable=unbalanced-tuple-unpacking
-    data = {
-        "build_type": globalenv.GetBuildType(),
-        "env_name": globalenv["PIOENV"],
-        "libsource_dirs": [
-            globalenv.subst(item) for item in globalenv.GetLibSourceDirs()
-        ],
-        "defines": dump_defines(projenv),
-        "includes": projenv.DumpIntegrationIncludes(),
-        "cc_flags": click.parser.split_arg_string(
-            _subst_cmd(projenv, "$CFLAGS $CCFLAGS $CPPFLAGS")
-        ),
-        "cxx_flags": click.parser.split_arg_string(
-            _subst_cmd(projenv, "$CXXFLAGS $CCFLAGS $CPPFLAGS")
-        ),
-        "cc_path": where_is_program(
-            globalenv.subst("$CC"), globalenv.subst("${ENV['PATH']}")
-        ),
-        "cxx_path": where_is_program(
-            globalenv.subst("$CXX"), globalenv.subst("${ENV['PATH']}")
-        ),
-        "gdb_path": where_is_program(
-            globalenv.subst("$GDB"), globalenv.subst("${ENV['PATH']}")
-        ),
-        "prog_path": globalenv.subst("$PROGPATH"),
-        "svd_path": dump_svd_path(globalenv),
-        "compiler_type": globalenv.GetCompilerType(),
-        "targets": globalenv.DumpTargets(),
-        "extra": dict(
-            flash_images=[
-                {"offset": item[0], "path": globalenv.subst(item[1])}
-                for item in globalenv.get("FLASH_EXTRA_IMAGES", [])
-            ]
-        ),
-    }
-    for key in ("IDE_EXTRA_DATA", "INTEGRATION_EXTRA_DATA"):
-        data["extra"].update(globalenv.get(key, {}))
-    return data
-
-
-def exists(_):
-    return True
-
-
-def generate(env):
-    env["IDE_EXTRA_DATA"] = {}  # legacy support
-    env["INTEGRATION_EXTRA_DATA"] = {}
-    env.AddMethod(IsIntegrationDump)
-    env.AddMethod(DumpIntegrationIncludes)
-    env.AddMethod(DumpIntegrationData)
-    return env

platformio/builder/tools/piomaxlen.py

@@ -1,96 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import hashlib
-import os
-import re
-
-from SCons.Platform import TempFileMunge  # pylint: disable=import-error
-from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
-from SCons.Subst import quote_spaces  # pylint: disable=import-error
-
-from platformio.compat import IS_WINDOWS, hashlib_encode_data
-
-# There are the next limits depending on a platform:
-# - Windows = 8192
-# - Unix    = 131072
-# We need ~512 characters for compiler and temporary file paths
-MAX_LINE_LENGTH = (8192 if IS_WINDOWS else 131072) - 512
-
-WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)")
-
-
-def tempfile_arg_esc_func(arg):
-    arg = quote_spaces(arg)
-    if not IS_WINDOWS:
-        return arg
-    # GCC requires double Windows slashes, let's use UNIX separator
-    return WINPATHSEP_RE.sub(r"/\1", arg)
-
-
-def long_sources_hook(env, sources):
-    _sources = str(sources).replace("\\", "/")
-    if len(str(_sources)) < MAX_LINE_LENGTH:
-        return sources
-
-    # fix space in paths
-    data = []
-    for line in _sources.split(".o "):
-        line = line.strip()
-        if not line.endswith(".o"):
-            line += ".o"
-        data.append('"%s"' % line)
-
-    return '@"%s"' % _file_long_data(env, " ".join(data))
-
-
-def _file_long_data(env, data):
-    build_dir = env.subst("$BUILD_DIR")
-    if not os.path.isdir(build_dir):
-        os.makedirs(build_dir)
-    tmp_file = os.path.join(
-        build_dir, "longcmd-%s" % hashlib.md5(hashlib_encode_data(data)).hexdigest()
-    )
-    if os.path.isfile(tmp_file):
-        return tmp_file
-    with open(tmp_file, mode="w", encoding="utf8") as fp:
-        fp.write(data)
-    return tmp_file
-
-
-def exists(env):
-    return "compiledb" not in COMMAND_LINE_TARGETS and not env.IsIntegrationDump()
-
-
-def generate(env):
-    if not exists(env):
-        return env
-    kwargs = dict(
-        _long_sources_hook=long_sources_hook,
-        TEMPFILE=TempFileMunge,
-        MAXLINELENGTH=MAX_LINE_LENGTH,
-        TEMPFILEARGESCFUNC=tempfile_arg_esc_func,
-        TEMPFILESUFFIX=".tmp",
-        TEMPFILEDIR="$BUILD_DIR",
-    )
-
-    for name in ("LINKCOM", "ASCOM", "ASPPCOM", "CCCOM", "CXXCOM"):
-        kwargs[name] = "${TEMPFILE('%s','$%sSTR')}" % (env.get(name), name)
-
-    kwargs["ARCOM"] = env.get("ARCOM", "").replace(
-        "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}"
-    )
-    env.Replace(**kwargs)
-
-    return env

platformio/builder/tools/piomisc.py

@@ -12,26 +12,217 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
+from __future__ import absolute_import
+
+import atexit
+import re
 import sys
+from os import environ, remove, walk
+from os.path import basename, isdir, isfile, join, realpath, relpath, sep
+from tempfile import mkstemp
+
+from SCons.Action import Action  # pylint: disable=import-error
+from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 
 from platformio import fs, util
+from platformio.compat import get_file_contents, glob_escape
+from platformio.managers.core import get_core_package_dir
 from platformio.proc import exec_command
 
 
+class InoToCPPConverter(object):
+
+    PROTOTYPE_RE = re.compile(
+        r"""^(
+        (?:template\<.*\>\s*)?      # template
+        ([a-z_\d\&]+\*?\s+){1,2}    # return type
+        ([a-z_\d]+\s*)              # name of prototype
+        \([a-z_,\.\*\&\[\]\s\d]*\)  # arguments
+        )\s*(\{|;)                  # must end with `{` or `;`
+        """, re.X | re.M | re.I)
+    DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I)
+    PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)"
+
+    def __init__(self, env):
+        self.env = env
+        self._main_ino = None
+
+    def is_main_node(self, contents):
+        return self.DETECTMAIN_RE.search(contents)
+
+    def convert(self, nodes):
+        contents = self.merge(nodes)
+        if not contents:
+            return None
+        return self.process(contents)
+
+    def merge(self, nodes):
+        assert nodes
+        lines = []
+        for node in nodes:
+            contents = get_file_contents(node.get_path())
+            _lines = [
+                '# 1 "%s"' % node.get_path().replace("\\", "/"), contents
+            ]
+            if self.is_main_node(contents):
+                lines = _lines + lines
+                self._main_ino = node.get_path()
+            else:
+                lines.extend(_lines)
+
+        if not self._main_ino:
+            self._main_ino = nodes[0].get_path()
+
+        return "\n".join(["#include <Arduino.h>"] + lines) if lines else None
+
+    def process(self, contents):
+        out_file = self._main_ino + ".cpp"
+        assert self._gcc_preprocess(contents, out_file)
+        contents = get_file_contents(out_file)
+        contents = self._join_multiline_strings(contents)
+        with open(out_file, "w") as fp:
+            fp.write(self.append_prototypes(contents))
+        return out_file
+
+    def _gcc_preprocess(self, contents, out_file):
+        tmp_path = mkstemp()[1]
+        with open(tmp_path, "w") as fp:
+            fp.write(contents)
+        env = self.env.Clone()
+        env.Append(CCFLAGS=["-x", "c++", "-E"])
+        cmd = env["CXXCOM"]\
+            .replace("$TARGET", '"%s"' % out_file)\
+            .replace("$SOURCES", '"%s"' % tmp_path)
+        env.Execute(
+            env.VerboseAction(cmd, "Converting " + basename(out_file[:-4])))
+        atexit.register(_delete_file, tmp_path)
+        return isfile(out_file)
+
+    def _join_multiline_strings(self, contents):
+        if "\\\n" not in contents:
+            return contents
+        newlines = []
+        linenum = 0
+        stropen = False
+        for line in contents.split("\n"):
+            _linenum = self._parse_preproc_line_num(line)
+            if _linenum is not None:
+                linenum = _linenum
+            else:
+                linenum += 1
+
+            if line.endswith("\\"):
+                if line.startswith('"'):
+                    stropen = True
+                    newlines.append(line[:-1])
+                    continue
+                elif stropen:
+                    newlines[len(newlines) - 1] += line[:-1]
+                    continue
+            elif stropen and line.endswith(('",', '";')):
+                newlines[len(newlines) - 1] += line
+                stropen = False
+                newlines.append('#line %d "%s"' %
+                                (linenum, self._main_ino.replace("\\", "/")))
+                continue
+
+            newlines.append(line)
+
+        return "\n".join(newlines)
+
+    @staticmethod
+    def _parse_preproc_line_num(line):
+        if not line.startswith("#"):
+            return None
+        tokens = line.split(" ", 3)
+        if len(tokens) > 2 and tokens[1].isdigit():
+            return int(tokens[1])
+        return None
+
+    def _parse_prototypes(self, contents):
+        prototypes = []
+        reserved_keywords = set(["if", "else", "while"])
+        for match in self.PROTOTYPE_RE.finditer(contents):
+            if (set([match.group(2).strip(),
+                     match.group(3).strip()]) & reserved_keywords):
+                continue
+            prototypes.append(match)
+        return prototypes
+
+    def _get_total_lines(self, contents):
+        total = 0
+        if contents.endswith("\n"):
+            contents = contents[:-1]
+        for line in contents.split("\n")[::-1]:
+            linenum = self._parse_preproc_line_num(line)
+            if linenum is not None:
+                return total + linenum
+            total += 1
+        return total
+
+    def append_prototypes(self, contents):
+        prototypes = self._parse_prototypes(contents) or []
+
+        # skip already declared prototypes
+        declared = set(
+            m.group(1).strip() for m in prototypes if m.group(4) == ";")
+        prototypes = [
+            m for m in prototypes if m.group(1).strip() not in declared
+        ]
+
+        if not prototypes:
+            return contents
+
+        prototype_names = set(m.group(3).strip() for m in prototypes)
+        split_pos = prototypes[0].start()
+        match_ptrs = re.search(
+            self.PROTOPTRS_TPLRE % ("|".join(prototype_names)),
+            contents[:split_pos], re.M)
+        if match_ptrs:
+            split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1
+
+        result = []
+        result.append(contents[:split_pos].strip())
+        result.append("%s;" % ";\n".join([m.group(1) for m in prototypes]))
+        result.append('#line %d "%s"' % (self._get_total_lines(
+            contents[:split_pos]), self._main_ino.replace("\\", "/")))
+        result.append(contents[split_pos:].strip())
+        return "\n".join(result)
+
+
+def ConvertInoToCpp(env):
+    src_dir = glob_escape(env.subst("$PROJECTSRC_DIR"))
+    ino_nodes = (env.Glob(join(src_dir, "*.ino")) +
+                 env.Glob(join(src_dir, "*.pde")))
+    if not ino_nodes:
+        return
+    c = InoToCPPConverter(env)
+    out_file = c.convert(ino_nodes)
+
+    atexit.register(_delete_file, out_file)
+
+
+def _delete_file(path):
+    try:
+        if isfile(path):
+            remove(path)
+    except:  # pylint: disable=bare-except
+        pass
+
+
 @util.memoized()
-def GetCompilerType(env):
+def _get_compiler_type(env):
     if env.subst("$CC").endswith("-gcc"):
         return "gcc"
     try:
-        sysenv = os.environ.copy()
-        sysenv["PATH"] = str(env["ENV"]["PATH"])
+        sysenv = environ.copy()
+        sysenv['PATH'] = str(env['ENV']['PATH'])
         result = exec_command([env.subst("$CC"), "-v"], env=sysenv)
     except OSError:
         return None
-    if result["returncode"] != 0:
+    if result['returncode'] != 0:
         return None
-    output = "".join([result["out"], result["err"]]).lower()
+    output = "".join([result['out'], result['err']]).lower()
     if "clang" in output and "LLVM" in output:
         return "clang"
     if "gcc" in output:
@@ -39,11 +230,16 @@ def GetCompilerType(env):
     return None
 
 
+def GetCompilerType(env):
+    return _get_compiler_type(env)
+
+
 def GetActualLDScript(env):
+
     def _lookup_in_ldpath(script):
         for d in env.get("LIBPATH", []):
-            path = os.path.join(env.subst(d), script)
-            if os.path.isfile(path):
+            path = join(env.subst(d), script)
+            if isfile(path):
                 return path
         return None
 
@@ -54,7 +250,7 @@ def GetActualLDScript(env):
         if f == "-T":
             script_in_next = True
             continue
-        if script_in_next:
+        elif script_in_next:
             script_in_next = False
             raw_script = f
         elif f.startswith("-Wl,-T"):
@@ -62,7 +258,7 @@ def GetActualLDScript(env):
         else:
             continue
         script = env.subst(raw_script.replace('"', "").strip())
-        if os.path.isfile(script):
+        if isfile(script):
             return script
         path = _lookup_in_ldpath(script)
         if path:
@@ -70,13 +266,12 @@ def GetActualLDScript(env):
 
     if script:
         sys.stderr.write(
-            "Error: Could not find '%s' LD script in LDPATH '%s'\n"
-            % (script, env.subst("$LIBPATH"))
-        )
+            "Error: Could not find '%s' LD script in LDPATH '%s'\n" %
+            (script, env.subst("$LIBPATH")))
         env.Exit(1)
 
     if not script and "LDSCRIPT_PATH" in env:
-        path = _lookup_in_ldpath(env["LDSCRIPT_PATH"])
+        path = _lookup_in_ldpath(env['LDSCRIPT_PATH'])
         if path:
             return path
 
@@ -84,43 +279,51 @@ def GetActualLDScript(env):
     env.Exit(1)
 
 
-def ConfigureDebugTarget(env):
-    def _cleanup_debug_flags(scope):
-        if scope not in env:
-            return
-        unflags = ["-Os", "-g"]
-        for level in [0, 1, 2, 3]:
-            for flag in ("O", "g", "ggdb"):
-                unflags.append("-%s%d" % (flag, level))
-        env[scope] = [f for f in env.get(scope, []) if f not in unflags]
+def VerboseAction(_, act, actstr):
+    if int(ARGUMENTS.get("PIOVERBOSE", 0)):
+        return act
+    return Action(act, actstr)
 
-    env.Append(CPPDEFINES=["__PLATFORMIO_BUILD_DEBUG__"])
 
-    for scope in ("ASFLAGS", "CCFLAGS", "LINKFLAGS"):
-        _cleanup_debug_flags(scope)
+def PioClean(env, clean_dir):
+    if not isdir(clean_dir):
+        print("Build environment is clean")
+        env.Exit(0)
+    clean_rel_path = relpath(clean_dir)
+    for root, _, files in walk(clean_dir):
+        for f in files:
+            dst = join(root, f)
+            remove(dst)
+            print("Removed %s" %
+                  (dst if clean_rel_path.startswith(".") else relpath(dst)))
+    print("Done cleaning")
+    fs.rmtree(clean_dir)
+    env.Exit(0)
 
-    debug_flags = env.ParseFlags(
-        env.get("PIODEBUGFLAGS")
-        if env.get("PIODEBUGFLAGS")
-        and not env.GetProjectOptions(as_dict=True).get("debug_build_flags")
-        else env.GetProjectOption("debug_build_flags")
-    )
 
-    env.MergeFlags(debug_flags)
-    optimization_flags = [
-        f for f in debug_flags.get("CCFLAGS", []) if f.startswith(("-O", "-g"))
-    ]
+def ProcessDebug(env):
+    if not env.subst("$PIODEBUGFLAGS"):
+        env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"])
+    env.Append(BUILD_FLAGS=list(env['PIODEBUGFLAGS']) +
+               ["-D__PLATFORMIO_BUILD_DEBUG__"])
+    unflags = ["-Os"]
+    for level in [0, 1, 2]:
+        for flag in ("O", "g", "ggdb"):
+            unflags.append("-%s%d" % (flag, level))
+    env.Append(BUILD_UNFLAGS=unflags)
 
-    if optimization_flags:
-        env.AppendUnique(
-            ASFLAGS=[
-                # skip -O flags for assembler
-                f
-                for f in optimization_flags
-                if f.startswith("-g")
-            ],
-            LINKFLAGS=optimization_flags,
-        )
+
+def ProcessTest(env):
+    env.Append(CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
+               CPPPATH=[join("$BUILD_DIR", "UnityTestLib")])
+    unitylib = env.BuildLibrary(join("$BUILD_DIR", "UnityTestLib"),
+                                get_core_package_dir("tool-unity"))
+    env.Prepend(LIBS=[unitylib])
+
+    src_filter = ["+<*.cpp>", "+<*.c>"]
+    if "PIOTEST_RUNNING_NAME" in env:
+        src_filter.append("+<%s%s>" % (env['PIOTEST_RUNNING_NAME'], sep))
+    env.Replace(PIOTEST_SRC_FILTER=src_filter)
 
 
 def GetExtraScripts(env, scope):
@@ -129,21 +332,24 @@ def GetExtraScripts(env, scope):
         if scope == "post" and ":" not in item:
             items.append(item)
         elif item.startswith("%s:" % scope):
-            items.append(item[len(scope) + 1 :])
+            items.append(item[len(scope) + 1:])
     if not items:
         return items
     with fs.cd(env.subst("$PROJECT_DIR")):
-        return [os.path.abspath(env.subst(item)) for item in items]
-
-
-def generate(env):
-    env.AddMethod(GetCompilerType)
-    env.AddMethod(GetActualLDScript)
-    env.AddMethod(ConfigureDebugTarget)
-    env.AddMethod(GetExtraScripts)
-    # bakward-compatibility with Zephyr build script
-    env.AddMethod(ConfigureDebugTarget, "ConfigureDebugFlags")
+        return [realpath(item) for item in items]
 
 
 def exists(_):
     return True
+
+
+def generate(env):
+    env.AddMethod(ConvertInoToCpp)
+    env.AddMethod(GetCompilerType)
+    env.AddMethod(GetActualLDScript)
+    env.AddMethod(VerboseAction)
+    env.AddMethod(PioClean)
+    env.AddMethod(ProcessDebug)
+    env.AddMethod(ProcessTest)
+    env.AddMethod(GetExtraScripts)
+    return env

platformio/builder/tools/pioplatform.py

@@ -12,88 +12,81 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
+from __future__ import absolute_import
+
 import sys
+from os.path import isdir, isfile, join
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
-from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
-from platformio import fs, util
-from platformio.compat import IS_MACOS, IS_WINDOWS
-from platformio.package.meta import PackageItem
-from platformio.package.version import get_original_version
-from platformio.platform.exception import UnknownBoard
-from platformio.platform.factory import PlatformFactory
+from platformio import exception, fs, util
+from platformio.compat import WINDOWS
+from platformio.managers.platform import PlatformFactory
 from platformio.project.config import ProjectOptions
 
 # pylint: disable=too-many-branches, too-many-locals
 
 
 @util.memoized()
-def _PioPlatform():
-    env = DefaultEnvironment()
-    p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"]))
-    p.configure_project_packages(env["PIOENV"], COMMAND_LINE_TARGETS)
+def PioPlatform(env):
+    variables = env.GetProjectOptions(as_dict=True)
+    if "framework" in variables:
+        # support PIO Core 3.0 dev/platforms
+        variables['pioframework'] = variables['framework']
+    p = PlatformFactory.newPlatform(env['PLATFORM_MANIFEST'])
+    p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
     return p
 
 
-def PioPlatform(_):
-    return _PioPlatform()
-
-
 def BoardConfig(env, board=None):
-    with fs.cd(env.subst("$PROJECT_DIR")):
-        try:
-            p = env.PioPlatform()
-            board = board or env.get("BOARD")
-            assert board, "BoardConfig: Board is not defined"
-            return p.board_config(board)
-        except (AssertionError, UnknownBoard) as exc:
-            sys.stderr.write("Error: %s\n" % str(exc))
-            env.Exit(1)
-    return None
+    p = env.PioPlatform()
+    try:
+        board = board or env.get("BOARD")
+        assert board, "BoardConfig: Board is not defined"
+        config = p.board_config(board)
+    except (AssertionError, exception.UnknownBoard) as e:
+        sys.stderr.write("Error: %s\n" % str(e))
+        env.Exit(1)
+    return config
 
 
 def GetFrameworkScript(env, framework):
     p = env.PioPlatform()
     assert p.frameworks and framework in p.frameworks
-    script_path = env.subst(p.frameworks[framework]["script"])
-    if not os.path.isfile(script_path):
-        script_path = os.path.join(p.get_dir(), script_path)
+    script_path = env.subst(p.frameworks[framework]['script'])
+    if not isfile(script_path):
+        script_path = join(p.get_dir(), script_path)
     return script_path
 
 
 def LoadPioPlatform(env):
     p = env.PioPlatform()
+    installed_packages = p.get_installed_packages()
 
     # Ensure real platform name
-    env["PIOPLATFORM"] = p.name
+    env['PIOPLATFORM'] = p.name
 
     # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH
-    for pkg in p.get_installed_packages():
-        type_ = p.get_package_type(pkg.metadata.name)
+    systype = util.get_systype()
+    for name in installed_packages:
+        type_ = p.get_package_type(name)
         if type_ not in ("toolchain", "uploader", "debugger"):
             continue
+        pkg_dir = p.get_package_dir(name)
         env.PrependENVPath(
             "PATH",
-            os.path.join(pkg.path, "bin")
-            if os.path.isdir(os.path.join(pkg.path, "bin"))
-            else pkg.path,
-        )
-        if (
-            not IS_WINDOWS
-            and os.path.isdir(os.path.join(pkg.path, "lib"))
-            and type_ != "toolchain"
-        ):
+            join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir)
+        if (not WINDOWS and isdir(join(pkg_dir, "lib"))
+                and type_ != "toolchain"):
             env.PrependENVPath(
-                "DYLD_LIBRARY_PATH" if IS_MACOS else "LD_LIBRARY_PATH",
-                os.path.join(pkg.path, "lib"),
-            )
+                "DYLD_LIBRARY_PATH"
+                if "darwin" in systype else "LD_LIBRARY_PATH",
+                join(pkg_dir, "lib"))
 
     # Platform specific LD Scripts
-    if os.path.isdir(os.path.join(p.get_dir(), "ldscripts")):
-        env.Prepend(LIBPATH=[os.path.join(p.get_dir(), "ldscripts")])
+    if isdir(join(p.get_dir(), "ldscripts")):
+        env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")])
 
     if "BOARD" not in env:
         return
@@ -101,27 +94,16 @@ def LoadPioPlatform(env):
     # update board manifest with overridden data from INI config
     board_config = env.BoardConfig()
     for option, value in env.GetProjectOptions():
-        if not option.startswith("board_"):
-            continue
-        option = option.lower()[6:]
-        try:
-            if isinstance(board_config.get(option), bool):
-                value = str(value).lower() in ("1", "yes", "true")
-            elif isinstance(board_config.get(option), int):
-                value = int(value)
-        except KeyError:
-            pass
-        board_config.update(option, value)
+        if option.startswith("board_"):
+            board_config.update(option.lower()[6:], value)
 
     # load default variables from board config
     for option_meta in ProjectOptions.values():
         if not option_meta.buildenvvar or option_meta.buildenvvar in env:
             continue
-        data_path = (
-            option_meta.name[6:]
-            if option_meta.name.startswith("board_")
-            else option_meta.name.replace("_", ".")
-        )
+        data_path = (option_meta.name[6:]
+                     if option_meta.name.startswith("board_") else
+                     option_meta.name.replace("_", "."))
         try:
             env[option_meta.buildenvvar] = board_config.get(data_path)
         except KeyError:
@@ -133,34 +115,25 @@ def LoadPioPlatform(env):
 
 def PrintConfiguration(env):  # pylint: disable=too-many-statements
     platform = env.PioPlatform()
-    pkg_metadata = PackageItem(platform.get_dir()).metadata
     board_config = env.BoardConfig() if "BOARD" in env else None
 
     def _get_configuration_data():
-        return (
-            None
-            if not board_config
-            else [
-                "CONFIGURATION:",
-                "https://docs.platformio.org/page/boards/%s/%s.html"
-                % (platform.name, board_config.id),
-            ]
-        )
+        return None if not board_config else [
+            "CONFIGURATION:",
+            "https://docs.platformio.org/page/boards/%s/%s.html" %
+            (platform.name, board_config.id)
+        ]
 
     def _get_plaform_data():
-        data = [
-            "PLATFORM: %s (%s)"
-            % (
-                platform.title,
-                pkg_metadata.version if pkg_metadata else platform.version,
-            )
-        ]
-        if (
-            int(ARGUMENTS.get("PIOVERBOSE", 0))
-            and pkg_metadata
-            and pkg_metadata.spec.external
-        ):
-            data.append("(%s)" % pkg_metadata.spec.uri)
+        data = ["PLATFORM: %s %s" % (platform.title, platform.version)]
+        src_manifest_path = platform.pm.get_src_manifest_path(
+            platform.get_dir())
+        if src_manifest_path:
+            src_manifest = fs.load_json(src_manifest_path)
+            if "version" in src_manifest:
+                data.append("#" + src_manifest['version'])
+            if int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                data.append("(%s)" % src_manifest['url'])
         if board_config:
             data.extend([">", board_config.get("name")])
         return data
@@ -178,23 +151,19 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
             return data
         ram = board_config.get("upload", {}).get("maximum_ram_size")
         flash = board_config.get("upload", {}).get("maximum_size")
-        data.append(
-            "%s RAM, %s Flash"
-            % (fs.humanize_file_size(ram), fs.humanize_file_size(flash))
-        )
+        data.append("%s RAM, %s Flash" %
+                    (fs.format_filesize(ram), fs.format_filesize(flash)))
         return data
 
     def _get_debug_data():
-        debug_tools = (
-            board_config.get("debug", {}).get("tools") if board_config else None
-        )
+        debug_tools = board_config.get(
+            "debug", {}).get("tools") if board_config else None
         if not debug_tools:
             return None
         data = [
-            "DEBUG:",
-            "Current",
-            "(%s)"
-            % board_config.get_debug_tool_name(env.GetProjectOption("debug_tool")),
+            "DEBUG:", "Current",
+            "(%s)" % board_config.get_debug_tool_name(
+                env.GetProjectOption("debug_tool"))
         ]
         onboard = []
         external = []
@@ -211,28 +180,28 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
 
     def _get_packages_data():
         data = []
-        for item in platform.dump_used_packages():
-            original_version = get_original_version(item["version"])
-            info = "%s @ %s" % (item["name"], item["version"])
+        for name, options in platform.packages.items():
+            if options.get("optional"):
+                continue
+            pkg_dir = platform.get_package_dir(name)
+            if not pkg_dir:
+                continue
+            manifest = platform.pm.load_manifest(pkg_dir)
+            original_version = util.get_original_version(manifest['version'])
+            info = "%s %s" % (manifest['name'], manifest['version'])
             extra = []
             if original_version:
                 extra.append(original_version)
-            if "src_url" in item and int(ARGUMENTS.get("PIOVERBOSE", 0)):
-                extra.append(item["src_url"])
+            if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)):
+                extra.append(manifest['__src_url'])
             if extra:
                 info += " (%s)" % ", ".join(extra)
             data.append(info)
-        if not data:
-            return None
-        return ["PACKAGES:"] + ["\n - %s" % d for d in sorted(data)]
+        return ["PACKAGES:", ", ".join(data)]
 
-    for data in (
-        _get_configuration_data(),
-        _get_plaform_data(),
-        _get_hardware_data(),
-        _get_debug_data(),
-        _get_packages_data(),
-    ):
+    for data in (_get_configuration_data(), _get_plaform_data(),
+                 _get_hardware_data(), _get_debug_data(),
+                 _get_packages_data()):
         if data and len(data) > 1:
             print(" ".join(data))
 

platformio/builder/tools/pioproject.py

@@ -12,34 +12,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio.compat import MISSING
-from platformio.project.config import ProjectConfig
+from __future__ import absolute_import
+
+from platformio.project.config import ProjectConfig, ProjectOptions
 
 
 def GetProjectConfig(env):
-    return ProjectConfig.get_instance(env["PROJECT_CONFIG"])
+    return ProjectConfig.get_instance(env['PROJECT_CONFIG'])
 
 
 def GetProjectOptions(env, as_dict=False):
-    return env.GetProjectConfig().items(env=env["PIOENV"], as_dict=as_dict)
+    return env.GetProjectConfig().items(env=env['PIOENV'], as_dict=as_dict)
 
 
-def GetProjectOption(env, option, default=MISSING):
-    return env.GetProjectConfig().get("env:" + env["PIOENV"], option, default)
+def GetProjectOption(env, option, default=None):
+    return env.GetProjectConfig().get("env:" + env['PIOENV'], option, default)
 
 
 def LoadProjectOptions(env):
-    config = env.GetProjectConfig()
-    section = "env:" + env["PIOENV"]
-    for option in config.options(section):
-        option_meta = config.find_option_meta(section, option)
-        if (
-            not option_meta
-            or not option_meta.buildenvvar
-            or option_meta.buildenvvar in env
-        ):
+    for option, value in env.GetProjectOptions():
+        option_meta = ProjectOptions.get("env." + option)
+        if (not option_meta or not option_meta.buildenvvar
+                or option_meta.buildenvvar in env):
             continue
-        env[option_meta.buildenvvar] = config.get(section, option)
+        env[option_meta.buildenvvar] = value
 
 
 def exists(_):

platformio/builder/tools/piosize.py

@@ -1,254 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=too-many-locals
-
-import json
-import sys
-from os import environ, makedirs, remove
-from os.path import isdir, join, splitdrive
-
-from elftools.elf.descriptions import describe_sh_flags
-from elftools.elf.elffile import ELFFile
-
-from platformio.compat import IS_WINDOWS
-from platformio.proc import exec_command
-
-
-def _run_tool(cmd, env, tool_args):
-    sysenv = environ.copy()
-    sysenv["PATH"] = str(env["ENV"]["PATH"])
-
-    build_dir = env.subst("$BUILD_DIR")
-    if not isdir(build_dir):
-        makedirs(build_dir)
-    tmp_file = join(build_dir, "size-data-longcmd.txt")
-
-    with open(tmp_file, mode="w", encoding="utf8") as fp:
-        fp.write("\n".join(tool_args))
-
-    cmd.append("@" + tmp_file)
-    result = exec_command(cmd, env=sysenv)
-    remove(tmp_file)
-
-    return result
-
-
-def _get_symbol_locations(env, elf_path, addrs):
-    if not addrs:
-        return {}
-    cmd = [env.subst("$CC").replace("-gcc", "-addr2line"), "-e", elf_path]
-    result = _run_tool(cmd, env, addrs)
-    locations = [line for line in result["out"].split("\n") if line]
-    assert len(addrs) == len(locations)
-
-    return dict(zip(addrs, [l.strip() for l in locations]))
-
-
-def _get_demangled_names(env, mangled_names):
-    if not mangled_names:
-        return {}
-    result = _run_tool(
-        [env.subst("$CC").replace("-gcc", "-c++filt")], env, mangled_names
-    )
-    demangled_names = [line for line in result["out"].split("\n") if line]
-    assert len(mangled_names) == len(demangled_names)
-
-    return dict(
-        zip(
-            mangled_names,
-            [dn.strip().replace("::__FUNCTION__", "") for dn in demangled_names],
-        )
-    )
-
-
-def _determine_section(sections, symbol_addr):
-    for section, info in sections.items():
-        if not _is_flash_section(info) and not _is_ram_section(info):
-            continue
-        if symbol_addr in range(info["start_addr"], info["start_addr"] + info["size"]):
-            return section
-    return "unknown"
-
-
-def _is_ram_section(section):
-    return (
-        section.get("type", "") in ("SHT_NOBITS", "SHT_PROGBITS")
-        and section.get("flags", "") == "WA"
-    )
-
-
-def _is_flash_section(section):
-    return section.get("type", "") == "SHT_PROGBITS" and "A" in section.get("flags", "")
-
-
-def _is_valid_symbol(symbol_name, symbol_type, symbol_address):
-    return symbol_name and symbol_address != 0 and symbol_type != "STT_NOTYPE"
-
-
-def _collect_sections_info(elffile):
-    sections = {}
-    for section in elffile.iter_sections():
-        if section.is_null() or section.name.startswith(".debug"):
-            continue
-
-        section_type = section["sh_type"]
-        section_flags = describe_sh_flags(section["sh_flags"])
-        section_size = section.data_size
-
-        sections[section.name] = {
-            "size": section_size,
-            "start_addr": section["sh_addr"],
-            "type": section_type,
-            "flags": section_flags,
-        }
-
-    return sections
-
-
-def _collect_symbols_info(env, elffile, elf_path, sections):
-    symbols = []
-
-    symbol_section = elffile.get_section_by_name(".symtab")
-    if symbol_section.is_null():
-        sys.stderr.write("Couldn't find symbol table. Is ELF file stripped?")
-        env.Exit(1)
-
-    sysenv = environ.copy()
-    sysenv["PATH"] = str(env["ENV"]["PATH"])
-
-    symbol_addrs = []
-    mangled_names = []
-    for s in symbol_section.iter_symbols():
-        symbol_info = s.entry["st_info"]
-        symbol_addr = s["st_value"]
-        symbol_size = s["st_size"]
-        symbol_type = symbol_info["type"]
-
-        if not _is_valid_symbol(s.name, symbol_type, symbol_addr):
-            continue
-
-        symbol = {
-            "addr": symbol_addr,
-            "bind": symbol_info["bind"],
-            "name": s.name,
-            "type": symbol_type,
-            "size": symbol_size,
-            "section": _determine_section(sections, symbol_addr),
-        }
-
-        if s.name.startswith("_Z"):
-            mangled_names.append(s.name)
-
-        symbol_addrs.append(hex(symbol_addr))
-        symbols.append(symbol)
-
-    symbol_locations = _get_symbol_locations(env, elf_path, symbol_addrs)
-    demangled_names = _get_demangled_names(env, mangled_names)
-    for symbol in symbols:
-        if symbol["name"].startswith("_Z"):
-            symbol["demangled_name"] = demangled_names.get(symbol["name"])
-        location = symbol_locations.get(hex(symbol["addr"]))
-        if not location or "?" in location:
-            continue
-        if IS_WINDOWS:
-            drive, tail = splitdrive(location)
-            location = join(drive.upper(), tail)
-        symbol["file"] = location
-        symbol["line"] = 0
-        if ":" in location:
-            file_, line = location.rsplit(":", 1)
-            if line.isdigit():
-                symbol["file"] = file_
-                symbol["line"] = int(line)
-    return symbols
-
-
-def _calculate_firmware_size(sections):
-    flash_size = ram_size = 0
-    for section_info in sections.values():
-        if _is_flash_section(section_info):
-            flash_size += section_info.get("size", 0)
-        if _is_ram_section(section_info):
-            ram_size += section_info.get("size", 0)
-
-    return ram_size, flash_size
-
-
-def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
-    data = {"device": {}, "memory": {}, "version": 1}
-
-    board = env.BoardConfig()
-    if board:
-        data["device"] = {
-            "mcu": board.get("build.mcu", ""),
-            "cpu": board.get("build.cpu", ""),
-            "frequency": board.get("build.f_cpu"),
-            "flash": int(board.get("upload.maximum_size", 0)),
-            "ram": int(board.get("upload.maximum_ram_size", 0)),
-        }
-        if data["device"]["frequency"] and data["device"]["frequency"].endswith("L"):
-            data["device"]["frequency"] = int(data["device"]["frequency"][0:-1])
-
-    elf_path = env.subst("$PIOMAINPROG")
-
-    with open(elf_path, "rb") as fp:
-        elffile = ELFFile(fp)
-
-        if not elffile.has_dwarf_info():
-            sys.stderr.write("Elf file doesn't contain DWARF information")
-            env.Exit(1)
-
-        sections = _collect_sections_info(elffile)
-        firmware_ram, firmware_flash = _calculate_firmware_size(sections)
-        data["memory"]["total"] = {
-            "ram_size": firmware_ram,
-            "flash_size": firmware_flash,
-            "sections": sections,
-        }
-
-        files = {}
-        for symbol in _collect_symbols_info(env, elffile, elf_path, sections):
-            file_path = symbol.get("file") or "unknown"
-            if not files.get(file_path, {}):
-                files[file_path] = {"symbols": [], "ram_size": 0, "flash_size": 0}
-
-            symbol_size = symbol.get("size", 0)
-            section = sections.get(symbol.get("section", ""), {})
-            if _is_ram_section(section):
-                files[file_path]["ram_size"] += symbol_size
-            if _is_flash_section(section):
-                files[file_path]["flash_size"] += symbol_size
-
-            files[file_path]["symbols"].append(symbol)
-
-        data["memory"]["files"] = []
-        for k, v in files.items():
-            file_data = {"path": k}
-            file_data.update(v)
-            data["memory"]["files"].append(file_data)
-
-    with open(
-        join(env.subst("$BUILD_DIR"), "sizedata.json"), mode="w", encoding="utf8"
-    ) as fp:
-        fp.write(json.dumps(data))
-
-
-def exists(_):
-    return True
-
-
-def generate(env):
-    env.AddMethod(DumpSizeData)
-    return env

platformio/builder/tools/piotarget.py

@@ -1,116 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-from SCons.Action import Action  # pylint: disable=import-error
-from SCons.Script import ARGUMENTS  # pylint: disable=import-error
-from SCons.Script import AlwaysBuild  # pylint: disable=import-error
-
-from platformio import compat, fs
-
-
-def VerboseAction(_, act, actstr):
-    if int(ARGUMENTS.get("PIOVERBOSE", 0)):
-        return act
-    return Action(act, actstr)
-
-
-def IsCleanTarget(env):
-    return env.GetOption("clean")
-
-
-def CleanProject(env, fullclean=False):
-    def _relpath(path):
-        if compat.IS_WINDOWS:
-            prefix = os.getcwd()[:2].lower()
-            if (
-                ":" not in prefix
-                or not path.lower().startswith(prefix)
-                or os.path.relpath(path).startswith("..")
-            ):
-                return path
-        return os.path.relpath(path)
-
-    def _clean_dir(path):
-        clean_rel_path = _relpath(path)
-        print(f"Removing {clean_rel_path}")
-        fs.rmtree(path)
-
-    build_dir = env.subst("$BUILD_DIR")
-    libdeps_dir = env.subst(os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))
-    if os.path.isdir(build_dir):
-        _clean_dir(build_dir)
-    else:
-        print("Build environment is clean")
-
-    if fullclean and os.path.isdir(libdeps_dir):
-        _clean_dir(libdeps_dir)
-
-    print("Done cleaning")
-
-
-def AddTarget(  # pylint: disable=too-many-arguments
-    env,
-    name,
-    dependencies,
-    actions,
-    title=None,
-    description=None,
-    group="General",
-    always_build=True,
-):
-    if "__PIO_TARGETS" not in env:
-        env["__PIO_TARGETS"] = {}
-    assert name not in env["__PIO_TARGETS"]
-    env["__PIO_TARGETS"][name] = dict(
-        name=name, title=title, description=description, group=group
-    )
-    target = env.Alias(name, dependencies, actions)
-    if always_build:
-        AlwaysBuild(target)
-    return target
-
-
-def AddPlatformTarget(env, *args, **kwargs):
-    return env.AddTarget(group="Platform", *args, **kwargs)
-
-
-def AddCustomTarget(env, *args, **kwargs):
-    return env.AddTarget(group="Custom", *args, **kwargs)
-
-
-def DumpTargets(env):
-    targets = env.get("__PIO_TARGETS") or {}
-    # pre-fill default targets if embedded dev-platform
-    if env.PioPlatform().is_embedded() and not any(
-        t["group"] == "Platform" for t in targets.values()
-    ):
-        targets["upload"] = dict(name="upload", group="Platform", title="Upload")
-    return list(targets.values())
-
-
-def exists(_):
-    return True
-
-
-def generate(env):
-    env.AddMethod(VerboseAction)
-    env.AddMethod(IsCleanTarget)
-    env.AddMethod(CleanProject)
-    env.AddMethod(AddTarget)
-    env.AddMethod(AddPlatformTarget)
-    env.AddMethod(AddCustomTarget)
-    env.AddMethod(DumpTargets)
-    return env

platformio/builder/tools/piotest.py

@@ -1,61 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-from platformio.builder.tools import piobuild
-from platformio.test.result import TestSuite
-from platformio.test.runners.factory import TestRunnerFactory
-
-
-def ConfigureTestTarget(env):
-    env.Append(
-        CPPDEFINES=["UNIT_TEST"],  # deprecated, use PIO_UNIT_TESTING
-        PIOTEST_SRC_FILTER=[f"+<*.{ext}>" for ext in piobuild.SRC_BUILD_EXT],
-    )
-    env.Prepend(CPPPATH=["$PROJECT_TEST_DIR"])
-
-    if "PIOTEST_RUNNING_NAME" in env:
-        test_name = env["PIOTEST_RUNNING_NAME"]
-        while True:
-            test_name = os.path.dirname(test_name)  # parent dir
-            # skip nested tests (user's side issue?)
-            if not test_name or os.path.basename(test_name).startswith("test_"):
-                break
-            env.Prepend(
-                PIOTEST_SRC_FILTER=[
-                    f"+<{test_name}{os.path.sep}*.{ext}>"
-                    for ext in piobuild.SRC_BUILD_EXT
-                ],
-                CPPPATH=[os.path.join("$PROJECT_TEST_DIR", test_name)],
-            )
-
-        env.Prepend(
-            PIOTEST_SRC_FILTER=[f"+<$PIOTEST_RUNNING_NAME{os.path.sep}>"],
-            CPPPATH=[os.path.join("$PROJECT_TEST_DIR", "$PIOTEST_RUNNING_NAME")],
-        )
-
-    test_runner = TestRunnerFactory.new(
-        TestSuite(env["PIOENV"], env.get("PIOTEST_RUNNING_NAME", "*")),
-        env.GetProjectConfig(),
-    )
-    test_runner.configure_build_env(env)
-
-
-def generate(env):
-    env.AddMethod(ConfigureTestTarget)
-
-
-def exists(_):
-    return True

platformio/builder/tools/pioupload.py

@@ -12,22 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=unused-argument
+from __future__ import absolute_import
 
-import os
 import re
 import sys
+from fnmatch import fnmatch
+from os import environ
+from os.path import isfile, join
 from shutil import copyfile
 from time import sleep
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from serial import Serial, SerialException
 
-from platformio import exception, fs
-from platformio.device.finder import SerialPortFinder, find_mbed_disk, is_pattern_port
-from platformio.device.list.util import list_serial_ports
+from platformio import exception, fs, util
+from platformio.compat import WINDOWS
 from platformio.proc import exec_command
 
+# pylint: disable=unused-argument
+
 
 def FlushSerialBuffer(env, port):
     s = Serial(env.subst(port))
@@ -57,9 +60,9 @@ def WaitForNewSerialPort(env, before):
     prev_port = env.subst("$UPLOAD_PORT")
     new_port = None
     elapsed = 0
-    before = [p["port"] for p in before]
+    before = [p['port'] for p in before]
     while elapsed < 5 and new_port is None:
-        now = [p["port"] for p in list_serial_ports()]
+        now = [p['port'] for p in util.get_serial_ports()]
         for p in now:
             if p not in before:
                 new_port = p
@@ -81,12 +84,10 @@ def WaitForNewSerialPort(env, before):
         sleep(1)
 
     if not new_port:
-        sys.stderr.write(
-            "Error: Couldn't find a board on the selected port. "
-            "Check that you have the correct port selected. "
-            "If it is correct, try pressing the board's reset "
-            "button after initiating the upload.\n"
-        )
+        sys.stderr.write("Error: Couldn't find a board on the selected port. "
+                         "Check that you have the correct port selected. "
+                         "If it is correct, try pressing the board's reset "
+                         "button after initiating the upload.\n")
         env.Exit(1)
 
     return new_port
@@ -94,29 +95,71 @@ def WaitForNewSerialPort(env, before):
 
 def AutodetectUploadPort(*args, **kwargs):
     env = args[0]
-    initial_port = env.subst("$UPLOAD_PORT")
-    upload_protocol = env.subst("$UPLOAD_PROTOCOL")
-    if initial_port and not is_pattern_port(initial_port):
-        print(env.subst("Using manually specified: $UPLOAD_PORT"))
+
+    def _get_pattern():
+        if "UPLOAD_PORT" not in env:
+            return None
+        if set(["*", "?", "[", "]"]) & set(env['UPLOAD_PORT']):
+            return env['UPLOAD_PORT']
+        return None
+
+    def _is_match_pattern(port):
+        pattern = _get_pattern()
+        if not pattern:
+            return True
+        return fnmatch(port, pattern)
+
+    def _look_for_mbed_disk():
+        msdlabels = ("mbed", "nucleo", "frdm", "microbit")
+        for item in util.get_logical_devices():
+            if item['path'].startswith("/net") or not _is_match_pattern(
+                    item['path']):
+                continue
+            mbed_pages = [
+                join(item['path'], n) for n in ("mbed.htm", "mbed.html")
+            ]
+            if any(isfile(p) for p in mbed_pages):
+                return item['path']
+            if item['name'] \
+                    and any(l in item['name'].lower() for l in msdlabels):
+                return item['path']
+        return None
+
+    def _look_for_serial_port():
+        port = None
+        board_hwids = []
+        upload_protocol = env.subst("$UPLOAD_PROTOCOL")
+        if "BOARD" in env and "build.hwids" in env.BoardConfig():
+            board_hwids = env.BoardConfig().get("build.hwids")
+        for item in util.get_serial_ports(filter_hwid=True):
+            if not _is_match_pattern(item['port']):
+                continue
+            port = item['port']
+            if upload_protocol.startswith("blackmagic"):
+                if WINDOWS and port.startswith("COM") and len(port) > 4:
+                    port = "\\\\.\\%s" % port
+                if "GDB" in item['description']:
+                    return port
+            for hwid in board_hwids:
+                hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
+                if hwid_str in item['hwid']:
+                    return port
+        return port
+
+    if "UPLOAD_PORT" in env and not _get_pattern():
+        print(env.subst("Use manually specified: $UPLOAD_PORT"))
         return
 
-    if upload_protocol == "mbed" or (
-        "mbed" in env.subst("$PIOFRAMEWORK") and not upload_protocol
-    ):
-        env.Replace(UPLOAD_PORT=find_mbed_disk(initial_port))
+    if (env.subst("$UPLOAD_PROTOCOL") == "mbed"
+            or ("mbed" in env.subst("$PIOFRAMEWORK")
+                and not env.subst("$UPLOAD_PROTOCOL"))):
+        env.Replace(UPLOAD_PORT=_look_for_mbed_disk())
     else:
         try:
             fs.ensure_udev_rules()
-        except exception.InvalidUdevRules as exc:
-            sys.stderr.write("\n%s\n\n" % exc)
-        env.Replace(
-            UPLOAD_PORT=SerialPortFinder(
-                board_config=env.BoardConfig() if "BOARD" in env else None,
-                upload_protocol=upload_protocol,
-                prefer_gdb_port="blackmagic" in upload_protocol,
-                verbose=int(ARGUMENTS.get("PIOVERBOSE", 0)),
-            ).find(initial_port)
-        )
+        except exception.InvalidUdevRules as e:
+            sys.stderr.write("\n%s\n\n" % e)
+        env.Replace(UPLOAD_PORT=_look_for_serial_port())
 
     if env.subst("$UPLOAD_PORT"):
         print(env.subst("Auto-detected: $UPLOAD_PORT"))
@@ -125,8 +168,7 @@ def AutodetectUploadPort(*args, **kwargs):
             "Error: Please specify `upload_port` for environment or use "
             "global `--upload-port` option.\n"
             "For some development platforms it can be a USB flash "
-            "drive (i.e. /media/<user>/<device name>)\n"
-        )
+            "drive (i.e. /media/<user>/<device name>)\n")
         env.Exit(1)
 
 
@@ -134,22 +176,19 @@ def UploadToDisk(_, target, source, env):
     assert "UPLOAD_PORT" in env
     progname = env.subst("$PROGNAME")
     for ext in ("bin", "hex"):
-        fpath = os.path.join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
-        if not os.path.isfile(fpath):
+        fpath = join(env.subst("$BUILD_DIR"), "%s.%s" % (progname, ext))
+        if not isfile(fpath):
             continue
-        copyfile(
-            fpath, os.path.join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext))
-        )
-    print(
-        "Firmware has been successfully uploaded.\n"
-        "(Some boards may require manual hard reset)"
-    )
+        copyfile(fpath,
+                 join(env.subst("$UPLOAD_PORT"), "%s.%s" % (progname, ext)))
+    print("Firmware has been successfully uploaded.\n"
+          "(Some boards may require manual hard reset)")
 
 
 def CheckUploadSize(_, target, source, env):
     check_conditions = [
         env.get("BOARD"),
-        env.get("SIZETOOL") or env.get("SIZECHECKCMD"),
+        env.get("SIZETOOL") or env.get("SIZECHECKCMD")
     ]
     if not all(check_conditions):
         return
@@ -159,11 +198,9 @@ def CheckUploadSize(_, target, source, env):
         return
 
     def _configure_defaults():
-        env.Replace(
-            SIZECHECKCMD="$SIZETOOL -B -d $SOURCES",
-            SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s",
-            SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+",
-        )
+        env.Replace(SIZECHECKCMD="$SIZETOOL -B -d $SOURCES",
+                    SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s",
+                    SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+")
 
     def _get_size_output():
         cmd = env.get("SIZECHECKCMD")
@@ -172,12 +209,12 @@ def CheckUploadSize(_, target, source, env):
         if not isinstance(cmd, list):
             cmd = cmd.split()
         cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
-        sysenv = os.environ.copy()
-        sysenv["PATH"] = str(env["ENV"]["PATH"])
+        sysenv = environ.copy()
+        sysenv['PATH'] = str(env['ENV']['PATH'])
         result = exec_command(env.subst(cmd), env=sysenv)
-        if result["returncode"] != 0:
+        if result['returncode'] != 0:
             return None
-        return result["out"].strip()
+        return result['out'].strip()
 
     def _calculate_size(output, pattern):
         if not output or not pattern:
@@ -197,12 +234,11 @@ def CheckUploadSize(_, target, source, env):
     def _format_availale_bytes(value, total):
         percent_raw = float(value) / float(total)
         blocks_per_progress = 10
-        used_blocks = min(
-            int(round(blocks_per_progress * percent_raw)), blocks_per_progress
-        )
+        used_blocks = int(round(blocks_per_progress * percent_raw))
+        if used_blocks > blocks_per_progress:
+            used_blocks = blocks_per_progress
         return "[{:{}}] {: 6.1%} (used {:d} bytes from {:d} bytes)".format(
-            "=" * used_blocks, blocks_per_progress, percent_raw, value, total
-        )
+            "=" * used_blocks, blocks_per_progress, percent_raw, value, total)
 
     if not env.get("SIZECHECKCMD") and not env.get("SIZEPROGREGEXP"):
         _configure_defaults()
@@ -210,11 +246,12 @@ def CheckUploadSize(_, target, source, env):
     program_size = _calculate_size(output, env.get("SIZEPROGREGEXP"))
     data_size = _calculate_size(output, env.get("SIZEDATAREGEXP"))
 
-    print('Advanced Memory Usage is available via "PlatformIO Home > Project Inspect"')
+    print("Memory Usage -> http://bit.ly/pio-memory-usage")
     if data_max_size and data_size > -1:
-        print("RAM:   %s" % _format_availale_bytes(data_size, data_max_size))
+        print("DATA:    %s" % _format_availale_bytes(data_size, data_max_size))
     if program_size > -1:
-        print("Flash: %s" % _format_availale_bytes(program_size, program_max_size))
+        print("PROGRAM: %s" %
+              _format_availale_bytes(program_size, program_max_size))
     if int(ARGUMENTS.get("PIOVERBOSE", 0)):
         print(output)
 
@@ -225,10 +262,9 @@ def CheckUploadSize(_, target, source, env):
     #         "than maximum allowed (%s bytes)\n" % (data_size, data_max_size))
     #     env.Exit(1)
     if program_size > program_max_size:
-        sys.stderr.write(
-            "Error: The program size (%d bytes) is greater "
-            "than maximum allowed (%s bytes)\n" % (program_size, program_max_size)
-        )
+        sys.stderr.write("Error: The program size (%d bytes) is greater "
+                         "than maximum allowed (%s bytes)\n" %
+                         (program_size, program_max_size))
         env.Exit(1)
 
 
@@ -236,7 +272,8 @@ def PrintUploadInfo(env):
     configured = env.subst("$UPLOAD_PROTOCOL")
     available = [configured] if configured else []
     if "BOARD" in env:
-        available.extend(env.BoardConfig().get("upload", {}).get("protocols", []))
+        available.extend(env.BoardConfig().get("upload",
+                                               {}).get("protocols", []))
     if available:
         print("AVAILABLE: %s" % ", ".join(sorted(set(available))))
     if configured:

platformio/builder/tools/piowinhooks.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from hashlib import md5
+from os import makedirs
+from os.path import isdir, isfile, join
+
+from platformio.compat import WINDOWS, hashlib_encode_data
+
+# Windows CLI has limit with command length to 8192
+# Leave 2000 chars for flags and other options
+MAX_SOURCES_LENGTH = 6000
+
+
+def long_sources_hook(env, sources):
+    _sources = str(sources).replace("\\", "/")
+    if len(str(_sources)) < MAX_SOURCES_LENGTH:
+        return sources
+
+    # fix space in paths
+    data = []
+    for line in _sources.split(".o "):
+        line = line.strip()
+        if not line.endswith(".o"):
+            line += ".o"
+        data.append('"%s"' % line)
+
+    return '@"%s"' % _file_long_data(env, " ".join(data))
+
+
+def long_incflags_hook(env, incflags):
+    _incflags = env.subst(incflags).replace("\\", "/")
+    if len(_incflags) < MAX_SOURCES_LENGTH:
+        return incflags
+
+    # fix space in paths
+    data = []
+    for line in _incflags.split(" -I"):
+        line = line.strip()
+        if not line.startswith("-I"):
+            line = "-I" + line
+        data.append('-I"%s"' % line[2:])
+
+    return '@"%s"' % _file_long_data(env, " ".join(data))
+
+
+def _file_long_data(env, data):
+    build_dir = env.subst("$BUILD_DIR")
+    if not isdir(build_dir):
+        makedirs(build_dir)
+    tmp_file = join(build_dir,
+                    "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest())
+    if isfile(tmp_file):
+        return tmp_file
+    with open(tmp_file, "w") as fp:
+        fp.write(data)
+    return tmp_file
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    if not WINDOWS:
+        return None
+
+    env.Replace(_long_sources_hook=long_sources_hook)
+    env.Replace(_long_incflags_hook=long_incflags_hook)
+    coms = {}
+    for key in ("ARCOM", "LINKCOM"):
+        coms[key] = env.get(key, "").replace(
+            "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}")
+    for key in ("_CCCOMCOM", "ASPPCOM"):
+        coms[key] = env.get(key, "").replace(
+            "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}")
+    env.Replace(**coms)
+
+    return env

platformio/builder/tools/platformio.py

@@ -0,0 +1,327 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import os
+import sys
+from os.path import basename, dirname, isdir, join, realpath
+
+from SCons import Builder, Util  # pylint: disable=import-error
+from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
+from SCons.Script import AlwaysBuild  # pylint: disable=import-error
+from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
+from SCons.Script import Export  # pylint: disable=import-error
+from SCons.Script import SConscript  # pylint: disable=import-error
+
+from platformio import fs
+from platformio.compat import string_types
+from platformio.util import pioversion_to_intstr
+
+SRC_HEADER_EXT = ["h", "hpp"]
+SRC_C_EXT = ["c", "cc", "cpp"]
+SRC_BUILD_EXT = SRC_C_EXT + ["S", "spp", "SPP", "sx", "s", "asm", "ASM"]
+SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % os.sep, "-<.svn%s>" % os.sep]
+
+
+def scons_patched_match_splitext(path, suffixes=None):
+    """Patch SCons Builder, append $OBJSUFFIX to the end of each target"""
+    tokens = Util.splitext(path)
+    if suffixes and tokens[1] and tokens[1] in suffixes:
+        return (path, tokens[1])
+    return tokens
+
+
+def _build_project_deps(env):
+    project_lib_builder = env.ConfigureProjectLibBuilder()
+
+    # prepend project libs to the beginning of list
+    env.Prepend(LIBS=project_lib_builder.build())
+    # prepend extra linker related options from libs
+    env.PrependUnique(
+        **{
+            key: project_lib_builder.env.get(key)
+            for key in ("LIBS", "LIBPATH", "LINKFLAGS")
+            if project_lib_builder.env.get(key)
+        })
+
+    projenv = env.Clone()
+
+    # CPPPATH from dependencies
+    projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH"))
+    # extra build flags from `platformio.ini`
+    projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS"))
+
+    if ("nobuild" not in COMMAND_LINE_TARGETS
+            and "arduino" in projenv.get("PIOFRAMEWORK", [])):
+        projenv.ConvertInoToCpp()
+
+    is_test = "__test" in COMMAND_LINE_TARGETS
+    if is_test:
+        projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR",
+                             "$PIOTEST_SRC_FILTER")
+    if not is_test or env.GetProjectOption("test_build_project_src", False):
+        projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR",
+                             env.get("SRC_FILTER"))
+
+    if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
+        sys.stderr.write(
+            "Error: Nothing to build. Please put your source code files "
+            "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR"))
+        env.Exit(1)
+
+    Export("projenv")
+
+
+def BuildProgram(env):
+
+    def _append_pio_macros():
+        env.AppendUnique(CPPDEFINES=[(
+            "PLATFORMIO",
+            int("{0:02d}{1:02d}{2:02d}".format(*pioversion_to_intstr())))])
+
+    _append_pio_macros()
+
+    env.PrintConfiguration()
+
+    # fix ASM handling under non case-sensitive OS
+    if not Util.case_sensitive_suffixes(".s", ".S"):
+        env.Replace(AS="$CC", ASCOM="$ASPPCOM")
+
+    if ("debug" in COMMAND_LINE_TARGETS
+            or env.GetProjectOption("build_type") == "debug"):
+        env.ProcessDebug()
+
+    # process extra flags from board
+    if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
+        env.ProcessFlags(env.BoardConfig().get("build.extra_flags"))
+
+    # apply user flags
+    env.ProcessFlags(env.get("BUILD_FLAGS"))
+
+    # process framework scripts
+    env.BuildFrameworks(env.get("PIOFRAMEWORK"))
+
+    # restore PIO macros if it was deleted by framework
+    _append_pio_macros()
+
+    # remove specified flags
+    env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
+
+    if "__test" in COMMAND_LINE_TARGETS:
+        env.ProcessTest()
+
+    # build project with dependencies
+    _build_project_deps(env)
+
+    # append into the beginning a main LD script
+    if (env.get("LDSCRIPT_PATH")
+            and not any("-Wl,-T" in f for f in env['LINKFLAGS'])):
+        env.Prepend(LINKFLAGS=["-T", "$LDSCRIPT_PATH"])
+
+    # enable "cyclic reference" for linker
+    if env.get("LIBS") and env.GetCompilerType() == "gcc":
+        env.Prepend(_LIBFLAGS="-Wl,--start-group ")
+        env.Append(_LIBFLAGS=" -Wl,--end-group")
+
+    program = env.Program(join("$BUILD_DIR", env.subst("$PROGNAME")),
+                          env['PIOBUILDFILES'])
+    env.Replace(PIOMAINPROG=program)
+
+    AlwaysBuild(
+        env.Alias(
+            "checkprogsize", program,
+            env.VerboseAction(env.CheckUploadSize,
+                              "Checking size $PIOMAINPROG")))
+
+    return program
+
+
+def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
+    if not isinstance(flags, list):
+        flags = [flags]
+    result = {}
+    for raw in flags:
+        for key, value in env.ParseFlags(str(raw)).items():
+            if key not in result:
+                result[key] = []
+            result[key].extend(value)
+
+    cppdefines = []
+    for item in result['CPPDEFINES']:
+        if not Util.is_Sequence(item):
+            cppdefines.append(item)
+            continue
+        name, value = item[:2]
+        if '\"' in value:
+            value = value.replace('\"', '\\\"')
+        elif value.isdigit():
+            value = int(value)
+        elif value.replace(".", "", 1).isdigit():
+            value = float(value)
+        cppdefines.append((name, value))
+    result['CPPDEFINES'] = cppdefines
+
+    # fix relative CPPPATH & LIBPATH
+    for k in ("CPPPATH", "LIBPATH"):
+        for i, p in enumerate(result.get(k, [])):
+            if isdir(p):
+                result[k][i] = realpath(p)
+
+    # fix relative path for "-include"
+    for i, f in enumerate(result.get("CCFLAGS", [])):
+        if isinstance(f, tuple) and f[0] == "-include":
+            result['CCFLAGS'][i] = (f[0], env.File(realpath(f[1].get_path())))
+
+    return result
+
+
+def ProcessFlags(env, flags):  # pylint: disable=too-many-branches
+    if not flags:
+        return
+    env.Append(**env.ParseFlagsExtended(flags))
+
+    # Cancel any previous definition of name, either built in or
+    # provided with a -U option // Issue #191
+    undefines = [
+        u for u in env.get("CCFLAGS", [])
+        if isinstance(u, string_types) and u.startswith("-U")
+    ]
+    if undefines:
+        for undef in undefines:
+            env['CCFLAGS'].remove(undef)
+            if undef[2:] in env['CPPDEFINES']:
+                env['CPPDEFINES'].remove(undef[2:])
+        env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines))
+
+
+def ProcessUnFlags(env, flags):
+    if not flags:
+        return
+    parsed = env.ParseFlagsExtended(flags)
+
+    # get all flags and copy them to each "*FLAGS" variable
+    all_flags = []
+    for key, unflags in parsed.items():
+        if key.endswith("FLAGS"):
+            all_flags.extend(unflags)
+    for key, unflags in parsed.items():
+        if key.endswith("FLAGS"):
+            parsed[key].extend(all_flags)
+
+    for key, unflags in parsed.items():
+        for unflag in unflags:
+            for current in env.get(key, []):
+                conditions = [
+                    unflag == current,
+                    isinstance(current, (tuple, list))
+                    and unflag[0] == current[0]
+                ]
+                if any(conditions):
+                    env[key].remove(current)
+
+
+def MatchSourceFiles(env, src_dir, src_filter=None):
+    src_filter = env.subst(src_filter) if src_filter else None
+    src_filter = src_filter or SRC_FILTER_DEFAULT
+    return fs.match_src_files(env.subst(src_dir), src_filter,
+                              SRC_BUILD_EXT + SRC_HEADER_EXT)
+
+
+def CollectBuildFiles(env,
+                      variant_dir,
+                      src_dir,
+                      src_filter=None,
+                      duplicate=False):
+    sources = []
+    variants = []
+
+    src_dir = env.subst(src_dir)
+    if src_dir.endswith(os.sep):
+        src_dir = src_dir[:-1]
+
+    for item in env.MatchSourceFiles(src_dir, src_filter):
+        _reldir = dirname(item)
+        _src_dir = join(src_dir, _reldir) if _reldir else src_dir
+        _var_dir = join(variant_dir, _reldir) if _reldir else variant_dir
+
+        if _var_dir not in variants:
+            variants.append(_var_dir)
+            env.VariantDir(_var_dir, _src_dir, duplicate)
+
+        if fs.path_endswith_ext(item, SRC_BUILD_EXT):
+            sources.append(env.File(join(_var_dir, basename(item))))
+
+    return sources
+
+
+def BuildFrameworks(env, frameworks):
+    if not frameworks:
+        return
+
+    if "BOARD" not in env:
+        sys.stderr.write("Please specify `board` in `platformio.ini` to use "
+                         "with '%s' framework\n" % ", ".join(frameworks))
+        env.Exit(1)
+
+    board_frameworks = env.BoardConfig().get("frameworks", [])
+    if frameworks == ["platformio"]:
+        if board_frameworks:
+            frameworks.insert(0, board_frameworks[0])
+        else:
+            sys.stderr.write(
+                "Error: Please specify `board` in `platformio.ini`\n")
+            env.Exit(1)
+
+    for f in frameworks:
+        if f == "arduino":
+            # Arduino IDE appends .o the end of filename
+            Builder.match_splitext = scons_patched_match_splitext
+
+        if f in board_frameworks:
+            SConscript(env.GetFrameworkScript(f), exports="env")
+        else:
+            sys.stderr.write(
+                "Error: This board doesn't support %s framework!\n" % f)
+            env.Exit(1)
+
+
+def BuildLibrary(env, variant_dir, src_dir, src_filter=None):
+    env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
+    return env.StaticLibrary(
+        env.subst(variant_dir),
+        env.CollectBuildFiles(variant_dir, src_dir, src_filter))
+
+
+def BuildSources(env, variant_dir, src_dir, src_filter=None):
+    nodes = env.CollectBuildFiles(variant_dir, src_dir, src_filter)
+    DefaultEnvironment().Append(
+        PIOBUILDFILES=[env.Object(node) for node in nodes])
+
+
+def exists(_):
+    return True
+
+
+def generate(env):
+    env.AddMethod(BuildProgram)
+    env.AddMethod(ParseFlagsExtended)
+    env.AddMethod(ProcessFlags)
+    env.AddMethod(ProcessUnFlags)
+    env.AddMethod(MatchSourceFiles)
+    env.AddMethod(CollectBuildFiles)
+    env.AddMethod(BuildFrameworks)
+    env.AddMethod(BuildLibrary)
+    env.AddMethod(BuildSources)
+    return env

platformio/cache.py

@@ -1,165 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import codecs
-import hashlib
-import os
-from time import time
-
-from platformio import app, fs
-from platformio.compat import hashlib_encode_data
-from platformio.package.lockfile import LockFile
-from platformio.project.helpers import get_project_cache_dir
-
-
-class ContentCache:
-    def __init__(self, namespace=None):
-        self.cache_dir = os.path.join(get_project_cache_dir(), namespace or "content")
-        self._db_path = os.path.join(self.cache_dir, "db.data")
-        self._lockfile = None
-        if not os.path.isdir(self.cache_dir):
-            os.makedirs(self.cache_dir)
-
-    def __enter__(self):
-        # cleanup obsolete items
-        self.delete()
-        return self
-
-    def __exit__(self, type_, value, traceback):
-        pass
-
-    @staticmethod
-    def key_from_args(*args):
-        h = hashlib.sha1()
-        for arg in args:
-            if arg:
-                h.update(hashlib_encode_data(arg))
-        return h.hexdigest()
-
-    def get_cache_path(self, key):
-        assert "/" not in key and "\\" not in key
-        key = str(key)
-        assert len(key) > 3
-        return os.path.join(self.cache_dir, key)
-
-    def get(self, key):
-        cache_path = self.get_cache_path(key)
-        if not os.path.isfile(cache_path):
-            return None
-        with codecs.open(cache_path, "rb", encoding="utf8") as fp:
-            return fp.read()
-
-    def set(self, key, data, valid):
-        if not app.get_setting("enable_cache"):
-            return False
-        cache_path = self.get_cache_path(key)
-        if os.path.isfile(cache_path):
-            self.delete(key)
-        if not data:
-            return False
-        tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400}
-        assert valid.endswith(tuple(tdmap))
-        expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1]))
-
-        if not self._lock_dbindex():
-            return False
-
-        if not os.path.isdir(os.path.dirname(cache_path)):
-            os.makedirs(os.path.dirname(cache_path))
-        try:
-            with codecs.open(cache_path, mode="wb", encoding="utf8") as fp:
-                fp.write(data)
-            with open(self._db_path, mode="a", encoding="utf8") as fp:
-                fp.write("%s=%s\n" % (str(expire_time), os.path.basename(cache_path)))
-        except UnicodeError:
-            if os.path.isfile(cache_path):
-                try:
-                    os.remove(cache_path)
-                except OSError:
-                    pass
-
-        return self._unlock_dbindex()
-
-    def delete(self, keys=None):
-        """Keys=None, delete expired items"""
-        if not os.path.isfile(self._db_path):
-            return None
-        if not keys:
-            keys = []
-        if not isinstance(keys, list):
-            keys = [keys]
-        paths_for_delete = [self.get_cache_path(k) for k in keys]
-        found = False
-        newlines = []
-        with open(self._db_path, encoding="utf8") as fp:
-            for line in fp.readlines():
-                line = line.strip()
-                if "=" not in line:
-                    continue
-                expire, fname = line.split("=")
-                path = os.path.join(self.cache_dir, fname)
-                try:
-                    if (
-                        time() < int(expire)
-                        and os.path.isfile(path)
-                        and path not in paths_for_delete
-                    ):
-                        newlines.append(line)
-                        continue
-                except ValueError:
-                    pass
-                found = True
-                if os.path.isfile(path):
-                    try:
-                        os.remove(path)
-                        if not os.listdir(os.path.dirname(path)):
-                            fs.rmtree(os.path.dirname(path))
-                    except OSError:
-                        pass
-
-        if found and self._lock_dbindex():
-            with open(self._db_path, mode="w", encoding="utf8") as fp:
-                fp.write("\n".join(newlines) + "\n")
-            self._unlock_dbindex()
-
-        return True
-
-    def clean(self):
-        if not os.path.isdir(self.cache_dir):
-            return
-        fs.rmtree(self.cache_dir)
-
-    def _lock_dbindex(self):
-        self._lockfile = LockFile(self.cache_dir)
-        try:
-            self._lockfile.acquire()
-        except:  # pylint: disable=bare-except
-            return False
-
-        return True
-
-    def _unlock_dbindex(self):
-        if self._lockfile:
-            self._lockfile.release()
-        return True
-
-
-#
-# Helpers
-#
-
-
-def cleanup_content_cache(namespace=None):
-    with ContentCache(namespace) as cc:
-        cc.clean()

platformio/check/__init__.py

@@ -1,13 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

platformio/check/cli.py

@@ -1,334 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
-# pylint: disable=redefined-builtin,too-many-statements
-
-import json
-import os
-import shutil
-from collections import Counter
-from os.path import dirname, isfile
-from time import time
-
-import click
-from tabulate import tabulate
-
-from platformio import app, exception, fs, util
-from platformio.check.defect import DefectItem
-from platformio.check.tools import CheckToolFactory
-from platformio.project.config import ProjectConfig
-from platformio.project.helpers import find_project_dir_above, get_project_dir
-
-
-@click.command("check", short_help="Static Code Analysis")
-@click.option("-e", "--environment", multiple=True)
-@click.option(
-    "-d",
-    "--project-dir",
-    default=os.getcwd,
-    type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True),
-)
-@click.option(
-    "-c",
-    "--project-conf",
-    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
-)
-@click.option("--pattern", multiple=True, hidden=True)
-@click.option("-f", "--src-filters", multiple=True)
-@click.option("--flags", multiple=True)
-@click.option(
-    "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values())
-)
-@click.option("-s", "--silent", is_flag=True)
-@click.option("-v", "--verbose", is_flag=True)
-@click.option("--json-output", is_flag=True)
-@click.option(
-    "--fail-on-defect",
-    multiple=True,
-    type=click.Choice(DefectItem.SEVERITY_LABELS.values()),
-)
-@click.option("--skip-packages", is_flag=True)
-def cli(
-    environment,
-    project_dir,
-    project_conf,
-    src_filters,
-    pattern,
-    flags,
-    severity,
-    silent,
-    verbose,
-    json_output,
-    fail_on_defect,
-    skip_packages,
-):
-    app.set_session_var("custom_project_conf", project_conf)
-
-    # find project directory on upper level
-    if isfile(project_dir):
-        project_dir = find_project_dir_above(project_dir)
-
-    results = []
-    with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(project_conf)
-        config.validate(environment)
-
-        default_envs = config.default_envs()
-        for envname in config.envs():
-            skipenv = any(
-                [
-                    environment and envname not in environment,
-                    not environment and default_envs and envname not in default_envs,
-                ]
-            )
-
-            env_options = config.items(env=envname, as_dict=True)
-            env_dump = []
-            for k, v in env_options.items():
-                if k not in ("platform", "framework", "board"):
-                    continue
-                env_dump.append(
-                    "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)
-                )
-
-            default_src_filters = [
-                "+<%s>" % os.path.basename(config.get("platformio", "src_dir")),
-                "+<%s>" % os.path.basename(config.get("platformio", "include_dir")),
-            ]
-
-            src_filters = (
-                src_filters
-                or pattern
-                or env_options.get(
-                    "check_src_filters",
-                    env_options.get("check_patterns", default_src_filters),
-                )
-            )
-
-            tool_options = dict(
-                verbose=verbose,
-                silent=silent,
-                src_filters=src_filters,
-                flags=flags or env_options.get("check_flags"),
-                severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]]
-                if silent
-                else severity or config.get("env:" + envname, "check_severity"),
-                skip_packages=skip_packages or env_options.get("check_skip_packages"),
-                platform_packages=env_options.get("platform_packages"),
-            )
-
-            for tool in config.get("env:" + envname, "check_tool"):
-                if skipenv:
-                    results.append({"env": envname, "tool": tool})
-                    continue
-                if not silent and not json_output:
-                    print_processing_header(tool, envname, env_dump)
-
-                ct = CheckToolFactory.new(
-                    tool, project_dir, config, envname, tool_options
-                )
-
-                result = {"env": envname, "tool": tool, "duration": time()}
-                rc = ct.check(
-                    on_defect_callback=None
-                    if (json_output or verbose)
-                    else lambda defect: click.echo(repr(defect))
-                )
-
-                result["defects"] = ct.get_defects()
-                result["duration"] = time() - result["duration"]
-
-                result["succeeded"] = rc == 0
-                if fail_on_defect:
-                    result["succeeded"] = rc == 0 and not any(
-                        DefectItem.SEVERITY_LABELS[d.severity] in fail_on_defect
-                        for d in result["defects"]
-                    )
-                result["stats"] = collect_component_stats(result)
-                results.append(result)
-
-                if verbose:
-                    click.echo("\n".join(repr(d) for d in result["defects"]))
-
-                if not json_output and not silent:
-                    if rc != 0:
-                        click.echo(
-                            "Error: %s failed to perform check! Please "
-                            "examine tool output in verbose mode." % tool
-                        )
-                    elif not result["defects"]:
-                        click.echo("No defects found")
-                    print_processing_footer(result)
-
-        if json_output:
-            click.echo(json.dumps(results_to_json(results)))
-        elif not silent:
-            print_check_summary(results, verbose=verbose)
-
-    # Reset custom project config
-    app.set_session_var("custom_project_conf", None)
-
-    command_failed = any(r.get("succeeded") is False for r in results)
-    if command_failed:
-        raise exception.ReturnErrorCode(1)
-
-
-def results_to_json(raw):
-    results = []
-    for item in raw:
-        if item.get("succeeded") is None:
-            continue
-        item.update(
-            {
-                "succeeded": bool(item.get("succeeded")),
-                "defects": [d.as_dict() for d in item.get("defects", [])],
-            }
-        )
-        results.append(item)
-
-    return results
-
-
-def print_processing_header(tool, envname, envdump):
-    click.echo(
-        "Checking %s > %s (%s)"
-        % (click.style(envname, fg="cyan", bold=True), tool, "; ".join(envdump))
-    )
-    terminal_width = shutil.get_terminal_size().columns
-    click.secho("-" * terminal_width, bold=True)
-
-
-def print_processing_footer(result):
-    is_failed = not result.get("succeeded")
-    util.print_labeled_bar(
-        "[%s] Took %.2f seconds"
-        % (
-            (
-                click.style("FAILED", fg="red", bold=True)
-                if is_failed
-                else click.style("PASSED", fg="green", bold=True)
-            ),
-            result["duration"],
-        ),
-        is_error=is_failed,
-    )
-
-
-def collect_component_stats(result):
-    components = {}
-
-    def _append_defect(component, defect):
-        if not components.get(component):
-            components[component] = Counter()
-        components[component].update({DefectItem.SEVERITY_LABELS[defect.severity]: 1})
-
-    for defect in result.get("defects", []):
-        component = dirname(defect.file) or defect.file
-        _append_defect(component, defect)
-
-        if component.lower().startswith(get_project_dir().lower()):
-            while os.sep in component:
-                component = dirname(component)
-                _append_defect(component, defect)
-
-    return components
-
-
-def print_defects_stats(results):
-    if not results:
-        return
-
-    component_stats = {}
-    for r in results:
-        for k, v in r.get("stats", {}).items():
-            if not component_stats.get(k):
-                component_stats[k] = Counter()
-            component_stats[k].update(r["stats"][k])
-
-    if not component_stats:
-        return
-
-    severity_labels = list(DefectItem.SEVERITY_LABELS.values())
-    severity_labels.reverse()
-    tabular_data = []
-    for k, v in component_stats.items():
-        tool_defect = [v.get(s, 0) for s in severity_labels]
-        tabular_data.append([k] + tool_defect)
-
-    total = ["Total"] + [sum(d) for d in list(zip(*tabular_data))[1:]]
-    tabular_data.sort()
-    tabular_data.append([])  # Empty line as delimiter
-    tabular_data.append(total)
-
-    headers = ["Component"]
-    headers.extend([label.upper() for label in severity_labels])
-    headers = [click.style(h, bold=True) for h in headers]
-    click.echo(tabulate(tabular_data, headers=headers, numalign="center"))
-    click.echo()
-
-
-def print_check_summary(results, verbose=False):
-    click.echo()
-
-    tabular_data = []
-    succeeded_nums = 0
-    failed_nums = 0
-    duration = 0
-
-    print_defects_stats(results)
-
-    for result in results:
-        duration += result.get("duration", 0)
-        if result.get("succeeded") is False:
-            failed_nums += 1
-            status_str = click.style("FAILED", fg="red")
-        elif result.get("succeeded") is None:
-            status_str = "IGNORED"
-            if not verbose:
-                continue
-        else:
-            succeeded_nums += 1
-            status_str = click.style("PASSED", fg="green")
-
-        tabular_data.append(
-            (
-                click.style(result["env"], fg="cyan"),
-                result["tool"],
-                status_str,
-                util.humanize_duration_time(result.get("duration")),
-            )
-        )
-
-    click.echo(
-        tabulate(
-            tabular_data,
-            headers=[
-                click.style(s, bold=True)
-                for s in ("Environment", "Tool", "Status", "Duration")
-            ],
-        ),
-        err=failed_nums,
-    )
-
-    util.print_labeled_bar(
-        "%s%d succeeded in %s"
-        % (
-            "%d failed, " % failed_nums if failed_nums else "",
-            succeeded_nums,
-            util.humanize_duration_time(duration),
-        ),
-        is_error=failed_nums,
-        fg="red" if failed_nums else "green",
-    )

platformio/check/defect.py

@@ -1,95 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-import click
-
-from platformio.exception import PlatformioException
-from platformio.project.helpers import get_project_dir
-
-# pylint: disable=too-many-instance-attributes, redefined-builtin
-# pylint: disable=too-many-arguments
-
-
-class DefectItem:
-    SEVERITY_HIGH = 1
-    SEVERITY_MEDIUM = 2
-    SEVERITY_LOW = 4
-    SEVERITY_LABELS = {4: "low", 2: "medium", 1: "high"}
-
-    def __init__(
-        self,
-        severity,
-        category,
-        message,
-        file=None,
-        line=0,
-        column=0,
-        id=None,
-        callstack=None,
-        cwe=None,
-    ):
-        assert severity in (self.SEVERITY_HIGH, self.SEVERITY_MEDIUM, self.SEVERITY_LOW)
-        self.severity = severity
-        self.category = category
-        self.message = message
-        self.line = int(line)
-        self.column = int(column)
-        self.callstack = callstack
-        self.cwe = cwe
-        self.id = id
-        self.file = file or "unknown"
-        if file.lower().startswith(get_project_dir().lower()):
-            self.file = os.path.relpath(file, get_project_dir())
-
-    def __repr__(self):
-        defect_color = None
-        if self.severity == self.SEVERITY_HIGH:
-            defect_color = "red"
-        elif self.severity == self.SEVERITY_MEDIUM:
-            defect_color = "yellow"
-
-        format_str = "{file}:{line}: [{severity}:{category}] {message} {id}"
-        return format_str.format(
-            severity=click.style(self.SEVERITY_LABELS[self.severity], fg=defect_color),
-            category=click.style(self.category.lower(), fg=defect_color),
-            file=click.style(self.file, bold=True),
-            message=self.message,
-            line=self.line,
-            id="%s" % "[%s]" % self.id if self.id else "",
-        )
-
-    def __or__(self, defect):
-        return self.severity | defect.severity
-
-    @staticmethod
-    def severity_to_int(label):
-        for key, value in DefectItem.SEVERITY_LABELS.items():
-            if label == value:
-                return key
-        raise PlatformioException("Unknown severity label -> %s" % label)
-
-    def as_dict(self):
-        return {
-            "severity": self.SEVERITY_LABELS[self.severity],
-            "category": self.category,
-            "message": self.message,
-            "file": os.path.abspath(self.file),
-            "line": self.line,
-            "column": self.column,
-            "callstack": self.callstack,
-            "id": self.id,
-            "cwe": self.cwe,
-        }

platformio/check/tools/__init__.py

@@ -1,33 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from platformio import exception
-from platformio.check.tools.clangtidy import ClangtidyCheckTool
-from platformio.check.tools.cppcheck import CppcheckCheckTool
-from platformio.check.tools.pvsstudio import PvsStudioCheckTool
-
-
-class CheckToolFactory:
-    @staticmethod
-    def new(tool, project_dir, config, envname, options):
-        cls = None
-        if tool == "cppcheck":
-            cls = CppcheckCheckTool
-        elif tool == "clangtidy":
-            cls = ClangtidyCheckTool
-        elif tool == "pvs-studio":
-            cls = PvsStudioCheckTool
-        else:
-            raise exception.PlatformioException("Unknown check tool `%s`" % tool)
-        return cls(project_dir, config, envname, options)

platformio/check/tools/base.py

@@ -1,267 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import tempfile
-
-import click
-
-from platformio import fs, proc
-from platformio.check.defect import DefectItem
-from platformio.package.manager.core import get_core_package_dir
-from platformio.package.meta import PackageSpec
-from platformio.project.helpers import load_build_metadata
-
-
-class CheckToolBase:  # pylint: disable=too-many-instance-attributes
-    def __init__(self, project_dir, config, envname, options):
-        self.config = config
-        self.envname = envname
-        self.options = options
-        self.project_dir = project_dir
-        self.cc_flags = []
-        self.cxx_flags = []
-        self.cpp_includes = []
-        self.cpp_defines = []
-        self.toolchain_defines = []
-        self._tmp_files = []
-        self.cc_path = None
-        self.cxx_path = None
-        self._defects = []
-        self._on_defect_callback = None
-        self._bad_input = False
-        self._load_cpp_data()
-
-        # detect all defects by default
-        if not self.options.get("severity"):
-            self.options["severity"] = [
-                DefectItem.SEVERITY_LOW,
-                DefectItem.SEVERITY_MEDIUM,
-                DefectItem.SEVERITY_HIGH,
-            ]
-        # cast to severity by ids
-        self.options["severity"] = [
-            s if isinstance(s, int) else DefectItem.severity_to_int(s)
-            for s in self.options["severity"]
-        ]
-
-    def _load_cpp_data(self):
-        data = load_build_metadata(self.project_dir, self.envname)
-        if not data:
-            return
-        self.cc_flags = data.get("cc_flags", [])
-        self.cxx_flags = data.get("cxx_flags", [])
-        self.cpp_includes = self._dump_includes(data.get("includes", {}))
-        self.cpp_defines = data.get("defines", [])
-        self.cc_path = data.get("cc_path")
-        self.cxx_path = data.get("cxx_path")
-        self.toolchain_defines = self._get_toolchain_defines()
-
-    def get_tool_dir(self, pkg_name):
-        for spec in self.options["platform_packages"] or []:
-            spec = PackageSpec(spec)
-            if spec.name == pkg_name:
-                return get_core_package_dir(pkg_name, spec=spec)
-        return get_core_package_dir(pkg_name)
-
-    def get_flags(self, tool):
-        result = []
-        flags = self.options.get("flags") or []
-        for flag in flags:
-            if ":" not in flag or flag.startswith("-"):
-                result.extend([f for f in flag.split(" ") if f])
-            elif flag.startswith("%s:" % tool):
-                result.extend([f for f in flag.split(":", 1)[1].split(" ") if f])
-
-        return result
-
-    def _get_toolchain_defines(self):
-        def _extract_defines(language, includes_file):
-            build_flags = self.cxx_flags if language == "c++" else self.cc_flags
-            defines = []
-            cmd = "echo | %s -x %s %s %s -dM -E -" % (
-                self.cc_path,
-                language,
-                " ".join(
-                    [f for f in build_flags if f.startswith(("-m", "-f", "-std"))]
-                ),
-                includes_file,
-            )
-            result = proc.exec_command(cmd, shell=True)
-
-            if result["returncode"] != 0:
-                click.echo("Warning: Failed to extract toolchain defines!")
-                if self.options.get("verbose"):
-                    click.echo(result["out"])
-                    click.echo(result["err"])
-
-            for line in result["out"].split("\n"):
-                tokens = line.strip().split(" ", 2)
-                if not tokens or tokens[0] != "#define":
-                    continue
-                if len(tokens) > 2:
-                    defines.append("%s=%s" % (tokens[1], tokens[2]))
-                else:
-                    defines.append(tokens[1])
-
-            return defines
-
-        incflags_file = self._long_includes_hook(self.cpp_includes)
-        return {lang: _extract_defines(lang, incflags_file) for lang in ("c", "c++")}
-
-    def _create_tmp_file(self, data):
-        with tempfile.NamedTemporaryFile("w", delete=False) as fp:
-            fp.write(data)
-            self._tmp_files.append(fp.name)
-            return fp.name
-
-    def _long_includes_hook(self, includes):
-        data = []
-        for inc in includes:
-            data.append('-I"%s"' % fs.to_unix_path(inc))
-
-        return '@"%s"' % self._create_tmp_file(" ".join(data))
-
-    @staticmethod
-    def _dump_includes(includes_map):
-        result = []
-        for includes in includes_map.values():
-            for include in includes:
-                if include not in result:
-                    result.append(include)
-        return result
-
-    @staticmethod
-    def is_flag_set(flag, flags):
-        return any(flag in f for f in flags)
-
-    def get_defects(self):
-        return self._defects
-
-    def configure_command(self):
-        raise NotImplementedError
-
-    def on_tool_output(self, line):
-        line = self.tool_output_filter(line)
-        if not line:
-            return
-
-        defect = self.parse_defect(line)
-
-        if not isinstance(defect, DefectItem):
-            if self.options.get("verbose"):
-                click.echo(line)
-            return
-
-        if defect.severity not in self.options["severity"]:
-            return
-
-        self._defects.append(defect)
-        if self._on_defect_callback:
-            self._on_defect_callback(defect)
-
-    @staticmethod
-    def tool_output_filter(line):
-        return line
-
-    @staticmethod
-    def parse_defect(raw_line):
-        return raw_line
-
-    def clean_up(self):
-        for f in self._tmp_files:
-            if os.path.isfile(f):
-                os.remove(f)
-
-    @staticmethod
-    def is_check_successful(cmd_result):
-        return cmd_result["returncode"] == 0
-
-    def execute_check_cmd(self, cmd):
-        result = proc.exec_command(
-            cmd,
-            stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
-            stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
-        )
-
-        if not self.is_check_successful(result):
-            click.echo(
-                "\nError: Failed to execute check command! Exited with code %d."
-                % result["returncode"]
-            )
-            if self.options.get("verbose"):
-                click.echo(result["out"])
-                click.echo(result["err"])
-            self._bad_input = True
-
-        return result
-
-    @staticmethod
-    def get_project_target_files(project_dir, src_filters):
-        c_extension = (".c",)
-        cpp_extensions = (".cc", ".cpp", ".cxx", ".ino")
-        header_extensions = (".h", ".hh", ".hpp", ".hxx")
-
-        result = {"c": [], "c++": [], "headers": []}
-
-        def _add_file(path):
-            if path.endswith(header_extensions):
-                result["headers"].append(os.path.abspath(path))
-            elif path.endswith(c_extension):
-                result["c"].append(os.path.abspath(path))
-            elif path.endswith(cpp_extensions):
-                result["c++"].append(os.path.abspath(path))
-
-        src_filters = normalize_src_filters(src_filters)
-        for f in fs.match_src_files(project_dir, src_filters):
-            _add_file(f)
-
-        return result
-
-    def check(self, on_defect_callback=None):
-        self._on_defect_callback = on_defect_callback
-        cmd = self.configure_command()
-        if cmd:
-            if self.options.get("verbose"):
-                click.echo(" ".join(cmd))
-
-            self.execute_check_cmd(cmd)
-
-        else:
-            if self.options.get("verbose"):
-                click.echo("Error: Couldn't configure command")
-            self._bad_input = True
-
-        self.clean_up()
-
-        return self._bad_input
-
-
-#
-# Helpers
-#
-
-
-def normalize_src_filters(src_filters):
-    def _normalize(src_filters):
-        return (
-            src_filters
-            if src_filters.startswith(("+<", "-<"))
-            else "+<%s>" % src_filters
-        )
-
-    if isinstance(src_filters, (list, tuple)):
-        return " ".join([_normalize(f) for f in src_filters])
-
-    return _normalize(src_filters)

platformio/check/tools/clangtidy.py

@@ -1,90 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-from os.path import join
-
-from platformio.check.defect import DefectItem
-from platformio.check.tools.base import CheckToolBase
-
-
-class ClangtidyCheckTool(CheckToolBase):
-    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
-        if not self.options.get("verbose") and "[clang-diagnostic-error]" in line:
-            return ""
-
-        if "[CommonOptionsParser]" in line:
-            self._bad_input = True
-            return line
-
-        if any(d in line for d in ("note: ", "error: ", "warning: ")):
-            return line
-
-        return ""
-
-    def parse_defect(self, raw_line):  # pylint: disable=arguments-differ
-        match = re.match(r"^(.*):(\d+):(\d+):\s+([^:]+):\s(.+)\[([^]]+)\]$", raw_line)
-        if not match:
-            return raw_line
-
-        file_, line, column, category, message, defect_id = match.groups()
-
-        severity = DefectItem.SEVERITY_LOW
-        if category == "error":
-            severity = DefectItem.SEVERITY_HIGH
-        elif category == "warning":
-            severity = DefectItem.SEVERITY_MEDIUM
-
-        return DefectItem(severity, category, message, file_, line, column, defect_id)
-
-    @staticmethod
-    def is_check_successful(cmd_result):
-        # Note: Clang-Tidy returns 1 for not critical compilation errors,
-        # so 0 and 1 are only acceptable values
-        return cmd_result["returncode"] < 2
-
-    def configure_command(self):
-        tool_path = join(self.get_tool_dir("tool-clangtidy"), "clang-tidy")
-
-        cmd = [tool_path, "--quiet"]
-        flags = self.get_flags("clangtidy")
-        if not (
-            self.is_flag_set("--checks", flags) or self.is_flag_set("--config", flags)
-        ):
-            cmd.append("--checks=*")
-
-        project_files = self.get_project_target_files(
-            self.project_dir, self.options["src_filters"]
-        )
-
-        src_files = []
-        for items in project_files.values():
-            src_files.extend(items)
-
-        cmd.extend(flags + src_files + ["--"])
-        cmd.extend(
-            ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines["c++"]]
-        )
-
-        includes = []
-        for inc in self.cpp_includes:
-            if self.options.get("skip_packages") and inc.lower().startswith(
-                self.config.get("platformio", "packages_dir").lower()
-            ):
-                continue
-            includes.append(inc)
-
-        cmd.extend(["-I%s" % inc for inc in includes])
-
-        return cmd

platformio/check/tools/cppcheck.py

@@ -1,271 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-import click
-
-from platformio import proc
-from platformio.check.defect import DefectItem
-from platformio.check.tools.base import CheckToolBase
-
-
-class CppcheckCheckTool(CheckToolBase):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._field_delimiter = "<&PIO&>"
-        self._buffer = ""
-        self.defect_fields = [
-            "severity",
-            "message",
-            "file",
-            "line",
-            "column",
-            "callstack",
-            "cwe",
-            "id",
-        ]
-
-    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
-        if (
-            not self.options.get("verbose")
-            and "--suppress=unmatchedSuppression:" in line
-        ):
-            return ""
-
-        if any(
-            msg in line
-            for msg in (
-                "No C or C++ source files found",
-                "unrecognized command line option",
-                "there was an internal error",
-            )
-        ):
-            self._bad_input = True
-
-        return line
-
-    def parse_defect(self, raw_line):  # pylint: disable=arguments-differ
-        if self._field_delimiter not in raw_line:
-            return None
-
-        self._buffer += raw_line
-        if any(f not in self._buffer for f in self.defect_fields):
-            return None
-
-        args = {}
-        for field in self._buffer.split(self._field_delimiter):
-            field = field.strip().replace('"', "")
-            name, value = field.split("=", 1)
-            args[name] = value
-
-        args["category"] = args["severity"]
-        if args["severity"] == "error":
-            args["severity"] = DefectItem.SEVERITY_HIGH
-        elif args["severity"] == "warning":
-            args["severity"] = DefectItem.SEVERITY_MEDIUM
-        else:
-            args["severity"] = DefectItem.SEVERITY_LOW
-
-        # Skip defects found in third-party software, but keep in mind that such defects
-        # might break checking process so defects from project files are not reported
-        breaking_defect_ids = ("preprocessorErrorDirective", "syntaxError")
-        if (
-            args.get("file", "")
-            .lower()
-            .startswith(self.config.get("platformio", "packages_dir").lower())
-        ):
-            if args["id"] in breaking_defect_ids:
-                if self.options.get("verbose"):
-                    click.echo(
-                        "Error: Found a breaking defect '%s' in %s:%s\n"
-                        "Please note: check results might not be valid!\n"
-                        "Try adding --skip-packages"
-                        % (args.get("message"), args.get("file"), args.get("line"))
-                    )
-                    click.echo()
-                self._bad_input = True
-            self._buffer = ""
-            return None
-
-        self._buffer = ""
-        return DefectItem(**args)
-
-    def configure_command(self, language, src_file):  # pylint: disable=arguments-differ
-        tool_path = os.path.join(self.get_tool_dir("tool-cppcheck"), "cppcheck")
-
-        cmd = [
-            tool_path,
-            "--addon-python=%s" % proc.get_pythonexe_path(),
-            "--error-exitcode=3",
-            "--verbose" if self.options.get("verbose") else "--quiet",
-        ]
-
-        cmd.append(
-            '--template="%s"'
-            % self._field_delimiter.join(
-                ["{0}={{{0}}}".format(f) for f in self.defect_fields]
-            )
-        )
-
-        flags = self.get_flags("cppcheck")
-        if not flags:
-            # by default user can suppress reporting individual defects
-            # directly in code // cppcheck-suppress warningID
-            cmd.append("--inline-suppr")
-        if not self.is_flag_set("--platform", flags):
-            cmd.append("--platform=unspecified")
-        if not self.is_flag_set("--enable", flags):
-            enabled_checks = [
-                "warning",
-                "style",
-                "performance",
-                "portability",
-                "unusedFunction",
-            ]
-            cmd.append("--enable=%s" % ",".join(enabled_checks))
-
-        if not self.is_flag_set("--language", flags):
-            cmd.append("--language=" + language)
-
-        build_flags = self.cxx_flags if language == "c++" else self.cc_flags
-
-        if not self.is_flag_set("--std", flags):
-            # Try to guess the standard version from the build flags
-            for flag in build_flags:
-                if "-std" in flag:
-                    cmd.append("-" + self.convert_language_standard(flag))
-
-        cmd.extend(
-            ["-D%s" % d for d in self.cpp_defines + self.toolchain_defines[language]]
-        )
-
-        cmd.extend(flags)
-
-        cmd.extend(
-            "--include=" + inc
-            for inc in self.get_forced_includes(build_flags, self.cpp_includes)
-        )
-        cmd.append("--includes-file=%s" % self._generate_inc_file())
-        cmd.append('"%s"' % src_file)
-
-        return cmd
-
-    @staticmethod
-    def get_forced_includes(build_flags, includes):
-        def _extract_filepath(flag, include_options, build_flags):
-            path = ""
-            for option in include_options:
-                if not flag.startswith(option):
-                    continue
-                if flag.split(option)[1].strip():
-                    path = flag.split(option)[1].strip()
-                elif build_flags.index(flag) + 1 < len(build_flags):
-                    path = build_flags[build_flags.index(flag) + 1]
-            return path
-
-        def _search_include_dir(filepath, include_paths):
-            for inc_path in include_paths:
-                path = os.path.join(inc_path, filepath)
-                if os.path.isfile(path):
-                    return path
-            return ""
-
-        result = []
-        include_options = ("-include", "-imacros")
-        for f in build_flags:
-            if f.startswith(include_options):
-                filepath = _extract_filepath(f, include_options, build_flags)
-                if not os.path.isabs(filepath):
-                    filepath = _search_include_dir(filepath, includes)
-                if os.path.isfile(filepath):
-                    result.append(filepath)
-
-        return result
-
-    def _generate_src_file(self, src_files):
-        return self._create_tmp_file("\n".join(src_files))
-
-    def _generate_inc_file(self):
-        result = []
-        for inc in self.cpp_includes:
-            if self.options.get("skip_packages") and inc.lower().startswith(
-                self.config.get("platformio", "packages_dir").lower()
-            ):
-                continue
-            result.append(inc)
-        return self._create_tmp_file("\n".join(result))
-
-    def clean_up(self):
-        super().clean_up()
-
-        # delete temporary dump files generated by addons
-        if not self.is_flag_set("--addon", self.get_flags("cppcheck")):
-            return
-
-        for files in self.get_project_target_files(
-            self.project_dir, self.options["src_filters"]
-        ).values():
-            for f in files:
-                dump_file = f + ".dump"
-                if os.path.isfile(dump_file):
-                    os.remove(dump_file)
-
-    @staticmethod
-    def is_check_successful(cmd_result):
-        # Cppcheck is configured to return '3' if a defect is found
-        return cmd_result["returncode"] in (0, 3)
-
-    @staticmethod
-    def convert_language_standard(flag):
-        cpp_standards_map = {
-            "0x": "11",
-            "1y": "14",
-            "1z": "17",
-            "2a": "20",
-        }
-
-        standard = flag[-2:]
-        # Note: GNU extensions are not supported and converted to regular standards
-        return flag.replace("gnu", "c").replace(
-            standard, cpp_standards_map.get(standard, standard)
-        )
-
-    def check(self, on_defect_callback=None):
-        self._on_defect_callback = on_defect_callback
-
-        project_files = self.get_project_target_files(
-            self.project_dir, self.options["src_filters"]
-        )
-        src_files_scope = ("c", "c++")
-        if not any(project_files[t] for t in src_files_scope):
-            click.echo("Error: Nothing to check.")
-            return True
-
-        for scope, files in project_files.items():
-            if scope not in src_files_scope:
-                continue
-            for src_file in files:
-                cmd = self.configure_command(scope, src_file)
-                if not cmd:
-                    self._bad_input = True
-                    continue
-                if self.options.get("verbose"):
-                    click.echo(" ".join(cmd))
-
-                self.execute_check_cmd(cmd)
-
-        self.clean_up()
-
-        return self._bad_input

platformio/check/tools/pvsstudio.py

@@ -1,251 +0,0 @@
-# Copyright (c) 2020-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import shutil
-import tempfile
-from xml.etree.ElementTree import fromstring
-
-import click
-
-from platformio import proc
-from platformio.check.defect import DefectItem
-from platformio.check.tools.base import CheckToolBase
-from platformio.compat import IS_WINDOWS
-
-
-class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-attributes
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._tmp_dir = tempfile.mkdtemp(prefix="piocheck")
-        self._tmp_preprocessed_file = self._generate_tmp_file_path() + ".i"
-        self._tmp_output_file = self._generate_tmp_file_path() + ".pvs"
-        self._tmp_cfg_file = self._generate_tmp_file_path() + ".cfg"
-        self._tmp_cmd_file = self._generate_tmp_file_path() + ".cmd"
-        self.tool_path = os.path.join(
-            self.get_tool_dir("tool-pvs-studio"),
-            "x64" if IS_WINDOWS else "bin",
-            "pvs-studio",
-        )
-
-        with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp:
-            fp.write(
-                "exclude-path = "
-                + self.config.get("platformio", "packages_dir").replace("\\", "/")
-            )
-
-        with open(self._tmp_cmd_file, mode="w", encoding="utf8") as fp:
-            fp.write(
-                " ".join(
-                    ['-I"%s"' % inc.replace("\\", "/") for inc in self.cpp_includes]
-                )
-            )
-
-    def tool_output_filter(self, line):  # pylint: disable=arguments-differ
-        if any(
-            err_msg in line.lower()
-            for err_msg in (
-                "license was not entered",
-                "license information is incorrect",
-            )
-        ):
-            self._bad_input = True
-        return line
-
-    def _process_defects(self, defects):
-        for defect in defects:
-            if not isinstance(defect, DefectItem):
-                return
-            if defect.severity not in self.options["severity"]:
-                return
-            self._defects.append(defect)
-            if self._on_defect_callback:
-                self._on_defect_callback(defect)
-
-    def _demangle_report(self, output_file):
-        converter_tool = os.path.join(
-            self.get_tool_dir("tool-pvs-studio"),
-            "HtmlGenerator" if IS_WINDOWS else os.path.join("bin", "plog-converter"),
-        )
-
-        cmd = (
-            converter_tool,
-            "-t",
-            "xml",
-            output_file,
-            "-m",
-            "cwe",
-            "-m",
-            "misra",
-            "-a",
-            # Enable all possible analyzers and defect levels
-            "GA:1,2,3;64:1,2,3;OP:1,2,3;CS:1,2,3;MISRA:1,2,3",
-            "--cerr",
-        )
-
-        result = proc.exec_command(cmd)
-        if result["returncode"] != 0:
-            click.echo(result["err"])
-            self._bad_input = True
-
-        return result["err"]
-
-    def parse_defects(self, output_file):
-        defects = []
-
-        report = self._demangle_report(output_file)
-        if not report:
-            self._bad_input = True
-            return []
-
-        try:
-            defects_data = fromstring(report)
-        except:  # pylint: disable=bare-except
-            click.echo("Error: Couldn't decode generated report!")
-            self._bad_input = True
-            return []
-
-        for table in defects_data.iter("PVS-Studio_Analysis_Log"):
-            message = table.find("Message").text
-            category = table.find("ErrorType").text
-            line = table.find("Line").text
-            file_ = table.find("File").text
-            defect_id = table.find("ErrorCode").text
-            cwe = table.find("CWECode")
-            cwe_id = None
-            if cwe is not None:
-                cwe_id = cwe.text.lower().replace("cwe-", "")
-            misra = table.find("MISRA")
-            if misra is not None:
-                message += " [%s]" % misra.text
-
-            severity = DefectItem.SEVERITY_LOW
-            if category == "error":
-                severity = DefectItem.SEVERITY_HIGH
-            elif category == "warning":
-                severity = DefectItem.SEVERITY_MEDIUM
-
-            defects.append(
-                DefectItem(
-                    severity, category, message, file_, line, id=defect_id, cwe=cwe_id
-                )
-            )
-
-        return defects
-
-    def configure_command(self, src_file):  # pylint: disable=arguments-differ
-        if os.path.isfile(self._tmp_output_file):
-            os.remove(self._tmp_output_file)
-
-        if not os.path.isfile(self._tmp_preprocessed_file):
-            click.echo("Error: Missing preprocessed file for '%s'" % src_file)
-            return ""
-
-        cmd = [
-            self.tool_path,
-            "--skip-cl-exe",
-            "yes",
-            "--language",
-            "C" if src_file.endswith(".c") else "C++",
-            "--preprocessor",
-            "gcc",
-            "--cfg",
-            self._tmp_cfg_file,
-            "--source-file",
-            src_file,
-            "--i-file",
-            self._tmp_preprocessed_file,
-            "--output-file",
-            self._tmp_output_file,
-        ]
-
-        flags = self.get_flags("pvs-studio")
-        if not self.is_flag_set("--platform", flags):
-            cmd.append("--platform=arm")
-        cmd.extend(flags)
-
-        return cmd
-
-    def _generate_tmp_file_path(self):
-        # pylint: disable=protected-access
-        return os.path.join(self._tmp_dir, next(tempfile._get_candidate_names()))
-
-    def _prepare_preprocessed_file(self, src_file):
-        if os.path.isfile(self._tmp_preprocessed_file):
-            os.remove(self._tmp_preprocessed_file)
-
-        flags = self.cxx_flags
-        compiler = self.cxx_path
-        if src_file.endswith(".c"):
-            flags = self.cc_flags
-            compiler = self.cc_path
-
-        cmd = [
-            compiler,
-            '"%s"' % src_file,
-            "-E",
-            "-o",
-            '"%s"' % self._tmp_preprocessed_file,
-        ]
-        cmd.extend([f for f in flags if f])
-        cmd.extend(['"-D%s"' % d.replace('"', '\\"') for d in self.cpp_defines])
-        cmd.append('@"%s"' % self._tmp_cmd_file)
-
-        # Explicitly specify C++ as the language used in .ino files
-        if src_file.endswith(".ino"):
-            cmd.insert(1, "-xc++")
-
-        result = proc.exec_command(" ".join(cmd), shell=True)
-        if result["returncode"] != 0 or result["err"]:
-            if self.options.get("verbose"):
-                click.echo(" ".join(cmd))
-            click.echo(result["err"])
-            self._bad_input = True
-
-    def clean_up(self):
-        super().clean_up()
-        if os.path.isdir(self._tmp_dir):
-            shutil.rmtree(self._tmp_dir)
-
-    @staticmethod
-    def is_check_successful(cmd_result):
-        return (
-            "license" not in cmd_result["err"].lower() and cmd_result["returncode"] == 0
-        )
-
-    def check(self, on_defect_callback=None):
-        self._on_defect_callback = on_defect_callback
-        for scope, files in self.get_project_target_files(
-            self.project_dir, self.options["src_filters"]
-        ).items():
-            if scope not in ("c", "c++"):
-                continue
-            for src_file in files:
-                self._prepare_preprocessed_file(src_file)
-                cmd = self.configure_command(src_file)
-                if self.options.get("verbose"):
-                    click.echo(" ".join(cmd))
-                if not cmd:
-                    self._bad_input = True
-                    continue
-
-                result = self.execute_check_cmd(cmd)
-                if result["returncode"] != 0:
-                    continue
-
-                self._process_defects(self.parse_defects(self._tmp_output_file))
-
-        self.clean_up()
-
-        return self._bad_input

platformio/cli.py

@@ -1,110 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import importlib
-from pathlib import Path
-
-import click
-
-
-class PlatformioCLI(click.MultiCommand):
-    leftover_args = []
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._pio_root_path = Path(__file__).parent
-        self._pio_cmd_aliases = dict(package="pkg")
-
-    def _find_pio_commands(self):
-        def _to_module_path(p):
-            return (
-                "platformio." + ".".join(p.relative_to(self._pio_root_path).parts)[:-3]
-            )
-
-        result = {}
-        for p in self._pio_root_path.rglob("cli.py"):
-            # skip this module
-            if p.parent == self._pio_root_path:
-                continue
-            cmd_name = p.parent.name
-            result[self._pio_cmd_aliases.get(cmd_name, cmd_name)] = _to_module_path(p)
-
-        # find legacy commands
-        for p in (self._pio_root_path / "commands").iterdir():
-            if p.name.startswith("_"):
-                continue
-            if (p / "command.py").is_file():
-                result[p.name] = _to_module_path(p / "command.py")
-            elif p.name.endswith(".py"):
-                result[p.name[:-3]] = _to_module_path(p)
-
-        return result
-
-    @staticmethod
-    def in_silence():
-        args = PlatformioCLI.leftover_args
-        return args and any(
-            [
-                args[0] == "debug" and "--interpreter" in " ".join(args),
-                args[0] == "upgrade",
-                "--json-output" in args,
-                "--version" in args,
-            ]
-        )
-
-    @classmethod
-    def reveal_cmd_path_args(cls, ctx):
-        result = []
-        group = ctx.command
-        args = cls.leftover_args[::]
-        while args:
-            cmd_name = args.pop(0)
-            next_group = group.get_command(ctx, cmd_name)
-            if next_group:
-                group = next_group
-                result.append(cmd_name)
-            if not hasattr(group, "get_command"):
-                break
-        return result
-
-    def invoke(self, ctx):
-        PlatformioCLI.leftover_args = ctx.args
-        if hasattr(ctx, "protected_args"):
-            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
-        return super().invoke(ctx)
-
-    def list_commands(self, ctx):
-        return sorted(list(self._find_pio_commands()))
-
-    def get_command(self, ctx, cmd_name):
-        commands = self._find_pio_commands()
-        if cmd_name not in commands:
-            return self._handle_obsolate_command(ctx, cmd_name)
-        module = importlib.import_module(commands[cmd_name])
-        return getattr(module, "cli")
-
-    @staticmethod
-    def _handle_obsolate_command(ctx, cmd_name):
-        # pylint: disable=import-outside-toplevel
-        if cmd_name == "init":
-            from platformio.project.commands.init import project_init_cmd
-
-            return project_init_cmd
-
-        if cmd_name == "package":
-            from platformio.package.cli import cli
-
-            return cli
-
-        raise click.UsageError('No such command "%s"' % cmd_name, ctx)

platformio/commands/__init__.py

@@ -11,3 +11,62 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+import os
+from os.path import dirname, isfile, join
+
+import click
+
+
+class PlatformioCLI(click.MultiCommand):
+
+    leftover_args = []
+
+    @staticmethod
+    def in_silence():
+        args = PlatformioCLI.leftover_args
+        return args and any([
+            args[0] == "debug" and "--interpreter" in " ".join(args),
+            args[0] == "upgrade", "--json-output" in args, "--version" in args
+        ])
+
+    def invoke(self, ctx):
+        PlatformioCLI.leftover_args = ctx.args
+        if hasattr(ctx, "protected_args"):
+            PlatformioCLI.leftover_args = ctx.protected_args + ctx.args
+        return super(PlatformioCLI, self).invoke(ctx)
+
+    def list_commands(self, ctx):
+        cmds = []
+        cmds_dir = dirname(__file__)
+        for name in os.listdir(cmds_dir):
+            if name.startswith("__init__"):
+                continue
+            if isfile(join(cmds_dir, name, "command.py")):
+                cmds.append(name)
+            elif name.endswith(".py"):
+                cmds.append(name[:-3])
+        cmds.sort()
+        return cmds
+
+    def get_command(self, ctx, cmd_name):
+        mod = None
+        try:
+            mod = __import__("platformio.commands." + cmd_name, None, None,
+                             ["cli"])
+        except ImportError:
+            try:
+                return self._handle_obsolate_command(cmd_name)
+            except AttributeError:
+                raise click.UsageError('No such command "%s"' % cmd_name, ctx)
+        return mod.cli
+
+    @staticmethod
+    def _handle_obsolate_command(name):
+        if name == "platforms":
+            from platformio.commands import platform
+            return platform.cli
+        if name == "serialports":
+            from platformio.commands import device
+            return device.cli
+        raise AttributeError()

platformio/commands/account.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=unused-argument
+
+import sys
+
+import click
+
+from platformio.managers.core import pioplus_call
+
+
+@click.group("account", short_help="Manage PIO Account")
+def cli():
+    pass
+
+
+@cli.command("register", short_help="Create new PIO Account")
+@click.option("-u", "--username")
+def account_register(**kwargs):
+    pioplus_call(sys.argv[1:])
+
+
+@cli.command("login", short_help="Log in to PIO Account")
+@click.option("-u", "--username")
+@click.option("-p", "--password")
+def account_login(**kwargs):
+    pioplus_call(sys.argv[1:])
+
+
+@cli.command("logout", short_help="Log out of PIO Account")
+def account_logout():
+    pioplus_call(sys.argv[1:])
+
+
+@cli.command("password", short_help="Change password")
+@click.option("--old-password")
+@click.option("--new-password")
+def account_password(**kwargs):
+    pioplus_call(sys.argv[1:])
+
+
+@cli.command("token", short_help="Get or regenerate Authentication Token")
+@click.option("-p", "--password")
+@click.option("--regenerate", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_token(**kwargs):
+    pioplus_call(sys.argv[1:])
+
+
+@cli.command("forgot", short_help="Forgot password")
+@click.option("-u", "--username")
+def account_forgot(**kwargs):
+    pioplus_call(sys.argv[1:])
+
+
+@cli.command("show", short_help="PIO Account information")
+@click.option("--offline", is_flag=True)
+@click.option("--json-output", is_flag=True)
+def account_show(**kwargs):
+    pioplus_call(sys.argv[1:])

platformio/commands/boards.py

@@ -13,16 +13,16 @@
 # limitations under the License.
 
 import json
-import shutil
 
 import click
 from tabulate import tabulate
 
 from platformio import fs
-from platformio.package.manager.platform import PlatformPackageManager
+from platformio.compat import dump_json_to_unicode
+from platformio.managers.platform import PlatformManager
 
 
-@click.command("boards", short_help="Board Explorer")
+@click.command("boards", short_help="Embedded Board Explorer")
 @click.argument("query", required=False)
 @click.option("--installed", is_flag=True)
 @click.option("--json-output", is_flag=True)
@@ -32,17 +32,14 @@ def cli(query, installed, json_output):  # pylint: disable=R0912
 
     grpboards = {}
     for board in _get_boards(installed):
-        if query and not any(
-            query.lower() in str(board.get(k, "")).lower()
-            for k in ("id", "name", "mcu", "vendor", "platform", "frameworks")
-        ):
+        if query and query.lower() not in json.dumps(board).lower():
             continue
-        if board["platform"] not in grpboards:
-            grpboards[board["platform"]] = []
-        grpboards[board["platform"]].append(board)
+        if board['platform'] not in grpboards:
+            grpboards[board['platform']] = []
+        grpboards[board['platform']].append(board)
 
-    terminal_width = shutil.get_terminal_size().columns
-    for platform, boards in sorted(grpboards.items()):
+    terminal_width, _ = click.get_terminal_size()
+    for (platform, boards) in sorted(grpboards.items()):
         click.echo("")
         click.echo("Platform: ", nl=False)
         click.secho(platform, bold=True)
@@ -53,25 +50,15 @@ def cli(query, installed, json_output):  # pylint: disable=R0912
 
 def print_boards(boards):
     click.echo(
-        tabulate(
-            [
-                (
-                    click.style(b["id"], fg="cyan"),
-                    b["mcu"],
-                    "%dMHz" % (b["fcpu"] / 1000000),
-                    fs.humanize_file_size(b["rom"]),
-                    fs.humanize_file_size(b["ram"]),
-                    b["name"],
-                )
-                for b in boards
-            ],
-            headers=["ID", "MCU", "Frequency", "Flash", "RAM", "Name"],
-        )
-    )
+        tabulate([(click.style(b['id'], fg="cyan"), b['mcu'], "%dMHz" %
+                   (b['fcpu'] / 1000000), fs.format_filesize(
+                       b['rom']), fs.format_filesize(b['ram']), b['name'])
+                  for b in boards],
+                 headers=["ID", "MCU", "Frequency", "Flash", "RAM", "Name"]))
 
 
 def _get_boards(installed=False):
-    pm = PlatformPackageManager()
+    pm = PlatformManager()
     return pm.get_installed_boards() if installed else pm.get_all_boards()
 
 
@@ -79,8 +66,8 @@ def _print_boards_json(query, installed=False):
     result = []
     for board in _get_boards(installed):
         if query:
-            search_data = "%s %s" % (board["id"], json.dumps(board).lower())
+            search_data = "%s %s" % (board['id'], json.dumps(board).lower())
             if query.lower() not in search_data.lower():
                 continue
         result.append(board)
-    click.echo(json.dumps(result))
+    click.echo(dump_json_to_unicode(result))

platformio/commands/ci.py

@@ -12,18 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import glob
-import os
-import shutil
-import tempfile
+from glob import glob
+from os import getenv, makedirs, remove
+from os.path import abspath, basename, expanduser, isdir, isfile, join
+from shutil import copyfile, copytree
+from tempfile import mkdtemp
 
 import click
 
-from platformio import fs
+from platformio import app, fs
+from platformio.commands.init import cli as cmd_init
+from platformio.commands.init import validate_boards
+from platformio.commands.run import cli as cmd_run
+from platformio.compat import glob_escape
 from platformio.exception import CIBuildEnvsEmpty
-from platformio.project.commands.init import project_init_cmd, validate_boards
 from platformio.project.config import ProjectConfig
-from platformio.run.cli import cli as cmd_run
 
 
 def validate_path(ctx, param, value):  # pylint: disable=unused-argument
@@ -31,71 +34,74 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
     value = list(value)
     for i, p in enumerate(value):
         if p.startswith("~"):
-            value[i] = fs.expanduser(p)
-        value[i] = os.path.abspath(value[i])
-        if not glob.glob(value[i], recursive=True):
+            value[i] = expanduser(p)
+        value[i] = abspath(value[i])
+        if not glob(value[i]):
             invalid_path = p
             break
     try:
         assert invalid_path is None
         return value
-    except AssertionError as exc:
-        raise click.BadParameter("Found invalid path: %s" % invalid_path) from exc
+    except AssertionError:
+        raise click.BadParameter("Found invalid path: %s" % invalid_path)
 
 
 @click.command("ci", short_help="Continuous Integration")
 @click.argument("src", nargs=-1, callback=validate_path)
-@click.option("-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY")
+@click.option("-l",
+              "--lib",
+              multiple=True,
+              callback=validate_path,
+              metavar="DIRECTORY")
 @click.option("--exclude", multiple=True)
-@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
-@click.option(
-    "--build-dir",
-    default=tempfile.mkdtemp,
-    type=click.Path(file_okay=False, dir_okay=True, writable=True),
-)
+@click.option("-b",
+              "--board",
+              multiple=True,
+              metavar="ID",
+              callback=validate_boards)
+@click.option("--build-dir",
+              default=mkdtemp,
+              type=click.Path(file_okay=False,
+                              dir_okay=True,
+                              writable=True,
+                              resolve_path=True))
 @click.option("--keep-build-dir", is_flag=True)
-@click.option(
-    "-c",
-    "--project-conf",
-    type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True),
-)
+@click.option("-c",
+              "--project-conf",
+              type=click.Path(exists=True,
+                              file_okay=True,
+                              dir_okay=False,
+                              readable=True,
+                              resolve_path=True))
 @click.option("-O", "--project-option", multiple=True)
-@click.option("-e", "--environment", "environments", multiple=True)
 @click.option("-v", "--verbose", is_flag=True)
 @click.pass_context
 def cli(  # pylint: disable=too-many-arguments, too-many-branches
-    ctx,
-    src,
-    lib,
-    exclude,
-    board,
-    build_dir,
-    keep_build_dir,
-    project_conf,
-    project_option,
-    environments,
-    verbose,
-):
-    if not src and os.getenv("PLATFORMIO_CI_SRC"):
-        src = validate_path(ctx, None, os.getenv("PLATFORMIO_CI_SRC").split(":"))
+        ctx, src, lib, exclude, board, build_dir, keep_build_dir, project_conf,
+        project_option, verbose):
+
+    if not src and getenv("PLATFORMIO_CI_SRC"):
+        src = validate_path(ctx, None, getenv("PLATFORMIO_CI_SRC").split(":"))
     if not src:
         raise click.BadParameter("Missing argument 'src'")
 
     try:
-        if not keep_build_dir and os.path.isdir(build_dir):
+        app.set_session_var("force_option", True)
+
+        if not keep_build_dir and isdir(build_dir):
             fs.rmtree(build_dir)
-        if not os.path.isdir(build_dir):
-            os.makedirs(build_dir)
+        if not isdir(build_dir):
+            makedirs(build_dir)
 
         for dir_name, patterns in dict(lib=lib, src=src).items():
             if not patterns:
                 continue
             contents = []
             for p in patterns:
-                contents += glob.glob(p, recursive=True)
-            _copy_contents(os.path.join(build_dir, dir_name), contents)
+                contents += glob(p)
+            _copy_contents(join(build_dir, dir_name), contents)
 
-        if project_conf and os.path.isfile(project_conf):
+        if project_conf and isfile(project_conf):
             _copy_project_conf(build_dir, project_conf)
         elif not board:
             raise CIBuildEnvsEmpty()
@@ -104,71 +110,64 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
             _exclude_contents(build_dir, exclude)
 
         # initialise project
-        ctx.invoke(
-            project_init_cmd,
-            project_dir=build_dir,
-            boards=board,
-            project_options=project_option,
-        )
+        ctx.invoke(cmd_init,
+                   project_dir=build_dir,
+                   board=board,
+                   project_option=project_option)
 
         # process project
-        ctx.invoke(
-            cmd_run, project_dir=build_dir, environment=environments, verbose=verbose
-        )
+        ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose)
     finally:
         if not keep_build_dir:
             fs.rmtree(build_dir)
 
 
-def _copy_contents(dst_dir, contents):  # pylint: disable=too-many-branches
+def _copy_contents(dst_dir, contents):
     items = {"dirs": set(), "files": set()}
 
     for path in contents:
-        if os.path.isdir(path):
-            items["dirs"].add(path)
-        elif os.path.isfile(path):
-            items["files"].add(path)
+        if isdir(path):
+            items['dirs'].add(path)
+        elif isfile(path):
+            items['files'].add(path)
 
-    dst_dir_name = os.path.basename(dst_dir)
+    dst_dir_name = basename(dst_dir)
 
-    if dst_dir_name == "src" and len(items["dirs"]) == 1:
-        if not os.path.isdir(dst_dir):
-            shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
+    if dst_dir_name == "src" and len(items['dirs']) == 1:
+        copytree(list(items['dirs']).pop(), dst_dir, symlinks=True)
     else:
-        if not os.path.isdir(dst_dir):
-            os.makedirs(dst_dir)
-        for d in items["dirs"]:
-            src_dst_dir = os.path.join(dst_dir, os.path.basename(d))
-            if not os.path.isdir(src_dst_dir):
-                shutil.copytree(d, src_dst_dir, symlinks=True)
+        if not isdir(dst_dir):
+            makedirs(dst_dir)
+        for d in items['dirs']:
+            copytree(d, join(dst_dir, basename(d)), symlinks=True)
 
-    if not items["files"]:
+    if not items['files']:
         return
 
     if dst_dir_name == "lib":
-        dst_dir = os.path.join(dst_dir, tempfile.mkdtemp(dir=dst_dir))
+        dst_dir = join(dst_dir, mkdtemp(dir=dst_dir))
 
-    for f in items["files"]:
-        dst_file = os.path.join(dst_dir, os.path.basename(f))
+    for f in items['files']:
+        dst_file = join(dst_dir, basename(f))
         if f == dst_file:
             continue
-        shutil.copyfile(f, dst_file)
+        copyfile(f, dst_file)
 
 
 def _exclude_contents(dst_dir, patterns):
     contents = []
     for p in patterns:
-        contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True)
+        contents += glob(join(glob_escape(dst_dir), p))
     for path in contents:
-        path = os.path.abspath(path)
-        if os.path.isdir(path):
+        path = abspath(path)
+        if isdir(path):
             fs.rmtree(path)
-        elif os.path.isfile(path):
-            os.remove(path)
+        elif isfile(path):
+            remove(path)
 
 
 def _copy_project_conf(build_dir, project_conf):
     config = ProjectConfig(project_conf, parse_extra=False)
     if config.has_section("platformio"):
         config.remove_section("platformio")
-    config.save(os.path.join(build_dir, "platformio.ini"))
+    config.save(join(build_dir, "platformio.ini"))

platformio/commands/debug/__init__.py

@@ -11,3 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from platformio.commands.debug.command import cli

platformio/commands/debug/client.py

@@ -0,0 +1,292 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+import signal
+import time
+from hashlib import sha1
+from os.path import abspath, basename, dirname, isdir, join, splitext
+from tempfile import mkdtemp
+
+from twisted.internet import protocol  # pylint: disable=import-error
+from twisted.internet import reactor  # pylint: disable=import-error
+from twisted.internet import stdio  # pylint: disable=import-error
+from twisted.internet import task  # pylint: disable=import-error
+
+from platformio import app, exception, fs, proc, util
+from platformio.commands.debug import helpers, initcfgs
+from platformio.commands.debug.process import BaseProcess
+from platformio.commands.debug.server import DebugServer
+from platformio.compat import hashlib_encode_data
+from platformio.project.helpers import get_project_cache_dir
+from platformio.telemetry import MeasurementProtocol
+
+LOG_FILE = None
+
+
+class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
+
+    PIO_SRC_NAME = ".pioinit"
+    INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed"
+
+    def __init__(self, project_dir, args, debug_options, env_options):
+        self.project_dir = project_dir
+        self.args = list(args)
+        self.debug_options = debug_options
+        self.env_options = env_options
+
+        self._debug_server = DebugServer(debug_options, env_options)
+        self._session_id = None
+
+        if not isdir(get_project_cache_dir()):
+            os.makedirs(get_project_cache_dir())
+        self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(),
+                                   prefix=".piodebug-")
+
+        self._target_is_run = False
+        self._last_server_activity = 0
+        self._auto_continue_timer = None
+
+    def spawn(self, gdb_path, prog_path):
+        session_hash = gdb_path + prog_path
+        self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest()
+        self._kill_previous_session()
+
+        patterns = {
+            "PROJECT_DIR": self.project_dir,
+            "PROG_PATH": prog_path,
+            "PROG_DIR": dirname(prog_path),
+            "PROG_NAME": basename(splitext(prog_path)[0]),
+            "DEBUG_PORT": self.debug_options['port'],
+            "UPLOAD_PROTOCOL": self.debug_options['upload_protocol'],
+            "INIT_BREAK": self.debug_options['init_break'] or "",
+            "LOAD_CMDS": "\n".join(self.debug_options['load_cmds'] or []),
+        }
+
+        self._debug_server.spawn(patterns)
+
+        if not patterns['DEBUG_PORT']:
+            patterns['DEBUG_PORT'] = self._debug_server.get_debug_port()
+        self.generate_pioinit(self._gdbsrc_dir, patterns)
+
+        # start GDB client
+        args = [
+            "piogdb",
+            "-q",
+            "--directory", self._gdbsrc_dir,
+            "--directory", self.project_dir,
+            "-l", "10"
+        ]  # yapf: disable
+        args.extend(self.args)
+        if not gdb_path:
+            raise exception.DebugInvalidOptions("GDB client is not configured")
+        gdb_data_dir = self._get_data_dir(gdb_path)
+        if gdb_data_dir:
+            args.extend(["--data-directory", gdb_data_dir])
+        args.append(patterns['PROG_PATH'])
+
+        return reactor.spawnProcess(self,
+                                    gdb_path,
+                                    args,
+                                    path=self.project_dir,
+                                    env=os.environ)
+
+    @staticmethod
+    def _get_data_dir(gdb_path):
+        if "msp430" in gdb_path:
+            return None
+        gdb_data_dir = abspath(join(dirname(gdb_path), "..", "share", "gdb"))
+        return gdb_data_dir if isdir(gdb_data_dir) else None
+
+    def generate_pioinit(self, dst_dir, patterns):
+        server_exe = (self.debug_options.get("server")
+                      or {}).get("executable", "").lower()
+        if "jlink" in server_exe:
+            cfg = initcfgs.GDB_JLINK_INIT_CONFIG
+        elif "st-util" in server_exe:
+            cfg = initcfgs.GDB_STUTIL_INIT_CONFIG
+        elif "mspdebug" in server_exe:
+            cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG
+        elif "qemu" in server_exe:
+            cfg = initcfgs.GDB_QEMU_INIT_CONFIG
+        elif self.debug_options['require_debug_port']:
+            cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG
+        else:
+            cfg = initcfgs.GDB_DEFAULT_INIT_CONFIG
+        commands = cfg.split("\n")
+
+        if self.debug_options['init_cmds']:
+            commands = self.debug_options['init_cmds']
+        commands.extend(self.debug_options['extra_cmds'])
+
+        if not any("define pio_reset_target" in cmd for cmd in commands):
+            commands = [
+                "define pio_reset_target",
+                "   echo Warning! Undefined pio_reset_target command\\n",
+                "   mon reset",
+                "end"
+            ] + commands  # yapf: disable
+        if not any("define pio_reset_halt_target" in cmd for cmd in commands):
+            commands = [
+                "define pio_reset_halt_target",
+                "   echo Warning! Undefined pio_reset_halt_target command\\n",
+                "   mon reset halt",
+                "end"
+            ] + commands  # yapf: disable
+        if not any("define pio_restart_target" in cmd for cmd in commands):
+            commands += [
+                "define pio_restart_target",
+                "   pio_reset_halt_target",
+                "   $INIT_BREAK",
+                "   %s" % ("continue" if patterns['INIT_BREAK'] else "next"),
+                "end"
+            ]  # yapf: disable
+
+        banner = [
+            "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n",
+            "echo PlatformIO: debug_tool = %s\\n" % self.debug_options['tool'],
+            "echo PlatformIO: Initializing remote target...\\n"
+        ]
+        footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER]
+        commands = banner + commands + footer
+
+        with open(join(dst_dir, self.PIO_SRC_NAME), "w") as fp:
+            fp.write("\n".join(self.apply_patterns(commands, patterns)))
+
+    def connectionMade(self):
+        self._lock_session(self.transport.pid)
+
+        p = protocol.Protocol()
+        p.dataReceived = self.onStdInData
+        stdio.StandardIO(p)
+
+    def onStdInData(self, data):
+        if LOG_FILE:
+            with open(LOG_FILE, "ab") as fp:
+                fp.write(data)
+
+        self._last_server_activity = time.time()
+
+        if b"-exec-run" in data:
+            if self._target_is_run:
+                token, _ = data.split(b"-", 1)
+                self.outReceived(token + b"^running\n")
+                return
+            data = data.replace(b"-exec-run", b"-exec-continue")
+
+        if b"-exec-continue" in data:
+            self._target_is_run = True
+        if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"):
+            # Allow terminating via SIGINT/CTRL+C
+            signal.signal(signal.SIGINT, signal.default_int_handler)
+            self.transport.write(b"pio_reset_target\n")
+        self.transport.write(data)
+
+    def processEnded(self, reason):  # pylint: disable=unused-argument
+        self._unlock_session()
+        if self._gdbsrc_dir and isdir(self._gdbsrc_dir):
+            fs.rmtree(self._gdbsrc_dir)
+        if self._debug_server:
+            self._debug_server.terminate()
+
+        reactor.stop()
+
+    def outReceived(self, data):
+        if LOG_FILE:
+            with open(LOG_FILE, "ab") as fp:
+                fp.write(data)
+
+        self._last_server_activity = time.time()
+        super(GDBClient, self).outReceived(data)
+        self._handle_error(data)
+        # go to init break automatically
+        if self.INIT_COMPLETED_BANNER.encode() in data:
+            self._auto_continue_timer = task.LoopingCall(
+                self._auto_exec_continue)
+            self._auto_continue_timer.start(0.1)
+
+    def errReceived(self, data):
+        super(GDBClient, self).errReceived(data)
+        self._handle_error(data)
+
+    def console_log(self, msg):
+        if helpers.is_mi_mode(self.args):
+            self.outReceived(('~"%s\\n"\n' % msg).encode())
+        else:
+            self.outReceived(("%s\n" % msg).encode())
+
+    def _auto_exec_continue(self):
+        auto_exec_delay = 0.5  # in seconds
+        if self._last_server_activity > (time.time() - auto_exec_delay):
+            return
+        if self._auto_continue_timer:
+            self._auto_continue_timer.stop()
+        self._auto_continue_timer = None
+
+        if not self.debug_options['init_break'] or self._target_is_run:
+            return
+        self.console_log(
+            "PlatformIO: Resume the execution to `debug_init_break = %s`" %
+            self.debug_options['init_break'])
+        self.console_log("PlatformIO: More configuration options -> "
+                         "http://bit.ly/pio-debug")
+        self.transport.write(b"0-exec-continue\n" if helpers.
+                             is_mi_mode(self.args) else b"continue\n")
+        self._target_is_run = True
+
+    def _handle_error(self, data):
+        if (self.PIO_SRC_NAME.encode() not in data
+                or b"Error in sourced" not in data):
+            return
+        configuration = {"debug": self.debug_options, "env": self.env_options}
+        exd = re.sub(r'\\(?!")', "/", json.dumps(configuration))
+        exd = re.sub(r'"(?:[a-z]\:)?((/[^"/]+)+)"',
+                     lambda m: '"%s"' % join(*m.group(1).split("/")[-2:]), exd,
+                     re.I | re.M)
+        mp = MeasurementProtocol()
+        mp['exd'] = "DebugGDBPioInitError: %s" % exd
+        mp['exf'] = 1
+        mp.send("exception")
+        self.transport.loseConnection()
+
+    def _kill_previous_session(self):
+        assert self._session_id
+        pid = None
+        with app.ContentCache() as cc:
+            pid = cc.get(self._session_id)
+            cc.delete(self._session_id)
+        if not pid:
+            return
+        if "windows" in util.get_systype():
+            kill = ["Taskkill", "/PID", pid, "/F"]
+        else:
+            kill = ["kill", pid]
+        try:
+            proc.exec_command(kill)
+        except:  # pylint: disable=bare-except
+            pass
+
+    def _lock_session(self, pid):
+        if not self._session_id:
+            return
+        with app.ContentCache() as cc:
+            cc.set(self._session_id, str(pid), "1h")
+
+    def _unlock_session(self):
+        if not self._session_id:
+            return
+        with app.ContentCache() as cc:
+            cc.delete(self._session_id)

platformio/commands/debug/command.py

@@ -0,0 +1,151 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-arguments, too-many-statements
+# pylint: disable=too-many-locals, too-many-branches
+
+import os
+import signal
+from os.path import isfile, join
+
+import click
+
+from platformio import exception, fs, proc, util
+from platformio.commands.debug import helpers
+from platformio.managers.core import inject_contrib_pysite
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import (is_platformio_project,
+                                        load_project_ide_data)
+
+
+@click.command("debug",
+               context_settings=dict(ignore_unknown_options=True),
+               short_help="PIO Unified Debugger")
+@click.option("-d",
+              "--project-dir",
+              default=os.getcwd,
+              type=click.Path(exists=True,
+                              file_okay=False,
+                              dir_okay=True,
+                              writable=True,
+                              resolve_path=True))
+@click.option("-c",
+              "--project-conf",
+              type=click.Path(exists=True,
+                              file_okay=True,
+                              dir_okay=False,
+                              readable=True,
+                              resolve_path=True))
+@click.option("--environment", "-e", metavar="<environment>")
+@click.option("--verbose", "-v", is_flag=True)
+@click.option("--interface", type=click.Choice(["gdb"]))
+@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
+@click.pass_context
+def cli(ctx, project_dir, project_conf, environment, verbose, interface,
+        __unprocessed):
+    # use env variables from Eclipse or CLion
+    for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"):
+        if is_platformio_project(project_dir):
+            break
+        if os.getenv(sysenv):
+            project_dir = os.getenv(sysenv)
+
+    with fs.cd(project_dir):
+        config = ProjectConfig.get_instance(
+            project_conf or join(project_dir, "platformio.ini"))
+        config.validate(envs=[environment] if environment else None)
+
+        env_name = environment or helpers.get_default_debug_env(config)
+        env_options = config.items(env=env_name, as_dict=True)
+        if not set(env_options.keys()) >= set(["platform", "board"]):
+            raise exception.ProjectEnvsNotAvailable()
+        debug_options = helpers.validate_debug_options(ctx, env_options)
+        assert debug_options
+
+    if not interface:
+        return helpers.predebug_project(ctx, project_dir, env_name, False,
+                                        verbose)
+
+    configuration = load_project_ide_data(project_dir, env_name)
+    if not configuration:
+        raise exception.DebugInvalidOptions(
+            "Could not load debug configuration")
+
+    if "--version" in __unprocessed:
+        result = proc.exec_command([configuration['gdb_path'], "--version"])
+        if result['returncode'] == 0:
+            return click.echo(result['out'])
+        raise exception.PlatformioException("\n".join(
+            [result['out'], result['err']]))
+
+    try:
+        fs.ensure_udev_rules()
+    except exception.InvalidUdevRules as e:
+        for line in str(e).split("\n") + [""]:
+            click.echo(
+                ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") %
+                line)
+
+    debug_options['load_cmds'] = helpers.configure_esp32_load_cmds(
+        debug_options, configuration)
+
+    rebuild_prog = False
+    preload = debug_options['load_cmds'] == ["preload"]
+    load_mode = debug_options['load_mode']
+    if load_mode == "always":
+        rebuild_prog = (
+            preload
+            or not helpers.has_debug_symbols(configuration['prog_path']))
+    elif load_mode == "modified":
+        rebuild_prog = (
+            helpers.is_prog_obsolete(configuration['prog_path'])
+            or not helpers.has_debug_symbols(configuration['prog_path']))
+    else:
+        rebuild_prog = not isfile(configuration['prog_path'])
+
+    if preload or (not rebuild_prog and load_mode != "always"):
+        # don't load firmware through debug server
+        debug_options['load_cmds'] = []
+
+    if rebuild_prog:
+        if helpers.is_mi_mode(__unprocessed):
+            click.echo('~"Preparing firmware for debugging...\\n"')
+            output = helpers.GDBBytesIO()
+            with util.capture_std_streams(output):
+                helpers.predebug_project(ctx, project_dir, env_name, preload,
+                                         verbose)
+            output.close()
+        else:
+            click.echo("Preparing firmware for debugging...")
+            helpers.predebug_project(ctx, project_dir, env_name, preload,
+                                     verbose)
+
+        # save SHA sum of newly created prog
+        if load_mode == "modified":
+            helpers.is_prog_obsolete(configuration['prog_path'])
+
+    if not isfile(configuration['prog_path']):
+        raise exception.DebugInvalidOptions("Program/firmware is missed")
+
+    # run debugging client
+    inject_contrib_pysite()
+    from platformio.commands.debug.client import GDBClient, reactor
+
+    client = GDBClient(project_dir, __unprocessed, debug_options, env_options)
+    client.spawn(configuration['gdb_path'], configuration['prog_path'])
+
+    signal.signal(signal.SIGINT, lambda *args, **kwargs: None)
+    reactor.run()
+
+    return True

platformio/commands/debug/helpers.py

@@ -0,0 +1,265 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import time
+from fnmatch import fnmatch
+from hashlib import sha1
+from io import BytesIO
+from os.path import isfile
+
+from platformio import exception, fs, util
+from platformio.commands.platform import \
+    platform_install as cmd_platform_install
+from platformio.commands.run import cli as cmd_run
+from platformio.managers.platform import PlatformFactory
+from platformio.project.config import ProjectConfig
+
+
+class GDBBytesIO(BytesIO):  # pylint: disable=too-few-public-methods
+
+    STDOUT = sys.stdout
+
+    def write(self, text):
+        if "\n" in text:
+            for line in text.strip().split("\n"):
+                self.STDOUT.write('~"%s\\n"\n' % line)
+        else:
+            self.STDOUT.write('~"%s"' % text)
+        self.STDOUT.flush()
+
+
+def is_mi_mode(args):
+    return "--interpreter" in " ".join(args)
+
+
+def get_default_debug_env(config):
+    default_envs = config.default_envs()
+    all_envs = config.envs()
+    for env in default_envs:
+        if config.get("env:" + env, "build_type") == "debug":
+            return env
+    for env in all_envs:
+        if config.get("env:" + env, "build_type") == "debug":
+            return env
+    return default_envs[0] if default_envs else all_envs[0]
+
+
+def predebug_project(ctx, project_dir, env_name, preload, verbose):
+    ctx.invoke(cmd_run,
+               project_dir=project_dir,
+               environment=[env_name],
+               target=["debug"] + (["upload"] if preload else []),
+               verbose=verbose)
+    if preload:
+        time.sleep(5)
+
+
+def validate_debug_options(cmd_ctx, env_options):
+
+    def _cleanup_cmds(items):
+        items = ProjectConfig.parse_multi_values(items)
+        return [
+            "$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items
+        ]
+
+    try:
+        platform = PlatformFactory.newPlatform(env_options['platform'])
+    except exception.UnknownPlatform:
+        cmd_ctx.invoke(cmd_platform_install,
+                       platforms=[env_options['platform']],
+                       skip_default_package=True)
+        platform = PlatformFactory.newPlatform(env_options['platform'])
+
+    board_config = platform.board_config(env_options['board'])
+    tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool"))
+    tool_settings = board_config.get("debug", {}).get("tools",
+                                                      {}).get(tool_name, {})
+    server_options = None
+
+    # specific server per a system
+    if isinstance(tool_settings.get("server", {}), list):
+        for item in tool_settings['server'][:]:
+            tool_settings['server'] = item
+            if util.get_systype() in item.get("system", []):
+                break
+
+    # user overwrites debug server
+    if env_options.get("debug_server"):
+        server_options = {
+            "cwd": None,
+            "executable": None,
+            "arguments": env_options.get("debug_server")
+        }
+        server_options['executable'] = server_options['arguments'][0]
+        server_options['arguments'] = server_options['arguments'][1:]
+    elif "server" in tool_settings:
+        server_package = tool_settings['server'].get("package")
+        server_package_dir = platform.get_package_dir(
+            server_package) if server_package else None
+        if server_package and not server_package_dir:
+            platform.install_packages(with_packages=[server_package],
+                                      skip_default_package=True,
+                                      silent=True)
+            server_package_dir = platform.get_package_dir(server_package)
+        server_options = dict(
+            cwd=server_package_dir if server_package else None,
+            executable=tool_settings['server'].get("executable"),
+            arguments=[
+                a.replace("$PACKAGE_DIR", server_package_dir)
+                if server_package_dir else a
+                for a in tool_settings['server'].get("arguments", [])
+            ])
+
+    extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds"))
+    extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds")))
+    result = dict(
+        tool=tool_name,
+        upload_protocol=env_options.get(
+            "upload_protocol",
+            board_config.get("upload", {}).get("protocol")),
+        load_cmds=_cleanup_cmds(
+            env_options.get(
+                "debug_load_cmds",
+                tool_settings.get("load_cmds",
+                                  tool_settings.get("load_cmd", "load")))),
+        load_mode=env_options.get("debug_load_mode",
+                                  tool_settings.get("load_mode", "always")),
+        init_break=env_options.get(
+            "debug_init_break", tool_settings.get("init_break",
+                                                  "tbreak main")),
+        init_cmds=_cleanup_cmds(
+            env_options.get("debug_init_cmds",
+                            tool_settings.get("init_cmds"))),
+        extra_cmds=extra_cmds,
+        require_debug_port=tool_settings.get("require_debug_port", False),
+        port=reveal_debug_port(
+            env_options.get("debug_port", tool_settings.get("port")),
+            tool_name, tool_settings),
+        server=server_options)
+    return result
+
+
+def configure_esp32_load_cmds(debug_options, configuration):
+    ignore_conds = [
+        debug_options['load_cmds'] != ["load"],
+        "xtensa-esp32" not in configuration.get("cc_path", ""),
+        not configuration.get("flash_extra_images"), not all([
+            isfile(item['path'])
+            for item in configuration.get("flash_extra_images")
+        ])
+    ]
+    if any(ignore_conds):
+        return debug_options['load_cmds']
+
+    mon_cmds = [
+        'monitor program_esp32 "{{{path}}}" {offset} verify'.format(
+            path=fs.to_unix_path(item['path']), offset=item['offset'])
+        for item in configuration.get("flash_extra_images")
+    ]
+    mon_cmds.append('monitor program_esp32 "{%s.bin}" 0x10000 verify' %
+                    fs.to_unix_path(configuration['prog_path'][:-4]))
+    return mon_cmds
+
+
+def has_debug_symbols(prog_path):
+    if not isfile(prog_path):
+        return False
+    matched = {
+        b".debug_info": False,
+        b".debug_abbrev": False,
+        b" -Og": False,
+        b" -g": False,
+        b"__PLATFORMIO_BUILD_DEBUG__": False
+    }
+    with open(prog_path, "rb") as fp:
+        last_data = b""
+        while True:
+            data = fp.read(1024)
+            if not data:
+                break
+            for pattern, found in matched.items():
+                if found:
+                    continue
+                if pattern in last_data + data:
+                    matched[pattern] = True
+            last_data = data
+    return all(matched.values())
+
+
+def is_prog_obsolete(prog_path):
+    prog_hash_path = prog_path + ".sha1"
+    if not isfile(prog_path):
+        return True
+    shasum = sha1()
+    with open(prog_path, "rb") as fp:
+        while True:
+            data = fp.read(1024)
+            if not data:
+                break
+            shasum.update(data)
+    new_digest = shasum.hexdigest()
+    old_digest = None
+    if isfile(prog_hash_path):
+        with open(prog_hash_path, "r") as fp:
+            old_digest = fp.read()
+    if new_digest == old_digest:
+        return False
+    with open(prog_hash_path, "w") as fp:
+        fp.write(new_digest)
+    return True
+
+
+def reveal_debug_port(env_debug_port, tool_name, tool_settings):
+
+    def _get_pattern():
+        if not env_debug_port:
+            return None
+        if set(["*", "?", "[", "]"]) & set(env_debug_port):
+            return env_debug_port
+        return None
+
+    def _is_match_pattern(port):
+        pattern = _get_pattern()
+        if not pattern:
+            return True
+        return fnmatch(port, pattern)
+
+    def _look_for_serial_port(hwids):
+        for item in util.get_serialports(filter_hwid=True):
+            if not _is_match_pattern(item['port']):
+                continue
+            port = item['port']
+            if tool_name.startswith("blackmagic"):
+                if "windows" in util.get_systype() and \
+                        port.startswith("COM") and len(port) > 4:
+                    port = "\\\\.\\%s" % port
+                if "GDB" in item['description']:
+                    return port
+            for hwid in hwids:
+                hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
+                if hwid_str in item['hwid']:
+                    return port
+        return None
+
+    if env_debug_port and not _get_pattern():
+        return env_debug_port
+    if not tool_settings.get("require_debug_port"):
+        return None
+
+    debug_port = _look_for_serial_port(tool_settings.get("hwids", []))
+    if not debug_port:
+        raise exception.DebugInvalidOptions(
+            "Please specify `debug_port` for environment")
+    return debug_port

platformio/commands/debug/initcfgs.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GDB_DEFAULT_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor reset halt
+end
+
+define pio_reset_target
+    monitor reset
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+pio_reset_halt_target
+$LOAD_CMDS
+monitor init
+pio_reset_halt_target
+"""
+
+GDB_STUTIL_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor halt
+    monitor reset
+end
+
+define pio_reset_target
+    monitor reset
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+pio_reset_halt_target
+$LOAD_CMDS
+pio_reset_halt_target
+"""
+
+GDB_JLINK_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor halt
+    monitor reset
+end
+
+define pio_reset_target
+    monitor reset
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+pio_reset_halt_target
+$LOAD_CMDS
+"""
+
+GDB_BLACKMAGIC_INIT_CONFIG = """
+define pio_reset_halt_target
+    set language c
+    set *0xE000ED0C = 0x05FA0004
+    set $busy = (*0xE000ED0C & 0x4)
+    while ($busy)
+        set $busy = (*0xE000ED0C & 0x4)
+    end
+    set language auto
+end
+
+define pio_reset_target
+    pio_reset_halt_target
+end
+
+target extended-remote $DEBUG_PORT
+monitor swdp_scan
+attach 1
+set mem inaccessible-by-default off
+$INIT_BREAK
+$LOAD_CMDS
+
+set language c
+set *0xE000ED0C = 0x05FA0004
+set $busy = (*0xE000ED0C & 0x4)
+while ($busy)
+    set $busy = (*0xE000ED0C & 0x4)
+end
+set language auto
+"""
+
+GDB_MSPDEBUG_INIT_CONFIG = """
+define pio_reset_halt_target
+end
+
+define pio_reset_target
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+monitor erase
+$LOAD_CMDS
+pio_reset_halt_target
+"""
+
+GDB_QEMU_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor system_reset
+end
+
+define pio_reset_target
+    pio_reset_halt_target
+end
+
+target extended-remote $DEBUG_PORT
+$INIT_BREAK
+$LOAD_CMDS
+pio_reset_halt_target
+"""

platformio/commands/debug/process.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import signal
+
+import click
+from twisted.internet import protocol  # pylint: disable=import-error
+
+from platformio import fs
+from platformio.compat import string_types
+from platformio.proc import get_pythonexe_path
+from platformio.project.helpers import get_project_core_dir
+
+LOG_FILE = None
+
+
+class BaseProcess(protocol.ProcessProtocol, object):
+
+    STDOUT_CHUNK_SIZE = 2048
+
+    COMMON_PATTERNS = {
+        "PLATFORMIO_HOME_DIR": get_project_core_dir(),
+        "PLATFORMIO_CORE_DIR": get_project_core_dir(),
+        "PYTHONEXE": get_pythonexe_path()
+    }
+
+    def apply_patterns(self, source, patterns=None):
+        _patterns = self.COMMON_PATTERNS.copy()
+        _patterns.update(patterns or {})
+
+        for key, value in _patterns.items():
+            if key.endswith(("_DIR", "_PATH")):
+                _patterns[key] = fs.to_unix_path(value)
+
+        def _replace(text):
+            for key, value in _patterns.items():
+                pattern = "$%s" % key
+                text = text.replace(pattern, value or "")
+            return text
+
+        if isinstance(source, string_types):
+            source = _replace(source)
+        elif isinstance(source, (list, dict)):
+            items = enumerate(source) if isinstance(source,
+                                                    list) else source.items()
+            for key, value in items:
+                if isinstance(value, string_types):
+                    source[key] = _replace(value)
+                elif isinstance(value, (list, dict)):
+                    source[key] = self.apply_patterns(value, patterns)
+
+        return source
+
+    def outReceived(self, data):
+        if LOG_FILE:
+            with open(LOG_FILE, "ab") as fp:
+                fp.write(data)
+        while data:
+            chunk = data[:self.STDOUT_CHUNK_SIZE]
+            click.echo(chunk, nl=False)
+            data = data[self.STDOUT_CHUNK_SIZE:]
+
+    @staticmethod
+    def errReceived(data):
+        if LOG_FILE:
+            with open(LOG_FILE, "ab") as fp:
+                fp.write(data)
+        click.echo(data, nl=False, err=True)
+
+    @staticmethod
+    def processEnded(_):
+        # Allow terminating via SIGINT/CTRL+C
+        signal.signal(signal.SIGINT, signal.default_int_handler)

platformio/commands/debug/server.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from os.path import isdir, isfile, join
+
+from twisted.internet import error  # pylint: disable=import-error
+from twisted.internet import reactor  # pylint: disable=import-error
+
+from platformio import exception, fs, util
+from platformio.commands.debug.process import BaseProcess
+from platformio.proc import where_is_program
+
+
+class DebugServer(BaseProcess):
+
+    def __init__(self, debug_options, env_options):
+        self.debug_options = debug_options
+        self.env_options = env_options
+
+        self._debug_port = None
+        self._transport = None
+        self._process_ended = False
+
+    def spawn(self, patterns):  # pylint: disable=too-many-branches
+        systype = util.get_systype()
+        server = self.debug_options.get("server")
+        if not server:
+            return None
+        server = self.apply_patterns(server, patterns)
+        server_executable = server['executable']
+        if not server_executable:
+            return None
+        if server['cwd']:
+            server_executable = join(server['cwd'], server_executable)
+        if ("windows" in systype and not server_executable.endswith(".exe")
+                and isfile(server_executable + ".exe")):
+            server_executable = server_executable + ".exe"
+
+        if not isfile(server_executable):
+            server_executable = where_is_program(server_executable)
+        if not isfile(server_executable):
+            raise exception.DebugInvalidOptions(
+                "\nCould not launch Debug Server '%s'. Please check that it "
+                "is installed and is included in a system PATH\n\n"
+                "See documentation or contact contact@platformio.org:\n"
+                "http://docs.platformio.org/page/plus/debugging.html\n" %
+                server_executable)
+
+        self._debug_port = ":3333"
+        openocd_pipe_allowed = all([
+            not self.debug_options['port'],
+            "openocd" in server_executable
+        ])  # yapf: disable
+        if openocd_pipe_allowed:
+            args = []
+            if server['cwd']:
+                args.extend(["-s", server['cwd']])
+            args.extend([
+                "-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"
+            ])
+            args.extend(server['arguments'])
+            str_args = " ".join(
+                [arg if arg.startswith("-") else '"%s"' % arg for arg in args])
+            self._debug_port = '| "%s" %s' % (server_executable, str_args)
+            self._debug_port = fs.to_unix_path(self._debug_port)
+        else:
+            env = os.environ.copy()
+            # prepend server "lib" folder to LD path
+            if ("windows" not in systype and server['cwd']
+                    and isdir(join(server['cwd'], "lib"))):
+                ld_key = ("DYLD_LIBRARY_PATH"
+                          if "darwin" in systype else "LD_LIBRARY_PATH")
+                env[ld_key] = join(server['cwd'], "lib")
+                if os.environ.get(ld_key):
+                    env[ld_key] = "%s:%s" % (env[ld_key],
+                                             os.environ.get(ld_key))
+            # prepend BIN to PATH
+            if server['cwd'] and isdir(join(server['cwd'], "bin")):
+                env['PATH'] = "%s%s%s" % (
+                    join(server['cwd'], "bin"), os.pathsep,
+                    os.environ.get("PATH", os.environ.get("Path", "")))
+
+            self._transport = reactor.spawnProcess(
+                self,
+                server_executable, [server_executable] + server['arguments'],
+                path=server['cwd'],
+                env=env)
+            if "mspdebug" in server_executable.lower():
+                self._debug_port = ":2000"
+            elif "jlink" in server_executable.lower():
+                self._debug_port = ":2331"
+            elif "qemu" in server_executable.lower():
+                self._debug_port = ":1234"
+
+        return self._transport
+
+    def get_debug_port(self):
+        return self._debug_port
+
+    def processEnded(self, reason):
+        self._process_ended = True
+        super(DebugServer, self).processEnded(reason)
+
+    def terminate(self):
+        if self._process_ended or not self._transport:
+            return
+        try:
+            self._transport.signalProcess("KILL")
+        except (OSError, error.ProcessExitedAlready):
+            pass

platformio/commands/device.py

@@ -0,0 +1,221 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from fnmatch import fnmatch
+from os import getcwd
+from os.path import join
+
+import click
+from serial.tools import miniterm
+
+from platformio import exception, util
+from platformio.compat import dump_json_to_unicode
+from platformio.project.config import ProjectConfig
+
+
+@click.group(short_help="Monitor device or list existing")
+def cli():
+    pass
+
+
+@cli.command("list", short_help="List devices")
+@click.option("--serial", is_flag=True, help="List serial ports, default")
+@click.option("--logical", is_flag=True, help="List logical devices")
+@click.option("--mdns", is_flag=True, help="List multicast DNS services")
+@click.option("--json-output", is_flag=True)
+def device_list(  # pylint: disable=too-many-branches
+        serial, logical, mdns, json_output):
+    if not logical and not mdns:
+        serial = True
+    data = {}
+    if serial:
+        data['serial'] = util.get_serial_ports()
+    if logical:
+        data['logical'] = util.get_logical_devices()
+    if mdns:
+        data['mdns'] = util.get_mdns_services()
+
+    single_key = list(data)[0] if len(list(data)) == 1 else None
+
+    if json_output:
+        return click.echo(
+            dump_json_to_unicode(data[single_key] if single_key else data))
+
+    titles = {
+        "serial": "Serial Ports",
+        "logical": "Logical Devices",
+        "mdns": "Multicast DNS Services"
+    }
+
+    for key, value in data.items():
+        if not single_key:
+            click.secho(titles[key], bold=True)
+            click.echo("=" * len(titles[key]))
+
+        if key == "serial":
+            for item in value:
+                click.secho(item['port'], fg="cyan")
+                click.echo("-" * len(item['port']))
+                click.echo("Hardware ID: %s" % item['hwid'])
+                click.echo("Description: %s" % item['description'])
+                click.echo("")
+
+        if key == "logical":
+            for item in value:
+                click.secho(item['path'], fg="cyan")
+                click.echo("-" * len(item['path']))
+                click.echo("Name: %s" % item['name'])
+                click.echo("")
+
+        if key == "mdns":
+            for item in value:
+                click.secho(item['name'], fg="cyan")
+                click.echo("-" * len(item['name']))
+                click.echo("Type: %s" % item['type'])
+                click.echo("IP: %s" % item['ip'])
+                click.echo("Port: %s" % item['port'])
+                if item['properties']:
+                    click.echo("Properties: %s" % ("; ".join([
+                        "%s=%s" % (k, v)
+                        for k, v in item['properties'].items()
+                    ])))
+                click.echo("")
+
+        if single_key:
+            click.echo("")
+
+    return True
+
+
+@cli.command("monitor", short_help="Monitor device (Serial)")
+@click.option("--port", "-p", help="Port, a number or a device name")
+@click.option("--baud", "-b", type=int, help="Set baud rate, default=9600")
+@click.option("--parity",
+              default="N",
+              type=click.Choice(["N", "E", "O", "S", "M"]),
+              help="Set parity, default=N")
+@click.option("--rtscts",
+              is_flag=True,
+              help="Enable RTS/CTS flow control, default=Off")
+@click.option("--xonxoff",
+              is_flag=True,
+              help="Enable software flow control, default=Off")
+@click.option("--rts",
+              default=None,
+              type=click.IntRange(0, 1),
+              help="Set initial RTS line state")
+@click.option("--dtr",
+              default=None,
+              type=click.IntRange(0, 1),
+              help="Set initial DTR line state")
+@click.option("--echo", is_flag=True, help="Enable local echo, default=Off")
+@click.option("--encoding",
+              default="UTF-8",
+              help="Set the encoding for the serial port (e.g. hexlify, "
+              "Latin1, UTF-8), default: UTF-8")
+@click.option("--filter", "-f", multiple=True, help="Add text transformation")
+@click.option("--eol",
+              default="CRLF",
+              type=click.Choice(["CR", "LF", "CRLF"]),
+              help="End of line mode, default=CRLF")
+@click.option("--raw",
+              is_flag=True,
+              help="Do not apply any encodings/transformations")
+@click.option("--exit-char",
+              type=int,
+              default=3,
+              help="ASCII code of special character that is used to exit "
+              "the application, default=3 (Ctrl+C)")
+@click.option("--menu-char",
+              type=int,
+              default=20,
+              help="ASCII code of special character that is used to "
+              "control miniterm (menu), default=20 (DEC)")
+@click.option("--quiet",
+              is_flag=True,
+              help="Diagnostics: suppress non-error messages, default=Off")
+@click.option("-d",
+              "--project-dir",
+              default=getcwd,
+              type=click.Path(exists=True,
+                              file_okay=False,
+                              dir_okay=True,
+                              resolve_path=True))
+@click.option(
+    "-e",
+    "--environment",
+    help="Load configuration from `platformio.ini` and specified environment")
+def device_monitor(**kwargs):  # pylint: disable=too-many-branches
+    env_options = {}
+    try:
+        env_options = get_project_options(kwargs['project_dir'],
+                                          kwargs['environment'])
+        for k in ("port", "speed", "rts", "dtr"):
+            k2 = "monitor_%s" % k
+            if k == "speed":
+                k = "baud"
+            if kwargs[k] is None and k2 in env_options:
+                kwargs[k] = env_options[k2]
+                if k != "port":
+                    kwargs[k] = int(kwargs[k])
+    except exception.NotPlatformIOProject:
+        pass
+
+    if not kwargs['port']:
+        ports = util.get_serial_ports(filter_hwid=True)
+        if len(ports) == 1:
+            kwargs['port'] = ports[0]['port']
+
+    sys.argv = ["monitor"] + env_options.get("monitor_flags", [])
+    for k, v in kwargs.items():
+        if k in ("port", "baud", "rts", "dtr", "environment", "project_dir"):
+            continue
+        k = "--" + k.replace("_", "-")
+        if k in env_options.get("monitor_flags", []):
+            continue
+        if isinstance(v, bool):
+            if v:
+                sys.argv.append(k)
+        elif isinstance(v, tuple):
+            for i in v:
+                sys.argv.extend([k, i])
+        else:
+            sys.argv.extend([k, str(v)])
+
+    if kwargs['port'] and (set(["*", "?", "[", "]"]) & set(kwargs['port'])):
+        for item in util.get_serial_ports():
+            if fnmatch(item['port'], kwargs['port']):
+                kwargs['port'] = item['port']
+                break
+
+    try:
+        miniterm.main(default_port=kwargs['port'],
+                      default_baudrate=kwargs['baud'] or 9600,
+                      default_rts=kwargs['rts'],
+                      default_dtr=kwargs['dtr'])
+    except Exception as e:
+        raise exception.MinitermException(e)
+
+
+def get_project_options(project_dir, environment=None):
+    config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
+    config.validate(envs=[environment] if environment else None)
+    if not environment:
+        default_envs = config.default_envs()
+        if default_envs:
+            environment = default_envs[0]
+        else:
+            environment = config.envs()[0]
+    return config.items(env=environment, as_dict=True)

platformio/commands/home/__init__.py

@@ -11,3 +11,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from platformio.commands.home.command import cli

platformio/commands/home/command.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mimetypes
+import socket
+from os.path import isdir
+
+import click
+
+from platformio import exception
+from platformio.managers.core import (get_core_package_dir,
+                                      inject_contrib_pysite)
+
+
+@click.command("home", short_help="PIO Home")
+@click.option("--port", type=int, default=8008, help="HTTP port, default=8008")
+@click.option(
+    "--host",
+    default="127.0.0.1",
+    help="HTTP host, default=127.0.0.1. "
+    "You can open PIO Home for inbound connections with --host=0.0.0.0")
+@click.option("--no-open", is_flag=True)  # pylint: disable=too-many-locals
+def cli(port, host, no_open):
+    # import contrib modules
+    inject_contrib_pysite()
+    # pylint: disable=import-error
+    from autobahn.twisted.resource import WebSocketResource
+    from twisted.internet import reactor
+    from twisted.web import server
+    # pylint: enable=import-error
+    from platformio.commands.home.rpc.handlers.app import AppRPC
+    from platformio.commands.home.rpc.handlers.ide import IDERPC
+    from platformio.commands.home.rpc.handlers.misc import MiscRPC
+    from platformio.commands.home.rpc.handlers.os import OSRPC
+    from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
+    from platformio.commands.home.rpc.handlers.project import ProjectRPC
+    from platformio.commands.home.rpc.server import JSONRPCServerFactory
+    from platformio.commands.home.web import WebRoot
+
+    factory = JSONRPCServerFactory()
+    factory.addHandler(AppRPC(), namespace="app")
+    factory.addHandler(IDERPC(), namespace="ide")
+    factory.addHandler(MiscRPC(), namespace="misc")
+    factory.addHandler(OSRPC(), namespace="os")
+    factory.addHandler(PIOCoreRPC(), namespace="core")
+    factory.addHandler(ProjectRPC(), namespace="project")
+
+    contrib_dir = get_core_package_dir("contrib-piohome")
+    if not isdir(contrib_dir):
+        raise exception.PlatformioException("Invalid path to PIO Home Contrib")
+
+    # Ensure PIO Home mimetypes are known
+    mimetypes.add_type("text/html", ".html")
+    mimetypes.add_type("text/css", ".css")
+    mimetypes.add_type("application/javascript", ".js")
+
+    root = WebRoot(contrib_dir)
+    root.putChild(b"wsrpc", WebSocketResource(factory))
+    site = server.Site(root)
+
+    # hook for `platformio-node-helpers`
+    if host == "__do_not_start__":
+        return
+
+    # if already started
+    already_started = False
+    socket.setdefaulttimeout(1)
+    try:
+        socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
+        already_started = True
+    except:  # pylint: disable=bare-except
+        pass
+
+    home_url = "http://%s:%d" % (host, port)
+    if not no_open:
+        if already_started:
+            click.launch(home_url)
+        else:
+            reactor.callLater(1, lambda: click.launch(home_url))
+
+    click.echo("\n".join([
+        "",
+        "  ___I_",
+        " /\\-_--\\   PlatformIO Home",
+        "/  \\_-__\\",
+        "|[]| [] |  %s" % home_url,
+        "|__|____|______________%s" % ("_" * len(host)),
+    ]))
+    click.echo("")
+    click.echo("Open PIO Home in your browser by this URL => %s" % home_url)
+
+    if already_started:
+        return
+
+    click.echo("PIO Home has been started. Press Ctrl+C to shutdown.")
+
+    reactor.listenTCP(port, site, interface=host)
+    reactor.run()

platformio/commands/home/helpers.py

@@ -0,0 +1,71 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=keyword-arg-before-vararg, arguments-differ
+
+import os
+import socket
+
+import requests
+from twisted.internet import defer  # pylint: disable=import-error
+from twisted.internet import reactor  # pylint: disable=import-error
+from twisted.internet import threads  # pylint: disable=import-error
+
+from platformio import util
+from platformio.proc import where_is_program
+
+
+class AsyncSession(requests.Session):
+
+    def __init__(self, n=None, *args, **kwargs):
+        if n:
+            pool = reactor.getThreadPool()
+            pool.adjustPoolsize(0, n)
+
+        super(AsyncSession, self).__init__(*args, **kwargs)
+
+    def request(self, *args, **kwargs):
+        func = super(AsyncSession, self).request
+        return threads.deferToThread(func, *args, **kwargs)
+
+    def wrap(self, *args, **kwargs):  # pylint: disable=no-self-use
+        return defer.ensureDeferred(*args, **kwargs)
+
+
+@util.memoized(expire="60s")
+def requests_session():
+    return AsyncSession(n=5)
+
+
+@util.memoized(expire="60s")
+def get_core_fullpath():
+    return where_is_program(
+        "platformio" + (".exe" if "windows" in util.get_systype() else ""))
+
+
+@util.memoized(expire="10s")
+def is_twitter_blocked():
+    ip = "104.244.42.1"
+    timeout = 2
+    try:
+        if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")):
+            requests.get("http://%s" % ip,
+                         allow_redirects=False,
+                         timeout=timeout)
+        else:
+            socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80))
+        return False
+    except:  # pylint: disable=bare-except
+        pass
+    return True

platformio/commands/home/rpc/handlers/app.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from os.path import expanduser, join
+
+from platformio import __version__, app, util
+from platformio.project.helpers import (get_project_core_dir,
+                                        is_platformio_project)
+
+
+class AppRPC(object):
+
+    APPSTATE_PATH = join(get_project_core_dir(), "homestate.json")
+
+    IGNORE_STORAGE_KEYS = [
+        "cid", "coreVersion", "coreSystype", "coreCaller", "coreSettings",
+        "homeDir", "projectsDir"
+    ]
+
+    @staticmethod
+    def load_state():
+        with app.State(AppRPC.APPSTATE_PATH, lock=True) as state:
+            storage = state.get("storage", {})
+
+            # base data
+            caller_id = app.get_session_var("caller_id")
+            storage['cid'] = app.get_cid()
+            storage['coreVersion'] = __version__
+            storage['coreSystype'] = util.get_systype()
+            storage['coreCaller'] = (str(caller_id).lower()
+                                     if caller_id else None)
+            storage['coreSettings'] = {
+                name: {
+                    "description": data['description'],
+                    "default_value": data['value'],
+                    "value": app.get_setting(name)
+                }
+                for name, data in app.DEFAULT_SETTINGS.items()
+            }
+
+            storage['homeDir'] = expanduser("~")
+            storage['projectsDir'] = storage['coreSettings']['projects_dir'][
+                'value']
+
+            # skip non-existing recent projects
+            storage['recentProjects'] = [
+                p for p in storage.get("recentProjects", [])
+                if is_platformio_project(p)
+            ]
+
+            state['storage'] = storage
+            state.modified = False  # skip saving extra fields
+            return state.as_dict()
+
+    @staticmethod
+    def get_state():
+        return AppRPC.load_state()
+
+    @staticmethod
+    def save_state(state):
+        with app.State(AppRPC.APPSTATE_PATH, lock=True) as s:
+            s.clear()
+            s.update(state)
+            storage = s.get("storage", {})
+            for k in AppRPC.IGNORE_STORAGE_KEYS:
+                if k in storage:
+                    del storage[k]
+        return True

platformio/commands/home/rpc/handlers/ide.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+import jsonrpc  # pylint: disable=import-error
+from twisted.internet import defer  # pylint: disable=import-error
+
+
+class IDERPC(object):
+
+    def __init__(self):
+        self._queue = {}
+
+    def send_command(self, command, params, sid=0):
+        if not self._queue.get(sid):
+            raise jsonrpc.exceptions.JSONRPCDispatchException(
+                code=4005, message="PIO Home IDE agent is not started")
+        while self._queue[sid]:
+            self._queue[sid].pop().callback({
+                "id": time.time(),
+                "method": command,
+                "params": params
+            })
+
+    def listen_commands(self, sid=0):
+        if sid not in self._queue:
+            self._queue[sid] = []
+        self._queue[sid].append(defer.Deferred())
+        return self._queue[sid][-1]
+
+    def open_project(self, project_dir, sid=0):
+        return self.send_command("open_project", project_dir, sid)

platformio/commands/home/rpc/handlers/misc.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import time
+
+from twisted.internet import defer, reactor  # pylint: disable=import-error
+
+from platformio import app
+from platformio.commands.home.rpc.handlers.os import OSRPC
+
+
+class MiscRPC(object):
+
+    def load_latest_tweets(self, username):
+        cache_key = "piohome_latest_tweets_" + str(username)
+        cache_valid = "7d"
+        with app.ContentCache() as cc:
+            cache_data = cc.get(cache_key)
+            if cache_data:
+                cache_data = json.loads(cache_data)
+                # automatically update cache in background every 12 hours
+                if cache_data['time'] < (time.time() - (3600 * 12)):
+                    reactor.callLater(5, self._preload_latest_tweets, username,
+                                      cache_key, cache_valid)
+                return cache_data['result']
+
+        result = self._preload_latest_tweets(username, cache_key, cache_valid)
+        return result
+
+    @staticmethod
+    @defer.inlineCallbacks
+    def _preload_latest_tweets(username, cache_key, cache_valid):
+        result = yield OSRPC.fetch_content(
+            "https://api.platformio.org/tweets/" + username)
+        result = json.loads(result)
+        with app.ContentCache() as cc:
+            cc.set(cache_key,
+                   json.dumps({
+                       "time": int(time.time()),
+                       "result": result
+                   }), cache_valid)
+        defer.returnValue(result)

platformio/commands/home/rpc/handlers/os.py

@@ -0,0 +1,152 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import codecs
+import glob
+import os
+import shutil
+from functools import cmp_to_key
+from os.path import expanduser, isdir, isfile, join
+
+import click
+from twisted.internet import defer  # pylint: disable=import-error
+
+from platformio import app, util
+from platformio.commands.home import helpers
+from platformio.compat import PY2, get_filesystem_encoding
+
+
+class OSRPC(object):
+
+    @staticmethod
+    @defer.inlineCallbacks
+    def fetch_content(uri, data=None, headers=None, cache_valid=None):
+        if not headers:
+            headers = {
+                "User-Agent":
+                ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) "
+                 "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 "
+                 "Safari/603.3.8")
+            }
+        cache_key = (app.ContentCache.key_from_args(uri, data)
+                     if cache_valid else None)
+        with app.ContentCache() as cc:
+            if cache_key:
+                result = cc.get(cache_key)
+                if result is not None:
+                    defer.returnValue(result)
+
+        # check internet before and resolve issue with 60 seconds timeout
+        util.internet_on(raise_exception=True)
+
+        session = helpers.requests_session()
+        if data:
+            r = yield session.post(uri, data=data, headers=headers)
+        else:
+            r = yield session.get(uri, headers=headers)
+
+        r.raise_for_status()
+        result = r.text
+        if cache_valid:
+            with app.ContentCache() as cc:
+                cc.set(cache_key, result, cache_valid)
+        defer.returnValue(result)
+
+    def request_content(self, uri, data=None, headers=None, cache_valid=None):
+        if uri.startswith('http'):
+            return self.fetch_content(uri, data, headers, cache_valid)
+        if not isfile(uri):
+            return None
+        with codecs.open(uri, encoding="utf-8") as fp:
+            return fp.read()
+
+    @staticmethod
+    def open_url(url):
+        return click.launch(url)
+
+    @staticmethod
+    def reveal_file(path):
+        return click.launch(
+            path.encode(get_filesystem_encoding()) if PY2 else path,
+            locate=True)
+
+    @staticmethod
+    def is_file(path):
+        return isfile(path)
+
+    @staticmethod
+    def is_dir(path):
+        return isdir(path)
+
+    @staticmethod
+    def make_dirs(path):
+        return os.makedirs(path)
+
+    @staticmethod
+    def rename(src, dst):
+        return os.rename(src, dst)
+
+    @staticmethod
+    def copy(src, dst):
+        return shutil.copytree(src, dst)
+
+    @staticmethod
+    def glob(pathnames, root=None):
+        if not isinstance(pathnames, list):
+            pathnames = [pathnames]
+        result = set()
+        for pathname in pathnames:
+            result |= set(
+                glob.glob(join(root, pathname) if root else pathname))
+        return list(result)
+
+    @staticmethod
+    def list_dir(path):
+
+        def _cmp(x, y):
+            if x[1] and not y[1]:
+                return -1
+            if not x[1] and y[1]:
+                return 1
+            if x[0].lower() > y[0].lower():
+                return 1
+            if x[0].lower() < y[0].lower():
+                return -1
+            return 0
+
+        items = []
+        if path.startswith("~"):
+            path = expanduser(path)
+        if not isdir(path):
+            return items
+        for item in os.listdir(path):
+            try:
+                item_is_dir = isdir(join(path, item))
+                if item_is_dir:
+                    os.listdir(join(path, item))
+                items.append((item, item_is_dir))
+            except OSError:
+                pass
+        return sorted(items, key=cmp_to_key(_cmp))
+
+    @staticmethod
+    def get_logical_devices():
+        items = []
+        for item in util.get_logical_devices():
+            if item['name']:
+                item['name'] = item['name']
+            items.append(item)
+        return items

platformio/commands/home/rpc/handlers/piocore.py

@@ -0,0 +1,162 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import json
+import os
+import sys
+from io import BytesIO, StringIO
+
+import click
+import jsonrpc  # pylint: disable=import-error
+from twisted.internet import defer  # pylint: disable=import-error
+from twisted.internet import threads  # pylint: disable=import-error
+from twisted.internet import utils  # pylint: disable=import-error
+
+from platformio import __main__, __version__, fs
+from platformio.commands.home import helpers
+from platformio.compat import (PY2, get_filesystem_encoding, is_bytes,
+                               string_types)
+
+try:
+    from thread import get_ident as thread_get_ident
+except ImportError:
+    from threading import get_ident as thread_get_ident
+
+
+class MultiThreadingStdStream(object):
+
+    def __init__(self, parent_stream):
+        self._buffers = {thread_get_ident(): parent_stream}
+
+    def __getattr__(self, name):
+        thread_id = thread_get_ident()
+        self._ensure_thread_buffer(thread_id)
+        return getattr(self._buffers[thread_id], name)
+
+    def _ensure_thread_buffer(self, thread_id):
+        if thread_id not in self._buffers:
+            self._buffers[thread_id] = BytesIO() if PY2 else StringIO()
+
+    def write(self, value):
+        thread_id = thread_get_ident()
+        self._ensure_thread_buffer(thread_id)
+        return self._buffers[thread_id].write(
+            value.decode() if is_bytes(value) else value)
+
+    def get_value_and_reset(self):
+        result = ""
+        try:
+            result = self.getvalue()
+            self.truncate(0)
+            self.seek(0)
+        except AttributeError:
+            pass
+        return result
+
+
+class PIOCoreRPC(object):
+
+    @staticmethod
+    def version():
+        return __version__
+
+    @staticmethod
+    def setup_multithreading_std_streams():
+        if isinstance(sys.stdout, MultiThreadingStdStream):
+            return
+        PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout)
+        PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr)
+        sys.stdout = PIOCoreRPC.thread_stdout
+        sys.stderr = PIOCoreRPC.thread_stderr
+
+    @staticmethod
+    def call(args, options=None):
+        return defer.maybeDeferred(PIOCoreRPC._call_generator, args, options)
+
+    @staticmethod
+    @defer.inlineCallbacks
+    def _call_generator(args, options=None):
+        for i, arg in enumerate(args):
+            if isinstance(arg, string_types):
+                args[i] = arg.encode(get_filesystem_encoding()) if PY2 else arg
+            else:
+                args[i] = str(arg)
+
+        to_json = "--json-output" in args
+
+        try:
+            if args and args[0] in ("account", "remote"):
+                result = yield PIOCoreRPC._call_subprocess(args, options)
+                defer.returnValue(PIOCoreRPC._process_result(result, to_json))
+            else:
+                result = yield PIOCoreRPC._call_inline(args, options)
+                try:
+                    defer.returnValue(
+                        PIOCoreRPC._process_result(result, to_json))
+                except ValueError:
+                    # fall-back to subprocess method
+                    result = yield PIOCoreRPC._call_subprocess(args, options)
+                    defer.returnValue(
+                        PIOCoreRPC._process_result(result, to_json))
+        except Exception as e:  # pylint: disable=bare-except
+            raise jsonrpc.exceptions.JSONRPCDispatchException(
+                code=4003, message="PIO Core Call Error", data=str(e))
+
+    @staticmethod
+    def _call_inline(args, options):
+        PIOCoreRPC.setup_multithreading_std_streams()
+        cwd = (options or {}).get("cwd") or os.getcwd()
+
+        def _thread_task():
+            with fs.cd(cwd):
+                exit_code = __main__.main(["-c"] + args)
+            return (PIOCoreRPC.thread_stdout.get_value_and_reset(),
+                    PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code)
+
+        return threads.deferToThread(_thread_task)
+
+    @staticmethod
+    def _call_subprocess(args, options):
+        cwd = (options or {}).get("cwd") or os.getcwd()
+        return utils.getProcessOutputAndValue(
+            helpers.get_core_fullpath(),
+            args,
+            path=cwd,
+            env={k: v
+                 for k, v in os.environ.items() if "%" not in k})
+
+    @staticmethod
+    def _process_result(result, to_json=False):
+        out, err, code = result
+        text = ("%s\n\n%s" % (out, err)).strip()
+        if code != 0:
+            raise Exception(text)
+        if not to_json:
+            return text
+        try:
+            return json.loads(out)
+        except ValueError as e:
+            click.secho("%s => `%s`" % (e, out), fg="red", err=True)
+            # if PIO Core prints unhandled warnings
+            for line in out.split("\n"):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    return json.loads(line)
+                except ValueError:
+                    pass
+            raise e