Merge branch 'release/v5.2.5'

* Removing inconsistent dot at README list

* Removing inconsistent dot at HISTORY file

* Removing inconsistent dot at CONTRIBUTING file

.coveragerc

@@ -1,21 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[report]
-# Regexes for lines to exclude from consideration
-exclude_lines =
-    pragma: no cover
-    def __repr__
-    raise AssertionError
-    raise NotImplementedError

.github/workflows/docs.yml

@@ -103,7 +103,7 @@ jobs:
       - name: Deploy to Github Pages
         uses: peaceiris/actions-gh-pages@v3
         with:
-          personal_token: ${{ secrets.PERSONAL_TOKEN }}
+          personal_token: ${{ secrets.DEPLOY_GH_DOCS_TOKEN }}
           external_repository: ${{ env.DOCS_REPO }}
           publish_dir: ./${{ env.DOCS_DIR }}
           commit_message: Sync Docs

.readthedocs.yml

@@ -1,12 +0,0 @@
-# See https://docs.readthedocs.io/en/stable/config-file/index.html
-
-version: 2
-
-sphinx:
-  configuration: docs/conf.py
-
-formats:
-  - pdf
-
-submodules:
-  include: all

CONTRIBUTING.md

@@ -3,7 +3,7 @@ Contributing
 
 To get started, <a href="https://cla-assistant.io/platformio/platformio-core">sign the Contributor License Agreement</a>.
 
-1. Fork the repository on GitHub.
+1. Fork the repository on GitHub
 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git`
 3. Run `pip install tox`
 4. Go to the root of project where is located `tox.ini` and run `tox -e py37`
@@ -18,4 +18,4 @@ To get started, <a href="https://cla-assistant.io/platformio/platformio-core">si
 8. Run the tests `make test`
 9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html)
 10. Commit changes to your forked repository
-11. Submit a Pull Request on GitHub.
+11. Submit a Pull Request on GitHub

HISTORY.rst

@@ -8,20 +8,59 @@ PlatformIO Core 5
 
 **A professional collaborative platform for embedded development**
 
+5.2.5 (2022-02-10)
+~~~~~~~~~~~~~~~~~~
+
+- Improved support for private packages in `PlatformIO Registry <https://registry.platformio.org/>`__
+- Improved checking of available Internet connection for IPv6-only workstations (`pull #4151 <https://github.com/platformio/platformio-core/pull/4151>`_)
+- Better detecting of default PlatformIO project directory on Linux OS (`pull #4158 <https://github.com/platformio/platformio-core/pull/4158>`_)
+- Respect disabling debugging server from "platformio.ini" passing an empty value to the `debug_server <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-server>`__ option
+- Fixed a "module 'asyncio' has no attribute 'run'" error when launching PIO Home using Python 3.6 (`issue #4169 <https://github.com/platformio/platformio-core/issues/4169>`_)
+
+5.2.4 (2021-12-15)
+~~~~~~~~~~~~~~~~~~
+
+- Added support for a new ``headers`` field in `library.json <https://docs.platformio.org/en/latest/librarymanager/config.html>`__ (declare a list of header files that can be included in a project source files using ``#include <...>`` directive)
+- Improved tab completion support for Bash, ZSH, and Fish shells (`issue #4114 <https://github.com/platformio/platformio-core/issues/4114>`_)
+- Improved support for projects located on a network share (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_, `issue #3926 <https://github.com/platformio/platformio-core/issues/3926>`_, `issue #4099 <https://github.com/platformio/platformio-core/issues/4099>`_)
+- Improved PIO Remote setup on credit-card sized computers (Raspberry Pi, BeagleBon, etc) (`issue #3865 <https://github.com/platformio/platformio-core/issues/3865>`_)
+- Upgraded build engine to the SCons 4.3 (`release notes <https://github.com/SCons/scons/blob/rel_4.3.0/CHANGES.txt>`__)
+- Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 <https://github.com/platformio/platformio-core/issues/4102>`_)
+- Fixed an issue with the NetBeans project generator when the path to PlatformIO contains a space (`issue #4096 <https://github.com/platformio/platformio-core/issues/4096>`_)
+- Fixed an issue when the system environment variable does not override a project configuration option (`issue #4125 <https://github.com/platformio/platformio-core/issues/4125>`_)
+- Fixed an issue when referencing ``*_dir`` option from a custom project configuration environment (`issue #4110 <https://github.com/platformio/platformio-core/issues/4110>`_)
+- Fixed an issue with the CLion template that generated a broken CMake file if user's home directory contained an unescaped backslash (`issue #4071 <https://github.com/platformio/platformio-core/issues/4071>`_)
+- Fixed an issue with wrong detecting Windows architecture when Python 32bit is used (`issue #4134 <https://github.com/platformio/platformio-core/issues/4134>`_)
+
+5.2.3 (2021-11-05)
+~~~~~~~~~~~~~~~~~~
+
+- Automatically synchronize active projects between IDE and `PlatformIO Home <https://docs.platformio.org/en/latest/home/index.html>`__
+- Added support for custom `device monitor filters <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#filters>`__ (`issue #3924 <https://github.com/platformio/platformio-core/issues/3924>`_)
+- Show human-readable message when infinite recursion is detected while processing `Interpolation of Values <https://docs.platformio.org/en/latest/projectconf/interpolation.html>`__ (`issue #3883 <https://github.com/platformio/platformio-core/issues/3883>`_)
+- Improved directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ configuration file (`issue #3934 <https://github.com/platformio/platformio-core/issues/3934>`_)
+- Ignore resolving of SCons variables (e.g., ``${(SOURCE.get_abspath())}``) when preprocessing interpolations (`issue #3933 <https://github.com/platformio/platformio-core/issues/3933>`_)
+- Added "inc" as a sign that it's the root of the library (`issue #4093 <https://github.com/platformio/platformio-core/issues/4093>`_)
+- Fixed an issue when the ``$PROJECT_DIR`` variable was not properly replaced in the `debug_server <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-server>`__ option (`issue #4086 <https://github.com/platformio/platformio-core/issues/4086>`_)
+- Fixed an issue when `PIO Remote <https://docs.platformio.org/en/latest/plus/pio-remote.html>`__ device monitor crashes on the first keypress (`issue #3832 <https://github.com/platformio/platformio-core/issues/3832>`_)
+- Fixed "Do not know how to make File target 'debug'" issue when debugging project using `CLion IDE <https://docs.platformio.org/en/latest/integration/ide/clion.html>`__ (`pull #4089 <https://github.com/platformio/platformio-core/issues/4089>`_)
+- Fixed "UnicodeEncodeError" when a build output contains non-ASCII characters (`issue #3971 <https://github.com/platformio/platformio-core/issues/3971>`_)
+- Fixed an issue when VSCode's debugger does not the honor default environment (`issue #4098 <https://github.com/platformio/platformio-core/issues/4098>`_)
+
 5.2.2 (2021-10-20)
 ~~~~~~~~~~~~~~~~~~
 
 - Override debugging firmware loading mode using ``--load-mode`` option for `pio debug <https://docs.platformio.org/en/latest/core/userguide/cmd_debug.html>`__ command
 - Added support for CLion IDE 2021.3 (`pull #4085 <https://github.com/platformio/platformio-core/issues/4085>`_)
 - Removed debugging "legacy Click" message from CLI (`issue #4083 <https://github.com/platformio/platformio-core/issues/4083>`_)
-- Fixed a "TypeError: sequence item 1: expected str instance, list found" issue when extending configuration option in `"platformio.ini" <https://docs.platformio.org/page/projectconf.html>`__ with the multi-line default value (`issue #4082 <https://github.com/platformio/platformio-core/issues/4082>`_)
+- Fixed a "TypeError: sequence item 1: expected str instance, list found" issue when extending configuration option in `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ with the multi-line default value (`issue #4082 <https://github.com/platformio/platformio-core/issues/4082>`_)
 
 5.2.1 (2021-10-11)
 ~~~~~~~~~~~~~~~~~~
 
 - Clean a build environment and installed library dependencies using a new ``cleanall`` target (`issue #4062 <https://github.com/platformio/platformio-core/issues/4062>`_)
-- Override a default library builder via a new ``builder`` field in a ``build`` group of `library.json <https://docs.platformio.org/page/librarymanager/config.html#build>`__ manifest (`issue #3957 <https://github.com/platformio/platformio-core/issues/3957>`_)
-- Updated `Cppcheck <https://docs.platformio.org/page/plus/check-tools/cppcheck.html>`__ v2.6 with new checks, increased reliability of advanced addons (MISRA/CERT) and various improvements
+- Override a default library builder via a new ``builder`` field in a ``build`` group of `library.json <https://docs.platformio.org/en/latest/librarymanager/config.html#build>`__ manifest (`issue #3957 <https://github.com/platformio/platformio-core/issues/3957>`_)
+- Updated `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.6 with new checks, increased reliability of advanced addons (MISRA/CERT) and various improvements
 - Handle the "test" folder as a part of CLion project (`issue #4005 <https://github.com/platformio/platformio-core/issues/4005>`_)
 - Improved handling of a library root based on "Conan" or "CMake" build systems (`issue #3887 <https://github.com/platformio/platformio-core/issues/3887>`_)
 - Fixed a "KeyError: Invalid board option 'build.cpu'" when using a precompiled library with a board that does not have a CPU field in the manifest (`issue #4056 <https://github.com/platformio/platformio-core/issues/4056>`_)
@@ -33,11 +72,11 @@ PlatformIO Core 5
 
 * **PlatformIO Debugging**
 
-  - Boosted `PlatformIO Debugging <https://docs.platformio.org/page/plus/debugging.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
-  - `Debug unit tests <https://docs.platformio.org/page/plus/debugging.html#debug-unit-tests>`__ created with `PlatformIO Unit Testing <https://docs.platformio.org/page/plus/unit-testing.html>`__ solution  (`issue #948 <https://github.com/platformio/platformio-core/issues/948>`_)
+  - Boosted `PlatformIO Debugging <https://docs.platformio.org/en/latest/plus/debugging.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
+  - `Debug unit tests <https://docs.platformio.org/en/latest/plus/debugging.html#debug-unit-tests>`__ created with `PlatformIO Unit Testing <https://docs.platformio.org/en/latest/plus/unit-testing.html>`__ solution  (`issue #948 <https://github.com/platformio/platformio-core/issues/948>`_)
   - Debug native (desktop) applications on a host machine (`issue #980 <https://github.com/platformio/platformio-core/issues/980>`_)
-  - Support debugging on Windows using Windows CMD/CLI (`pio debug <https://docs.platformio.org/page/core/userguide/cmd_debug.html>`__) (`issue #3793 <https://github.com/platformio/platformio-core/issues/3793>`_)
-  - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern <https://docs.platformio.org/page/projectconf/section_env_debug.html#debug-server-ready-pattern>`__ option
+  - Support debugging on Windows using Windows CMD/CLI (`pio debug <https://docs.platformio.org/en/latest/core/userguide/cmd_debug.html>`__) (`issue #3793 <https://github.com/platformio/platformio-core/issues/3793>`_)
+  - Configure a custom pattern to determine when debugging server is started with a new `debug_server_ready_pattern <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-server-ready-pattern>`__ option
   - Fixed an issue with silent hanging when a custom debug server is not found (`issue #3756 <https://github.com/platformio/platformio-core/issues/3756>`_)
 
 * **Package Management**
@@ -49,7 +88,7 @@ PlatformIO Core 5
     * Check for duplicates and used version
     * Validate package manifest
 
-  - Added a new option ``--non-interactive`` to `pio package publish <https://docs.platformio.org/page/core/userguide/package/cmd_publish.html>`__ command
+  - Added a new option ``--non-interactive`` to `pio package publish <https://docs.platformio.org/en/latest/core/userguide/package/cmd_publish.html>`__ command
 
 * **Build System**
 
@@ -62,9 +101,9 @@ PlatformIO Core 5
 
   - Updated analysis tools:
 
-    * `Clang-Tidy <https://docs.platformio.org/page/plus/check-tools/clang-tidy.html>`__ v12.0.1 with new modules and extended checks list
-    * `Cppcheck <https://docs.platformio.org/page/plus/check-tools/cppcheck.html>`__ v2.5.0 with improved code analysis and MISRA improvements
-    * `PVS-Studio <https://docs.platformio.org/page/plus/check-tools/pvs-studio.html>`__ v7.14 with support for intermodular analysis, improved MISRA support and new diagnostics
+    * `Clang-Tidy <https://docs.platformio.org/en/latest/plus/check-tools/clang-tidy.html>`__ v12.0.1 with new modules and extended checks list
+    * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.5.0 with improved code analysis and MISRA improvements
+    * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.14 with support for intermodular analysis, improved MISRA support and new diagnostics
 
 * **Miscellaneous**
 
@@ -76,7 +115,7 @@ PlatformIO Core 5
 ~~~~~~~~~~~~~~~~~~
 
 * Fixed a "The command line is too long" issue with a linking process on Windows (`issue #3827 <https://github.com/platformio/platformio-core/issues/3827>`_)
-* Fixed an issue with `device monitor <https://docs.platformio.org/page/core/userguide/device/cmd_monitor.html>`__ when the "send_on_enter" filter didn't send EOL chars (`issue #3787 <https://github.com/platformio/platformio-core/issues/3787>`_)
+* Fixed an issue with `device monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html>`__ when the "send_on_enter" filter didn't send EOL chars (`issue #3787 <https://github.com/platformio/platformio-core/issues/3787>`_)
 * Fixed an issue with silent mode when unwanted data is printed to stdout (`issue #3837 <https://github.com/platformio/platformio-core/issues/3837>`_)
 * Fixed an issue when code inspection fails with "Bad JSON" (`issue #3790 <https://github.com/platformio/platformio-core/issues/3790>`_)
 * Fixed an issue with overriding user-specified debugging configuration information in VSCode (`issue #3824 <https://github.com/platformio/platformio-core/issues/3824>`_)
@@ -86,8 +125,8 @@ PlatformIO Core 5
 
 * **PlatformIO Home**
 
-  - Boosted `PlatformIO Home <https://docs.platformio.org/page/home/index.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
-  - Added a new ``--session-id`` option to `pio home <https://docs.platformio.org/page/core/userguide/cmd_home.html>`__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 <https://github.com/platformio/platformio-core/issues/3397>`_)
+  - Boosted `PlatformIO Home <https://docs.platformio.org/en/latest/home/index.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
+  - Added a new ``--session-id`` option to `pio home <https://docs.platformio.org/en/latest/core/userguide/cmd_home.html>`__ command that helps to keep PlatformIO Home isolated from other instances and protect from 3rd party access (`issue #3397 <https://github.com/platformio/platformio-core/issues/3397>`_)
 
 * **Build System**
 
@@ -97,28 +136,28 @@ PlatformIO Core 5
 
 * **Package Management**
 
-  - New options for `pio system prune <https://docs.platformio.org/page/core/userguide/system/cmd_prune.html>`__ command:
+  - New options for `pio system prune <https://docs.platformio.org/en/latest/core/userguide/system/cmd_prune.html>`__ command:
 
     + ``--dry-run`` option to show data that will be removed
     + ``--core-packages`` option to remove unnecessary core packages
     + ``--platform-packages`` option to remove unnecessary development platform packages (`issue #923 <https://github.com/platformio/platformio-core/issues/923>`_)
 
-  - Added new `check_prune_system_threshold <https://docs.platformio.org/page/core/userguide/cmd_settings.html#check-prune-system-threshold>`__ setting
+  - Added new `check_prune_system_threshold <https://docs.platformio.org/en/latest/core/userguide/cmd_settings.html#check-prune-system-threshold>`__ setting
   - Disabled automatic removal of unnecessary development platform packages (`issue #3708 <https://github.com/platformio/platformio-core/issues/3708>`_, `issue #3770 <https://github.com/platformio/platformio-core/issues/3770>`_)
   - Fixed an issue when unnecessary packages were removed in  ``update --dry-run`` mode (`issue #3809 <https://github.com/platformio/platformio-core/issues/3809>`_)
   - Fixed a "ValueError: Invalid simple block" when uninstalling a package with a custom name and external source (`issue #3816 <https://github.com/platformio/platformio-core/issues/3816>`_)
 
 * **Debugging**
 
-  - Configure a custom debug adapter speed using a new `debug_speed <https://docs.platformio.org/page/projectconf/section_env_debug.html#debug-speed>`__ option (`issue #3799 <https://github.com/platformio/platformio-core/issues/3799>`_)
+  - Configure a custom debug adapter speed using a new `debug_speed <https://docs.platformio.org/en/latest/projectconf/section_env_debug.html#debug-speed>`__ option (`issue #3799 <https://github.com/platformio/platformio-core/issues/3799>`_)
   - Handle debugging server's "ready_pattern" in "stderr" output
 
 * **Miscellaneous**
 
-  - Improved listing of `multicast DNS services <https://docs.platformio.org/page/core/userguide/device/cmd_list.html>`_
+  - Improved listing of `multicast DNS services <https://docs.platformio.org/en/latest/core/userguide/device/cmd_list.html>`_
   - Fixed a "UnicodeDecodeError: 'utf-8' codec can't decode byte" when using J-Link for firmware uploading on Linux (`issue #3804 <https://github.com/platformio/platformio-core/issues/3804>`_)
   - Fixed an issue with a compiler driver for ".ccls" language server (`issue #3808 <https://github.com/platformio/platformio-core/issues/3808>`_)
-  - Fixed an issue when `pio device monitor --eol <https://docs.platformio.org/page/core/userguide/device/cmd_monitor.html#cmdoption-pio-device-monitor-eol>`__ and "send_on_enter" filter do not work properly (`issue #3787 <https://github.com/platformio/platformio-core/issues/3787>`_)
+  - Fixed an issue when `pio device monitor --eol <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#cmdoption-pio-device-monitor-eol>`__ and "send_on_enter" filter do not work properly (`issue #3787 <https://github.com/platformio/platformio-core/issues/3787>`_)
 
 5.0.4 (2020-12-30)
 ~~~~~~~~~~~~~~~~~~
@@ -127,8 +166,8 @@ PlatformIO Core 5
 - Improved ".ccls" configuration file for Emacs, Vim, and Sublime Text integrations
 - Updated analysis tools:
 
-  * `Cppcheck <https://docs.platformio.org/page/plus/check-tools/cppcheck.html>`__ v2.3 with improved C++ parser and several new MISRA rules
-  * `PVS-Studio <https://docs.platformio.org/page/plus/check-tools/pvs-studio.html>`__ v7.11 with new diagnostics and updated mass suppression mechanism
+  * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.3 with improved C++ parser and several new MISRA rules
+  * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.11 with new diagnostics and updated mass suppression mechanism
 
 - Show a warning message about deprecated support for Python 2 and Python 3.5
 - Do not provide "intelliSenseMode" option when generating configuration for VSCode C/C++ extension
@@ -138,12 +177,12 @@ PlatformIO Core 5
 5.0.3 (2020-11-12)
 ~~~~~~~~~~~~~~~~~~
 
-- Added an error selector for `Sublime Text <https://docs.platformio.org/page/integration/ide/sublimetext.html>`__ build runner (`issue #3733 <https://github.com/platformio/platformio-core/issues/3733>`_)
+- Added an error selector for `Sublime Text <https://docs.platformio.org/en/latest/integration/ide/sublimetext.html>`__ build runner (`issue #3733 <https://github.com/platformio/platformio-core/issues/3733>`_)
 - Generate a working "projectEnvName" for PlatformIO IDE's debugger for VSCode
 - Force VSCode's intelliSenseMode to "gcc-x64" when GCC toolchain is used
 - Print ignored test suites and environments in the test summary report only in verbose mode (`issue #3726 <https://github.com/platformio/platformio-core/issues/3726>`_)
 - Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 <https://github.com/platformio/platformio-core/issues/3662>`_)
-- Fixed an issue when `pio package pack <https://docs.platformio.org/page/core/userguide/package/cmd_pack.html>`__ ignores some folders (`issue #3730 <https://github.com/platformio/platformio-core/issues/3730>`_)
+- Fixed an issue when `pio package pack <https://docs.platformio.org/en/latest/core/userguide/package/cmd_pack.html>`__ ignores some folders (`issue #3730 <https://github.com/platformio/platformio-core/issues/3730>`_)
 
 5.0.2 (2020-10-30)
 ~~~~~~~~~~~~~~~~~~
@@ -151,21 +190,21 @@ PlatformIO Core 5
 - Initialize a new project or update the existing passing working environment name and its options (`issue #3686 <https://github.com/platformio/platformio-core/issues/3686>`_)
 - Automatically build PlatformIO Core extra Python dependencies on a host machine if they are missed in the registry (`issue #3700 <https://github.com/platformio/platformio-core/issues/3700>`_)
 - Improved "core.call" RPC for PlatformIO Home (`issue #3671 <https://github.com/platformio/platformio-core/issues/3671>`_)
-- Fixed a "PermissionError: [WinError 5]" on Windows when an external repository is used with `lib_deps <https://docs.platformio.org/page/projectconf/section_env_library.html#lib-deps>`__ option (`issue #3664 <https://github.com/platformio/platformio-core/issues/3664>`_)
+- Fixed a "PermissionError: [WinError 5]" on Windows when an external repository is used with `lib_deps <https://docs.platformio.org/en/latest/projectconf/section_env_library.html#lib-deps>`__ option (`issue #3664 <https://github.com/platformio/platformio-core/issues/3664>`_)
 - Fixed a "KeyError: 'versions'" when dependency does not exist in the registry (`issue #3666 <https://github.com/platformio/platformio-core/issues/3666>`_)
 - Fixed an issue with GCC linker when "native" dev-platform is used in pair with library dependencies (`issue #3669 <https://github.com/platformio/platformio-core/issues/3669>`_)
 - Fixed an "AssertionError: ensure_dir_exists" when checking library updates from simultaneous subprocesses (`issue #3677 <https://github.com/platformio/platformio-core/issues/3677>`_)
-- Fixed an issue when `pio package publish <https://docs.platformio.org/page/core/userguide/package/cmd_publish.html>`__ command removes original archive after submitting to the registry (`issue #3716 <https://github.com/platformio/platformio-core/issues/3716>`_)
-- Fixed an issue when multiple `pio lib install <https://docs.platformio.org/page/core/userguide/lib/cmd_install.html>`__ command with the same local library results in duplicates in ``lib_deps`` (`issue #3715 <https://github.com/platformio/platformio-core/issues/3715>`_)
-- Fixed an issue with a "wrong" timestamp in device monitor output using `"time" filter <https://docs.platformio.org/page/core/userguide/device/cmd_monitor.html#filters>`__ (`issue #3712 <https://github.com/platformio/platformio-core/issues/3712>`_)
+- Fixed an issue when `pio package publish <https://docs.platformio.org/en/latest/core/userguide/package/cmd_publish.html>`__ command removes original archive after submitting to the registry (`issue #3716 <https://github.com/platformio/platformio-core/issues/3716>`_)
+- Fixed an issue when multiple `pio lib install <https://docs.platformio.org/en/latest/core/userguide/lib/cmd_install.html>`__ command with the same local library results in duplicates in ``lib_deps`` (`issue #3715 <https://github.com/platformio/platformio-core/issues/3715>`_)
+- Fixed an issue with a "wrong" timestamp in device monitor output using `"time" filter <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html#filters>`__ (`issue #3712 <https://github.com/platformio/platformio-core/issues/3712>`_)
 
 5.0.1 (2020-09-10)
 ~~~~~~~~~~~~~~~~~~
 
-- Added support for "owner" requirement when declaring ``dependencies`` using `library.json <https://docs.platformio.org/page/librarymanager/config.html#dependencies>`__
-- Fixed an issue when using a custom git/ssh package with `platform_packages <https://docs.platformio.org/page/projectconf/section_env_platform.html#platform-packages>`__ option (`issue #3624 <https://github.com/platformio/platformio-core/issues/3624>`_)
-- Fixed an issue with "ImportError: cannot import name '_get_backend' from 'cryptography.hazmat.backends'" when using `Remote Development <https://docs.platformio.org/page/plus/pio-remote.html>`__ on RaspberryPi device (`issue #3652 <https://github.com/platformio/platformio-core/issues/3652>`_)
-- Fixed an issue when `pio package unpublish <https://docs.platformio.org/page/core/userguide/package/cmd_unpublish.html>`__ command crashes (`issue #3660 <https://github.com/platformio/platformio-core/issues/3660>`_)
+- Added support for "owner" requirement when declaring ``dependencies`` using `library.json <https://docs.platformio.org/en/latest/librarymanager/config.html#dependencies>`__
+- Fixed an issue when using a custom git/ssh package with `platform_packages <https://docs.platformio.org/en/latest/projectconf/section_env_platform.html#platform-packages>`__ option (`issue #3624 <https://github.com/platformio/platformio-core/issues/3624>`_)
+- Fixed an issue with "ImportError: cannot import name '_get_backend' from 'cryptography.hazmat.backends'" when using `Remote Development <https://docs.platformio.org/en/latest/plus/pio-remote.html>`__ on RaspberryPi device (`issue #3652 <https://github.com/platformio/platformio-core/issues/3652>`_)
+- Fixed an issue when `pio package unpublish <https://docs.platformio.org/en/latest/core/userguide/package/cmd_unpublish.html>`__ command crashes (`issue #3660 <https://github.com/platformio/platformio-core/issues/3660>`_)
 - Fixed an issue when the package manager tries to install a built-in library from the registry (`issue #3662 <https://github.com/platformio/platformio-core/issues/3662>`_)
 - Fixed an issue with incorrect value for C++ language standard in IDE projects when an in-progress language standard is used (`issue #3653 <https://github.com/platformio/platformio-core/issues/3653>`_)
 - Fixed an issue with "Invalid simple block (semantic_version)" from library dependency that refs to an external source (repository, ZIP/Tar archives) (`issue #3658 <https://github.com/platformio/platformio-core/issues/3658>`_)
@@ -174,7 +213,7 @@ PlatformIO Core 5
 5.0.0 (2020-09-03)
 ~~~~~~~~~~~~~~~~~~
 
-Please check `Migration guide from 4.x to 5.0 <https://docs.platformio.org/page/core/migration.html>`__.
+Please check `Migration guide from 4.x to 5.0 <https://docs.platformio.org/en/latest/core/migration.html>`__.
 
 * Integration with the new **PlatformIO Trusted Registry**
 
@@ -189,19 +228,19 @@ Please check `Migration guide from 4.x to 5.0 <https://docs.platformio.org/page/
   - Built-in fine-grained access control (role-based, teams, organizations)
   - New CLI commands:
 
-    * `pio package <https://docs.platformio.org/page/core/userguide/package/index.html>`__ – manage packages in the registry
-    * `pio access <https://docs.platformio.org/page/core/userguide/access/index.html>`__ – manage package access for users, teams, and maintainers
+    * `pio package <https://docs.platformio.org/en/latest/core/userguide/package/index.html>`__ – manage packages in the registry
+    * `pio access <https://docs.platformio.org/en/latest/core/userguide/access/index.html>`__ – manage package access for users, teams, and maintainers
 
 * Integration with the new **Account Management System**
 
-  - `Manage organizations <https://docs.platformio.org/page/core/userguide/org/index.html>`__
-  - `Manage teams and team memberships <https://docs.platformio.org/page/core/userguide/team/index.html>`__
+  - `Manage organizations <https://docs.platformio.org/en/latest/core/userguide/org/index.html>`__
+  - `Manage teams and team memberships <https://docs.platformio.org/en/latest/core/userguide/team/index.html>`__
 
 * New **Package Management System**
 
   - Integrated PlatformIO Core with the new PlatformIO Registry
   - Support for owner-based dependency declaration (resolves name conflicts) (`issue #1824 <https://github.com/platformio/platformio-core/issues/1824>`_)
-  - Automatically save dependencies to `"platformio.ini" <https://docs.platformio.org/page/projectconf.html>`__ when installing using PlatformIO CLI (`issue #2964 <https://github.com/platformio/platformio-core/issues/2964>`_)
+  - Automatically save dependencies to `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__ when installing using PlatformIO CLI (`issue #2964 <https://github.com/platformio/platformio-core/issues/2964>`_)
   - Follow SemVer complaint version constraints when checking library updates `issue #1281 <https://github.com/platformio/platformio-core/issues/1281>`_)
   - Dropped support for "packageRepositories" section in "platform.json" manifest (please publish packages directly to the registry)
 
@@ -209,29 +248,29 @@ Please check `Migration guide from 4.x to 5.0 <https://docs.platformio.org/page/
 
   - Upgraded build engine to the `SCons 4.0 - a next-generation software construction tool <https://scons.org/>`__
 
-    * `Configuration files are Python scripts <https://docs.platformio.org/page/projectconf/advanced_scripting.html>`__ – use the power of a real programming language to solve build problems
+    * `Configuration files are Python scripts <https://docs.platformio.org/en/latest/projectconf/advanced_scripting.html>`__ – use the power of a real programming language to solve build problems
     * Built-in reliable and automatic dependency analysis
     * Improved support for parallel builds
-    * Ability to `share built files in a cache <https://docs.platformio.org/page/projectconf/section_platformio.html#projectconf-pio-build-cache-dir>`__ to speed up multiple builds
+    * Ability to `share built files in a cache <https://docs.platformio.org/en/latest/projectconf/section_platformio.html#projectconf-pio-build-cache-dir>`__ to speed up multiple builds
 
-  - New `Custom Targets <https://docs.platformio.org/page/projectconf/advanced_scripting.html#custom-targets>`__
+  - New `Custom Targets <https://docs.platformio.org/en/latest/projectconf/advanced_scripting.html#custom-targets>`__
 
     * Pre/Post processing based on dependent sources (another target, source file, etc.)
     * Command launcher with own arguments
-    * Launch command with custom options declared in `"platformio.ini" <https://docs.platformio.org/page/projectconf.html>`__
+    * Launch command with custom options declared in `"platformio.ini" <https://docs.platformio.org/en/latest/projectconf.html>`__
     * Python callback as a target (use the power of Python interpreter and PlatformIO Build API)
-    * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets <https://docs.platformio.org/page/core/userguide/cmd_run.html#cmdoption-platformio-run-list-targets>`__ command (`issue #3544 <https://github.com/platformio/platformio-core/issues/3544>`_)
+    * List available project targets (including dev-platform specific and custom targets) with a new `pio run --list-targets <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html#cmdoption-platformio-run-list-targets>`__ command (`issue #3544 <https://github.com/platformio/platformio-core/issues/3544>`_)
 
   - Enable "cyclic reference" for GCC linker only for the embedded dev-platforms (`issue #3570 <https://github.com/platformio/platformio-core/issues/3570>`_)
-  - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) <https://docs.platformio.org/page/librarymanager/ldf.html#dependency-finder-mode>`__ for Arduino library when "library.property" has "depends" field (`issue #3607 <https://github.com/platformio/platformio-core/issues/3607>`_)
+  - Automatically enable LDF dependency `chain+ mode (evaluates C/C++ Preprocessor conditional syntax) <https://docs.platformio.org/en/latest/librarymanager/ldf.html#dependency-finder-mode>`__ for Arduino library when "library.property" has "depends" field (`issue #3607 <https://github.com/platformio/platformio-core/issues/3607>`_)
   - Fixed an issue with improper processing of source files added via multiple Build Middlewares (`issue #3531 <https://github.com/platformio/platformio-core/issues/3531>`_)
   - Fixed an issue with the ``clean`` target on Windows when project and build directories are located on different logical drives (`issue #3542 <https://github.com/platformio/platformio-core/issues/3542>`_)
 
 * **Project Management**
 
-  - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci <https://docs.platformio.org/page/core/userguide/cmd_ci.html>`__, `src_filter <https://docs.platformio.org/page/projectconf/section_env_build.html#src-filter>`__, `check_patterns <https://docs.platformio.org/page/projectconf/section_env_check.html#check-patterns>`__, `library.json > srcFilter <https://docs.platformio.org/page/librarymanager/config.html#srcfilter>`__). Python 3.5+ is required
-  - Added a new ``-e, --environment`` option to `pio project init <https://docs.platformio.org/page/core/userguide/project/cmd_init.html#cmdoption-platformio-project-init-e>`__ command that helps to update a PlatformIO project using the existing environment
-  - Dump build system data intended for IDE extensions/plugins using a new `pio project data <https://docs.platformio.org/page/core/userguide/project/cmd_data.html>`__ command
+  - Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`pio ci <https://docs.platformio.org/en/latest/core/userguide/cmd_ci.html>`__, `src_filter <https://docs.platformio.org/en/latest/projectconf/section_env_build.html#src-filter>`__, `check_patterns <https://docs.platformio.org/en/latest/projectconf/section_env_check.html#check-patterns>`__, `library.json > srcFilter <https://docs.platformio.org/en/latest/librarymanager/config.html#srcfilter>`__). Python 3.5+ is required
+  - Added a new ``-e, --environment`` option to `pio project init <https://docs.platformio.org/en/latest/core/userguide/project/cmd_init.html#cmdoption-platformio-project-init-e>`__ command that helps to update a PlatformIO project using the existing environment
+  - Dump build system data intended for IDE extensions/plugins using a new `pio project data <https://docs.platformio.org/en/latest/core/userguide/project/cmd_data.html>`__ command
   - Do not generate ".travis.yml" for a new project, let the user have a choice
 
 * **Unit Testing**
@@ -244,19 +283,19 @@ Please check `Migration guide from 4.x to 5.0 <https://docs.platformio.org/page/
 
   - Updated analysis tools:
 
-    * `Cppcheck <https://docs.platformio.org/page/plus/check-tools/cppcheck.html>`__ v2.1 with a new "soundy" analysis option and improved code parser
-    * `PVS-Studio <https://docs.platformio.org/page/plus/check-tools/pvs-studio.html>`__ v7.09 with a new file list analysis mode and an extended list of analysis diagnostics
+    * `Cppcheck <https://docs.platformio.org/en/latest/plus/check-tools/cppcheck.html>`__ v2.1 with a new "soundy" analysis option and improved code parser
+    * `PVS-Studio <https://docs.platformio.org/en/latest/plus/check-tools/pvs-studio.html>`__ v7.09 with a new file list analysis mode and an extended list of analysis diagnostics
 
   - Added Cppcheck package for ARM-based single-board computers (`issue #3559 <https://github.com/platformio/platformio-core/issues/3559>`_)
   - Fixed an issue with PIO Check when a defect with a multiline error message is not reported in verbose mode (`issue #3631 <https://github.com/platformio/platformio-core/issues/3631>`_)
 
 * **Miscellaneous**
 
-  - Display system-wide information using a new `pio system info <https://docs.platformio.org/page/core/userguide/system/cmd_info.html>`__ command (`issue #3521 <https://github.com/platformio/platformio-core/issues/3521>`_)
-  - Remove unused data using a new `pio system prune <https://docs.platformio.org/page/core/userguide/system/cmd_prune.html>`__ command (`issue #3522 <https://github.com/platformio/platformio-core/issues/3522>`_)
+  - Display system-wide information using a new `pio system info <https://docs.platformio.org/en/latest/core/userguide/system/cmd_info.html>`__ command (`issue #3521 <https://github.com/platformio/platformio-core/issues/3521>`_)
+  - Remove unused data using a new `pio system prune <https://docs.platformio.org/en/latest/core/userguide/system/cmd_prune.html>`__ command (`issue #3522 <https://github.com/platformio/platformio-core/issues/3522>`_)
   - Show ignored project environments only in the verbose mode (`issue #3641 <https://github.com/platformio/platformio-core/issues/3641>`_)
   - Do not escape compiler arguments in VSCode template on Windows
-  - Drop support for Python 2 and 3.5.
+  - Drop support for Python 2 and 3.5
 
 .. _release_notes_4:
 

README.rst

@@ -1,5 +1,5 @@
-PlatformIO
-==========
+PlatformIO Core
+===============
 
 .. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg
     :target: https://docs.platformio.org/page/core/index.html
@@ -17,11 +17,12 @@ PlatformIO
     :target: https://pypi.python.org/pypi/platformio/
     :alt:  License
 .. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg
-   :alt: Community Labs
+   :alt: PlatformIO Labs
    :target: https://piolabs.com/?utm_source=github&utm_medium=core
 
-**Quick Links:** `Web <https://platformio.org?utm_source=github&utm_medium=core>`_ |
+**Quick Links:** `Homepage <https://platformio.org?utm_source=github&utm_medium=core>`_ |
 `PlatformIO IDE <https://platformio.org/platformio-ide?utm_source=github&utm_medium=core>`_ |
+`Registry <https://registry.platformio.org?utm_source=github&utm_medium=core>`_ |
 `Project Examples <https://github.com/platformio/platformio-examples/>`__ |
 `Docs <https://docs.platformio.org?utm_source=github&utm_medium=core>`_ |
 `Donate <https://platformio.org/donate?utm_source=github&utm_medium=core>`_ |
@@ -43,7 +44,7 @@ PlatformIO
 * Cross-platform IDE and Unified Debugger
 * Static Code Analyzer and Remote Unit Testing
 * Multi-platform and Multi-architecture Build System
-* Firmware File Explorer and Memory Inspection.
+* Firmware File Explorer and Memory Inspection
 
 Get Started
 -----------
@@ -70,66 +71,9 @@ Solutions
 Registry
 --------
 
-* `Libraries <https://platformio.org/lib?utm_source=github&utm_medium=core>`_
-* `Development Platforms <https://platformio.org/platforms?utm_source=github&utm_medium=core>`_
-* `Frameworks <https://platformio.org/frameworks?utm_source=github&utm_medium=core>`_
-* `Embedded Boards <https://platformio.org/boards?utm_source=github&utm_medium=core>`_
-
-Development Platforms
----------------------
-
-* `Aceinna IMU <https://platformio.org/platforms/aceinna_imu?utm_source=github&utm_medium=core>`_
-* `ASR Microelectronics ASR605x <https://platformio.org/platforms/asrmicro650x?utm_source=github&utm_medium=core>`_
-* `Atmel AVR <https://platformio.org/platforms/atmelavr?utm_source=github&utm_medium=core>`_
-* `Atmel SAM <https://platformio.org/platforms/atmelsam?utm_source=github&utm_medium=core>`_
-* `Espressif 32 <https://platformio.org/platforms/espressif32?utm_source=github&utm_medium=core>`_
-* `Espressif 8266 <https://platformio.org/platforms/espressif8266?utm_source=github&utm_medium=core>`_
-* `Freescale Kinetis <https://platformio.org/platforms/freescalekinetis?utm_source=github&utm_medium=core>`_
-* `Infineon XMC <https://platformio.org/platforms/infineonxmc?utm_source=github&utm_medium=core>`_
-* `Intel ARC32 <https://platformio.org/platforms/intel_arc32?utm_source=github&utm_medium=core>`_
-* `Intel MCS-51 (8051) <https://platformio.org/platforms/intel_mcs51?utm_source=github&utm_medium=core>`_
-* `Kendryte K210 <https://platformio.org/platforms/kendryte210?utm_source=github&utm_medium=core>`_
-* `Lattice iCE40 <https://platformio.org/platforms/lattice_ice40?utm_source=github&utm_medium=core>`_
-* `Maxim 32 <https://platformio.org/platforms/maxim32?utm_source=github&utm_medium=core>`_
-* `Microchip PIC32 <https://platformio.org/platforms/microchippic32?utm_source=github&utm_medium=core>`_
-* `Nordic nRF51 <https://platformio.org/platforms/nordicnrf51?utm_source=github&utm_medium=core>`_
-* `Nordic nRF52 <https://platformio.org/platforms/nordicnrf52?utm_source=github&utm_medium=core>`_
-* `Nuclei <https://platformio.org/platforms/nuclei?utm_source=github&utm_medium=core>`_
-* `NXP LPC <https://platformio.org/platforms/nxplpc?utm_source=github&utm_medium=core>`_
-* `RISC-V <https://platformio.org/platforms/riscv?utm_source=github&utm_medium=core>`_
-* `RISC-V GAP <https://platformio.org/platforms/riscv_gap?utm_source=github&utm_medium=core>`_
-* `Shakti <https://platformio.org/platforms/shakti?utm_source=github&utm_medium=core>`_
-* `Silicon Labs EFM32 <https://platformio.org/platforms/siliconlabsefm32?utm_source=github&utm_medium=core>`_
-* `ST STM32 <https://platformio.org/platforms/ststm32?utm_source=github&utm_medium=core>`_
-* `ST STM8 <https://platformio.org/platforms/ststm8?utm_source=github&utm_medium=core>`_
-* `Teensy <https://platformio.org/platforms/teensy?utm_source=github&utm_medium=core>`_
-* `TI MSP430 <https://platformio.org/platforms/timsp430?utm_source=github&utm_medium=core>`_
-* `TI Tiva <https://platformio.org/platforms/titiva?utm_source=github&utm_medium=core>`_
-* `WIZNet W7500 <https://platformio.org/platforms/wiznet7500?utm_source=github&utm_medium=core>`_
-
-Frameworks
-----------
-
-* `Arduino <https://platformio.org/frameworks/arduino?utm_source=github&utm_medium=core>`_
-* `CMSIS <https://platformio.org/frameworks/cmsis?utm_source=github&utm_medium=core>`_
-* `ESP-IDF <https://platformio.org/frameworks/espidf?utm_source=github&utm_medium=core>`_
-* `ESP8266 Non-OS SDK <https://platformio.org/frameworks/esp8266-nonos-sdk?utm_source=github&utm_medium=core>`_
-* `ESP8266 RTOS SDK <https://platformio.org/frameworks/esp8266-rtos-sdk?utm_source=github&utm_medium=core>`_
-* `Freedom E SDK <https://platformio.org/frameworks/freedom-e-sdk?utm_source=github&utm_medium=core>`_
-* `GigaDevice GD32V SDK <https://platformio.org/frameworks/gd32vf103-sdk?utm_source=github&utm_medium=core>`_
-* `Kendryte Standalone SDK <https://platformio.org/frameworks/kendryte-standalone-sdk?utm_source=github&utm_medium=core>`_
-* `Kendryte FreeRTOS SDK <https://platformio.org/frameworks/kendryte-freertos-sdk?utm_source=github&utm_medium=core>`_
-* `libOpenCM3 <https://platformio.org/frameworks/libopencm3?utm_source=github&utm_medium=core>`_
-* `Mbed <https://platformio.org/frameworks/mbed?utm_source=github&utm_medium=core>`_
-* `Nuclei SDK <https://platformio.org/frameworks/nuclei-sdk?utm_source=github&utm_medium=core>`_
-* `PULP OS <https://platformio.org/frameworks/pulp-os?utm_source=github&utm_medium=core>`_
-* `Pumbaa <https://platformio.org/frameworks/pumbaa?utm_source=github&utm_medium=core>`_
-* `Shakti SDK <https://platformio.org/frameworks/shakti-sdk?utm_source=github&utm_medium=core>`_
-* `Simba <https://platformio.org/frameworks/simba?utm_source=github&utm_medium=core>`_
-* `SPL <https://platformio.org/frameworks/spl?utm_source=github&utm_medium=core>`_
-* `STM32Cube <https://platformio.org/frameworks/stm32cube?utm_source=github&utm_medium=core>`_
-* `WiringPi <https://platformio.org/frameworks/wiringpi?utm_source=github&utm_medium=core>`_
-* `Zephyr <https://platformio.org/frameworks/zephyr?utm_source=github&utm_medium=core>`_
+* `Libraries <https://registry.platformio.org/search?t=library&utm_source=github&utm_medium=core>`_
+* `Development Platforms <https://registry.platformio.org/search?t=platform&utm_source=github&utm_medium=core>`_
+* `Development Tools <https://registry.platformio.org/search?t=tool&utm_source=github&utm_medium=core>`_
 
 Contributing
 ------------

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (5, 2, 2)
+VERSION = (5, 2, 5)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"
@@ -47,10 +47,10 @@ __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413"
 __default_requests_timeout__ = (10, None)  # (connect, read)
 
 __core_packages__ = {
-    "contrib-piohome": "~3.3.4",
+    "contrib-piohome": "~3.4.1",
     "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
     "tool-unity": "~1.20500.0",
-    "tool-scons": "~4.40200.0",
+    "tool-scons": "~4.40300.0",
     "tool-cppcheck": "~1.260.0",
     "tool-clangtidy": "~1.120001.0",
     "tool-pvs-studio": "~7.14.0",

platformio/__main__.py

@@ -24,13 +24,6 @@ from platformio import __version__, exception
 from platformio.commands import PlatformioCLI
 from platformio.compat import IS_CYGWIN, ensure_python3
 
-try:
-    import click_completion  # pylint: disable=import-error
-
-    click_completion.init()
-except:  # pylint: disable=bare-except
-    pass
-
 
 @click.command(
     cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"])
@@ -74,7 +67,6 @@ try:
     def process_result(ctx, result, *_, **__):
         _process_result(ctx, result)
 
-
 except (AttributeError, TypeError):  # legacy support for CLick > 8.0.1
 
     @cli.resultcallback()

platformio/app.py

@@ -21,17 +21,17 @@ import os
 import platform
 import socket
 import uuid
-from os.path import dirname, isdir, isfile, join, realpath
 
 from platformio import __version__, exception, fs, proc
 from platformio.compat import IS_WINDOWS, hashlib_encode_data
 from platformio.package.lockfile import LockFile
-from platformio.project.helpers import get_default_projects_dir, get_project_core_dir
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import get_default_projects_dir
 
 
 def projects_dir_validate(projects_dir):
-    assert isdir(projects_dir)
-    return realpath(projects_dir)
+    assert os.path.isdir(projects_dir)
+    return os.path.abspath(projects_dir)
 
 
 DEFAULT_SETTINGS = {
@@ -64,7 +64,7 @@ DEFAULT_SETTINGS = {
         "value": True,
     },
     "enable_telemetry": {
-        "description": ("Telemetry service <http://bit.ly/pio-telemetry> (Yes/No)"),
+        "description": ("Telemetry service <https://bit.ly/pio-telemetry> (Yes/No)"),
         "value": True,
     },
     "force_verbose": {
@@ -91,7 +91,10 @@ class State(object):
         self.path = path
         self.lock = lock
         if not self.path:
-            self.path = join(get_project_core_dir(), "appstate.json")
+            core_dir = ProjectConfig.get_instance().get("platformio", "core_dir")
+            if not os.path.isdir(core_dir):
+                os.makedirs(core_dir)
+            self.path = os.path.join(core_dir, "appstate.json")
         self._storage = {}
         self._lockfile = None
         self.modified = False
@@ -99,7 +102,7 @@ class State(object):
     def __enter__(self):
         try:
             self._lock_state_file()
-            if isfile(self.path):
+            if os.path.isfile(self.path):
                 self._storage = fs.load_json(self.path)
             assert isinstance(self._storage, dict)
         except (
@@ -117,7 +120,7 @@ class State(object):
                 with open(self.path, mode="w", encoding="utf8") as fp:
                     fp.write(json.dumps(self._storage))
             except IOError:
-                raise exception.HomeDirPermissionsError(get_project_core_dir())
+                raise exception.HomeDirPermissionsError(os.path.dirname(self.path))
         self._unlock_state_file()
 
     def _lock_state_file(self):
@@ -127,7 +130,7 @@ class State(object):
         try:
             self._lockfile.acquire()
         except IOError:
-            raise exception.HomeDirPermissionsError(dirname(self.path))
+            raise exception.HomeDirPermissionsError(os.path.dirname(self.path))
 
     def _unlock_state_file(self):
         if hasattr(self, "_lockfile") and self._lockfile:
@@ -250,29 +253,14 @@ def is_disabled_progressbar():
 
 
 def get_cid():
-    # pylint: disable=import-outside-toplevel
-    from platformio.clients.http import fetch_remote_content
-
     cid = get_state_item("cid")
     if cid:
         return cid
     uid = None
-    if os.getenv("C9_UID"):
-        uid = os.getenv("C9_UID")
+    if os.getenv("GITHUB_USER"):
+        uid = os.getenv("GITHUB_USER")
     elif os.getenv("GITPOD_GIT_USER_NAME"):
         uid = os.getenv("GITPOD_GIT_USER_NAME")
-    elif os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")):
-        try:
-            uid = json.loads(
-                fetch_remote_content(
-                    "{api}/user?token={token}".format(
-                        api=os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")),
-                        token=os.getenv("USER_TOKEN"),
-                    )
-                )
-            ).get("id")
-        except:  # pylint: disable=bare-except
-            pass
     if not uid:
         uid = uuid.getnode()
     cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest())

platformio/builder/main.py

@@ -109,34 +109,39 @@ env.Replace(
 config = env.GetProjectConfig()
 env.Replace(
     PROJECT_DIR=get_project_dir(),
-    PROJECT_CORE_DIR=config.get_optional_dir("core"),
-    PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"),
-    PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"),
-    PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"),
-    PROJECT_INCLUDE_DIR=config.get_optional_dir("include"),
-    PROJECT_SRC_DIR=config.get_optional_dir("src"),
-    PROJECTSRC_DIR=config.get_optional_dir("src"),  # legacy for dev/platform
-    PROJECT_TEST_DIR=config.get_optional_dir("test"),
-    PROJECT_DATA_DIR=config.get_optional_dir("data"),
-    PROJECTDATA_DIR=config.get_optional_dir("data"),  # legacy for dev/platform
-    PROJECT_BUILD_DIR=config.get_optional_dir("build"),
-    BUILD_CACHE_DIR=config.get_optional_dir("build_cache"),
+    PROJECT_CORE_DIR=config.get("platformio", "core_dir"),
+    PROJECT_PACKAGES_DIR=config.get("platformio", "packages_dir"),
+    PROJECT_WORKSPACE_DIR=config.get("platformio", "workspace_dir"),
+    PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"),
+    PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"),
+    PROJECT_SRC_DIR=config.get("platformio", "src_dir"),
+    PROJECTSRC_DIR="$PROJECT_SRC_DIR",  # legacy for dev/platform
+    PROJECT_TEST_DIR=config.get("platformio", "test_dir"),
+    PROJECT_DATA_DIR=config.get("platformio", "data_dir"),
+    PROJECTDATA_DIR="$PROJECT_DATA_DIR",  # legacy for dev/platform
+    PROJECT_BUILD_DIR=config.get("platformio", "build_dir"),
+    BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"),
     LIBSOURCE_DIRS=[
-        config.get_optional_dir("lib"),
+        config.get("platformio", "lib_dir"),
         os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
-        config.get_optional_dir("globallib"),
+        config.get("platformio", "globallib_dir"),
     ],
 )
 
-if (
-    compat.IS_WINDOWS
-    and sys.version_info >= (3, 8)
-    and env["PROJECT_DIR"].startswith("\\\\")
-):
+if int(ARGUMENTS.get("ISATTY", 0)):
+    # pylint: disable=protected-access
+    click._compat.isatty = lambda stream: True
+
+if compat.IS_WINDOWS and sys.version_info >= (3, 8) and os.getcwd().startswith("\\\\"):
+    click.secho("!!! WARNING !!!\t\t" * 3, fg="red")
     click.secho(
-        "There is a known issue with Python 3.8+ and mapped network drives on "
-        "Windows.\nSee a solution at:\n"
-        "https://github.com/platformio/platformio-core/issues/3417",
+        "Your project is located on a mapped network drive but the "
+        "current command-line shell does not support the UNC paths.",
+        fg="yellow",
+    )
+    click.secho(
+        "Please move your project to a physical drive or check this workaround: "
+        "https://bit.ly/3kuU5mP\n",
         fg="yellow",
     )
 
@@ -145,10 +150,6 @@ if env.subst("$BUILD_CACHE_DIR"):
         os.makedirs(env.subst("$BUILD_CACHE_DIR"))
     env.CacheDir("$BUILD_CACHE_DIR")
 
-if int(ARGUMENTS.get("ISATTY", 0)):
-    # pylint: disable=protected-access
-    click._compat.isatty = lambda stream: True
-
 is_clean_all = "cleanall" in COMMAND_LINE_TARGETS
 if env.GetOption("clean") or is_clean_all:
     env.PioClean(is_clean_all)

platformio/builder/tools/pioide.py

@@ -32,14 +32,14 @@ def _dump_includes(env):
         env.subst("$PROJECT_SRC_DIR"),
     ]
     includes["build"].extend(
-        [os.path.realpath(env.subst(item)) for item in env.get("CPPPATH", [])]
+        [os.path.abspath(env.subst(item)) for item in env.get("CPPPATH", [])]
     )
 
     # installed libs
     includes["compatlib"] = []
     for lb in env.GetLibBuilders():
         includes["compatlib"].extend(
-            [os.path.realpath(inc) for inc in lb.get_include_dirs()]
+            [os.path.abspath(inc) for inc in lb.get_include_dirs()]
         )
 
     # includes from toolchains
@@ -56,14 +56,12 @@ def _dump_includes(env):
             os.path.join(toolchain_dir, "*", "include*"),
         ]
         for g in toolchain_incglobs:
-            includes["toolchain"].extend(
-                [os.path.realpath(inc) for inc in glob.glob(g)]
-            )
+            includes["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)])
 
     # include Unity framework if there are tests in project
     includes["unity"] = []
     auto_install_unity = False
-    test_dir = env.GetProjectConfig().get_optional_dir("test")
+    test_dir = env.GetProjectConfig().get("platformio", "test_dir")
     if os.path.isdir(test_dir) and os.listdir(test_dir) != ["README"]:
         auto_install_unity = True
     unity_dir = get_core_package_dir(
@@ -132,7 +130,7 @@ def _dump_defines(env):
 def _get_svd_path(env):
     svd_path = env.GetProjectOption("debug_svd_path")
     if svd_path:
-        return os.path.realpath(svd_path)
+        return os.path.abspath(svd_path)
 
     if "BOARD" not in env:
         return None
@@ -147,7 +145,7 @@ def _get_svd_path(env):
     # default file from ./platform/misc/svd folder
     p = env.PioPlatform()
     if os.path.isfile(os.path.join(p.get_dir(), "misc", "svd", svd_path)):
-        return os.path.realpath(os.path.join(p.get_dir(), "misc", "svd", svd_path))
+        return os.path.abspath(os.path.join(p.get_dir(), "misc", "svd", svd_path))
     return None
 
 

platformio/builder/tools/piolib.py

@@ -32,7 +32,7 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
 from platformio import exception, fs, util
 from platformio.builder.tools import platformio as piotool
-from platformio.clients.http import InternetIsOffline
+from platformio.clients.http import HTTPClientError, InternetIsOffline
 from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
 from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.library import LibraryPackageManager
@@ -125,7 +125,7 @@ class LibBuilderBase(object):
     def __init__(self, env, path, manifest=None, verbose=False):
         self.env = env.Clone()
         self.envorigin = env.Clone()
-        self.path = os.path.realpath(env.subst(path))
+        self.path = os.path.abspath(env.subst(path))
         self.verbose = verbose
 
         try:
@@ -290,7 +290,7 @@ class LibBuilderBase(object):
             if self.extra_script:
                 self.env.SConscriptChdir(1)
                 self.env.SConscript(
-                    os.path.realpath(self.extra_script),
+                    os.path.abspath(self.extra_script),
                     exports={"env": self.env, "pio_lib_builder": self},
                 )
             self.env.ProcessUnFlags(self.build_unflags)
@@ -750,14 +750,14 @@ class PlatformIOLibBuilder(LibBuilderBase):
     def include_dir(self):
         if "includeDir" in self._manifest.get("build", {}):
             with fs.cd(self.path):
-                return os.path.realpath(self._manifest.get("build").get("includeDir"))
+                return os.path.abspath(self._manifest.get("build").get("includeDir"))
         return LibBuilderBase.include_dir.fget(self)  # pylint: disable=no-member
 
     @property
     def src_dir(self):
         if "srcDir" in self._manifest.get("build", {}):
             with fs.cd(self.path):
-                return os.path.realpath(self._manifest.get("build").get("srcDir"))
+                return os.path.abspath(self._manifest.get("build").get("srcDir"))
         return LibBuilderBase.src_dir.fget(self)  # pylint: disable=no-member
 
     @property
@@ -939,7 +939,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
             try:
                 lm.install(spec)
                 did_install = True
-            except (UnknownPackageError, InternetIsOffline) as e:
+            except (HTTPClientError, UnknownPackageError, InternetIsOffline) as e:
                 click.secho("Warning! %s" % e, fg="yellow")
 
         # reset cache
@@ -1024,7 +1024,7 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
     found_incompat = False
 
     for storage_dir in env.GetLibSourceDirs():
-        storage_dir = os.path.realpath(storage_dir)
+        storage_dir = os.path.abspath(storage_dir)
         if not os.path.isdir(storage_dir):
             continue
         for item in sorted(os.listdir(storage_dir)):
@@ -1095,7 +1095,7 @@ def ConfigureProjectLibBuilder(env):
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
     ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project)  # pylint: disable=no-member
 
-    click.echo("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf")
+    click.echo("LDF: Library Dependency Finder -> https://bit.ly/configure-pio-ldf")
     click.echo(
         "LDF Modes: Finder ~ %s, Compatibility ~ %s"
         % (ldf_mode, project.lib_compat_mode)

platformio/builder/tools/piomisc.py

@@ -376,7 +376,7 @@ def GetExtraScripts(env, scope):
     if not items:
         return items
     with fs.cd(env.subst("$PROJECT_DIR")):
-        return [os.path.realpath(item) for item in items]
+        return [os.path.abspath(item) for item in items]
 
 
 def exists(_):

platformio/builder/tools/platformio.py

@@ -207,12 +207,12 @@ def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
     for k in ("CPPPATH", "LIBPATH"):
         for i, p in enumerate(result.get(k, [])):
             if os.path.isdir(p):
-                result[k][i] = os.path.realpath(p)
+                result[k][i] = os.path.abspath(p)
 
     # fix relative path for "-include"
     for i, f in enumerate(result.get("CCFLAGS", [])):
         if isinstance(f, tuple) and f[0] == "-include":
-            result["CCFLAGS"][i] = (f[0], env.File(os.path.realpath(f[1].get_path())))
+            result["CCFLAGS"][i] = (f[0], env.File(os.path.abspath(f[1].get_path())))
 
     return result
 

platformio/clients/account.py

@@ -16,7 +16,7 @@ import os
 import time
 
 from platformio import __accounts_api__, app
-from platformio.clients.http import HTTPClient
+from platformio.clients.http import HTTPClient, HTTPClientError
 from platformio.exception import PlatformioException
 
 
@@ -61,13 +61,33 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         del account[key]
         app.set_state_item("account", account)
 
-    def send_auth_request(self, *args, **kwargs):
-        headers = kwargs.get("headers", {})
-        if "Authorization" not in headers:
-            token = self.fetch_authentication_token()
-            headers["Authorization"] = "Bearer %s" % token
-        kwargs["headers"] = headers
-        return self.fetch_json_data(*args, **kwargs)
+    def fetch_json_data(self, *args, **kwargs):
+        try:
+            return super(AccountClient, self).fetch_json_data(*args, **kwargs)
+        except HTTPClientError as exc:
+            raise AccountError(exc) from exc
+
+    def fetch_authentication_token(self):
+        if os.environ.get("PLATFORMIO_AUTH_TOKEN"):
+            return os.environ.get("PLATFORMIO_AUTH_TOKEN")
+        auth = app.get_state_item("account", {}).get("auth", {})
+        if auth.get("access_token") and auth.get("access_token_expire"):
+            if auth.get("access_token_expire") > time.time():
+                return auth.get("access_token")
+            if auth.get("refresh_token"):
+                try:
+                    data = self.fetch_json_data(
+                        "post",
+                        "/v1/login",
+                        headers={
+                            "Authorization": "Bearer %s" % auth.get("refresh_token")
+                        },
+                    )
+                    app.set_state_item("account", data)
+                    return data.get("auth").get("access_token")
+                except AccountError:
+                    self.delete_local_session()
+        raise AccountNotAuthorized()
 
     def login(self, username, password):
         try:
@@ -119,10 +139,11 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         return True
 
     def change_password(self, old_password, new_password):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/password",
             data={"old_password": old_password, "new_password": new_password},
+            x_with_authorization=True,
         )
 
     def registration(
@@ -150,10 +171,11 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         )
 
     def auth_token(self, password, regenerate):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/token",
             data={"password": password, "regenerate": 1 if regenerate else 0},
+            x_with_authorization=True,
         ).get("auth_token")
 
     def forgot_password(self, username):
@@ -164,18 +186,20 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         )
 
     def get_profile(self):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "get",
             "/v1/profile",
+            x_with_authorization=True,
         )
 
     def update_profile(self, profile, current_password):
         profile["current_password"] = current_password
         self.delete_local_state("summary")
-        response = self.send_auth_request(
+        response = self.fetch_json_data(
             "put",
             "/v1/profile",
             data=profile,
+            x_with_authorization=True,
         )
         return response
 
@@ -193,9 +217,10 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
                     "username": account.get("username"),
                 }
             }
-        result = self.send_auth_request(
+        result = self.fetch_json_data(
             "get",
             "/v1/summary",
+            x_with_authorization=True,
         )
         account["summary"] = dict(
             profile=result.get("profile"),
@@ -211,119 +236,121 @@ class AccountClient(HTTPClient):  # pylint:disable=too-many-public-methods
         return self.get_account_info(offline=True).get("profile").get("username")
 
     def destroy_account(self):
-        return self.send_auth_request("delete", "/v1/account")
+        return self.fetch_json_data(
+            "delete",
+            "/v1/account",
+            x_with_authorization=True,
+        )
 
     def create_org(self, orgname, email, displayname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/orgs",
             data={"orgname": orgname, "email": email, "displayname": displayname},
+            x_with_authorization=True,
         )
 
     def get_org(self, orgname):
-        return self.send_auth_request("get", "/v1/orgs/%s" % orgname)
+        return self.fetch_json_data(
+            "get",
+            "/v1/orgs/%s" % orgname,
+            x_with_authorization=True,
+        )
 
     def list_orgs(self):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "get",
             "/v1/orgs",
+            x_with_authorization=True,
         )
 
     def update_org(self, orgname, data):
-        return self.send_auth_request(
-            "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v}
+        return self.fetch_json_data(
+            "put",
+            "/v1/orgs/%s" % orgname,
+            data={k: v for k, v in data.items() if v},
+            x_with_authorization=True,
         )
 
     def destroy_org(self, orgname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s" % orgname,
+            x_with_authorization=True,
         )
 
     def add_org_owner(self, orgname, username):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/orgs/%s/owners" % orgname,
             data={"username": username},
+            x_with_authorization=True,
         )
 
     def list_org_owners(self, orgname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "get",
             "/v1/orgs/%s/owners" % orgname,
+            x_with_authorization=True,
         )
 
     def remove_org_owner(self, orgname, username):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s/owners" % orgname,
             data={"username": username},
+            x_with_authorization=True,
         )
 
     def create_team(self, orgname, teamname, description):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/orgs/%s/teams" % orgname,
             data={"name": teamname, "description": description},
+            x_with_authorization=True,
         )
 
     def destroy_team(self, orgname, teamname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s/teams/%s" % (orgname, teamname),
+            x_with_authorization=True,
         )
 
     def get_team(self, orgname, teamname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "get",
             "/v1/orgs/%s/teams/%s" % (orgname, teamname),
+            x_with_authorization=True,
         )
 
     def list_teams(self, orgname):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "get",
             "/v1/orgs/%s/teams" % orgname,
+            x_with_authorization=True,
         )
 
     def update_team(self, orgname, teamname, data):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "put",
             "/v1/orgs/%s/teams/%s" % (orgname, teamname),
             data={k: v for k, v in data.items() if v},
+            x_with_authorization=True,
         )
 
     def add_team_member(self, orgname, teamname, username):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "post",
             "/v1/orgs/%s/teams/%s/members" % (orgname, teamname),
             data={"username": username},
+            x_with_authorization=True,
         )
 
     def remove_team_member(self, orgname, teamname, username):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "delete",
             "/v1/orgs/%s/teams/%s/members" % (orgname, teamname),
             data={"username": username},
+            x_with_authorization=True,
         )
-
-    def fetch_authentication_token(self):
-        if os.environ.get("PLATFORMIO_AUTH_TOKEN"):
-            return os.environ.get("PLATFORMIO_AUTH_TOKEN")
-        auth = app.get_state_item("account", {}).get("auth", {})
-        if auth.get("access_token") and auth.get("access_token_expire"):
-            if auth.get("access_token_expire") > time.time():
-                return auth.get("access_token")
-            if auth.get("refresh_token"):
-                try:
-                    data = self.fetch_json_data(
-                        "post",
-                        "/v1/login",
-                        headers={
-                            "Authorization": "Bearer %s" % auth.get("refresh_token")
-                        },
-                    )
-                    app.set_state_item("account", data)
-                    return data.get("auth").get("access_token")
-                except AccountError:
-                    self.delete_local_session()
-        raise AccountNotAuthorized()

platformio/clients/http.py

@@ -21,7 +21,7 @@ import requests.adapters
 from requests.packages.urllib3.util.retry import Retry  # pylint:disable=import-error
 
 from platformio import __check_internet_hosts__, __default_requests_timeout__, app, util
-from platformio.cache import ContentCache
+from platformio.cache import ContentCache, cleanup_content_cache
 from platformio.exception import PlatformioException, UserSideException
 
 try:
@@ -117,6 +117,21 @@ class HTTPClient(object):
         # check Internet before and resolve issue with 60 seconds timeout
         ensure_internet_on(raise_exception=True)
 
+        headers = kwargs.get("headers", {})
+        with_authorization = (
+            kwargs.pop("x_with_authorization")
+            if "x_with_authorization" in kwargs
+            else False
+        )
+        if with_authorization and "Authorization" not in headers:
+            # pylint: disable=import-outside-toplevel
+            from platformio.clients.account import AccountClient
+
+            headers["Authorization"] = (
+                "Bearer %s" % AccountClient().fetch_authentication_token()
+            )
+        kwargs["headers"] = headers
+
         # set default timeout
         if "timeout" not in kwargs:
             kwargs["timeout"] = __default_requests_timeout__
@@ -134,7 +149,9 @@ class HTTPClient(object):
                     raise HTTPClientError(str(e))
 
     def fetch_json_data(self, method, path, **kwargs):
-        cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None
+        if method != "get":
+            cleanup_content_cache("http")
+        cache_valid = kwargs.pop("x_cache_valid") if "x_cache_valid" in kwargs else None
         if not cache_valid:
             return self._parse_json_response(self.send_request(method, path, **kwargs))
         cache_key = ContentCache.key_from_args(
@@ -179,8 +196,9 @@ def _internet_on():
                     continue
                 requests.get("http://%s" % host, allow_redirects=False, timeout=timeout)
                 return True
-            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            s.connect((host, 80))
+            # try to resolve `host` for both AF_INET and AF_INET6, and then try to connect
+            # to all possible addresses (IPv4 and IPv6) in turn until a connection succeeds:
+            s = socket.create_connection((host, 80))
             s.close()
             return True
         except:  # pylint: disable=bare-except

platformio/clients/registry.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from platformio import __registry_api__, fs
-from platformio.clients.account import AccountClient
+from platformio.clients.account import AccountClient, AccountError
 from platformio.clients.http import HTTPClient, HTTPClientError
 
 # pylint: disable=too-many-arguments
@@ -23,19 +23,29 @@ class RegistryClient(HTTPClient):
     def __init__(self):
         super(RegistryClient, self).__init__(__registry_api__)
 
-    def send_auth_request(self, *args, **kwargs):
-        headers = kwargs.get("headers", {})
-        if "Authorization" not in headers:
-            token = AccountClient().fetch_authentication_token()
-            headers["Authorization"] = "Bearer %s" % token
-        kwargs["headers"] = headers
-        return self.fetch_json_data(*args, **kwargs)
+    @staticmethod
+    def allowed_private_packages():
+        private_permissions = set(
+            [
+                "service.registry.publish-private-tool",
+                "service.registry.publish-private-platform",
+                "service.registry.publish-private-library",
+            ]
+        )
+        try:
+            info = AccountClient().get_account_info() or {}
+            for item in info.get("packages", []):
+                if set(item.keys()) & private_permissions:
+                    return True
+        except AccountError:
+            pass
+        return False
 
     def publish_package(  # pylint: disable=redefined-builtin
         self, owner, type, archive_path, released_at=None, private=False, notify=True
     ):
         with open(archive_path, "rb") as fp:
-            return self.send_auth_request(
+            return self.fetch_json_data(
                 "post",
                 "/v3/packages/%s/%s" % (owner, type),
                 params={
@@ -50,6 +60,7 @@ class RegistryClient(HTTPClient):
                     ),
                 },
                 data=fp,
+                x_with_authorization=True,
             )
 
     def unpublish_package(  # pylint: disable=redefined-builtin
@@ -58,36 +69,40 @@ class RegistryClient(HTTPClient):
         path = "/v3/packages/%s/%s/%s" % (owner, type, name)
         if version:
             path += "/" + version
-        return self.send_auth_request(
-            "delete",
-            path,
-            params={"undo": 1 if undo else 0},
+        return self.fetch_json_data(
+            "delete", path, params={"undo": 1 if undo else 0}, x_with_authorization=True
         )
 
     def update_resource(self, urn, private):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "put",
             "/v3/resources/%s" % urn,
             data={"private": int(private)},
+            x_with_authorization=True,
         )
 
     def grant_access_for_resource(self, urn, client, level):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "put",
             "/v3/resources/%s/access" % urn,
             data={"client": client, "level": level},
+            x_with_authorization=True,
         )
 
     def revoke_access_from_resource(self, urn, client):
-        return self.send_auth_request(
+        return self.fetch_json_data(
             "delete",
             "/v3/resources/%s/access" % urn,
             data={"client": client},
+            x_with_authorization=True,
         )
 
     def list_resources(self, owner):
-        return self.send_auth_request(
-            "get", "/v3/resources", params={"owner": owner} if owner else None
+        return self.fetch_json_data(
+            "get",
+            "/v3/resources",
+            params={"owner": owner} if owner else None,
+            x_with_authorization=True,
         )
 
     def list_packages(self, query=None, filters=None, page=None):
@@ -117,7 +132,11 @@ class RegistryClient(HTTPClient):
         if page:
             params["page"] = int(page)
         return self.fetch_json_data(
-            "get", "/v3/packages", params=params, cache_valid="1h"
+            "get",
+            "/v3/search",
+            params=params,
+            x_cache_valid="1h",
+            x_with_authorization=self.allowed_private_packages(),
         )
 
     def get_package(self, type_, owner, name, version=None):
@@ -128,7 +147,8 @@ class RegistryClient(HTTPClient):
                     type=type_, owner=owner.lower(), name=name.lower()
                 ),
                 params=dict(version=version) if version else None,
-                cache_valid="1h",
+                x_cache_valid="1h",
+                x_with_authorization=self.allowed_private_packages(),
             )
         except HTTPClientError as e:
             if e.response is not None and e.response.status_code == 404:

platformio/commands/access.py

@@ -134,6 +134,14 @@ def access_list(owner, urn_type, json_output):
         table_data = []
         table_data.append(("URN:", resource.get("urn")))
         table_data.append(("Owner:", resource.get("owner")))
+        table_data.append(
+            (
+                "Access:",
+                click.style("Private", fg="red")
+                if resource.get("private", False)
+                else "Public",
+            )
+        )
         table_data.append(
             (
                 "Access level(s):",

platformio/commands/check/command.py

@@ -106,8 +106,8 @@ def cli(
                 )
 
             default_patterns = [
-                config.get_optional_dir("src"),
-                config.get_optional_dir("include"),
+                config.get("platformio", "src_dir"),
+                config.get("platformio", "include_dir"),
             ]
             tool_options = dict(
                 verbose=verbose,

platformio/commands/check/defect.py

@@ -86,7 +86,7 @@ class DefectItem(object):
             "severity": self.SEVERITY_LABELS[self.severity],
             "category": self.category,
             "message": self.message,
-            "file": os.path.realpath(self.file),
+            "file": os.path.abspath(self.file),
             "line": self.line,
             "column": self.column,
             "callstack": self.callstack,

platformio/commands/check/tools/base.py

@@ -201,11 +201,11 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
 
         def _add_file(path):
             if path.endswith(header_extensions):
-                result["headers"].append(os.path.realpath(path))
+                result["headers"].append(os.path.abspath(path))
             elif path.endswith(c_extension):
-                result["c"].append(os.path.realpath(path))
+                result["c"].append(os.path.abspath(path))
             elif path.endswith(cpp_extensions):
-                result["c++"].append(os.path.realpath(path))
+                result["c++"].append(os.path.abspath(path))
 
         for pattern in patterns:
             for item in glob.glob(pattern, recursive=True):

platformio/commands/check/tools/clangtidy.py

@@ -77,7 +77,7 @@ class ClangtidyCheckTool(CheckToolBase):
         includes = []
         for inc in self.cpp_includes:
             if self.options.get("skip_packages") and inc.lower().startswith(
-                self.config.get_optional_dir("packages").lower()
+                self.config.get("platformio", "packages_dir").lower()
             ):
                 continue
             includes.append(inc)

platformio/commands/check/tools/cppcheck.py

@@ -84,7 +84,7 @@ class CppcheckCheckTool(CheckToolBase):
         if (
             args.get("file", "")
             .lower()
-            .startswith(self.config.get_optional_dir("packages").lower())
+            .startswith(self.config.get("platformio", "packages_dir").lower())
         ):
             if args["id"] in breaking_defect_ids:
                 if self.options.get("verbose"):
@@ -201,7 +201,7 @@ class CppcheckCheckTool(CheckToolBase):
         result = []
         for inc in self.cpp_includes:
             if self.options.get("skip_packages") and inc.lower().startswith(
-                self.config.get_optional_dir("packages").lower()
+                self.config.get("platformio", "packages_dir").lower()
             ):
                 continue
             result.append(inc)

platformio/commands/check/tools/pvsstudio.py

@@ -43,7 +43,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
         with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp:
             fp.write(
                 "exclude-path = "
-                + self.config.get_optional_dir("packages").replace("\\", "/")
+                + self.config.get("platformio", "packages_dir").replace("\\", "/")
             )
 
         with open(self._tmp_cmd_file, mode="w", encoding="utf8") as fp:

platformio/commands/ci.py

@@ -33,7 +33,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
     for i, p in enumerate(value):
         if p.startswith("~"):
             value[i] = fs.expanduser(p)
-        value[i] = os.path.realpath(value[i])
+        value[i] = os.path.abspath(value[i])
         if not glob.glob(value[i], recursive=True):
             invalid_path = p
             break
@@ -162,7 +162,7 @@ def _exclude_contents(dst_dir, patterns):
     for p in patterns:
         contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True)
     for path in contents:
-        path = os.path.realpath(path)
+        path = os.path.abspath(path)
         if os.path.isdir(path):
             fs.rmtree(path)
         elif os.path.isfile(path):

platformio/commands/device/command.py

@@ -172,28 +172,21 @@ def device_list(  # pylint: disable=too-many-branches
     help="Load configuration from `platformio.ini` and specified environment",
 )
 def device_monitor(**kwargs):  # pylint: disable=too-many-branches
-    # load default monitor filters
-    filters_dir = os.path.join(fs.get_source_dir(), "commands", "device", "filters")
-    for name in os.listdir(filters_dir):
-        if not name.endswith(".py"):
-            continue
-        device_helpers.load_monitor_filter(
-            os.path.join(filters_dir, name), options=kwargs
-        )
-
     project_options = {}
+    platform = None
     try:
         with fs.cd(kwargs["project_dir"]):
             project_options = device_helpers.get_project_options(kwargs["environment"])
-        kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options)
+            kwargs = device_helpers.apply_project_monitor_options(
+                kwargs, project_options
+            )
+            if "platform" in project_options:
+                platform = PlatformFactory.new(project_options["platform"])
     except NotPlatformIOProjectError:
         pass
 
-    platform = None
-    if "platform" in project_options:
-        with fs.cd(kwargs["project_dir"]):
-            platform = PlatformFactory.new(project_options["platform"])
-            device_helpers.register_platform_filters(platform, options=kwargs)
+    with fs.cd(kwargs["project_dir"]):
+        device_helpers.register_filters(platform=platform, options=kwargs)
 
     if not kwargs["port"]:
         ports = util.get_serial_ports(filter_hwid=True)
@@ -231,7 +224,7 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
             "--- Available filters and text transformations: %s"
             % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys()))
         )
-        click.echo("--- More details at http://bit.ly/pio-monitor-filters")
+        click.echo("--- More details at https://bit.ly/pio-monitor-filters")
     try:
         miniterm.main(
             default_port=kwargs["port"],

platformio/commands/device/helpers.py

@@ -20,6 +20,7 @@ from serial.tools import miniterm
 from platformio import fs
 from platformio.commands.device import DeviceMonitorFilter
 from platformio.compat import get_object_members, load_python_module
+from platformio.package.manager.tool import ToolPackageManager
 from platformio.project.config import ProjectConfig
 
 
@@ -92,15 +93,40 @@ def load_monitor_filter(path, options=None):
     return True
 
 
-def register_platform_filters(platform, options=None):
-    monitor_dir = os.path.join(platform.get_dir(), "monitor")
+def load_monitor_filters(monitor_dir, prefix=None, options=None):
     if not os.path.isdir(monitor_dir):
         return
-
     for name in os.listdir(monitor_dir):
-        if not name.startswith("filter_") or not name.endswith(".py"):
+        if (prefix and not name.startswith(prefix)) or not name.endswith(".py"):
             continue
         path = os.path.join(monitor_dir, name)
         if not os.path.isfile(path):
             continue
         load_monitor_filter(path, options)
+
+
+def register_filters(platform=None, options=None):
+    # project filters
+    load_monitor_filters(
+        ProjectConfig.get_instance().get("platformio", "monitor_dir"),
+        prefix="filter_",
+        options=options,
+    )
+    # platform filters
+    if platform:
+        load_monitor_filters(
+            os.path.join(platform.get_dir(), "monitor"),
+            prefix="filter_",
+            options=options,
+        )
+    # load package filters
+    pm = ToolPackageManager()
+    for pkg in pm.get_installed():
+        load_monitor_filters(
+            os.path.join(pkg.path, "monitor"), prefix="filter_", options=options
+        )
+    # default filters
+    load_monitor_filters(
+        os.path.join(fs.get_source_dir(), "commands", "device", "filters"),
+        options=options,
+    )

platformio/commands/home/rpc/handlers/app.py

@@ -14,16 +14,16 @@
 
 from __future__ import absolute_import
 
-from os.path import join
+import os
+from pathlib import Path
 
 from platformio import __version__, app, fs, util
-from platformio.project.helpers import get_project_core_dir, is_platformio_project
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import is_platformio_project
 
 
 class AppRPC:
 
-    APPSTATE_PATH = join(get_project_core_dir(), "homestate.json")
-
     IGNORE_STORAGE_KEYS = [
         "cid",
         "coreVersion",
@@ -34,9 +34,16 @@ class AppRPC:
         "projectsDir",
     ]
 
+    @staticmethod
+    def get_state_path():
+        core_dir = ProjectConfig.get_instance().get("platformio", "core_dir")
+        if not os.path.isdir(core_dir):
+            os.makedirs(core_dir)
+        return os.path.join(core_dir, "homestate.json")
+
     @staticmethod
     def load_state():
-        with app.State(AppRPC.APPSTATE_PATH, lock=True) as state:
+        with app.State(AppRPC.get_state_path(), lock=True) as state:
             storage = state.get("storage", {})
 
             # base data
@@ -58,9 +65,13 @@ class AppRPC:
             storage["projectsDir"] = storage["coreSettings"]["projects_dir"]["value"]
 
             # skip non-existing recent projects
-            storage["recentProjects"] = [
-                p for p in storage.get("recentProjects", []) if is_platformio_project(p)
-            ]
+            storage["recentProjects"] = list(
+                set(
+                    str(Path(p).resolve())
+                    for p in storage.get("recentProjects", [])
+                    if is_platformio_project(p)
+                )
+            )
 
             state["storage"] = storage
             state.modified = False  # skip saving extra fields
@@ -72,7 +83,7 @@ class AppRPC:
 
     @staticmethod
     def save_state(state):
-        with app.State(AppRPC.APPSTATE_PATH, lock=True) as s:
+        with app.State(AppRPC.get_state_path(), lock=True) as s:
             s.clear()
             s.update(state)
             storage = s.get("storage", {})

platformio/commands/home/rpc/handlers/ide.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import time
+from pathlib import Path
 
 from ajsonrpc.core import JSONRPC20DispatchException
 
@@ -20,29 +21,65 @@ from platformio.compat import aio_get_running_loop
 
 
 class IDERPC:
+
+    COMMAND_TIMEOUT = 1.5  # in seconds
+
     def __init__(self):
-        self._queue = {}
+        self._ide_queue = []
+        self._cmd_queue = {}
 
-    def send_command(self, sid, command, params):
-        if not self._queue.get(sid):
-            raise JSONRPC20DispatchException(
-                code=4005, message="PIO Home IDE agent is not started"
-            )
-        while self._queue[sid]:
-            self._queue[sid].pop().set_result(
-                {"id": time.time(), "method": command, "params": params}
-            )
+    async def listen_commands(self):
+        f = aio_get_running_loop().create_future()
+        self._ide_queue.append(f)
+        self._process_commands()
+        return await f
 
-    async def listen_commands(self, sid=0):
-        if sid not in self._queue:
-            self._queue[sid] = []
-        self._queue[sid].append(aio_get_running_loop().create_future())
-        return await self._queue[sid][-1]
-
-    def open_project(self, sid, project_dir):
-        return self.send_command(sid, "open_project", project_dir)
-
-    def open_text_document(self, sid, path, line=None, column=None):
-        return self.send_command(
-            sid, "open_text_document", dict(path=path, line=line, column=column)
+    async def send_command(self, command, params=None):
+        cmd_id = f"ide-{command}-{time.time()}"
+        self._cmd_queue[cmd_id] = {
+            "method": command,
+            "params": params,
+            "time": time.time(),
+            "future": aio_get_running_loop().create_future(),
+        }
+        self._process_commands()
+        # in case if IDE agent has not been started
+        aio_get_running_loop().call_later(
+            self.COMMAND_TIMEOUT + 0.1, self._process_commands
         )
+        return await self._cmd_queue[cmd_id]["future"]
+
+    def on_command_result(self, cmd_id, value):
+        if cmd_id not in self._cmd_queue:
+            return
+        if self._cmd_queue[cmd_id]["method"] == "get_pio_project_dirs":
+            value = [str(Path(p).resolve()) for p in value]
+        self._cmd_queue[cmd_id]["future"].set_result(value)
+        del self._cmd_queue[cmd_id]
+
+    def _process_commands(self):
+        for cmd_id in list(self._cmd_queue):
+            cmd_data = self._cmd_queue[cmd_id]
+            if cmd_data["future"].done():
+                del self._cmd_queue[cmd_id]
+                continue
+
+            if (
+                not self._ide_queue
+                and (time.time() - cmd_data["time"]) > self.COMMAND_TIMEOUT
+            ):
+                cmd_data["future"].set_exception(
+                    JSONRPC20DispatchException(
+                        code=4005, message="PIO Home IDE agent is not started"
+                    )
+                )
+                continue
+
+            while self._ide_queue:
+                self._ide_queue.pop().set_result(
+                    {
+                        "id": cmd_id,
+                        "method": cmd_data["method"],
+                        "params": cmd_data["params"],
+                    }
+                )

platformio/commands/home/rpc/handlers/project.py

@@ -23,10 +23,10 @@ from ajsonrpc.core import JSONRPC20DispatchException
 from platformio import exception, fs
 from platformio.commands.home.rpc.handlers.app import AppRPC
 from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
-from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.project.config import ProjectConfig
 from platformio.project.exception import ProjectError
+from platformio.project.generator import ProjectGenerator
 from platformio.project.helpers import get_project_dir, is_platformio_project
 from platformio.project.options import get_config_options_schema
 
@@ -81,7 +81,7 @@ class ProjectRPC:
             data["description"] = config.get("platformio", "description")
             data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", []))
 
-            libdeps_dir = config.get_optional_dir("libdeps")
+            libdeps_dir = config.get("platformio", "libdeps_dir")
             for section in config.sections():
                 if not section.startswith("env:"):
                     continue
@@ -93,7 +93,7 @@ class ProjectRPC:
             # skip non existing folders and resolve full path
             for key in ("envLibdepsDirs", "libExtraDirs"):
                 data[key] = [
-                    fs.expanduser(d) if d.startswith("~") else os.path.realpath(d)
+                    fs.expanduser(d) if d.startswith("~") else os.path.abspath(d)
                     for d in data[key]
                     if os.path.isdir(d)
                 ]
@@ -253,7 +253,7 @@ class ProjectRPC:
 
         with fs.cd(project_dir):
             config = ProjectConfig()
-            src_dir = config.get_optional_dir("src")
+            src_dir = config.get("platformio", "src_dir")
             main_path = os.path.join(
                 src_dir, "main.%s" % ("cpp" if is_cpp_project else "c")
             )
@@ -308,7 +308,7 @@ class ProjectRPC:
         )
         with fs.cd(project_dir):
             config = ProjectConfig()
-            src_dir = config.get_optional_dir("src")
+            src_dir = config.get("platformio", "src_dir")
             if os.path.isdir(src_dir):
                 fs.rmtree(src_dir)
             shutil.copytree(arduino_project_dir, src_dir, symlinks=True)

platformio/commands/home/rpc/server.py

@@ -35,7 +35,7 @@ class JSONRPCServerFactoryBase:
     def __call__(self, *args, **kwargs):
         raise NotImplementedError
 
-    def addObjectHandler(self, handler, namespace):
+    def add_object_handler(self, handler, namespace):
         self.manager.dispatcher.add_object(handler, prefix="%s." % namespace)
 
     def on_client_connect(self):
@@ -92,6 +92,6 @@ class WebSocketJSONRPCServer(WebSocketEndpoint):
     async def _handle_rpc(self, websocket, data):
         # pylint: disable=no-member
         response = await self.factory.manager.get_response_for_payload(data)
-        if response.error:
+        if response.error and response.error.data:
             click.secho("Error: %s" % response.error.data, fg="red", err=True)
         await websocket.send_text(self.factory.manager.serialize(response.body))

platformio/commands/home/run.py

@@ -65,13 +65,13 @@ def run_server(host, port, no_open, shutdown_timeout, home_url):
         raise PlatformioException("Invalid path to PIO Home Contrib")
 
     ws_rpc_factory = WebSocketJSONRPCServerFactory(shutdown_timeout)
-    ws_rpc_factory.addObjectHandler(AccountRPC(), namespace="account")
-    ws_rpc_factory.addObjectHandler(AppRPC(), namespace="app")
-    ws_rpc_factory.addObjectHandler(IDERPC(), namespace="ide")
-    ws_rpc_factory.addObjectHandler(MiscRPC(), namespace="misc")
-    ws_rpc_factory.addObjectHandler(OSRPC(), namespace="os")
-    ws_rpc_factory.addObjectHandler(PIOCoreRPC(), namespace="core")
-    ws_rpc_factory.addObjectHandler(ProjectRPC(), namespace="project")
+    ws_rpc_factory.add_object_handler(AccountRPC(), namespace="account")
+    ws_rpc_factory.add_object_handler(AppRPC(), namespace="app")
+    ws_rpc_factory.add_object_handler(IDERPC(), namespace="ide")
+    ws_rpc_factory.add_object_handler(MiscRPC(), namespace="misc")
+    ws_rpc_factory.add_object_handler(OSRPC(), namespace="os")
+    ws_rpc_factory.add_object_handler(PIOCoreRPC(), namespace="core")
+    ws_rpc_factory.add_object_handler(ProjectRPC(), namespace="project")
 
     path = urlparse(home_url).path
     routes = [

platformio/commands/lib/command.py

@@ -43,7 +43,7 @@ CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps"
 
 
 def get_project_global_lib_dir():
-    return ProjectConfig.get_instance().get_optional_dir("globallib")
+    return ProjectConfig.get_instance().get("platformio", "globallib_dir")
 
 
 @click.group(short_help="Library manager")
@@ -111,7 +111,7 @@ def cli(ctx, **options):
                 os.path.join(storage_dir, "platformio.ini")
             )
             config.validate(options["environment"], silent=in_silence)
-            libdeps_dir = config.get_optional_dir("libdeps")
+            libdeps_dir = config.get("platformio", "libdeps_dir")
             for env in config.envs():
                 if options["environment"] and env not in options["environment"]:
                     continue
@@ -355,7 +355,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         "get",
         "/v2/lib/search",
         params=dict(query=" ".join(query), page=page),
-        cache_valid="1d",
+        x_cache_valid="1d",
     )
 
     if json_output:
@@ -408,7 +408,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
             "get",
             "/v2/lib/search",
             params=dict(query=" ".join(query), page=int(result["page"]) + 1),
-            cache_valid="1d",
+            x_cache_valid="1d",
         )
 
 
@@ -440,7 +440,9 @@ def lib_show(library, json_output):
     lm = LibraryPackageManager()
     lib_id = lm.reveal_registry_package_id(library, silent=json_output)
     regclient = lm.get_registry_client_instance()
-    lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h")
+    lib = regclient.fetch_json_data(
+        "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h"
+    )
     if json_output:
         return click.echo(json.dumps(lib))
 
@@ -535,7 +537,7 @@ def lib_register(config_url):  # pylint: disable=unused-argument
 @click.option("--json-output", is_flag=True)
 def lib_stats(json_output):
     regclient = LibraryPackageManager().get_registry_client_instance()
-    result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h")
+    result = regclient.fetch_json_data("get", "/v2/lib/stats", x_cache_valid="1h")
 
     if json_output:
         return click.echo(json.dumps(result))

platformio/commands/package.py

@@ -191,6 +191,12 @@ def package_publish(  # pylint: disable=too-many-arguments, too-many-locals
                 abort=True,
             )
 
+        click.secho(
+            "The package publishing may take some time depending "
+            "on your Internet connection and the package size.",
+            fg="yellow",
+        )
+        click.echo("Publishing...")
         response = RegistryClient().publish_package(
             owner, type_, archive_path, released_at, private, notify
         )

platformio/commands/platform.py

@@ -61,7 +61,7 @@ def platform_frameworks(query, json_output):
     regclient = PlatformPackageManager().get_registry_client_instance()
     frameworks = []
     for framework in regclient.fetch_json_data(
-        "get", "/v2/frameworks", cache_valid="1d"
+        "get", "/v2/frameworks", x_cache_valid="1d"
     ):
         if query == "all":
             query = ""
@@ -354,7 +354,7 @@ def _print_platforms(platforms):
 
 def _get_registry_platforms():
     regclient = PlatformPackageManager().get_registry_client_instance()
-    return regclient.fetch_json_data("get", "/v2/platforms", cache_valid="1d")
+    return regclient.fetch_json_data("get", "/v2/platforms", x_cache_valid="1d")
 
 
 def _get_platform_data(*args, **kwargs):

platformio/commands/project.py

@@ -22,11 +22,11 @@ from tabulate import tabulate
 
 from platformio import fs
 from platformio.commands.platform import platform_install as cli_platform_install
-from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.platform.exception import UnknownBoard
 from platformio.project.config import ProjectConfig
 from platformio.project.exception import NotPlatformIOProjectError
+from platformio.project.generator import ProjectGenerator
 from platformio.project.helpers import is_platformio_project, load_project_ide_data
 
 
@@ -191,10 +191,7 @@ def project_init(
                 os.path.join(project_dir, "platformio.ini")
             )
         config.validate()
-        pg = ProjectGenerator(
-            config, environment or get_best_envname(config, board), ide
-        )
-        pg.generate()
+        ProjectGenerator(config, environment, ide, board).generate()
 
     if is_new_project:
         init_cvs_ignore(project_dir)
@@ -226,10 +223,10 @@ def init_base_project(project_dir):
         config = ProjectConfig()
         config.save()
         dir_to_readme = [
-            (config.get_optional_dir("src"), None),
-            (config.get_optional_dir("include"), init_include_readme),
-            (config.get_optional_dir("lib"), init_lib_readme),
-            (config.get_optional_dir("test"), init_test_readme),
+            (config.get("platformio", "src_dir"), None),
+            (config.get("platformio", "include_dir"), init_include_readme),
+            (config.get("platformio", "lib_dir"), init_lib_readme),
+            (config.get("platformio", "test_dir"), init_test_readme),
         ]
         for (path, cb) in dir_to_readme:
             if os.path.isdir(path):
@@ -441,23 +438,3 @@ def update_project_env(project_dir, environment, project_option):
         config.set(section, _name.strip(), _value.strip())
 
     config.save()
-
-
-def get_best_envname(config, board_ids=None):
-    envname = None
-    default_envs = config.default_envs()
-    if default_envs:
-        envname = default_envs[0]
-        if not board_ids:
-            return envname
-
-    for env in config.envs():
-        if not board_ids:
-            return env
-        if not envname:
-            envname = env
-        items = config.items(env=env, as_dict=True)
-        if "board" in items and items.get("board") in board_ids:
-            return env
-
-    return envname

platformio/commands/remote/client/agent_service.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import os
-from os.path import getatime, getmtime, isdir, isfile, join
 
 from twisted.logger import LogLevel  # pylint: disable=import-error
 from twisted.spread import pb  # pylint: disable=import-error
@@ -25,15 +24,16 @@ from platformio.commands.remote.ac.serial import SerialPortAsyncCmd
 from platformio.commands.remote.client.base import RemoteClientBase
 from platformio.project.config import ProjectConfig
 from platformio.project.exception import NotPlatformIOProjectError
-from platformio.project.helpers import get_project_core_dir
 
 
 class RemoteAgentService(RemoteClientBase):
     def __init__(self, name, share, working_dir=None):
         RemoteClientBase.__init__(self)
         self.log_level = LogLevel.info
-        self.working_dir = working_dir or join(get_project_core_dir(), "remote")
-        if not isdir(self.working_dir):
+        self.working_dir = working_dir or os.path.join(
+            ProjectConfig.get_instance().get("platformio", "core_dir"), "remote"
+        )
+        if not os.path.isdir(self.working_dir):
             os.makedirs(self.working_dir)
         if name:
             self.name = str(name)[:50]
@@ -138,14 +138,14 @@ class RemoteAgentService(RemoteClientBase):
         self, command, options
     ):
         assert options and "project_id" in options
-        project_dir = join(self.working_dir, "projects", options["project_id"])
-        origin_pio_ini = join(project_dir, "platformio.ini")
-        back_pio_ini = join(project_dir, "platformio.ini.bak")
+        project_dir = os.path.join(self.working_dir, "projects", options["project_id"])
+        origin_pio_ini = os.path.join(project_dir, "platformio.ini")
+        back_pio_ini = os.path.join(project_dir, "platformio.ini.bak")
 
         # remove insecure project options
         try:
             conf = ProjectConfig(origin_pio_ini)
-            if isfile(back_pio_ini):
+            if os.path.isfile(back_pio_ini):
                 os.remove(back_pio_ini)
             os.rename(origin_pio_ini, back_pio_ini)
             # cleanup
@@ -159,7 +159,10 @@ class RemoteAgentService(RemoteClientBase):
             conf.save(origin_pio_ini)
 
             # restore A/M times
-            os.utime(origin_pio_ini, (getatime(back_pio_ini), getmtime(back_pio_ini)))
+            os.utime(
+                origin_pio_ini,
+                (os.path.getatime(back_pio_ini), os.path.getmtime(back_pio_ini)),
+            )
         except NotPlatformIOProjectError as e:
             raise pb.Error(str(e))
 
@@ -194,8 +197,8 @@ class RemoteAgentService(RemoteClientBase):
             paused_acs.append(ac)
 
         def _cb_on_end():
-            if isfile(back_pio_ini):
-                if isfile(origin_pio_ini):
+            if os.path.isfile(back_pio_ini):
+                if os.path.isfile(origin_pio_ini):
                     os.remove(origin_pio_ini)
                 os.rename(back_pio_ini, origin_pio_ini)
             for ac in paused_acs:

platformio/commands/remote/client/device_monitor.py

@@ -84,7 +84,7 @@ class DeviceMonitorClient(  # pylint: disable=too-many-instance-attributes
         self._ac_id = None
         self._d_acread = None
         self._d_acwrite = None
-        self._acwrite_buffer = ""
+        self._acwrite_buffer = b""
 
     def agent_pool_ready(self):
         d = task.deferLater(
@@ -226,7 +226,7 @@ class DeviceMonitorClient(  # pylint: disable=too-many-instance-attributes
             return
 
         data = self._acwrite_buffer
-        self._acwrite_buffer = ""
+        self._acwrite_buffer = b""
         try:
             d = self.agentpool.callRemote("acwrite", self._agent_id, self._ac_id, data)
             d.addCallback(self.cb_acwrite_result)
@@ -237,4 +237,4 @@ class DeviceMonitorClient(  # pylint: disable=too-many-instance-attributes
     def cb_acwrite_result(self, result):
         assert result > 0
         if self._acwrite_buffer:
-            self.acwrite_data("")
+            self.acwrite_data(b"")

platformio/commands/remote/client/run_or_test.py

@@ -69,8 +69,8 @@ class RunOrTestClient(AsyncClientBase):
                 os.path.join(self.options["project_dir"], "platformio.ini")
             )
             psync.add_item(cfg.path, "platformio.ini")
-            psync.add_item(cfg.get_optional_dir("shared"), "shared")
-            psync.add_item(cfg.get_optional_dir("boards"), "boards")
+            psync.add_item(cfg.get("platformio", "shared_dir"), "shared")
+            psync.add_item(cfg.get("platformio", "boards_dir"), "boards")
 
             if self.options["force_remote"]:
                 self._add_project_source_items(cfg, psync)
@@ -78,26 +78,26 @@ class RunOrTestClient(AsyncClientBase):
                 self._add_project_binary_items(cfg, psync)
 
             if self.command == "test":
-                psync.add_item(cfg.get_optional_dir("test"), "test")
+                psync.add_item(cfg.get("platformio", "test_dir"), "test")
 
     def _add_project_source_items(self, cfg, psync):
-        psync.add_item(cfg.get_optional_dir("lib"), "lib")
+        psync.add_item(cfg.get("platformio", "lib_dir"), "lib")
         psync.add_item(
-            cfg.get_optional_dir("include"),
+            cfg.get("platformio", "include_dir"),
             "include",
             cb_filter=self._cb_tarfile_filter,
         )
         psync.add_item(
-            cfg.get_optional_dir("src"), "src", cb_filter=self._cb_tarfile_filter
+            cfg.get("platformio", "src_dir"), "src", cb_filter=self._cb_tarfile_filter
         )
         if set(["buildfs", "uploadfs", "uploadfsota"]) & set(
             self.options.get("target", [])
         ):
-            psync.add_item(cfg.get_optional_dir("data"), "data")
+            psync.add_item(cfg.get("platformio", "data_dir"), "data")
 
     @staticmethod
     def _add_project_binary_items(cfg, psync):
-        build_dir = cfg.get_optional_dir("build")
+        build_dir = cfg.get("platformio", "build_dir")
         for env_name in os.listdir(build_dir):
             env_dir = os.path.join(build_dir, env_name)
             if not os.path.isdir(env_dir):

platformio/commands/remote/command.py

@@ -37,7 +37,7 @@ from platformio.project.exception import NotPlatformIOProjectError
 @click.pass_context
 def cli(ctx, agent):
     ctx.obj = agent
-    inject_contrib_pysite(verify_openssl=True)
+    inject_contrib_pysite()
 
 
 @cli.group("agent", short_help="Start a new agent or list active")

platformio/commands/run/command.py

@@ -101,7 +101,7 @@ def cli(
 
         # clean obsolete build dir
         if not disable_auto_clean:
-            build_dir = config.get_optional_dir("build")
+            build_dir = config.get("platformio", "build_dir")
             try:
                 clean_build_dir(build_dir, config)
             except:  # pylint: disable=bare-except

platformio/commands/run/helpers.py

@@ -23,8 +23,8 @@ from platformio.project.helpers import compute_project_checksum, get_project_dir
 
 def handle_legacy_libdeps(project_dir, config):
     legacy_libdeps_dir = join(project_dir, ".piolibdeps")
-    if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get_optional_dir(
-        "libdeps"
+    if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get(
+        "platformio", "libdeps_dir"
     ):
         return
     if not config.has_section("env"):

platformio/commands/system/command.py

@@ -14,7 +14,6 @@
 
 import json
 import platform
-import subprocess
 import sys
 
 import click
@@ -22,6 +21,7 @@ from tabulate import tabulate
 
 from platformio import __version__, compat, fs, proc, util
 from platformio.commands.system.completion import (
+    ShellType,
     get_completion_install_path,
     install_completion_code,
     uninstall_completion_code,
@@ -64,7 +64,7 @@ def system_info(json_output):
     }
     data["core_dir"] = {
         "title": "PlatformIO Core Directory",
-        "value": project_config.get_optional_dir("core"),
+        "value": project_config.get("platformio", "core_dir"),
     }
     data["platformio_exe"] = {
         "title": "PlatformIO Core Executable",
@@ -88,7 +88,7 @@ def system_info(json_output):
         "title": "Tools & Toolchains",
         "value": len(
             ToolPackageManager(
-                project_config.get_optional_dir("packages")
+                project_config.get("platformio", "packages_dir")
             ).get_installed()
         ),
     }
@@ -150,23 +150,11 @@ def system_prune(force, dry_run, cache, core_packages, platform_packages):
 
 @cli.group("completion", short_help="Shell completion support")
 def completion():
-    # pylint: disable=import-error,import-outside-toplevel
-    try:
-        import click_completion  # pylint: disable=unused-import,unused-variable
-    except ImportError:
-        click.echo("Installing dependent packages...")
-        subprocess.check_call(
-            [proc.get_pythonexe_path(), "-m", "pip", "install", "click-completion"],
-        )
+    pass
 
 
 @completion.command("install", short_help="Install shell completion files/code")
-@click.option(
-    "--shell",
-    default=None,
-    type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]),
-    help="The shell type, default=auto",
-)
+@click.argument("shell", type=click.Choice([t.value for t in ShellType]))
 @click.option(
     "--path",
     type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
@@ -174,26 +162,18 @@ def completion():
     "The standard installation path is used by default.",
 )
 def completion_install(shell, path):
-
-    import click_completion  # pylint: disable=import-outside-toplevel,import-error
-
-    shell = shell or click_completion.get_auto_shell()
+    shell = ShellType(shell)
     path = path or get_completion_install_path(shell)
     install_completion_code(shell, path)
     click.echo(
         "PlatformIO CLI completion has been installed for %s shell to %s \n"
         "Please restart a current shell session."
-        % (click.style(shell, fg="cyan"), click.style(path, fg="blue"))
+        % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue"))
     )
 
 
 @completion.command("uninstall", short_help="Uninstall shell completion files/code")
-@click.option(
-    "--shell",
-    default=None,
-    type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]),
-    help="The shell type, default=auto",
-)
+@click.argument("shell", type=click.Choice([t.value for t in ShellType]))
 @click.option(
     "--path",
     type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True),
@@ -201,14 +181,11 @@ def completion_install(shell, path):
     "The standard installation path is used by default.",
 )
 def completion_uninstall(shell, path):
-
-    import click_completion  # pylint: disable=import-outside-toplevel,import-error
-
-    shell = shell or click_completion.get_auto_shell()
+    shell = ShellType(shell)
     path = path or get_completion_install_path(shell)
     uninstall_completion_code(shell, path)
     click.echo(
         "PlatformIO CLI completion has been uninstalled for %s shell from %s \n"
         "Please restart a current shell session."
-        % (click.style(shell, fg="cyan"), click.style(path, fg="blue"))
+        % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue"))
     )

platformio/commands/system/completion.py

@@ -13,61 +13,75 @@
 # limitations under the License.
 
 import os
-import subprocess
+from enum import Enum
 
 import click
 
+from platformio.compat import IS_MACOS
+
+
+class ShellType(Enum):
+    FISH = "fish"
+    ZSH = "zsh"
+    BASH = "bash"
+
 
 def get_completion_install_path(shell):
     home_dir = os.path.expanduser("~")
     prog_name = click.get_current_context().find_root().info_name
-    if shell == "fish":
+    if shell == ShellType.FISH:
         return os.path.join(
             home_dir, ".config", "fish", "completions", "%s.fish" % prog_name
         )
-    if shell == "bash":
-        return os.path.join(home_dir, ".bash_completion")
-    if shell == "zsh":
+    if shell == ShellType.ZSH:
         return os.path.join(home_dir, ".zshrc")
-    if shell == "powershell":
-        return subprocess.check_output(
-            ["powershell", "-NoProfile", "echo $profile"]
-        ).strip()
+    if shell == ShellType.BASH:
+        return os.path.join(home_dir, ".bash_completion")
+    raise click.ClickException("%s is not supported." % shell)
+
+
+def get_completion_code(shell):
+    if shell == ShellType.FISH:
+        return "eval (env _PIO_COMPLETE=fish_source pio)"
+    if shell == ShellType.ZSH:
+        code = "autoload -Uz compinit\ncompinit\n" if IS_MACOS else ""
+        return code + 'eval "$(_PIO_COMPLETE=zsh_source pio)"'
+    if shell == ShellType.BASH:
+        return 'eval "$(_PIO_COMPLETE=bash_source pio)"'
     raise click.ClickException("%s is not supported." % shell)
 
 
 def is_completion_code_installed(shell, path):
-    if shell == "fish" or not os.path.exists(path):
+    if shell == ShellType.FISH or not os.path.exists(path):
         return False
-
-    import click_completion  # pylint: disable=import-error,import-outside-toplevel
-
     with open(path, encoding="utf8") as fp:
-        return click_completion.get_code(shell=shell) in fp.read()
+        return get_completion_code(shell) in fp.read()
 
 
 def install_completion_code(shell, path):
-    import click_completion  # pylint: disable=import-error,import-outside-toplevel
-
     if is_completion_code_installed(shell, path):
         return None
-
-    return click_completion.install(shell=shell, path=path, append=shell != "fish")
+    append = shell != ShellType.FISH
+    with open(path, mode="a" if append else "w", encoding="utf8") as fp:
+        if append:
+            fp.write("\n\n# Begin: PlatformIO Core completion support\n")
+        fp.write(get_completion_code(shell))
+        if append:
+            fp.write("\n# End: PlatformIO Core completion support\n\n")
+    return True
 
 
 def uninstall_completion_code(shell, path):
     if not os.path.exists(path):
         return True
-    if shell == "fish":
+    if shell == ShellType.FISH:
         os.remove(path)
         return True
 
-    import click_completion  # pylint: disable=import-error,import-outside-toplevel
-
     with open(path, "r+", encoding="utf8") as fp:
         contents = fp.read()
         fp.seek(0)
         fp.truncate()
-        fp.write(contents.replace(click_completion.get_code(shell=shell), ""))
+        fp.write(contents.replace(get_completion_code(shell), ""))
 
     return True

platformio/commands/test/helpers.py

@@ -18,7 +18,7 @@ from platformio import exception
 
 
 def get_test_names(config):
-    test_dir = config.get_optional_dir("test")
+    test_dir = config.get("platformio", "test_dir")
     if not os.path.isdir(test_dir):
         raise exception.TestDirNotExists(test_dir)
     names = []

platformio/commands/test/native.py

@@ -31,7 +31,7 @@ class NativeTestProcessor(TestProcessorBase):
         return self.run()
 
     def run(self):
-        build_dir = self.options["project_config"].get_optional_dir("build")
+        build_dir = self.options["project_config"].get("platformio", "build_dir")
         result = proc.exec_command(
             [join(build_dir, self.env_name, "program")],
             stdout=LineBufferedAsyncPipe(self.on_run_out),

platformio/commands/test/processor.py

@@ -124,7 +124,7 @@ class TestProcessorBase(object):
     def build_or_upload(self, target):
         if not self._output_file_generated:
             self.generate_output_file(
-                self.options["project_config"].get_optional_dir("test")
+                self.options["project_config"].get("platformio", "test_dir")
             )
             self._output_file_generated = True
 

platformio/debug/config/base.py

@@ -20,7 +20,7 @@ from platformio.compat import string_types
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.debug.helpers import reveal_debug_port
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import get_project_core_dir, load_project_ide_data
+from platformio.project.helpers import load_project_ide_data
 from platformio.project.options import ProjectOptions
 
 
@@ -153,7 +153,14 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
         raise DebugInvalidOptionsError("Could not load a build configuration")
 
     def _configure_server(self):
+        # user disabled server in platformio.ini
+        if "debug_server" in self.env_options and not self.env_options.get(
+            "debug_server"
+        ):
+            return None
+
         result = None
+
         # specific server per a system
         if isinstance(self.tool_settings.get("server", {}), list):
             for item in self.tool_settings["server"][:]:
@@ -208,9 +215,9 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
     def reveal_patterns(self, source, recursive=True):
         program_path = self.program_path or ""
         patterns = {
-            "PLATFORMIO_CORE_DIR": get_project_core_dir(),
+            "PLATFORMIO_CORE_DIR": self.project_config.get("platformio", "core_dir"),
             "PYTHONEXE": proc.get_pythonexe_path(),
-            "PROJECT_DIR": self.project_config.path,
+            "PROJECT_DIR": os.getcwd(),
             "PROG_PATH": program_path,
             "PROG_DIR": os.path.dirname(program_path),
             "PROG_NAME": os.path.basename(os.path.splitext(program_path)[0]),

platformio/debug/process/gdb.py

@@ -61,7 +61,7 @@ class GDBClientProcess(DebugClientProcess):
     def _get_data_dir(gdb_path):
         if "msp430" in gdb_path:
             return None
-        gdb_data_dir = os.path.realpath(
+        gdb_data_dir = os.path.abspath(
             os.path.join(os.path.dirname(gdb_path), "..", "share", "gdb")
         )
         return gdb_data_dir if os.path.isdir(gdb_data_dir) else None
@@ -98,7 +98,7 @@ class GDBClientProcess(DebugClientProcess):
             ]
 
         banner = [
-            "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n",
+            "echo PlatformIO Unified Debugger -> https://bit.ly/pio-debug\\n",
             "echo PlatformIO: debug_tool = %s\\n" % self.debug_config.tool_name,
             "echo PlatformIO: Initializing remote target...\\n",
         ]
@@ -157,7 +157,7 @@ class GDBClientProcess(DebugClientProcess):
             % self.debug_config.init_break
         )
         self.console_log(
-            "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n"
+            "PlatformIO: More configuration options -> https://bit.ly/pio-debug\n"
         )
         if self.debug_config.platform.is_embedded():
             self.transport.get_pipe_transport(0).write(

platformio/fs.py

@@ -24,7 +24,7 @@ import sys
 
 import click
 
-from platformio import exception
+from platformio import exception, proc
 from platformio.compat import IS_WINDOWS
 
 
@@ -41,7 +41,7 @@ class cd(object):
 
 
 def get_source_dir():
-    curpath = os.path.realpath(__file__)
+    curpath = os.path.abspath(__file__)
     if not os.path.isfile(curpath):
         for p in sys.path:
             if os.path.isfile(os.path.join(p, __file__)):
@@ -119,7 +119,7 @@ def ensure_udev_rules():
     if not any(os.path.isfile(p) for p in installed_rules):
         raise exception.MissedUdevRules
 
-    origin_path = os.path.realpath(
+    origin_path = os.path.abspath(
         os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules")
     )
     if not os.path.isfile(origin_path):
@@ -181,6 +181,25 @@ def to_unix_path(path):
     return re.sub(r"[\\]+", "/", path)
 
 
+def normalize_path(path):
+    path = os.path.abspath(path)
+    if not IS_WINDOWS or not path.startswith("\\\\"):
+        return path
+    try:
+        result = proc.exec_command(["net", "use"])
+        if result["returncode"] != 0:
+            return path
+        share_re = re.compile(r"\s([A-Z]\:)\s+(\\\\[^\s]+)")
+        for line in result["out"].split("\n"):
+            share = share_re.search(line)
+            if not share:
+                continue
+            path = path.replace(share.group(2), share.group(1))
+    except OSError:
+        pass
+    return path
+
+
 def expanduser(path):
     """
     Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE

platformio/ide/__init__.py

@@ -1,13 +0,0 @@
-# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

platformio/maintenance.py

@@ -35,7 +35,6 @@ from platformio.package.manager.tool import ToolPackageManager
 from platformio.package.meta import PackageSpec
 from platformio.package.version import pepver_to_semver
 from platformio.platform.factory import PlatformFactory
-from platformio.proc import is_container
 
 
 def on_platformio_start(ctx, force, caller):
@@ -78,17 +77,12 @@ def set_caller(caller=None):
     caller = caller or os.getenv("PLATFORMIO_CALLER")
     if caller:
         return app.set_session_var("caller_id", caller)
-    if os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"):
+    if os.getenv("CODESPACES"):
+        caller = "codespaces"
+    elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"):
         caller = "vscode"
-    elif os.getenv("GITPOD_INSTANCE_ID") or os.getenv("GITPOD_WORKSPACE_URL"):
+    elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"):
         caller = "gitpod"
-    elif is_container():
-        if os.getenv("C9_UID"):
-            caller = "C9"
-        elif os.getenv("USER") == "cabox":
-            caller = "CA"
-        elif os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")):
-            caller = "Che"
     return app.set_session_var("caller_id", caller)
 
 

platformio/package/lockfile.py

@@ -48,7 +48,7 @@ class LockFile(object):
     def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY):
         self.timeout = timeout
         self.delay = delay
-        self._lock_path = os.path.realpath(path) + ".lock"
+        self._lock_path = os.path.abspath(path) + ".lock"
         self._fp = None
 
     def _lock(self):

platformio/package/manager/_download.py

@@ -76,7 +76,7 @@ class PackageManagerDownloadMixin(object):
                             raise_error = True
                     if raise_error:
                         self.print_message(
-                            "Error: Please read http://bit.ly/package-manager-ioerror",
+                            "Error: Please read https://bit.ly/package-manager-ioerror",
                             fg="red",
                             err=True,
                         )

platformio/package/manager/_registry.py

@@ -54,6 +54,7 @@ class RegistryFileMirrorIterator(object):
             params=dict(bypass=",".join(self._visited_mirrors))
             if self._visited_mirrors
             else None,
+            x_with_authorization=RegistryClient.allowed_private_packages(),
         )
         stop_conditions = [
             response.status_code not in (302, 307),
@@ -180,7 +181,7 @@ class PackageManageRegistryMixin(object):
             )
         self.print_message(
             "Please specify detailed REQUIREMENTS using package owner and version "
-            "(showed above) to avoid name conflicts",
+            "(shown above) to avoid name conflicts",
             fg="yellow",
         )
 

platformio/package/manager/base.py

@@ -252,9 +252,9 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods
         # external "URL" mismatch
         if spec.external:
             # local folder mismatch
-            if os.path.realpath(spec.url) == os.path.realpath(pkg.path) or (
+            if os.path.abspath(spec.url) == os.path.abspath(pkg.path) or (
                 spec.url.startswith("file://")
-                and os.path.realpath(pkg.path) == os.path.realpath(spec.url[7:])
+                and os.path.abspath(pkg.path) == os.path.abspath(spec.url[7:])
             ):
                 return True
             if spec.url != pkg.metadata.spec.url:

platformio/package/manager/core.py

@@ -20,6 +20,7 @@ import sys
 from datetime import date
 
 from platformio import __core_packages__, exception, fs, util
+from platformio.exception import UserSideException
 from platformio.package.exception import UnknownPackageError
 from platformio.package.manager.tool import ToolPackageManager
 from platformio.package.meta import PackageItem, PackageSpec
@@ -101,7 +102,7 @@ def remove_unnecessary_core_packages(dry_run=False):
     return candidates
 
 
-def inject_contrib_pysite(verify_openssl=False):
+def inject_contrib_pysite():
     # pylint: disable=import-outside-toplevel
     from site import addsitedir
 
@@ -119,12 +120,10 @@ def inject_contrib_pysite(verify_openssl=False):
     addsitedir(contrib_pysite_dir)
     sys.path.insert(0, contrib_pysite_dir)
 
-    if not verify_openssl:
-        return True
-
     try:
         # pylint: disable=import-error,unused-import,unused-variable
         from OpenSSL import SSL
+
     except:  # pylint: disable=bare-except
         build_contrib_pysite_package(contrib_pysite_dir)
 
@@ -152,8 +151,15 @@ def build_contrib_pysite_package(target_dir, with_metadata=True):
     ]
     if "linux" in systype:
         args.extend(["--no-binary", ":all:"])
-    for dep in get_contrib_pysite_deps():
-        subprocess.check_call(args + [dep])
+    try:
+        subprocess.run(args + get_contrib_pysite_deps(), check=True)
+    except subprocess.CalledProcessError as exc:
+        if "linux" in systype:
+            raise UserSideException(
+                "\n\nPlease ensure that the next packages are installed:\n\n"
+                "sudo apt install python3-dev libffi-dev libssl-dev\n"
+            )
+        raise exc
 
     # build manifests
     with open(
@@ -206,31 +212,15 @@ def build_contrib_pysite_package(target_dir, with_metadata=True):
 
 
 def get_contrib_pysite_deps():
-    sys_type = util.get_systype()
-    py_version = "%d%d" % (sys.version_info.major, sys.version_info.minor)
-
-    twisted_version = "20.3.0"
+    twisted_version = "21.7.0"
     result = [
+        # twisted[tls], see setup.py for %twisted_version%
         "twisted == %s" % twisted_version,
+        # pyopenssl depends on it, use RUST-less version
+        "cryptography >= 3.3, < 35.0.0",
+        "pyopenssl >= 16.0.0, <= 21.0.0",
+        "service_identity >= 18.1.0, <= 21.1.0",
     ]
-
-    # twisted[tls], see setup.py for %twisted_version%
-    result.extend(
-        ["pyopenssl >= 16.0.0", "service_identity >= 18.1.0", "idna >= 0.6, != 2.3"]
-    )
-
-    if "windows" in sys_type:
-        result.append("pypiwin32 == 223")
-        # workaround for twisted wheels
-        twisted_wheel = (
-            "https://download.lfd.uci.edu/pythonlibs/x2tqcw5k/Twisted-"
-            "%s-cp%s-cp%s-win%s.whl"
-            % (
-                twisted_version,
-                py_version,
-                py_version,
-                "_amd64" if "amd64" in sys_type else "32",
-            )
-        )
-        result[0] = twisted_wheel
+    if "windows" in util.get_systype():
+        result.append("pywin32 != 226")
     return result

platformio/package/manager/library.py

@@ -21,13 +21,15 @@ from platformio.package.exception import (
 )
 from platformio.package.manager.base import BasePackageManager
 from platformio.package.meta import PackageItem, PackageSpec, PackageType
-from platformio.project.helpers import get_project_global_lib_dir
+from platformio.project.config import ProjectConfig
 
 
 class LibraryPackageManager(BasePackageManager):  # pylint: disable=too-many-ancestors
     def __init__(self, package_dir=None):
         super(LibraryPackageManager, self).__init__(
-            PackageType.LIBRARY, package_dir or get_project_global_lib_dir()
+            PackageType.LIBRARY,
+            package_dir
+            or ProjectConfig.get_instance().get("platformio", "globallib_dir"),
         )
 
     @property
@@ -60,7 +62,7 @@ class LibraryPackageManager(BasePackageManager):  # pylint: disable=too-many-anc
 
     @staticmethod
     def find_library_root(path):
-        root_dir_signs = set(["include", "Include", "src", "Src"])
+        root_dir_signs = set(["include", "Include", "inc", "Inc", "src", "Src"])
         root_file_signs = set(
             [
                 "conanfile.py",  # Conan-based library

platformio/package/manager/platform.py

@@ -31,7 +31,7 @@ class PlatformPackageManager(BasePackageManager):  # pylint: disable=too-many-an
         self.config = ProjectConfig.get_instance()
         super(PlatformPackageManager, self).__init__(
             PackageType.PLATFORM,
-            package_dir or self.config.get_optional_dir("platforms"),
+            package_dir or self.config.get("platformio", "platforms_dir"),
         )
 
     @property
@@ -140,7 +140,7 @@ class PlatformPackageManager(BasePackageManager):  # pylint: disable=too-many-an
 
     def get_registered_boards(self):
         return self.get_registry_client_instance().fetch_json_data(
-            "get", "/v2/boards", cache_valid="1d"
+            "get", "/v2/boards", x_cache_valid="1d"
         )
 
     def get_all_boards(self):

platformio/package/manager/tool.py

@@ -20,7 +20,7 @@ from platformio.project.config import ProjectConfig
 class ToolPackageManager(BasePackageManager):  # pylint: disable=too-many-ancestors
     def __init__(self, package_dir=None):
         if not package_dir:
-            package_dir = ProjectConfig.get_instance().get_optional_dir("packages")
+            package_dir = ProjectConfig.get_instance().get("platformio", "packages_dir")
         super(ToolPackageManager, self).__init__(PackageType.TOOL, package_dir)
 
     @property

platformio/package/manifest/parser.py

@@ -167,7 +167,7 @@ class BaseManifestParser(object):
         return self._data
 
     @staticmethod
-    def str_to_list(value, sep=",", lowercase=True):
+    def str_to_list(value, sep=",", lowercase=False, unique=False):
         if isinstance(value, string_types):
             value = value.split(sep)
         assert isinstance(value, list)
@@ -178,6 +178,8 @@ class BaseManifestParser(object):
                 continue
             if lowercase:
                 item = item.lower()
+            if unique and item in result:
+                continue
             result.append(item)
         return result
 
@@ -323,12 +325,16 @@ class LibraryJsonManifestParser(BaseManifestParser):
         # normalize Union[str, list] fields
         for k in ("keywords", "platforms", "frameworks"):
             if k in data:
-                data[k] = self.str_to_list(data[k], sep=",")
+                data[k] = self.str_to_list(
+                    data[k], sep=",", lowercase=True, unique=True
+                )
 
+        if "headers" in data:
+            data["headers"] = self.str_to_list(data["headers"], sep=",", unique=True)
         if "authors" in data:
             data["authors"] = self._parse_authors(data["authors"])
         if "platforms" in data:
-            data["platforms"] = self._parse_platforms(data["platforms"]) or None
+            data["platforms"] = self._fix_platforms(data["platforms"]) or None
         if "export" in data:
             data["export"] = self._parse_export(data["export"])
         if "dependencies" in data:
@@ -361,15 +367,11 @@ class LibraryJsonManifestParser(BaseManifestParser):
         return [self.cleanup_author(author) for author in raw]
 
     @staticmethod
-    def _parse_platforms(raw):
-        assert isinstance(raw, list)
-        result = []
-        # renamed platforms
-        for item in raw:
-            if item == "espressif":
-                item = "espressif8266"
-            result.append(item)
-        return result
+    def _fix_platforms(items):
+        assert isinstance(items, list)
+        if "espressif" in items:
+            items[items.index("espressif")] = "espressif8266"
+        return items
 
     @staticmethod
     def _parse_export(raw):
@@ -430,7 +432,9 @@ class ModuleJsonManifestParser(BaseManifestParser):
         if "dependencies" in data:
             data["dependencies"] = self._parse_dependencies(data["dependencies"])
         if "keywords" in data:
-            data["keywords"] = self.str_to_list(data["keywords"], sep=",")
+            data["keywords"] = self.str_to_list(
+                data["keywords"], sep=",", lowercase=True, unique=True
+            )
         return data
 
     def _parse_authors(self, raw):
@@ -475,11 +479,13 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
                 homepage=homepage,
                 repository=repository or None,
                 description=self._parse_description(data),
-                platforms=self._parse_platforms(data) or ["*"],
-                keywords=self._parse_keywords(data),
+                platforms=self._parse_platforms(data) or None,
+                keywords=self._parse_keywords(data) or None,
                 export=self._parse_export(),
             )
         )
+        if "includes" in data:
+            data["headers"] = self.str_to_list(data["includes"], sep=",", unique=True)
         if "author" in data:
             data["authors"] = self._parse_authors(data)
             for key in ("author", "maintainer"):
@@ -511,22 +517,24 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
         for k in ("sentence", "paragraph"):
             if k in properties and properties[k] not in lines:
                 lines.append(properties[k])
-        if len(lines) == 2 and not lines[0].endswith("."):
-            lines[0] += "."
+        if len(lines) == 2:
+            if not lines[0].endswith("."):
+                lines[0] += "."
+            if len(lines[0]) + len(lines[1]) >= 1000:
+                del lines[1]
         return " ".join(lines)
 
-    @staticmethod
-    def _parse_keywords(properties):
-        result = []
-        for item in re.split(r"[\s/]+", properties.get("category", "uncategorized")):
-            item = item.strip()
-            if not item:
-                continue
-            result.append(item.lower())
-        return result
+    def _parse_keywords(self, properties):
+        return self.str_to_list(
+            re.split(
+                r"[\s/]+",
+                properties.get("category", ""),
+            ),
+            lowercase=True,
+            unique=True,
+        )
 
-    @staticmethod
-    def _parse_platforms(properties):
+    def _parse_platforms(self, properties):
         result = []
         platforms_map = {
             "avr": "atmelavr",
@@ -547,7 +555,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
                 return ["*"]
             if arch in platforms_map:
                 result.append(platforms_map[arch])
-        return result
+        return self.str_to_list(result, lowercase=True, unique=True)
 
     def _parse_authors(self, properties):
         if "author" not in properties:
@@ -600,7 +608,6 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
         return None
 
     def _parse_export(self):
-        result = {"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]}
         include = None
         if self.remote_url:
             url_attrs = urlparse(self.remote_url)
@@ -613,8 +620,8 @@ class LibraryPropertiesManifestParser(BaseManifestParser):
                     or None
                 )
         if include:
-            result["include"] = [include]
-        return result
+            return dict(include=[include])
+        return None
 
     @staticmethod
     def _parse_dependencies(raw):
@@ -643,24 +650,31 @@ class PlatformJsonManifestParser(BaseManifestParser):
     def parse(self, contents):
         data = json.loads(contents)
         if "keywords" in data:
-            data["keywords"] = self.str_to_list(data["keywords"], sep=",")
+            data["keywords"] = self.str_to_list(
+                data["keywords"], sep=",", lowercase=True, unique=True
+            )
         if "frameworks" in data:
-            data["frameworks"] = self._parse_frameworks(data["frameworks"])
+            data["frameworks"] = (
+                self.str_to_list(
+                    list(data["frameworks"].keys()), lowercase=True, unique=True
+                )
+                if isinstance(data["frameworks"], dict)
+                else None
+            )
         if "packages" in data:
             data["dependencies"] = self._parse_dependencies(data["packages"])
         return data
 
-    @staticmethod
-    def _parse_frameworks(raw):
-        if not isinstance(raw, dict):
-            return None
-        return [name.lower() for name in raw.keys()]
-
     @staticmethod
     def _parse_dependencies(raw):
-        return [
-            dict(name=name, version=opts.get("version")) for name, opts in raw.items()
-        ]
+        result = []
+        for name, opts in raw.items():
+            item = {"name": name}
+            for k in ("owner", "version"):
+                if k in opts:
+                    item[k] = opts[k]
+            result.append(item)
+        return result
 
 
 class PackageJsonManifestParser(BaseManifestParser):
@@ -669,22 +683,21 @@ class PackageJsonManifestParser(BaseManifestParser):
     def parse(self, contents):
         data = json.loads(contents)
         if "keywords" in data:
-            data["keywords"] = self.str_to_list(data["keywords"], sep=",")
+            data["keywords"] = self.str_to_list(
+                data["keywords"], sep=",", lowercase=True, unique=True
+            )
         data = self._parse_system(data)
         data = self._parse_homepage(data)
         data = self._parse_repository(data)
         return data
 
-    @staticmethod
-    def _parse_system(data):
+    def _parse_system(self, data):
         if "system" not in data:
             return data
         if data["system"] in ("*", ["*"], "all"):
             del data["system"]
             return data
-        if not isinstance(data["system"], list):
-            data["system"] = [data["system"]]
-        data["system"] = [s.strip().lower() for s in data["system"]]
+        data["system"] = self.str_to_list(data["system"], lowercase=True, unique=True)
         return data
 
     @staticmethod

platformio/package/manifest/schema.py

@@ -33,7 +33,6 @@ if MARSHMALLOW_2:
     class CompatSchema(Schema):
         pass
 
-
 else:
 
     class CompatSchema(Schema):
@@ -89,7 +88,7 @@ class StrictListField(fields.List):
 class AuthorSchema(StrictSchema):
     name = fields.Str(required=True, validate=validate.Length(min=1, max=100))
     email = fields.Email(validate=validate.Length(min=1, max=50))
-    maintainer = fields.Bool(default=False)
+    maintainer = fields.Bool(dump_default=False)
     url = fields.Url(validate=validate.Length(min=1, max=255))
 
 
@@ -209,6 +208,13 @@ class ManifestSchema(BaseSchema):
             ]
         )
     )
+    headers = StrictListField(
+        fields.Str(
+            validate=[
+                validate.Length(min=1, max=255),
+            ]
+        )
+    )
 
     # platform.json specific
     title = fields.Str(validate=validate.Length(min=1, max=100))
@@ -253,7 +259,7 @@ class ManifestSchema(BaseSchema):
     @staticmethod
     @memoized(expire="1h")
     def load_spdx_licenses():
-        version = "3.14"
+        version = "3.16"
         spdx_data_url = (
             "https://raw.githubusercontent.com/spdx/license-list-data/"
             "v%s/json/licenses.json" % version

platformio/package/pack.py

@@ -22,7 +22,11 @@ import tempfile
 from platformio import fs
 from platformio.compat import IS_WINDOWS
 from platformio.package.exception import PackageException, UserSideException
-from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory
+from platformio.package.manifest.parser import (
+    LibraryPropertiesManifestParser,
+    ManifestFileType,
+    ManifestParserFactory,
+)
 from platformio.package.manifest.schema import ManifestSchema
 from platformio.package.meta import PackageItem
 from platformio.package.unpack import FileUnpacker
@@ -42,6 +46,8 @@ class PackagePacker(object):
         ".vscode",
         ".cache",
         "**/.cache",
+        "**/__pycache__",
+        "**/*.pyc",
         # VCS
         ".git/",
         ".hg/",
@@ -49,16 +55,26 @@ class PackagePacker(object):
     ]
     EXCLUDE_EXTRA = [
         # Tests
-        "tests?",
+        "test",
+        "tests",
         # Docs
         "doc",
         "docs",
         "mkdocs",
+        "doxygen",
+        "*.doxyfile",
+        "html",
+        "media",
         "**/*.[pP][dD][fF]",
-        "**/*.[dD][oO][cC]?",
-        "**/*.[pP][pP][tT]?",
+        "**/*.[dD][oO][cC]",
+        "**/*.[dD][oO][cC][xX]",
+        "**/*.[pP][pP][tT]",
+        "**/*.[pP][pP][tT][xX]",
+        "**/*.[xX][lL][sS]",
+        "**/*.[xX][lL][sS][xX]",
         "**/*.[dD][oO][xX]",
-        "**/*.[hH][tT][mM]?",
+        "**/*.[hH][tT][mM]",
+        "**/*.[hH][tT][mM][lL]",
         "**/*.[tT][eE][xX]",
         "**/*.[jJ][sS]",
         "**/*.[cC][sS][sS]",
@@ -74,14 +90,13 @@ class PackagePacker(object):
         "**/*.[mM][pP][34]",
         "**/*.[pP][sS][dD]",
         "**/*.[wW][aA][wW]",
+        "**/*.sqlite",
     ]
     EXCLUDE_LIBRARY_EXTRA = [
         "assets",
         "extra",
+        "extras",
         "resources",
-        "html",
-        "media",
-        "doxygen",
         "**/build/",
         "**/*.flat",
         "**/*.[jJ][aA][rR]",
@@ -96,6 +111,7 @@ class PackagePacker(object):
     def __init__(self, package, manifest_uri=None):
         self.package = package
         self.manifest_uri = manifest_uri
+        self.manifest_parser = None
 
     @staticmethod
     def get_archive_name(name, version, system=None):
@@ -127,7 +143,8 @@ class PackagePacker(object):
                 src = tmp_dir
 
             src = self.find_source_root(src)
-            manifest = self.load_manifest(src)
+            self.manifest_parser = ManifestParserFactory.new_from_dir(src)
+            manifest = ManifestSchema().load_manifest(self.manifest_parser.as_dict())
             filename = self.get_archive_name(
                 manifest["name"],
                 manifest["version"],
@@ -143,11 +160,6 @@ class PackagePacker(object):
         finally:
             shutil.rmtree(tmp_dir)
 
-    @staticmethod
-    def load_manifest(src):
-        mp = ManifestParserFactory.new_from_dir(src)
-        return ManifestSchema().load_manifest(mp.as_dict())
-
     def find_source_root(self, src):
         if self.manifest_uri:
             mp = (
@@ -213,7 +225,9 @@ class PackagePacker(object):
         # exclude items declared in manifest
         result += ["-<%s>" % p for p in exclude or []]
         # apply extra excludes if no custom "export" field in manifest
-        if not include and not exclude:
+        if (not include and not exclude) or isinstance(
+            self.manifest_parser, LibraryPropertiesManifestParser
+        ):
             result += ["-<%s>" % p for p in exclude_extra]
         # automatically include manifests
         result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT]

platformio/package/unpack.py

@@ -27,7 +27,7 @@ class ExtractArchiveItemError(PackageException):
 
     MESSAGE = (
         "Could not extract `{0}` to `{1}`. Try to disable antivirus "
-        "tool or check this solution -> http://bit.ly/faq-package-manager"
+        "tool or check this solution -> https://bit.ly/faq-package-manager"
     )
 
 

platformio/platform/_run.py

@@ -110,6 +110,8 @@ class PlatformRunMixin(object):
             args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value)))
 
         proc.copy_pythonpath_to_osenv()
+        # force SCons output to Unicode
+        os.environ["PYTHONIOENCODING"] = "utf-8"
 
         if targets and "menuconfig" in targets:
             return proc.exec_command(
@@ -187,7 +189,7 @@ class PlatformRunMixin(object):
             filename=filename,
             filename_styled=click.style(filename, fg="cyan"),
             link=click.style(
-                "https://platformio.org/lib/search?query=header:%s"
+                "https://registry.platformio.org/search?q=header:%s"
                 % quote(filename, safe=""),
                 fg="blue",
             ),

platformio/platform/base.py

@@ -45,7 +45,7 @@ class PlatformBase(  # pylint: disable=too-many-instance-attributes,too-many-pub
         self._custom_packages = None
 
         self.config = ProjectConfig.get_instance()
-        self.pm = ToolPackageManager(self.config.get_optional_dir("packages"))
+        self.pm = ToolPackageManager(self.config.get("platformio", "packages_dir"))
 
     @property
     def name(self):
@@ -145,8 +145,8 @@ class PlatformBase(  # pylint: disable=too-many-instance-attributes,too-many-pub
             self._BOARDS_CACHE[board_id] = config
 
         bdirs = [
-            self.config.get_optional_dir("boards"),
-            os.path.join(self.config.get_optional_dir("core"), "boards"),
+            self.config.get("platformio", "boards_dir"),
+            os.path.join(self.config.get("platformio", "core_dir"), "boards"),
             os.path.join(self.get_dir(), "boards"),
         ]
 

platformio/proc.py

@@ -30,7 +30,9 @@ from platformio.compat import (
 class AsyncPipeBase(object):
     def __init__(self):
         self._fd_read, self._fd_write = os.pipe()
-        self._pipe_reader = os.fdopen(self._fd_read, errors="backslashreplace")
+        self._pipe_reader = os.fdopen(
+            self._fd_read, encoding="utf-8", errors="backslashreplace"
+        )
         self._buffer = ""
         self._thread = Thread(target=self.run)
         self._thread.start()

platformio/project/config.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import glob
-import hashlib
 import json
 import os
 import re
@@ -21,7 +20,7 @@ import re
 import click
 
 from platformio import fs
-from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
+from platformio.compat import string_types
 from platformio.project import exception
 from platformio.project.options import ProjectOptions
 
@@ -49,7 +48,7 @@ MISSING = object()
 class ProjectConfigBase(object):
 
     INLINE_COMMENT_RE = re.compile(r"\s+;.*$")
-    VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}")
+    VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}")
 
     expand_interpolations = True
     warnings = []
@@ -272,11 +271,13 @@ class ProjectConfigBase(object):
                 if value == MISSING:
                     value = ""
                 value += ("\n" if value else "") + envvar_value
-            elif envvar_value and value == MISSING:
+            elif envvar_value:
                 value = envvar_value
 
         if value == MISSING:
             value = default if default != MISSING else option_meta.default
+        if callable(value):
+            value = value()
         if value == MISSING:
             return None
 
@@ -295,7 +296,12 @@ class ProjectConfigBase(object):
         section, option = match.group(1), match.group(2)
         if section == "sysenv":
             return os.getenv(option)
-        value = self.getraw(section, option)
+        try:
+            value = self.getraw(section, option)
+        except RecursionError:
+            raise exception.ProjectOptionValueError(
+                "Infinite recursion has been detected", option, section
+            )
         if isinstance(value, list):
             return "\n".join(value)
         return value
@@ -311,6 +317,8 @@ class ProjectConfigBase(object):
         if not option_meta:
             return value
 
+        if option_meta.validate:
+            value = option_meta.validate(value)
         if option_meta.multiple:
             value = self.parse_multi_values(value or [])
         try:
@@ -354,70 +362,12 @@ class ProjectConfigBase(object):
 
 
 class ProjectConfigDirsMixin(object):
-    def _get_core_dir(self, exists=False):
-        default = ProjectOptions["platformio.core_dir"].default
-        core_dir = self.get("platformio", "core_dir")
-        win_core_dir = None
-        if IS_WINDOWS and core_dir == default:
-            win_core_dir = os.path.splitdrive(core_dir)[0] + "\\.platformio"
-            if os.path.isdir(win_core_dir):
-                core_dir = win_core_dir
-
-        if exists and not os.path.isdir(core_dir):
-            try:
-                os.makedirs(core_dir)
-            except OSError as e:
-                if win_core_dir:
-                    os.makedirs(win_core_dir)
-                    core_dir = win_core_dir
-                else:
-                    raise e
-
-        return core_dir
-
-    def get_optional_dir(self, name, exists=False):
-        if not ProjectOptions.get("platformio.%s_dir" % name):
-            raise ValueError("Unknown optional directory -> " + name)
-
-        if name == "core":
-            result = self._get_core_dir(exists)
-        else:
-            result = self.get("platformio", name + "_dir")
-
-        if result is None:
-            return None
-
-        project_dir = os.getcwd()
-
-        # patterns
-        if "$PROJECT_HASH" in result:
-            result = result.replace(
-                "$PROJECT_HASH",
-                "%s-%s"
-                % (
-                    os.path.basename(project_dir),
-                    hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
-                ),
-            )
-
-        if "$PROJECT_DIR" in result:
-            result = result.replace("$PROJECT_DIR", project_dir)
-        if "$PROJECT_CORE_DIR" in result:
-            result = result.replace("$PROJECT_CORE_DIR", self.get_optional_dir("core"))
-        if "$PROJECT_WORKSPACE_DIR" in result:
-            result = result.replace(
-                "$PROJECT_WORKSPACE_DIR", self.get_optional_dir("workspace")
-            )
-
-        if result.startswith("~"):
-            result = fs.expanduser(result)
-
-        result = os.path.realpath(result)
-
-        if exists and not os.path.isdir(result):
-            os.makedirs(result)
-
-        return result
+    def get_optional_dir(self, name):
+        """
+        Deprecated, used by platformio-node-helpers.project.observer.fetchLibDirs
+        PlatformIO IDE for Atom depends on platformio-node-helpers@~7.2.0
+        """
+        return self.get("platformio", f"{name}_dir")
 
 
 class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin):

platformio/project/generator.py

@@ -24,15 +24,34 @@ from platformio.project.helpers import load_project_ide_data
 
 
 class ProjectGenerator(object):
-    def __init__(self, config, env_name, ide):
+    def __init__(self, config, env_name, ide, board_ids=None):
         self.config = config
         self.project_dir = os.path.dirname(config.path)
-        self.env_name = str(env_name)
+        self.original_env_name = env_name
+        self.env_name = str(env_name or self.get_best_envname(board_ids))
         self.ide = str(ide)
 
+    def get_best_envname(self, board_ids=None):
+        envname = None
+        default_envs = self.config.default_envs()
+        if default_envs:
+            envname = default_envs[0]
+            if not board_ids:
+                return envname
+
+        for env in self.config.envs():
+            if not board_ids:
+                return env
+            if not envname:
+                envname = env
+            items = self.config.items(env=env, as_dict=True)
+            if "board" in items and items.get("board") in board_ids:
+                return env
+        return envname
+
     @staticmethod
     def get_supported_ides():
-        tpls_dir = os.path.join(fs.get_source_dir(), "ide", "tpls")
+        tpls_dir = os.path.join(fs.get_source_dir(), "project", "tpls")
         return sorted(
             [
                 d
@@ -61,8 +80,9 @@ class ProjectGenerator(object):
             "systype": util.get_systype(),
             "project_name": os.path.basename(self.project_dir),
             "project_dir": self.project_dir,
+            "original_env_name": self.original_env_name,
             "env_name": self.env_name,
-            "user_home_dir": os.path.realpath(fs.expanduser("~")),
+            "user_home_dir": os.path.abspath(fs.expanduser("~")),
             "platformio_path": sys.argv[0]
             if os.path.isfile(sys.argv[0])
             else where_is_program("platformio"),
@@ -79,11 +99,11 @@ class ProjectGenerator(object):
             tpl_vars.update(
                 {
                     "src_files": self.get_src_files(),
-                    "project_src_dir": self.config.get_optional_dir("src"),
-                    "project_lib_dir": self.config.get_optional_dir("lib"),
-                    "project_test_dir": self.config.get_optional_dir("test"),
+                    "project_src_dir": self.config.get("platformio", "src_dir"),
+                    "project_lib_dir": self.config.get("platformio", "lib_dir"),
+                    "project_test_dir": self.config.get("platformio", "test_dir"),
                     "project_libdeps_dir": os.path.join(
-                        self.config.get_optional_dir("libdeps"), self.env_name
+                        self.config.get("platformio", "libdeps_dir"), self.env_name
                     ),
                 }
             )
@@ -103,14 +123,16 @@ class ProjectGenerator(object):
     def get_src_files(self):
         result = []
         with fs.cd(self.project_dir):
-            for root, _, files in os.walk(self.config.get_optional_dir("src")):
+            for root, _, files in os.walk(self.config.get("platformio", "src_dir")):
                 for f in files:
-                    result.append(os.path.relpath(os.path.join(root, f)))
+                    result.append(
+                        os.path.relpath(os.path.join(os.path.realpath(root), f))
+                    )
         return result
 
     def get_tpls(self):
         tpls = []
-        tpls_dir = os.path.join(fs.get_source_dir(), "ide", "tpls", self.ide)
+        tpls_dir = os.path.join(fs.get_source_dir(), "project", "tpls", self.ide)
         for root, _, files in os.walk(tpls_dir):
             for f in files:
                 if not f.endswith(".tpl"):

platformio/project/helpers.py

@@ -14,83 +14,60 @@
 
 import json
 import os
+import subprocess
 from hashlib import sha1
-from os import walk
-from os.path import dirname, isdir, isfile, join
 
 from click.testing import CliRunner
 
 from platformio import __version__, exception, fs
-from platformio.compat import IS_WINDOWS, hashlib_encode_data
+from platformio.compat import IS_MACOS, IS_WINDOWS, hashlib_encode_data
 from platformio.project.config import ProjectConfig
 
 
 def get_project_dir():
-    return os.getcwd()
+    return fs.normalize_path(os.getcwd())
 
 
 def is_platformio_project(project_dir=None):
     if not project_dir:
         project_dir = get_project_dir()
-    return isfile(join(project_dir, "platformio.ini"))
+    return os.path.isfile(os.path.join(project_dir, "platformio.ini"))
 
 
 def find_project_dir_above(path):
-    if isfile(path):
-        path = dirname(path)
+    if os.path.isfile(path):
+        path = os.path.dirname(path)
     if is_platformio_project(path):
         return path
-    if isdir(dirname(path)):
-        return find_project_dir_above(dirname(path))
+    if os.path.isdir(os.path.dirname(path)):
+        return find_project_dir_above(os.path.dirname(path))
     return None
 
 
-def get_project_core_dir():
-    """Deprecated, use ProjectConfig.get_optional_dir("core") instead"""
-    return ProjectConfig.get_instance(
-        join(get_project_dir(), "platformio.ini")
-    ).get_optional_dir("core", exists=True)
+def get_project_all_lib_dirs():
+    """Used by platformio-node-helpers.project.observer.fetchLibDirs"""
+    config = ProjectConfig.get_instance()
+    libdeps_dir = config.get("platformio", "libdeps_dir")
+    result = [
+        config.get("platformio", "globallib_dir"),
+        config.get("platformio", "lib_dir"),
+        libdeps_dir,
+    ]
+    if not os.path.isdir(libdeps_dir):
+        return result
+    for d in os.listdir(libdeps_dir):
+        if os.path.isdir(os.path.join(libdeps_dir, d)):
+            result.append(os.path.join(libdeps_dir, d))
+    return result
 
 
 def get_project_cache_dir():
-    """Deprecated, use ProjectConfig.get_optional_dir("cache") instead"""
-    return ProjectConfig.get_instance(
-        join(get_project_dir(), "platformio.ini")
-    ).get_optional_dir("cache")
-
-
-def get_project_global_lib_dir():
-    """
-    Deprecated, use ProjectConfig.get_optional_dir("globallib") instead
-    "platformio-node-helpers" depends on it
-    """
-    return ProjectConfig.get_instance(
-        join(get_project_dir(), "platformio.ini")
-    ).get_optional_dir("globallib")
-
-
-def get_project_lib_dir():
-    """
-    Deprecated, use ProjectConfig.get_optional_dir("lib") instead
-    "platformio-node-helpers" depends on it
-    """
-    return ProjectConfig.get_instance(
-        join(get_project_dir(), "platformio.ini")
-    ).get_optional_dir("lib")
-
-
-def get_project_libdeps_dir():
-    """
-    Deprecated, use ProjectConfig.get_optional_dir("libdeps") instead
-    "platformio-node-helpers" depends on it
-    """
-    return ProjectConfig.get_instance(
-        join(get_project_dir(), "platformio.ini")
-    ).get_optional_dir("libdeps")
+    """Deprecated, use ProjectConfig.get("platformio", "cache_dir") instead"""
+    return ProjectConfig.get_instance().get("platformio", "cache_dir")
 
 
 def get_default_projects_dir():
-    docs_dir = join(fs.expanduser("~"), "Documents")
+    docs_dir = os.path.join(fs.expanduser("~"), "Documents")
     try:
         assert IS_WINDOWS
         import ctypes.wintypes  # pylint: disable=import-outside-toplevel
@@ -99,8 +76,16 @@ def get_default_projects_dir():
         ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf)
         docs_dir = buf.value
     except:  # pylint: disable=bare-except
-        pass
-    return join(docs_dir, "PlatformIO", "Projects")
+        if not IS_MACOS:
+            try:
+                docs_dir = (
+                    subprocess.check_output(["xdg-user-dir", "DOCUMENTS"])
+                    .decode("utf-8")
+                    .strip()
+                )
+            except FileNotFoundError:  # command not found
+                pass
+    return os.path.join(docs_dir, "PlatformIO", "Projects")
 
 
 def compute_project_checksum(config):
@@ -113,16 +98,16 @@ def compute_project_checksum(config):
     # project file structure
     check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
     for d in (
-        config.get_optional_dir("include"),
-        config.get_optional_dir("src"),
-        config.get_optional_dir("lib"),
+        config.get("platformio", "include_dir"),
+        config.get("platformio", "src_dir"),
+        config.get("platformio", "lib_dir"),
     ):
-        if not isdir(d):
+        if not os.path.isdir(d):
             continue
         chunks = []
-        for root, _, files in walk(d):
+        for root, _, files in os.walk(d):
             for f in files:
-                path = join(root, f)
+                path = os.path.join(root, f)
                 if path.endswith(check_suffixes):
                     chunks.append(path)
         if not chunks:
@@ -171,8 +156,8 @@ def _load_project_ide_data(project_dir, env_names):
 
 def _load_cached_project_ide_data(project_dir, env_names):
     build_dir = ProjectConfig.get_instance(
-        join(project_dir, "platformio.ini")
-    ).get_optional_dir("build")
+        os.path.join(project_dir, "platformio.ini")
+    ).get("platformio", "build_dir")
     result = {}
     for name in env_names:
         if not os.path.isfile(os.path.join(build_dir, name, "idedata.json")):

platformio/project/options.py

@@ -14,12 +14,14 @@
 
 # pylint: disable=redefined-builtin, too-many-arguments
 
+import hashlib
 import os
 from collections import OrderedDict
 
 import click
 
 from platformio import fs
+from platformio.compat import IS_WINDOWS, hashlib_encode_data
 
 
 class ConfigOption(object):  # pylint: disable=too-many-instance-attributes
@@ -35,6 +37,7 @@ class ConfigOption(object):  # pylint: disable=too-many-instance-attributes
         buildenvvar=None,
         oldnames=None,
         default=None,
+        validate=None,
     ):
         self.scope = scope
         self.group = group
@@ -46,6 +49,7 @@ class ConfigOption(object):  # pylint: disable=too-many-instance-attributes
         self.buildenvvar = buildenvvar
         self.oldnames = oldnames
         self.default = default
+        self.validate = validate
 
     def as_dict(self):
         result = dict(
@@ -56,17 +60,15 @@ class ConfigOption(object):  # pylint: disable=too-many-instance-attributes
             type="string",
             multiple=self.multiple,
             sysenvvar=self.sysenvvar,
-            default=self.default,
+            default=self.default() if callable(self.default) else self.default,
         )
         if isinstance(self.type, click.ParamType):
             result["type"] = self.type.name
-
         if isinstance(self.type, (click.IntRange, click.FloatRange)):
             result["min"] = self.type.min
             result["max"] = self.type.max
         if isinstance(self.type, click.Choice):
             result["choices"] = self.type.choices
-
         return result
 
 
@@ -78,6 +80,52 @@ def ConfigEnvOption(*args, **kwargs):
     return ConfigOption("env", *args, **kwargs)
 
 
+def calculate_path_hash(path):
+    return "%s-%s" % (
+        os.path.basename(path),
+        hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10],
+    )
+
+
+def expand_dir_templates(path):
+    project_dir = os.getcwd()
+    tpls = {
+        "$PROJECT_DIR": lambda: project_dir,
+        "$PROJECT_HASH": lambda: calculate_path_hash(project_dir),
+    }
+    done = False
+    while not done:
+        done = True
+        for tpl, cb in tpls.items():
+            if tpl not in path:
+                continue
+            path = path.replace(tpl, cb())
+            done = False
+    return path
+
+
+def validate_dir(path):
+    if not path:
+        return path
+    # if not all values expanded, ignore validation
+    if "${" in path and "}" in path:
+        return path
+    if path.startswith("~"):
+        path = fs.expanduser(path)
+    if "$" in path:
+        path = expand_dir_templates(path)
+    return fs.normalize_path(path)
+
+
+def get_default_core_dir():
+    path = os.path.join(fs.expanduser("~"), ".platformio")
+    if IS_WINDOWS:
+        win_core_dir = os.path.splitdrive(path)[0] + "\\.platformio"
+        if os.path.isdir(win_core_dir):
+            return win_core_dir
+    return path
+
+
 ProjectOptions = OrderedDict(
     [
         ("%s.%s" % (option.scope, option.name), option)
@@ -120,7 +168,8 @@ ProjectOptions = OrderedDict(
                 ),
                 oldnames=["home_dir"],
                 sysenvvar="PLATFORMIO_CORE_DIR",
-                default=os.path.join(fs.expanduser("~"), ".platformio"),
+                default=get_default_core_dir,
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -130,7 +179,8 @@ ProjectOptions = OrderedDict(
                     "Finder (LDF) looks for global libraries"
                 ),
                 sysenvvar="PLATFORMIO_GLOBALLIB_DIR",
-                default=os.path.join("$PROJECT_CORE_DIR", "lib"),
+                default=os.path.join("${platformio.core_dir}", "lib"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -140,7 +190,8 @@ ProjectOptions = OrderedDict(
                     "platforms"
                 ),
                 sysenvvar="PLATFORMIO_PLATFORMS_DIR",
-                default=os.path.join("$PROJECT_CORE_DIR", "platforms"),
+                default=os.path.join("${platformio.core_dir}", "platforms"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -149,7 +200,8 @@ ProjectOptions = OrderedDict(
                     "A location where PlatformIO Core keeps installed packages"
                 ),
                 sysenvvar="PLATFORMIO_PACKAGES_DIR",
-                default=os.path.join("$PROJECT_CORE_DIR", "packages"),
+                default=os.path.join("${platformio.core_dir}", "packages"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -160,7 +212,8 @@ ProjectOptions = OrderedDict(
                     "other service information)"
                 ),
                 sysenvvar="PLATFORMIO_CACHE_DIR",
-                default=os.path.join("$PROJECT_CORE_DIR", ".cache"),
+                default=os.path.join("${platformio.core_dir}", ".cache"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -171,6 +224,7 @@ ProjectOptions = OrderedDict(
                     "build environments"
                 ),
                 sysenvvar="PLATFORMIO_BUILD_CACHE_DIR",
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -182,6 +236,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_WORKSPACE_DIR",
                 default=os.path.join("$PROJECT_DIR", ".pio"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -192,7 +247,8 @@ ProjectOptions = OrderedDict(
                     "and other cached information"
                 ),
                 sysenvvar="PLATFORMIO_BUILD_DIR",
-                default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"),
+                default=os.path.join("${platformio.workspace_dir}", "build"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -202,7 +258,8 @@ ProjectOptions = OrderedDict(
                     "dependencies declared via `lib_deps` option"
                 ),
                 sysenvvar="PLATFORMIO_LIBDEPS_DIR",
-                default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"),
+                default=os.path.join("${platformio.workspace_dir}", "libdeps"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -213,6 +270,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_INCLUDE_DIR",
                 default=os.path.join("$PROJECT_DIR", "include"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -223,6 +281,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_SRC_DIR",
                 default=os.path.join("$PROJECT_DIR", "src"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -230,6 +289,7 @@ ProjectOptions = OrderedDict(
                 description="A storage for the custom/private project libraries",
                 sysenvvar="PLATFORMIO_LIB_DIR",
                 default=os.path.join("$PROJECT_DIR", "lib"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -240,6 +300,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_DATA_DIR",
                 default=os.path.join("$PROJECT_DIR", "data"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -250,13 +311,23 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_TEST_DIR",
                 default=os.path.join("$PROJECT_DIR", "test"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
                 name="boards_dir",
-                description="A global storage for custom board manifests",
+                description="A storage for custom board manifests",
                 sysenvvar="PLATFORMIO_BOARDS_DIR",
                 default=os.path.join("$PROJECT_DIR", "boards"),
+                validate=validate_dir,
+            ),
+            ConfigPlatformioOption(
+                group="directory",
+                name="monitor_dir",
+                description="A storage for custom monitor filters",
+                sysenvvar="PLATFORMIO_MONITOR_DIR",
+                default=os.path.join("$PROJECT_DIR", "monitor"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -267,6 +338,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_SHARED_DIR",
                 default=os.path.join("$PROJECT_DIR", "shared"),
+                validate=validate_dir,
             ),
             #
             # [env]

platformio/project/tpls/clion/CMakeLists.txt.tpl

@@ -26,7 +26,7 @@ add_custom_target(
 
 add_custom_target(
     Debug ALL
-    COMMAND platformio -c clion run --target debug "$<$<NOT:$<CONFIG:All>>:-e${CMAKE_BUILD_TYPE}>"
+    COMMAND platformio -c clion debug "$<$<NOT:$<CONFIG:All>>:-e${CMAKE_BUILD_TYPE}>"
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 

platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -8,15 +8,14 @@
 % import os
 % import re
 %
-% from platformio.compat import WINDOWS
-% from platformio.project.helpers import (load_project_ide_data)
+% from platformio.project.helpers import load_project_ide_data
 %
 % def _normalize_path(path):
 %   if project_dir in path:
 %     path = path.replace(project_dir, "${CMAKE_CURRENT_LIST_DIR}")
 %   elif user_home_dir in path:
 %     if "windows" in systype:
-%       path = path.replace(user_home_dir, "$ENV{HOMEDRIVE}$ENV{HOMEPATH}")
+%       path = path.replace(user_home_dir, "${ENV_HOME_PATH}")
 %     else:
 %       path = path.replace(user_home_dir, "$ENV{HOME}")
 %     end
@@ -40,7 +39,7 @@
 % end
 %
 % def _get_lib_dirs(envname):
-%   env_libdeps_dir = os.path.join(config.get_optional_dir("libdeps"), envname)
+%   env_libdeps_dir = os.path.join(config.get("platformio", "libdeps_dir"), envname)
 %   env_lib_extra_dirs = config.get("env:" + envname, "lib_extra_dirs", [])
 %   return _fix_lib_dirs([env_libdeps_dir] + env_lib_extra_dirs)
 % end
@@ -54,6 +53,11 @@ set(CMAKE_CONFIGURATION_TYPES "{{ ";".join(envs) }};" CACHE STRING "Build Types
 set(CMAKE_CONFIGURATION_TYPES "{{ env_name }}" CACHE STRING "Build Types reflect PlatformIO Environments" FORCE)
 % end
 
+# Convert "Home Directory" that may contain unescaped backslashes on Windows
+% if "windows" in systype:
+file(TO_CMAKE_PATH $ENV{HOMEDRIVE}$ENV{HOMEPATH} ENV_HOME_PATH)
+% end
+
 % if svd_path:
 set(CLION_SVD_FILE_PATH "{{ _normalize_path(svd_path) }}" CACHE FILEPATH "Peripheral Registers Definitions File" FORCE)
 % end
@@ -75,7 +79,7 @@ set(CMAKE_CXX_STANDARD {{ cxx_stds[-1] }})
 
 if (CMAKE_BUILD_TYPE MATCHES "{{ env_name }}")
 % for define in defines:
-    add_definitions(-D{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}})
+    add_definitions(-D{{!re.sub(r"([\"\(\)\ #])", r"\\\1", define)}})
 % end
 
 % for include in filter_includes(includes):
@@ -99,7 +103,7 @@ endif()
 % for env, data in ide_data.items():
 if (CMAKE_BUILD_TYPE MATCHES "{{ env }}")
 %   for define in data["defines"]:
-    add_definitions(-D{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}})
+    add_definitions(-D{{!re.sub(r"([\"\(\)\ #])", r"\\\1", define)}})
 %   end
 
 %   for include in filter_includes(data["includes"]):

platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl

@@ -30,8 +30,8 @@
       <makefileType>
         <makeTool>
           <buildCommandWorkingDir>.</buildCommandWorkingDir>
-          <buildCommand>{{platformio_path}} -f -c netbeans run</buildCommand>
-          <cleanCommand>{{platformio_path}} -f -c netbeans run --target clean</cleanCommand>
+          <buildCommand>"{{platformio_path}}" -f -c netbeans run</buildCommand>
+          <cleanCommand>"{{platformio_path}}" -f -c netbeans run --target clean</cleanCommand>
           <executablePath></executablePath>
           <cTool>
             % cleaned_includes = filter_includes(includes)