Improved PlatformIO directory interpolation (${platformio.***_dir}) in “platformio.ini” configuration file // Resolve #3934

HISTORY.rst

@@ -11,8 +11,10 @@ PlatformIO Core 5
 5.2.3 (2021-??-??)
 ~~~~~~~~~~~~~~~~~~
 
+- Improved PlatformIO directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" <https://docs.platformio.org/page/projectconf.html>`__ configuration file (`issue #3934 <https://github.com/platformio/platformio-core/issues/3934>`_)
 - Fixed an issue when the "$PROJECT_DIR" gets the full path to "platformio.ini", not the directory name (`issue #4086 <https://github.com/platformio/platformio-core/issues/4086>`_)
 
+
 5.2.2 (2021-10-20)
 ~~~~~~~~~~~~~~~~~~
 

platformio/app.py

@@ -21,17 +21,17 @@ import os
 import platform
 import socket
 import uuid
-from os.path import dirname, isdir, isfile, join, realpath
 
 from platformio import __version__, exception, fs, proc
 from platformio.compat import IS_WINDOWS, hashlib_encode_data
 from platformio.package.lockfile import LockFile
-from platformio.project.helpers import get_default_projects_dir, get_project_core_dir
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import get_default_projects_dir
 
 
 def projects_dir_validate(projects_dir):
-    assert isdir(projects_dir)
+    assert os.path.isdir(projects_dir)
-    return realpath(projects_dir)
+    return os.path.realpath(projects_dir)
 
 
 DEFAULT_SETTINGS = {
@@ -91,7 +91,10 @@ class State(object):
         self.path = path
         self.lock = lock
         if not self.path:
-            self.path = join(get_project_core_dir(), "appstate.json")
+            core_dir = ProjectConfig.get_instance().get("platformio", "core_dir")
+            if not os.path.isdir(core_dir):
+                os.makedirs(core_dir)
+            self.path = os.path.join(core_dir, "appstate.json")
         self._storage = {}
         self._lockfile = None
         self.modified = False
@@ -99,7 +102,7 @@ class State(object):
     def __enter__(self):
         try:
             self._lock_state_file()
-            if isfile(self.path):
+            if os.path.isfile(self.path):
                 self._storage = fs.load_json(self.path)
             assert isinstance(self._storage, dict)
         except (
@@ -117,7 +120,7 @@ class State(object):
                 with open(self.path, mode="w", encoding="utf8") as fp:
                     fp.write(json.dumps(self._storage))
             except IOError:
-                raise exception.HomeDirPermissionsError(get_project_core_dir())
+                raise exception.HomeDirPermissionsError(os.path.dirname(self.path))
         self._unlock_state_file()
 
     def _lock_state_file(self):
@@ -127,7 +130,7 @@ class State(object):
         try:
             self._lockfile.acquire()
         except IOError:
-            raise exception.HomeDirPermissionsError(dirname(self.path))
+            raise exception.HomeDirPermissionsError(os.path.dirname(self.path))
 
     def _unlock_state_file(self):
         if hasattr(self, "_lockfile") and self._lockfile:

platformio/builder/main.py

@@ -109,22 +109,22 @@ env.Replace(
 config = env.GetProjectConfig()
 env.Replace(
     PROJECT_DIR=get_project_dir(),
-    PROJECT_CORE_DIR=config.get_optional_dir("core"),
+    PROJECT_CORE_DIR=config.get("platformio", "core_dir"),
-    PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"),
+    PROJECT_PACKAGES_DIR=config.get("platformio", "packages_dir"),
-    PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"),
+    PROJECT_WORKSPACE_DIR=config.get("platformio", "workspace_dir"),
-    PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"),
+    PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"),
-    PROJECT_INCLUDE_DIR=config.get_optional_dir("include"),
+    PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"),
-    PROJECT_SRC_DIR=config.get_optional_dir("src"),
+    PROJECT_SRC_DIR=config.get("platformio", "src_dir"),
-    PROJECTSRC_DIR=config.get_optional_dir("src"),  # legacy for dev/platform
+    PROJECTSRC_DIR=config.get("platformio", "src_dir"),  # legacy for dev/platform
-    PROJECT_TEST_DIR=config.get_optional_dir("test"),
+    PROJECT_TEST_DIR=config.get("platformio", "test_dir"),
-    PROJECT_DATA_DIR=config.get_optional_dir("data"),
+    PROJECT_DATA_DIR=config.get("platformio", "data_dir"),
-    PROJECTDATA_DIR=config.get_optional_dir("data"),  # legacy for dev/platform
+    PROJECTDATA_DIR=config.get("platformio", "data_dir"),  # legacy for dev/platform
-    PROJECT_BUILD_DIR=config.get_optional_dir("build"),
+    PROJECT_BUILD_DIR=config.get("platformio", "build_dir"),
-    BUILD_CACHE_DIR=config.get_optional_dir("build_cache"),
+    BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"),
     LIBSOURCE_DIRS=[
-        config.get_optional_dir("lib"),
+        config.get("platformio", "lib_dir"),
         os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
-        config.get_optional_dir("globallib"),
+        config.get("platformio", "globallib_dir"),
     ],
 )
 

platformio/builder/tools/pioide.py

@@ -63,7 +63,7 @@ def _dump_includes(env):
     # include Unity framework if there are tests in project
     includes["unity"] = []
     auto_install_unity = False
-    test_dir = env.GetProjectConfig().get_optional_dir("test")
+    test_dir = env.GetProjectConfig().get("platformio", "test_dir")
     if os.path.isdir(test_dir) and os.listdir(test_dir) != ["README"]:
         auto_install_unity = True
     unity_dir = get_core_package_dir(

platformio/commands/check/command.py

@@ -106,8 +106,8 @@ def cli(
                 )
 
             default_patterns = [
-                config.get_optional_dir("src"),
+                config.get("platformio", "src_dir"),
-                config.get_optional_dir("include"),
+                config.get("platformio", "include_dir"),
             ]
             tool_options = dict(
                 verbose=verbose,

platformio/commands/check/tools/clangtidy.py

@@ -77,7 +77,7 @@ class ClangtidyCheckTool(CheckToolBase):
         includes = []
         for inc in self.cpp_includes:
             if self.options.get("skip_packages") and inc.lower().startswith(
-                self.config.get_optional_dir("packages").lower()
+                self.config.get("platformio", "packages_dir").lower()
             ):
                 continue
             includes.append(inc)

platformio/commands/check/tools/cppcheck.py

@@ -84,7 +84,7 @@ class CppcheckCheckTool(CheckToolBase):
         if (
             args.get("file", "")
             .lower()
-            .startswith(self.config.get_optional_dir("packages").lower())
+            .startswith(self.config.get("platformio", "packages_dir").lower())
         ):
             if args["id"] in breaking_defect_ids:
                 if self.options.get("verbose"):
@@ -201,7 +201,7 @@ class CppcheckCheckTool(CheckToolBase):
         result = []
         for inc in self.cpp_includes:
             if self.options.get("skip_packages") and inc.lower().startswith(
-                self.config.get_optional_dir("packages").lower()
+                self.config.get("platformio", "packages_dir").lower()
             ):
                 continue
             result.append(inc)

platformio/commands/check/tools/pvsstudio.py

@@ -43,7 +43,7 @@ class PvsStudioCheckTool(CheckToolBase):  # pylint: disable=too-many-instance-at
         with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp:
             fp.write(
                 "exclude-path = "
-                + self.config.get_optional_dir("packages").replace("\\", "/")
+                + self.config.get("platformio", "packages_dir").replace("\\", "/")
             )
 
         with open(self._tmp_cmd_file, mode="w", encoding="utf8") as fp:

platformio/commands/home/rpc/handlers/app.py

@@ -14,16 +14,15 @@
 
 from __future__ import absolute_import
 
-from os.path import join
+import os
 
 from platformio import __version__, app, fs, util
-from platformio.project.helpers import get_project_core_dir, is_platformio_project
+from platformio.project.config import ProjectConfig
+from platformio.project.helpers import is_platformio_project
 
 
 class AppRPC:
 
-    APPSTATE_PATH = join(get_project_core_dir(), "homestate.json")
-
     IGNORE_STORAGE_KEYS = [
         "cid",
         "coreVersion",
@@ -34,9 +33,16 @@ class AppRPC:
         "projectsDir",
     ]
 
+    @staticmethod
+    def get_state_path():
+        core_dir = ProjectConfig.get_instance().get("platformio", "core_dir")
+        if not os.path.isdir(core_dir):
+            os.makedirs(core_dir)
+        return os.path.join(core_dir, "homestate.json")
+
     @staticmethod
     def load_state():
-        with app.State(AppRPC.APPSTATE_PATH, lock=True) as state:
+        with app.State(AppRPC.get_state_path(), lock=True) as state:
             storage = state.get("storage", {})
 
             # base data
@@ -72,7 +78,7 @@ class AppRPC:
 
     @staticmethod
     def save_state(state):
-        with app.State(AppRPC.APPSTATE_PATH, lock=True) as s:
+        with app.State(AppRPC.get_state_path(), lock=True) as s:
             s.clear()
             s.update(state)
             storage = s.get("storage", {})

platformio/commands/home/rpc/handlers/project.py

@@ -81,7 +81,7 @@ class ProjectRPC:
             data["description"] = config.get("platformio", "description")
             data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", []))
 
-            libdeps_dir = config.get_optional_dir("libdeps")
+            libdeps_dir = config.get("platformio", "libdeps_dir")
             for section in config.sections():
                 if not section.startswith("env:"):
                     continue
@@ -253,7 +253,7 @@ class ProjectRPC:
 
         with fs.cd(project_dir):
             config = ProjectConfig()
-            src_dir = config.get_optional_dir("src")
+            src_dir = config.get("platformio", "src_dir")
             main_path = os.path.join(
                 src_dir, "main.%s" % ("cpp" if is_cpp_project else "c")
             )
@@ -308,7 +308,7 @@ class ProjectRPC:
         )
         with fs.cd(project_dir):
             config = ProjectConfig()
-            src_dir = config.get_optional_dir("src")
+            src_dir = config.get("platformio", "src_dir")
             if os.path.isdir(src_dir):
                 fs.rmtree(src_dir)
             shutil.copytree(arduino_project_dir, src_dir, symlinks=True)

platformio/commands/lib/command.py

@@ -43,7 +43,7 @@ CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps"
 
 
 def get_project_global_lib_dir():
-    return ProjectConfig.get_instance().get_optional_dir("globallib")
+    return ProjectConfig.get_instance().get("platformio", "globallib_dir")
 
 
 @click.group(short_help="Library manager")
@@ -111,7 +111,7 @@ def cli(ctx, **options):
                 os.path.join(storage_dir, "platformio.ini")
             )
             config.validate(options["environment"], silent=in_silence)
-            libdeps_dir = config.get_optional_dir("libdeps")
+            libdeps_dir = config.get("platformio", "libdeps_dir")
             for env in config.envs():
                 if options["environment"] and env not in options["environment"]:
                     continue

platformio/commands/project.py

@@ -226,10 +226,10 @@ def init_base_project(project_dir):
         config = ProjectConfig()
         config.save()
         dir_to_readme = [
-            (config.get_optional_dir("src"), None),
+            (config.get("platformio", "src_dir"), None),
-            (config.get_optional_dir("include"), init_include_readme),
+            (config.get("platformio", "include_dir"), init_include_readme),
-            (config.get_optional_dir("lib"), init_lib_readme),
+            (config.get("platformio", "lib_dir"), init_lib_readme),
-            (config.get_optional_dir("test"), init_test_readme),
+            (config.get("platformio", "test_dir"), init_test_readme),
         ]
         for (path, cb) in dir_to_readme:
             if os.path.isdir(path):

platformio/commands/remote/client/agent_service.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import os
-from os.path import getatime, getmtime, isdir, isfile, join
 
 from twisted.logger import LogLevel  # pylint: disable=import-error
 from twisted.spread import pb  # pylint: disable=import-error
@@ -25,15 +24,16 @@ from platformio.commands.remote.ac.serial import SerialPortAsyncCmd
 from platformio.commands.remote.client.base import RemoteClientBase
 from platformio.project.config import ProjectConfig
 from platformio.project.exception import NotPlatformIOProjectError
-from platformio.project.helpers import get_project_core_dir
 
 
 class RemoteAgentService(RemoteClientBase):
     def __init__(self, name, share, working_dir=None):
         RemoteClientBase.__init__(self)
         self.log_level = LogLevel.info
-        self.working_dir = working_dir or join(get_project_core_dir(), "remote")
+        self.working_dir = working_dir or os.path.join(
-        if not isdir(self.working_dir):
+            ProjectConfig.get_instance().get("platformio", "core_dir"), "remote"
+        )
+        if not os.path.isdir(self.working_dir):
             os.makedirs(self.working_dir)
         if name:
             self.name = str(name)[:50]
@@ -138,14 +138,14 @@ class RemoteAgentService(RemoteClientBase):
         self, command, options
     ):
         assert options and "project_id" in options
-        project_dir = join(self.working_dir, "projects", options["project_id"])
+        project_dir = os.path.join(self.working_dir, "projects", options["project_id"])
-        origin_pio_ini = join(project_dir, "platformio.ini")
+        origin_pio_ini = os.path.join(project_dir, "platformio.ini")
-        back_pio_ini = join(project_dir, "platformio.ini.bak")
+        back_pio_ini = os.path.join(project_dir, "platformio.ini.bak")
 
         # remove insecure project options
         try:
             conf = ProjectConfig(origin_pio_ini)
-            if isfile(back_pio_ini):
+            if os.path.isfile(back_pio_ini):
                 os.remove(back_pio_ini)
             os.rename(origin_pio_ini, back_pio_ini)
             # cleanup
@@ -159,7 +159,10 @@ class RemoteAgentService(RemoteClientBase):
             conf.save(origin_pio_ini)
 
             # restore A/M times
-            os.utime(origin_pio_ini, (getatime(back_pio_ini), getmtime(back_pio_ini)))
+            os.utime(
+                origin_pio_ini,
+                (os.path.getatime(back_pio_ini), os.path.getmtime(back_pio_ini)),
+            )
         except NotPlatformIOProjectError as e:
             raise pb.Error(str(e))
 
@@ -194,8 +197,8 @@ class RemoteAgentService(RemoteClientBase):
             paused_acs.append(ac)
 
         def _cb_on_end():
-            if isfile(back_pio_ini):
+            if os.path.isfile(back_pio_ini):
-                if isfile(origin_pio_ini):
+                if os.path.isfile(origin_pio_ini):
                     os.remove(origin_pio_ini)
                 os.rename(back_pio_ini, origin_pio_ini)
             for ac in paused_acs:

platformio/commands/remote/client/run_or_test.py

@@ -69,8 +69,8 @@ class RunOrTestClient(AsyncClientBase):
                 os.path.join(self.options["project_dir"], "platformio.ini")
             )
             psync.add_item(cfg.path, "platformio.ini")
-            psync.add_item(cfg.get_optional_dir("shared"), "shared")
+            psync.add_item(cfg.get("platformio", "shared_dir"), "shared")
-            psync.add_item(cfg.get_optional_dir("boards"), "boards")
+            psync.add_item(cfg.get("platformio", "boards_dir"), "boards")
 
             if self.options["force_remote"]:
                 self._add_project_source_items(cfg, psync)
@@ -78,26 +78,26 @@ class RunOrTestClient(AsyncClientBase):
                 self._add_project_binary_items(cfg, psync)
 
             if self.command == "test":
-                psync.add_item(cfg.get_optional_dir("test"), "test")
+                psync.add_item(cfg.get("platformio", "test_dir"), "test")
 
     def _add_project_source_items(self, cfg, psync):
-        psync.add_item(cfg.get_optional_dir("lib"), "lib")
+        psync.add_item(cfg.get("platformio", "lib_dir"), "lib")
         psync.add_item(
-            cfg.get_optional_dir("include"),
+            cfg.get("platformio", "include_dir"),
             "include",
             cb_filter=self._cb_tarfile_filter,
         )
         psync.add_item(
-            cfg.get_optional_dir("src"), "src", cb_filter=self._cb_tarfile_filter
+            cfg.get("platformio", "src_dir"), "src", cb_filter=self._cb_tarfile_filter
         )
         if set(["buildfs", "uploadfs", "uploadfsota"]) & set(
             self.options.get("target", [])
         ):
-            psync.add_item(cfg.get_optional_dir("data"), "data")
+            psync.add_item(cfg.get("platformio", "data_dir"), "data")
 
     @staticmethod
     def _add_project_binary_items(cfg, psync):
-        build_dir = cfg.get_optional_dir("build")
+        build_dir = cfg.get("platformio", "build_dir")
         for env_name in os.listdir(build_dir):
             env_dir = os.path.join(build_dir, env_name)
             if not os.path.isdir(env_dir):

platformio/commands/run/command.py

@@ -101,7 +101,7 @@ def cli(
 
         # clean obsolete build dir
         if not disable_auto_clean:
-            build_dir = config.get_optional_dir("build")
+            build_dir = config.get("platformio", "build_dir")
             try:
                 clean_build_dir(build_dir, config)
             except:  # pylint: disable=bare-except

platformio/commands/run/helpers.py

@@ -23,8 +23,8 @@ from platformio.project.helpers import compute_project_checksum, get_project_dir
 
 def handle_legacy_libdeps(project_dir, config):
     legacy_libdeps_dir = join(project_dir, ".piolibdeps")
-    if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get_optional_dir(
+    if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get(
-        "libdeps"
+        "platformio", "libdeps_dir"
     ):
         return
     if not config.has_section("env"):

platformio/commands/system/command.py

@@ -64,7 +64,7 @@ def system_info(json_output):
     }
     data["core_dir"] = {
         "title": "PlatformIO Core Directory",
-        "value": project_config.get_optional_dir("core"),
+        "value": project_config.get("platformio", "core_dir"),
     }
     data["platformio_exe"] = {
         "title": "PlatformIO Core Executable",
@@ -88,7 +88,7 @@ def system_info(json_output):
         "title": "Tools & Toolchains",
         "value": len(
             ToolPackageManager(
-                project_config.get_optional_dir("packages")
+                project_config.get("platformio", "packages_dir")
             ).get_installed()
         ),
     }

platformio/commands/test/helpers.py

@@ -18,7 +18,7 @@ from platformio import exception
 
 
 def get_test_names(config):
-    test_dir = config.get_optional_dir("test")
+    test_dir = config.get("platformio", "test_dir")
     if not os.path.isdir(test_dir):
         raise exception.TestDirNotExists(test_dir)
     names = []

platformio/commands/test/native.py

@@ -31,7 +31,7 @@ class NativeTestProcessor(TestProcessorBase):
         return self.run()
 
     def run(self):
-        build_dir = self.options["project_config"].get_optional_dir("build")
+        build_dir = self.options["project_config"].get("platformio", "build_dir")
         result = proc.exec_command(
             [join(build_dir, self.env_name, "program")],
             stdout=LineBufferedAsyncPipe(self.on_run_out),

platformio/commands/test/processor.py

@@ -124,7 +124,7 @@ class TestProcessorBase(object):
     def build_or_upload(self, target):
         if not self._output_file_generated:
             self.generate_output_file(
-                self.options["project_config"].get_optional_dir("test")
+                self.options["project_config"].get("platformio", "test_dir")
             )
             self._output_file_generated = True
 

platformio/debug/config/base.py

@@ -20,7 +20,7 @@ from platformio.compat import string_types
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.debug.helpers import reveal_debug_port
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import get_project_core_dir, load_project_ide_data
+from platformio.project.helpers import load_project_ide_data
 from platformio.project.options import ProjectOptions
 
 
@@ -208,7 +208,7 @@ class DebugConfigBase:  # pylint: disable=too-many-instance-attributes
     def reveal_patterns(self, source, recursive=True):
         program_path = self.program_path or ""
         patterns = {
-            "PLATFORMIO_CORE_DIR": get_project_core_dir(),
+            "PLATFORMIO_CORE_DIR": self.project_config.get("platformio", "core_dir"),
             "PYTHONEXE": proc.get_pythonexe_path(),
             "PROJECT_DIR": os.getcwd(),
             "PROG_PATH": program_path,

platformio/ide/projectgenerator.py

@@ -79,11 +79,11 @@ class ProjectGenerator(object):
             tpl_vars.update(
                 {
                     "src_files": self.get_src_files(),
-                    "project_src_dir": self.config.get_optional_dir("src"),
+                    "project_src_dir": self.config.get("platformio", "src_dir"),
-                    "project_lib_dir": self.config.get_optional_dir("lib"),
+                    "project_lib_dir": self.config.get("platformio", "lib_dir"),
-                    "project_test_dir": self.config.get_optional_dir("test"),
+                    "project_test_dir": self.config.get("platformio", "test_dir"),
                     "project_libdeps_dir": os.path.join(
-                        self.config.get_optional_dir("libdeps"), self.env_name
+                        self.config.get("platformio", "libdeps_dir"), self.env_name
                     ),
                 }
             )
@@ -103,7 +103,7 @@ class ProjectGenerator(object):
     def get_src_files(self):
         result = []
         with fs.cd(self.project_dir):
-            for root, _, files in os.walk(self.config.get_optional_dir("src")):
+            for root, _, files in os.walk(self.config.get("platformio", "src_dir")):
                 for f in files:
                     result.append(os.path.relpath(os.path.join(root, f)))
         return result

platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -40,7 +40,7 @@
 % end
 %
 % def _get_lib_dirs(envname):
-%   env_libdeps_dir = os.path.join(config.get_optional_dir("libdeps"), envname)
+%   env_libdeps_dir = os.path.join(config.get("platformio", "libdeps_dir"), envname)
 %   env_lib_extra_dirs = config.get("env:" + envname, "lib_extra_dirs", [])
 %   return _fix_lib_dirs([env_libdeps_dir] + env_lib_extra_dirs)
 % end

platformio/package/manager/platform.py

@@ -31,7 +31,7 @@ class PlatformPackageManager(BasePackageManager):  # pylint: disable=too-many-an
         self.config = ProjectConfig.get_instance()
         super(PlatformPackageManager, self).__init__(
             PackageType.PLATFORM,
-            package_dir or self.config.get_optional_dir("platforms"),
+            package_dir or self.config.get("platformio", "platforms_dir"),
         )
 
     @property

platformio/package/manager/tool.py

@@ -20,7 +20,7 @@ from platformio.project.config import ProjectConfig
 class ToolPackageManager(BasePackageManager):  # pylint: disable=too-many-ancestors
     def __init__(self, package_dir=None):
         if not package_dir:
-            package_dir = ProjectConfig.get_instance().get_optional_dir("packages")
+            package_dir = ProjectConfig.get_instance().get("platformio", "packages_dir")
         super(ToolPackageManager, self).__init__(PackageType.TOOL, package_dir)
 
     @property

platformio/platform/base.py

@@ -45,7 +45,7 @@ class PlatformBase(  # pylint: disable=too-many-instance-attributes,too-many-pub
         self._custom_packages = None
 
         self.config = ProjectConfig.get_instance()
-        self.pm = ToolPackageManager(self.config.get_optional_dir("packages"))
+        self.pm = ToolPackageManager(self.config.get("platformio", "packages_dir"))
 
     @property
     def name(self):
@@ -145,8 +145,8 @@ class PlatformBase(  # pylint: disable=too-many-instance-attributes,too-many-pub
             self._BOARDS_CACHE[board_id] = config
 
         bdirs = [
-            self.config.get_optional_dir("boards"),
+            self.config.get("platformio", "boards_dir"),
-            os.path.join(self.config.get_optional_dir("core"), "boards"),
+            os.path.join(self.config.get("platformio", "core_dir"), "boards"),
             os.path.join(self.get_dir(), "boards"),
         ]
 

platformio/project/config.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import glob
-import hashlib
 import json
 import os
 import re
@@ -21,7 +20,7 @@ import re
 import click
 
 from platformio import fs
-from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types
+from platformio.compat import string_types
 from platformio.project import exception
 from platformio.project.options import ProjectOptions
 
@@ -311,8 +310,11 @@ class ProjectConfigBase(object):
         if not option_meta:
             return value
 
+        if option_meta.validate:
+            value = option_meta.validate(value)
         if option_meta.multiple:
             value = self.parse_multi_values(value or [])
+
         try:
             return self.cast_to(value, option_meta.type)
         except click.BadParameter as e:
@@ -353,74 +355,7 @@ class ProjectConfigBase(object):
         return True
 
 
-class ProjectConfigDirsMixin(object):
+class ProjectConfig(ProjectConfigBase):
-    def _get_core_dir(self, exists=False):
-        default = ProjectOptions["platformio.core_dir"].default
-        core_dir = self.get("platformio", "core_dir")
-        win_core_dir = None
-        if IS_WINDOWS and core_dir == default:
-            win_core_dir = os.path.splitdrive(core_dir)[0] + "\\.platformio"
-            if os.path.isdir(win_core_dir):
-                core_dir = win_core_dir
-
-        if exists and not os.path.isdir(core_dir):
-            try:
-                os.makedirs(core_dir)
-            except OSError as e:
-                if win_core_dir:
-                    os.makedirs(win_core_dir)
-                    core_dir = win_core_dir
-                else:
-                    raise e
-
-        return core_dir
-
-    def get_optional_dir(self, name, exists=False):
-        if not ProjectOptions.get("platformio.%s_dir" % name):
-            raise ValueError("Unknown optional directory -> " + name)
-
-        if name == "core":
-            result = self._get_core_dir(exists)
-        else:
-            result = self.get("platformio", name + "_dir")
-
-        if result is None:
-            return None
-
-        project_dir = os.getcwd()
-
-        # patterns
-        if "$PROJECT_HASH" in result:
-            result = result.replace(
-                "$PROJECT_HASH",
-                "%s-%s"
-                % (
-                    os.path.basename(project_dir),
-                    hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
-                ),
-            )
-
-        if "$PROJECT_DIR" in result:
-            result = result.replace("$PROJECT_DIR", project_dir)
-        if "$PROJECT_CORE_DIR" in result:
-            result = result.replace("$PROJECT_CORE_DIR", self.get_optional_dir("core"))
-        if "$PROJECT_WORKSPACE_DIR" in result:
-            result = result.replace(
-                "$PROJECT_WORKSPACE_DIR", self.get_optional_dir("workspace")
-            )
-
-        if result.startswith("~"):
-            result = fs.expanduser(result)
-
-        result = os.path.realpath(result)
-
-        if exists and not os.path.isdir(result):
-            os.makedirs(result)
-
-        return result
-
-
-class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin):
 
     _instances = {}
 

platformio/project/helpers.py

@@ -15,8 +15,6 @@
 import json
 import os
 from hashlib import sha1
-from os import walk
-from os.path import dirname, isdir, isfile, join
 
 from click.testing import CliRunner
 
@@ -32,65 +30,50 @@ def get_project_dir():
 def is_platformio_project(project_dir=None):
     if not project_dir:
         project_dir = get_project_dir()
-    return isfile(join(project_dir, "platformio.ini"))
+    return os.path.isfile(os.path.join(project_dir, "platformio.ini"))
 
 
 def find_project_dir_above(path):
-    if isfile(path):
+    if os.path.isfile(path):
-        path = dirname(path)
+        path = os.path.dirname(path)
     if is_platformio_project(path):
         return path
-    if isdir(dirname(path)):
+    if os.path.isdir(os.path.dirname(path)):
-        return find_project_dir_above(dirname(path))
+        return find_project_dir_above(os.path.dirname(path))
     return None
 
 
-def get_project_core_dir():
-    """Deprecated, use ProjectConfig.get_optional_dir("core") instead"""
-    return ProjectConfig.get_instance(
-        join(get_project_dir(), "platformio.ini")
-    ).get_optional_dir("core", exists=True)
-
-
 def get_project_cache_dir():
-    """Deprecated, use ProjectConfig.get_optional_dir("cache") instead"""
+    """Deprecated, use ProjectConfig.get("platformio", "cache_dir") instead"""
-    return ProjectConfig.get_instance(
+    return ProjectConfig.get_instance().get("platformio", "cache_dir")
-        join(get_project_dir(), "platformio.ini")
-    ).get_optional_dir("cache")
 
 
 def get_project_global_lib_dir():
     """
-    Deprecated, use ProjectConfig.get_optional_dir("globallib") instead
+    Deprecated, use ProjectConfig.get("platformio", "globallib_dir") instead
     "platformio-node-helpers" depends on it
     """
-    return ProjectConfig.get_instance(
+    return ProjectConfig.get_instance().get("platformio", "globallib_dir")
-        join(get_project_dir(), "platformio.ini")
-    ).get_optional_dir("globallib")
 
 
 def get_project_lib_dir():
     """
-    Deprecated, use ProjectConfig.get_optional_dir("lib") instead
+    Deprecated, use ProjectConfig.get("platformio", "lib_dir") instead
     "platformio-node-helpers" depends on it
     """
-    return ProjectConfig.get_instance(
+    return ProjectConfig.get_instance().get("platformio", "lib_dir")
-        join(get_project_dir(), "platformio.ini")
-    ).get_optional_dir("lib")
 
 
 def get_project_libdeps_dir():
     """
-    Deprecated, use ProjectConfig.get_optional_dir("libdeps") instead
+    Deprecated, use ProjectConfig.get("platformio", "libdeps_dir") instead
     "platformio-node-helpers" depends on it
     """
-    return ProjectConfig.get_instance(
+    return ProjectConfig.get_instance().get("platformio", "libdeps_dir")
-        join(get_project_dir(), "platformio.ini")
-    ).get_optional_dir("libdeps")
 
 
 def get_default_projects_dir():
-    docs_dir = join(fs.expanduser("~"), "Documents")
+    docs_dir = os.path.join(fs.expanduser("~"), "Documents")
     try:
         assert IS_WINDOWS
         import ctypes.wintypes  # pylint: disable=import-outside-toplevel
@@ -100,7 +83,7 @@ def get_default_projects_dir():
         docs_dir = buf.value
     except:  # pylint: disable=bare-except
         pass
-    return join(docs_dir, "PlatformIO", "Projects")
+    return os.path.join(docs_dir, "PlatformIO", "Projects")
 
 
 def compute_project_checksum(config):
@@ -113,16 +96,16 @@ def compute_project_checksum(config):
     # project file structure
     check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
     for d in (
-        config.get_optional_dir("include"),
+        config.get("platformio", "include_dir"),
-        config.get_optional_dir("src"),
+        config.get("platformio", "src_dir"),
-        config.get_optional_dir("lib"),
+        config.get("platformio", "lib_dir"),
     ):
-        if not isdir(d):
+        if not os.path.isdir(d):
             continue
         chunks = []
-        for root, _, files in walk(d):
+        for root, _, files in os.walk(d):
             for f in files:
-                path = join(root, f)
+                path = os.path.join(root, f)
                 if path.endswith(check_suffixes):
                     chunks.append(path)
         if not chunks:
@@ -171,8 +154,8 @@ def _load_project_ide_data(project_dir, env_names):
 
 def _load_cached_project_ide_data(project_dir, env_names):
     build_dir = ProjectConfig.get_instance(
-        join(project_dir, "platformio.ini")
+        os.path.join(project_dir, "platformio.ini")
-    ).get_optional_dir("build")
+    ).get("platformio", "build_dir")
     result = {}
     for name in env_names:
         if not os.path.isfile(os.path.join(build_dir, name, "idedata.json")):

platformio/project/options.py

@@ -14,12 +14,14 @@
 
 # pylint: disable=redefined-builtin, too-many-arguments
 
+import hashlib
 import os
 from collections import OrderedDict
 
 import click
 
 from platformio import fs
+from platformio.compat import IS_WINDOWS, hashlib_encode_data
 
 
 class ConfigOption(object):  # pylint: disable=too-many-instance-attributes
@@ -35,6 +37,7 @@ class ConfigOption(object):  # pylint: disable=too-many-instance-attributes
         buildenvvar=None,
         oldnames=None,
         default=None,
+        validate=None,
     ):
         self.scope = scope
         self.group = group
@@ -46,6 +49,7 @@ class ConfigOption(object):  # pylint: disable=too-many-instance-attributes
         self.buildenvvar = buildenvvar
         self.oldnames = oldnames
         self.default = default
+        self.validate = validate
 
     def as_dict(self):
         result = dict(
@@ -60,13 +64,11 @@ class ConfigOption(object):  # pylint: disable=too-many-instance-attributes
         )
         if isinstance(self.type, click.ParamType):
             result["type"] = self.type.name
-
         if isinstance(self.type, (click.IntRange, click.FloatRange)):
             result["min"] = self.type.min
             result["max"] = self.type.max
         if isinstance(self.type, click.Choice):
             result["choices"] = self.type.choices
-
         return result
 
 
@@ -78,6 +80,50 @@ def ConfigEnvOption(*args, **kwargs):
     return ConfigOption("env", *args, **kwargs)
 
 
+def calculate_path_hash(path):
+    return "%s-%s" % (
+        os.path.basename(path),
+        hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10],
+    )
+
+
+def expand_dir_templates(path):
+    project_dir = os.getcwd()
+    tpls = {
+        "$PROJECT_DIR": lambda: project_dir,
+        "$PROJECT_HASH": lambda: calculate_path_hash(project_dir),
+    }
+    done = False
+    while not done:
+        done = True
+        for tpl, cb in tpls.items():
+            if tpl not in path:
+                continue
+            path = path.replace(tpl, cb())
+            done = False
+    return path
+
+
+def validate_dir(path):
+    if not path:
+        return path
+    if path.startswith("~"):
+        path = fs.expanduser(path)
+    if "$" in path:
+        path = expand_dir_templates(path)
+    return os.path.realpath(path)
+
+
+def validate_core_dir(path):
+    default = ProjectOptions["platformio.core_dir"].default
+    win_core_dir = None
+    if IS_WINDOWS and path == default:
+        win_core_dir = os.path.splitdrive(path)[0] + "\\.platformio"
+        if os.path.isdir(win_core_dir):
+            path = win_core_dir
+    return validate_dir(path)
+
+
 ProjectOptions = OrderedDict(
     [
         ("%s.%s" % (option.scope, option.name), option)
@@ -121,6 +167,7 @@ ProjectOptions = OrderedDict(
                 oldnames=["home_dir"],
                 sysenvvar="PLATFORMIO_CORE_DIR",
                 default=os.path.join(fs.expanduser("~"), ".platformio"),
+                validate=validate_core_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -130,7 +177,8 @@ ProjectOptions = OrderedDict(
                     "Finder (LDF) looks for global libraries"
                 ),
                 sysenvvar="PLATFORMIO_GLOBALLIB_DIR",
-                default=os.path.join("$PROJECT_CORE_DIR", "lib"),
+                default=os.path.join("${platformio.core_dir}", "lib"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -140,7 +188,8 @@ ProjectOptions = OrderedDict(
                     "platforms"
                 ),
                 sysenvvar="PLATFORMIO_PLATFORMS_DIR",
-                default=os.path.join("$PROJECT_CORE_DIR", "platforms"),
+                default=os.path.join("${platformio.core_dir}", "platforms"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -149,7 +198,8 @@ ProjectOptions = OrderedDict(
                     "A location where PlatformIO Core keeps installed packages"
                 ),
                 sysenvvar="PLATFORMIO_PACKAGES_DIR",
-                default=os.path.join("$PROJECT_CORE_DIR", "packages"),
+                default=os.path.join("${platformio.core_dir}", "packages"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -160,7 +210,8 @@ ProjectOptions = OrderedDict(
                     "other service information)"
                 ),
                 sysenvvar="PLATFORMIO_CACHE_DIR",
-                default=os.path.join("$PROJECT_CORE_DIR", ".cache"),
+                default=os.path.join("${platformio.core_dir}", ".cache"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -171,6 +222,7 @@ ProjectOptions = OrderedDict(
                     "build environments"
                 ),
                 sysenvvar="PLATFORMIO_BUILD_CACHE_DIR",
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -182,6 +234,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_WORKSPACE_DIR",
                 default=os.path.join("$PROJECT_DIR", ".pio"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -192,7 +245,8 @@ ProjectOptions = OrderedDict(
                     "and other cached information"
                 ),
                 sysenvvar="PLATFORMIO_BUILD_DIR",
-                default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"),
+                default=os.path.join("${platformio.workspace_dir}", "build"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -202,7 +256,8 @@ ProjectOptions = OrderedDict(
                     "dependencies declared via `lib_deps` option"
                 ),
                 sysenvvar="PLATFORMIO_LIBDEPS_DIR",
-                default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"),
+                default=os.path.join("${platformio.workspace_dir}", "libdeps"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -213,6 +268,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_INCLUDE_DIR",
                 default=os.path.join("$PROJECT_DIR", "include"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -223,6 +279,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_SRC_DIR",
                 default=os.path.join("$PROJECT_DIR", "src"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -230,6 +287,7 @@ ProjectOptions = OrderedDict(
                 description="A storage for the custom/private project libraries",
                 sysenvvar="PLATFORMIO_LIB_DIR",
                 default=os.path.join("$PROJECT_DIR", "lib"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -240,6 +298,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_DATA_DIR",
                 default=os.path.join("$PROJECT_DIR", "data"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -250,6 +309,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_TEST_DIR",
                 default=os.path.join("$PROJECT_DIR", "test"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -257,6 +317,7 @@ ProjectOptions = OrderedDict(
                 description="A global storage for custom board manifests",
                 sysenvvar="PLATFORMIO_BOARDS_DIR",
                 default=os.path.join("$PROJECT_DIR", "boards"),
+                validate=validate_dir,
             ),
             ConfigPlatformioOption(
                 group="directory",
@@ -267,6 +328,7 @@ ProjectOptions = OrderedDict(
                 ),
                 sysenvvar="PLATFORMIO_SHARED_DIR",
                 default=os.path.join("$PROJECT_DIR", "shared"),
+                validate=validate_dir,
             ),
             #
             # [env]

tests/package/test_manager.py

@@ -486,5 +486,5 @@ def test_update_without_metadata(isolated_pio_core, tmpdir_factory):
     # update
     lm = LibraryPackageManager(str(storage_dir))
     new_pkg = lm.update(pkg, silent=True)
-    assert len(lm.get_installed()) == 3
+    assert len(lm.get_installed()) == 4
     assert new_pkg.metadata.spec.owner == "ottowinter"

tests/test_examples.py

@@ -67,7 +67,7 @@ def pytest_generate_tests(metafunc):
 def test_run(pioproject_dir):
     with fs.cd(pioproject_dir):
         config = ProjectConfig()
-        build_dir = config.get_optional_dir("build")
+        build_dir = config.get("platformio", "build_dir")
         if os.path.isdir(build_dir):
             fs.rmtree(build_dir)
 

tests/test_projectconf.py

@@ -18,12 +18,15 @@ import os
 
 import pytest
 
+from platformio import fs
 from platformio.project.config import ConfigParser, ProjectConfig
 from platformio.project.exception import InvalidProjectConfError, UnknownEnvNamesError
+from platformio.project.options import calculate_path_hash
 
 BASE_CONFIG = """
 [platformio]
 env_default = base, extra_2
+build_dir = ~/tmp/pio-$PROJECT_HASH
 extra_configs =
   extra_envs.ini
   extra_debug.ini
@@ -83,17 +86,23 @@ lib_install = 574
 build_flags = ${custom.debug_flags} ${custom.extra_flags}
 lib_ignore = ${env.lib_ignore}, Lib3
 upload_port = /dev/extra_2/port
+debug_server = ${custom.debug_server}
 """
 
 EXTRA_DEBUG_CONFIG = """
 # Override original "custom.debug_flags"
 [custom]
 debug_flags = -D DEBUG=1
+debug_server =
+    ${platformio.packages_dir}/tool-openocd/openocd
+    --help
 
 [env:extra_2]
 build_flags = -Og
 """
 
+DEFAULT_CORE_DIR = os.path.join(fs.expanduser("~"), ".platformio")
+
 
 @pytest.fixture(scope="module")
 def config(tmpdir_factory):
@@ -124,7 +133,7 @@ def test_warnings(config):
 
 
 def test_defaults(config):
-    assert config.get_optional_dir("core") == os.path.join(
+    assert config.get("platformio", "core_dir") == os.path.join(
         os.path.expanduser("~"), ".platformio"
     )
     assert config.get("strict_ldf", "lib_deps", ["Empty"]) == ["Empty"]
@@ -272,6 +281,14 @@ def test_getraw_value(config):
     assert config.getraw("env", "monitor_speed") == "9600"
     assert config.getraw("env:test_extends", "monitor_speed") == "115200"
 
+    # dir options
+    packages_dir = os.path.join(DEFAULT_CORE_DIR, "packages")
+    assert config.get("platformio", "packages_dir") == packages_dir
+    assert (
+        config.getraw("custom", "debug_server")
+        == f"\n{packages_dir}/tool-openocd/openocd\n--help"
+    )
+
 
 def test_get_value(config):
     assert config.get("custom", "debug_flags") == "-D DEBUG=1"
@@ -293,6 +310,15 @@ def test_get_value(config):
         "-D CUSTOM_DEBUG_FLAG",
     ]
 
+    # dir options
+    assert config.get("platformio", "packages_dir") == os.path.join(
+        DEFAULT_CORE_DIR, "packages"
+    )
+    assert config.get("env:extra_2", "debug_server") == [
+        os.path.join(DEFAULT_CORE_DIR, "packages", "tool-openocd", "openocd"),
+        "--help",
+    ]
+
 
 def test_items(config):
     assert config.items("custom") == [
@@ -300,6 +326,11 @@ def test_items(config):
         ("lib_flags", "-lc -lm"),
         ("extra_flags", ""),
         ("lib_ignore", "LibIgnoreCustom"),
+        (
+            "debug_server",
+            "\n%s/tool-openocd/openocd\n--help"
+            % os.path.join(DEFAULT_CORE_DIR, "packages"),
+        ),
     ]
     assert config.items(env="base") == [
         ("build_flags", ["-D DEBUG=1"]),
@@ -326,6 +357,13 @@ def test_items(config):
         ("build_flags", ["-Og"]),
         ("lib_ignore", ["LibIgnoreCustom", "Lib3"]),
         ("upload_port", "/dev/extra_2/port"),
+        (
+            "debug_server",
+            [
+                "%s/tool-openocd/openocd" % os.path.join(DEFAULT_CORE_DIR, "packages"),
+                "--help",
+            ],
+        ),
         ("monitor_speed", 9600),
         ("custom_monitor_speed", "115200"),
         ("lib_deps", ["Lib1", "Lib2"]),
@@ -426,6 +464,11 @@ def test_dump(tmpdir_factory):
         (
             "platformio",
             [
+                (
+                    "build_dir",
+                    "%s-%s"
+                    % (fs.expanduser("~/tmp/pio"), calculate_path_hash(os.getcwd())),
+                ),
                 ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]),
                 ("default_envs", ["base", "extra_2"]),
             ],