Fixed an issue when configuration file options partly ignored when `--project-conf` // Resolve #3034 (#3055)

* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` // Resolve #3034

* Py2 compatible makedirs

* Fix circle dependency

* Fix broken import in test examples

* Fix history

* Remove YAPF markers

* PyLint fix

* Fix invalid project conf path

* Move PIO Core to the root on Windows, issue with long CPPPATHs

* Respect global PLATFORMIO_BUILD_CACHE_DIR env var

* Fix Appveyor paths

* Minor changes

.appveyor.yml

@@ -6,14 +6,15 @@ platform:
 environment:
   matrix:
     - TOXENV: "py27"
-      PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P2_{build}
+      PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P2_{build}
 
     - TOXENV: "py36"
-      PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P3_{build}
+      PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P3_{build}
 
 install:
   - cmd: git submodule update --init --recursive
   - cmd: SET PATH=C:\MinGW\bin;%PATH%
+  - cmd: SET PLATFORMIO_CORE_DIR=C:\.pio
   - cmd: pip install --force-reinstall tox
 
 test_script:

.pylintrc

@@ -11,4 +11,5 @@ disable=
     too-few-public-methods,
     useless-object-inheritance,
     useless-import-alias,
-    fixme
+    fixme,
+    bad-option-value

HISTORY.rst

@@ -23,6 +23,7 @@ PlatformIO Core 4.0
 * Added ``--no-ansi`` flag for `PIO Core <http://docs.platformio.org/page/userguide/index.html>`__ to disable ANSI control characters
 * Fixed an issue with project generator for `CLion IDE <http://docs.platformio.org/page/ide/clion.html>`__ when 2 environments were used (`issue #2824 <https://github.com/platformio/platformio-core/issues/2824>`_)
 * Fixed default PIO Unified Debugger configuration for `J-Link probe <http://docs.platformio.org/page/plus/debug-tools/jlink.html>`__
+* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 <https://github.com/platformio/platformio-core/issues/3034>`_)
 
 4.0.3 (2019-08-30)
 ~~~~~~~~~~~~~~~~~~

platformio/__main__.py

@@ -70,7 +70,7 @@ def configure():
     # https://urllib3.readthedocs.org
     # /en/latest/security.html#insecureplatformwarning
     try:
-        import urllib3
+        import urllib3  # pylint: disable=import-outside-toplevel
 
         urllib3.disable_warnings()
     except (AttributeError, ImportError):

platformio/app.py

@@ -17,7 +17,7 @@ import hashlib
 import os
 import uuid
 from os import environ, getenv, listdir, remove
-from os.path import abspath, dirname, expanduser, isdir, isfile, join
+from os.path import abspath, dirname, isdir, isfile, join
 from time import time
 
 import requests
@@ -25,21 +25,11 @@ import requests
 from platformio import exception, fs, lockfile
 from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data
 from platformio.proc import is_ci
-from platformio.project.helpers import get_project_cache_dir, get_project_core_dir
+from platformio.project.helpers import (
-
+    get_default_projects_dir,
-
+    get_project_cache_dir,
-def get_default_projects_dir():
+    get_project_core_dir,
-    docs_dir = join(expanduser("~"), "Documents")
+)
-    try:
-        assert WINDOWS
-        import ctypes.wintypes
-
-        buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH)
-        ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf)
-        docs_dir = buf.value
-    except:  # pylint: disable=bare-except
-        pass
-    return join(docs_dir, "PlatformIO", "Projects")
 
 
 def projects_dir_validate(projects_dir):
@@ -88,7 +78,12 @@ DEFAULT_SETTINGS = {
     },
 }
 
-SESSION_VARS = {"command_ctx": None, "force_option": False, "caller_id": None}
+SESSION_VARS = {
+    "command_ctx": None,
+    "force_option": False,
+    "caller_id": None,
+    "custom_project_conf": None,
+}
 
 
 class State(object):
@@ -415,6 +410,6 @@ def get_cid():
         uid = uuid.getnode()
     cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest())
     cid = str(cid)
-    if WINDOWS or os.getuid() > 0:  # yapf: disable pylint: disable=no-member
+    if WINDOWS or os.getuid() > 0:  # pylint: disable=no-member
         set_state_item("cid", cid)
     return cid

platformio/builder/main.py

@@ -31,7 +31,7 @@ from platformio import fs
 from platformio.compat import PY2, dump_json_to_unicode
 from platformio.managers.platform import PlatformBase
 from platformio.proc import get_pythonexe_path
-from platformio.project import helpers as project_helpers
+from platformio.project.helpers import get_project_dir
 
 AllowSubstExceptions(NameError)
 
@@ -44,7 +44,7 @@ clivars.AddVariables(
     ("PIOENV",),
     ("PIOTEST_RUNNING_NAME",),
     ("UPLOAD_PORT",),
-)  # yapf: disable
+)
 
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
@@ -67,26 +67,10 @@ DEFAULT_ENV_OPTIONS = dict(
     # Propagating External Environment
     ENV=environ,
     UNIX_TIME=int(time()),
-    PROJECT_DIR=project_helpers.get_project_dir(),
+    BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"),
-    PROJECTCORE_DIR=project_helpers.get_project_core_dir(),
+    BUILD_SRC_DIR=join("$BUILD_DIR", "src"),
-    PROJECTPACKAGES_DIR=project_helpers.get_project_packages_dir(),
+    BUILD_TEST_DIR=join("$BUILD_DIR", "test"),
-    PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(),
-    PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(),
-    PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(),
-    PROJECTSRC_DIR=project_helpers.get_project_src_dir(),
-    PROJECTTEST_DIR=project_helpers.get_project_test_dir(),
-    PROJECTDATA_DIR=project_helpers.get_project_data_dir(),
-    PROJECTBUILD_DIR=project_helpers.get_project_build_dir(),
-    BUILDCACHE_DIR=project_helpers.get_project_optional_dir("build_cache_dir"),
-    BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"),
-    BUILDSRC_DIR=join("$BUILD_DIR", "src"),
-    BUILDTEST_DIR=join("$BUILD_DIR", "test"),
     LIBPATH=["$BUILD_DIR"],
-    LIBSOURCE_DIRS=[
-        project_helpers.get_project_lib_dir(),
-        join("$PROJECTLIBDEPS_DIR", "$PIOENV"),
-        project_helpers.get_project_global_lib_dir(),
-    ],
     PROGNAME="program",
     PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
     PYTHONEXE=get_pythonexe_path(),
@@ -110,10 +94,33 @@ env.Replace(
     }
 )
 
-if env.subst("$BUILDCACHE_DIR"):
+# Setup project optional directories
-    if not isdir(env.subst("$BUILDCACHE_DIR")):
+config = env.GetProjectConfig()
-        makedirs(env.subst("$BUILDCACHE_DIR"))
+env.Replace(
-    env.CacheDir("$BUILDCACHE_DIR")
+    PROJECT_DIR=get_project_dir(),
+    PROJECT_CORE_DIR=config.get_optional_dir("core"),
+    PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"),
+    PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"),
+    PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"),
+    PROJECT_INCLUDE_DIR=config.get_optional_dir("include"),
+    PROJECT_SRC_DIR=config.get_optional_dir("src"),
+    PROJECTSRC_DIR=config.get_optional_dir("src"),  # legacy for dev/platform
+    PROJECT_TEST_DIR=config.get_optional_dir("test"),
+    PROJECT_DATA_DIR=config.get_optional_dir("data"),
+    PROJECTDATA_DIR=config.get_optional_dir("data"),  # legacy for dev/platform
+    PROJECT_BUILD_DIR=config.get_optional_dir("build"),
+    BUILD_CACHE_DIR=config.get_optional_dir("build_cache"),
+    LIBSOURCE_DIRS=[
+        config.get_optional_dir("lib"),
+        join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
+        config.get_optional_dir("globallib"),
+    ],
+)
+
+if env.subst("$BUILD_CACHE_DIR"):
+    if not isdir(env.subst("$BUILD_CACHE_DIR")):
+        makedirs(env.subst("$BUILD_CACHE_DIR"))
+    env.CacheDir("$BUILD_CACHE_DIR")
 
 if int(ARGUMENTS.get("ISATTY", 0)):
     # pylint: disable=protected-access

platformio/builder/tools/pioide.py

@@ -54,7 +54,7 @@ def _dump_includes(env, projenv):
     if unity_dir:
         includes.append(unity_dir)
 
-    includes.extend([env.subst("$PROJECTINCLUDE_DIR"), env.subst("$PROJECTSRC_DIR")])
+    includes.extend([env.subst("$PROJECT_INCLUDE_DIR"), env.subst("$PROJECT_SRC_DIR")])
 
     # remove duplicates
     result = []

platformio/builder/tools/piolib.py

@@ -601,8 +601,7 @@ class MbedLibBuilder(LibBuilderBase):
             mbed_config_path = join(self.env.subst(p), "mbed_config.h")
             if isfile(mbed_config_path):
                 break
-            else:
+            mbed_config_path = None
-                mbed_config_path = None
         if not mbed_config_path:
             return None
 
@@ -821,16 +820,16 @@ class ProjectAsLibBuilder(LibBuilderBase):
 
     @property
     def include_dir(self):
-        include_dir = self.env.subst("$PROJECTINCLUDE_DIR")
+        include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
         return include_dir if isdir(include_dir) else None
 
     @property
     def src_dir(self):
-        return self.env.subst("$PROJECTSRC_DIR")
+        return self.env.subst("$PROJECT_SRC_DIR")
 
     def get_include_dirs(self):
         include_dirs = []
-        project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR")
+        project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR")
         if isdir(project_include_dir):
             include_dirs.append(project_include_dir)
         for include_dir in LibBuilderBase.get_include_dirs(self):
@@ -845,9 +844,9 @@ class ProjectAsLibBuilder(LibBuilderBase):
         if "__test" in COMMAND_LINE_TARGETS:
             items.extend(
                 [
-                    join("$PROJECTTEST_DIR", item)
+                    join("$PROJECT_TEST_DIR", item)
                     for item in self.env.MatchSourceFiles(
-                        "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER"
+                        "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
                     )
                 ]
             )
@@ -896,7 +895,7 @@ class ProjectAsLibBuilder(LibBuilderBase):
                 not_found_uri.append(uri)
 
         did_install = False
-        lm = LibraryManager(self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV")))
+        lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV")))
         for uri in not_found_uri:
             try:
                 lm.install(uri)

platformio/builder/tools/piomisc.py

@@ -117,7 +117,7 @@ class InoToCPPConverter(object):
                     stropen = True
                     newlines.append(line[:-1])
                     continue
-                elif stropen:
+                if stropen:
                     newlines[len(newlines) - 1] += line[:-1]
                     continue
             elif stropen and line.endswith(('",', '";')):
@@ -199,7 +199,7 @@ class InoToCPPConverter(object):
 
 
 def ConvertInoToCpp(env):
-    src_dir = glob_escape(env.subst("$PROJECTSRC_DIR"))
+    src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR"))
     ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde"))
     if not ino_nodes:
         return
@@ -256,7 +256,7 @@ def GetActualLDScript(env):
         if f == "-T":
             script_in_next = True
             continue
-        elif script_in_next:
+        if script_in_next:
             script_in_next = False
             raw_script = f
         elif f.startswith("-Wl,-T"):
@@ -309,7 +309,7 @@ def PioClean(env, clean_dir):
     env.Exit(0)
 
 
-def ProcessDebug(env):
+def ConfigureDebugTarget(env):
     if not env.subst("$PIODEBUGFLAGS"):
         env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"])
     env.Append(
@@ -322,7 +322,7 @@ def ProcessDebug(env):
     env.Append(BUILD_UNFLAGS=unflags)
 
 
-def ProcessTest(env):
+def ConfigureTestTarget(env):
     env.Append(
         CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"],
         CPPPATH=[join("$BUILD_DIR", "UnityTestLib")],
@@ -361,7 +361,7 @@ def generate(env):
     env.AddMethod(GetActualLDScript)
     env.AddMethod(VerboseAction)
     env.AddMethod(PioClean)
-    env.AddMethod(ProcessDebug)
+    env.AddMethod(ConfigureDebugTarget)
-    env.AddMethod(ProcessTest)
+    env.AddMethod(ConfigureTestTarget)
     env.AddMethod(GetExtraScripts)
     return env

platformio/builder/tools/platformio.py

@@ -67,15 +67,17 @@ def _build_project_deps(env):
     is_test = "__test" in COMMAND_LINE_TARGETS
     if is_test:
         projenv.BuildSources(
-            "$BUILDTEST_DIR", "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER"
+            "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER"
         )
     if not is_test or env.GetProjectOption("test_build_project_src", False):
-        projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", env.get("SRC_FILTER"))
+        projenv.BuildSources(
+            "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER")
+        )
 
     if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS:
         sys.stderr.write(
             "Error: Nothing to build. Please put your source code files "
-            "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR")
+            "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR")
         )
         env.Exit(1)
 
@@ -102,7 +104,7 @@ def BuildProgram(env):
         env.Replace(AS="$CC", ASCOM="$ASPPCOM")
 
     if "debug" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "debug":
-        env.ProcessDebug()
+        env.ConfigureDebugTarget()
 
     # process extra flags from board
     if "BOARD" in env and "build.extra_flags" in env.BoardConfig():
@@ -121,7 +123,7 @@ def BuildProgram(env):
     env.ProcessUnFlags(env.get("BUILD_UNFLAGS"))
 
     if "__test" in COMMAND_LINE_TARGETS:
-        env.ProcessTest()
+        env.ConfigureTestTarget()
 
     # build project with dependencies
     _build_project_deps(env)

platformio/check/tools/cppcheck.py

@@ -19,7 +19,6 @@ from tempfile import NamedTemporaryFile
 from platformio.check.defect import DefectItem
 from platformio.check.tools.base import CheckToolBase
 from platformio.managers.core import get_core_package_dir
-from platformio.project.helpers import get_project_core_dir
 
 
 class CppcheckCheckTool(CheckToolBase):
@@ -110,7 +109,7 @@ class CppcheckCheckTool(CheckToolBase):
         cmd.append("--file-list=%s" % self._generate_src_file())
         cmd.append("--includes-file=%s" % self._generate_inc_file())
 
-        core_dir = get_project_core_dir()
+        core_dir = self.config.get_optional_dir("core")
         cmd.append("--suppress=*:%s*" % core_dir)
         cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir)
 

platformio/commands/check.py

@@ -17,23 +17,18 @@
 
 import os
 from collections import Counter
-from os.path import basename, dirname, isfile, join
+from os.path import basename, dirname, isfile
 from time import time
 
 import click
 from tabulate import tabulate
 
-from platformio import exception, fs, util
+from platformio import app, exception, fs, util
 from platformio.check.defect import DefectItem
 from platformio.check.tools import CheckToolFactory
 from platformio.compat import dump_json_to_unicode
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (
+from platformio.project.helpers import find_project_dir_above, get_project_dir
-    find_project_dir_above,
-    get_project_dir,
-    get_project_include_dir,
-    get_project_src_dir,
-)
 
 
 @click.command("check", short_help="Run a static analysis tool on code")
@@ -72,15 +67,15 @@ def cli(
     verbose,
     json_output,
 ):
+    app.set_session_var("custom_project_conf", project_conf)
+
     # find project directory on upper level
     if isfile(project_dir):
         project_dir = find_project_dir_above(project_dir)
 
     results = []
     with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(
+        config = ProjectConfig.get_instance(project_conf)
-            project_conf or join(project_dir, "platformio.ini")
-        )
         config.validate(environment)
 
         default_envs = config.default_envs()
@@ -103,7 +98,10 @@ def cli(
 
             default_filter = [
                 "+<%s/>" % basename(d)
-                for d in (get_project_src_dir(), get_project_include_dir())
+                for d in (
+                    config.get_optional_dir("src"),
+                    config.get_optional_dir("include"),
+                )
             ]
 
             tool_options = dict(

platformio/commands/debug.py

@@ -17,11 +17,11 @@
 
 import os
 import signal
-from os.path import isfile, join
+from os.path import isfile
 
 import click
 
-from platformio import exception, fs, proc, util
+from platformio import app, exception, fs, proc, util
 from platformio.debug import helpers
 from platformio.managers.core import inject_contrib_pysite
 from platformio.project.config import ProjectConfig
@@ -54,6 +54,8 @@ from platformio.project.helpers import is_platformio_project, load_project_ide_d
 @click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED)
 @click.pass_context
 def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unprocessed):
+    app.set_session_var("custom_project_conf", project_conf)
+
     # use env variables from Eclipse or CLion
     for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"):
         if is_platformio_project(project_dir):
@@ -62,9 +64,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro
             project_dir = os.getenv(sysenv)
 
     with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(
+        config = ProjectConfig.get_instance(project_conf)
-            project_conf or join(project_dir, "platformio.ini")
-        )
         config.validate(envs=[environment] if environment else None)
 
         env_name = environment or helpers.get_default_debug_env(config)

platformio/commands/device.py

@@ -15,12 +15,11 @@
 import sys
 from fnmatch import fnmatch
 from os import getcwd
-from os.path import join
 
 import click
 from serial.tools import miniterm
 
-from platformio import exception, util
+from platformio import exception, fs, util
 from platformio.compat import dump_json_to_unicode
 from platformio.project.config import ProjectConfig
 
@@ -175,7 +174,8 @@ def device_list(  # pylint: disable=too-many-branches
 def device_monitor(**kwargs):  # pylint: disable=too-many-branches
     env_options = {}
     try:
-        env_options = get_project_options(kwargs["project_dir"], kwargs["environment"])
+        with fs.cd(kwargs["project_dir"]):
+            env_options = get_project_options(kwargs["environment"])
         for k in ("port", "speed", "rts", "dtr"):
             k2 = "monitor_%s" % k
             if k == "speed":
@@ -225,8 +225,8 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
         raise exception.MinitermException(e)
 
 
-def get_project_options(project_dir, environment=None):
+def get_project_options(environment=None):
-    config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
+    config = ProjectConfig.get_instance()
     config.validate(envs=[environment] if environment else None)
     if not environment:
         default_envs = config.default_envs()

platformio/commands/init.py

@@ -24,13 +24,7 @@ from platformio.commands.platform import platform_install as cli_platform_instal
 from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.managers.platform import PlatformManager
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (
+from platformio.project.helpers import is_platformio_project
-    get_project_include_dir,
-    get_project_lib_dir,
-    get_project_src_dir,
-    get_project_test_dir,
-    is_platformio_project,
-)
 
 
 def validate_boards(ctx, param, value):  # pylint: disable=W0613
@@ -133,13 +127,14 @@ def cli(
 
 
 def init_base_project(project_dir):
-    ProjectConfig(join(project_dir, "platformio.ini")).save()
     with fs.cd(project_dir):
+        config = ProjectConfig()
+        config.save()
         dir_to_readme = [
-            (get_project_src_dir(), None),
+            (config.get_optional_dir("src"), None),
-            (get_project_include_dir(), init_include_readme),
+            (config.get_optional_dir("include"), init_include_readme),
-            (get_project_lib_dir(), init_lib_readme),
+            (config.get_optional_dir("lib"), init_lib_readme),
-            (get_project_test_dir(), init_test_readme),
+            (config.get_optional_dir("test"), init_test_readme),
         ]
         for (path, cb) in dir_to_readme:
             if isdir(path):

platformio/commands/lib.py

@@ -21,18 +21,13 @@ import click
 import semantic_version
 from tabulate import tabulate
 
-from platformio import exception, fs, util
+from platformio import exception, util
 from platformio.commands import PlatformioCLI
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib
 from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (
+from platformio.project.helpers import get_project_dir, is_platformio_project
-    get_project_dir,
-    get_project_global_lib_dir,
-    get_project_libdeps_dir,
-    is_platformio_project,
-)
 
 try:
     from urllib.parse import quote
@@ -45,6 +40,10 @@ CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs"
 CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps"
 
 
+def get_project_global_lib_dir():
+    return ProjectConfig.get_instance().get_optional_dir("globallib")
+
+
 @click.group(short_help="Library Manager")
 @click.option(
     "-d",
@@ -105,10 +104,9 @@ def cli(ctx, **options):
         if not is_platformio_project(storage_dir):
             ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
             continue
-        with fs.cd(storage_dir):
-            libdeps_dir = get_project_libdeps_dir()
         config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini"))
         config.validate(options["environment"], silent=in_silence)
+        libdeps_dir = config.get_optional_dir("libdeps")
         for env in config.envs():
             if options["environment"] and env not in options["environment"]:
                 continue
@@ -336,7 +334,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
         click.secho(" *", fg="green")
         click.secho("For example: DS*, PCA*, DHT* and etc.\n", fg="yellow")
         click.echo(
-            "For more examples and advanced search syntax, " "please use documentation:"
+            "For more examples and advanced search syntax, please use documentation:"
         )
         click.secho(
             "https://docs.platformio.org/page/userguide/lib/cmd_search.html\n",

platformio/commands/platform.py

@@ -331,7 +331,7 @@ def platform_uninstall(platforms):
     for platform in platforms:
         if pm.uninstall(platform):
             click.secho(
-                "The platform '%s' has been successfully " "uninstalled!" % platform,
+                "The platform '%s' has been successfully uninstalled!" % platform,
                 fg="green",
             )
 

platformio/commands/run.py

@@ -14,16 +14,16 @@
 
 from multiprocessing import cpu_count
 from os import getcwd
-from os.path import isfile, join
+from os.path import isfile
 from time import time
 
 import click
 from tabulate import tabulate
 
-from platformio import exception, fs, util
+from platformio import app, exception, fs, util
 from platformio.commands.device import device_monitor as cmd_device_monitor
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import find_project_dir_above, get_project_build_dir
+from platformio.project.helpers import find_project_dir_above
 from platformio.run.helpers import clean_build_dir, handle_legacy_libdeps
 from platformio.run.processor import EnvironmentProcessor
 from platformio.test.processor import CTX_META_TEST_IS_RUNNING
@@ -81,6 +81,8 @@ def cli(
     verbose,
     disable_auto_clean,
 ):
+    app.set_session_var("custom_project_conf", project_conf)
+
     # find project directory on upper level
     if isfile(project_dir):
         project_dir = find_project_dir_above(project_dir)
@@ -88,20 +90,18 @@ def cli(
     is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta
 
     with fs.cd(project_dir):
-        config = ProjectConfig.get_instance(
+        config = ProjectConfig.get_instance(project_conf)
-            project_conf or join(project_dir, "platformio.ini")
-        )
         config.validate(environment)
 
         # clean obsolete build dir
         if not disable_auto_clean:
+            build_dir = config.get_optional_dir("build")
             try:
-                clean_build_dir(get_project_build_dir(), config)
+                clean_build_dir(build_dir, config)
             except:  # pylint: disable=bare-except
                 click.secho(
                     "Can not remove temporary directory `%s`. Please remove "
-                    "it manually to avoid build issues"
+                    "it manually to avoid build issues" % build_dir,
-                    % get_project_build_dir(force=True),
                     fg="yellow",
                 )
 

platformio/commands/test.py

@@ -22,9 +22,8 @@ from time import time
 import click
 from tabulate import tabulate
 
-from platformio import exception, fs, util
+from platformio import app, exception, fs, util
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import get_project_test_dir
 from platformio.test.embedded import EmbeddedTestProcessor
 from platformio.test.native import NativeTestProcessor
 
@@ -97,17 +96,17 @@ def cli(  # pylint: disable=redefined-builtin
     monitor_dtr,
     verbose,
 ):
+    app.set_session_var("custom_project_conf", project_conf)
+
     with fs.cd(project_dir):
-        test_dir = get_project_test_dir()
+        config = ProjectConfig.get_instance(project_conf)
+        config.validate(envs=environment)
+
+        test_dir = config.get_optional_dir("test")
         if not isdir(test_dir):
             raise exception.TestDirNotExists(test_dir)
         test_names = get_test_names(test_dir)
 
-        config = ProjectConfig.get_instance(
-            project_conf or join(project_dir, "platformio.ini")
-        )
-        config.validate(envs=environment)
-
         click.echo("Verbose mode can be enabled via `-v, --verbose` option")
         click.secho("Collected %d items" % len(test_names), bold=True)
 

platformio/commands/upgrade.py

@@ -92,7 +92,7 @@ WARNING! Don't use `sudo` for the rest PlatformIO commands.
 def get_pip_package(to_develop):
     if not to_develop:
         return "platformio"
-    dl_url = "https://github.com/platformio/" "platformio-core/archive/develop.zip"
+    dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip"
     cache_dir = get_project_cache_dir()
     if not os.path.isdir(cache_dir):
         os.makedirs(cache_dir)

platformio/debug/client.py

@@ -91,7 +91,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
             self.project_dir,
             "-l",
             "10",
-        ]  # yapf: disable
+        ]
         args.extend(self.args)
         if not gdb_path:
             raise exception.DebugInvalidOptions("GDB client is not configured")
@@ -139,14 +139,14 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
                 "   echo Warning! Undefined pio_reset_target command\\n",
                 "   mon reset",
                 "end",
-            ] + commands  # yapf: disable
+            ] + commands
         if not any("define pio_reset_halt_target" in cmd for cmd in commands):
             commands = [
                 "define pio_reset_halt_target",
                 "   echo Warning! Undefined pio_reset_halt_target command\\n",
                 "   mon reset halt",
                 "end",
-            ] + commands  # yapf: disable
+            ] + commands
         if not any("define pio_restart_target" in cmd for cmd in commands):
             commands += [
                 "define pio_restart_target",
@@ -154,7 +154,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
                 "   $INIT_BREAK",
                 "   %s" % ("continue" if patterns["INIT_BREAK"] else "next"),
                 "end",
-            ]  # yapf: disable
+            ]
 
         banner = [
             "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n",
@@ -243,7 +243,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
             % self.debug_options["init_break"]
         )
         self.console_log(
-            "PlatformIO: More configuration options -> " "http://bit.ly/pio-debug"
+            "PlatformIO: More configuration options -> http://bit.ly/pio-debug"
         )
         self.transport.write(
             b"0-exec-continue\n" if helpers.is_mi_mode(self.args) else b"continue\n"

platformio/debug/server.py

@@ -64,7 +64,7 @@ class DebugServer(BaseProcess):
         self._debug_port = ":3333"
         openocd_pipe_allowed = all(
             [not self.debug_options["port"], "openocd" in server_executable]
-        )  # yapf: disable
+        )
         if openocd_pipe_allowed:
             args = []
             if server["cwd"]:

platformio/exception.py

@@ -148,7 +148,7 @@ class FDSizeMismatch(PlatformIOPackageException):
 class FDSHASumMismatch(PlatformIOPackageException):
 
     MESSAGE = (
-        "The 'sha1' sum '{0}' of downloaded file '{1}' " "is not equal to remote '{2}'"
+        "The 'sha1' sum '{0}' of downloaded file '{1}' is not equal to remote '{2}'"
     )
 
 

platformio/home/rpc/handlers/project.py

@@ -28,20 +28,16 @@ from platformio.home.rpc.handlers.piocore import PIOCoreRPC
 from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.managers.platform import PlatformManager
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (
+from platformio.project.helpers import is_platformio_project
-    get_project_libdeps_dir,
-    get_project_src_dir,
-    is_platformio_project,
-)
 
 
 class ProjectRPC(object):
     @staticmethod
     def _get_projects(project_dirs=None):
-        def _get_project_data(project_dir):
+        def _get_project_data():
             data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []}
-            config = ProjectConfig(join(project_dir, "platformio.ini"))
+            config = ProjectConfig()
-            libdeps_dir = get_project_libdeps_dir()
+            libdeps_dir = config.get_optional_dir("libdeps")
 
             data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", []))
 
@@ -76,7 +72,7 @@ class ProjectRPC(object):
             boards = []
             try:
                 with fs.cd(project_dir):
-                    data = _get_project_data(project_dir)
+                    data = _get_project_data()
             except exception.PlatformIOProjectException:
                 continue
 
@@ -178,9 +174,10 @@ class ProjectRPC(object):
                     "",
                     "void loop() {",
                     "  // put your main code here, to run repeatedly:",
-                    "}" "",
+                    "}",
+                    "",
                 ]
-            )  # yapf: disable
+            )
         elif framework == "mbed":
             main_content = "\n".join(
                 [
@@ -196,11 +193,12 @@ class ProjectRPC(object):
                     "}",
                     "",
                 ]
-            )  # yapf: disable
+            )
         if not main_content:
             return project_dir
         with fs.cd(project_dir):
-            src_dir = get_project_src_dir()
+            config = ProjectConfig()
+            src_dir = config.get_optional_dir("src")
             main_path = join(src_dir, "main.cpp")
             if isfile(main_path):
                 return project_dir
@@ -258,7 +256,8 @@ class ProjectRPC(object):
     @staticmethod
     def _finalize_arduino_import(_, project_dir, arduino_project_dir):
         with fs.cd(project_dir):
-            src_dir = get_project_src_dir()
+            config = ProjectConfig()
+            src_dir = config.get_optional_dir("src")
             if isdir(src_dir):
                 fs.rmtree(src_dir)
             shutil.copytree(arduino_project_dir, src_dir)

platformio/ide/projectgenerator.py

@@ -23,12 +23,7 @@ from platformio import fs, util
 from platformio.compat import get_file_contents
 from platformio.proc import where_is_program
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (
+from platformio.project.helpers import load_project_ide_data
-    get_project_lib_dir,
-    get_project_libdeps_dir,
-    get_project_src_dir,
-    load_project_ide_data,
-)
 
 
 class ProjectGenerator(object):
@@ -76,7 +71,7 @@ class ProjectGenerator(object):
             else where_is_program("platformio"),
             "env_path": os.getenv("PATH"),
             "env_pathsep": os.pathsep,
-        }  # yapf: disable
+        }
 
         # default env configuration
         tpl_vars.update(self.config.items(env=self.env_name, as_dict=True))
@@ -87,13 +82,13 @@ class ProjectGenerator(object):
             tpl_vars.update(
                 {
                     "src_files": self.get_src_files(),
-                    "project_src_dir": get_project_src_dir(),
+                    "project_src_dir": self.config.get_optional_dir("src"),
-                    "project_lib_dir": get_project_lib_dir(),
+                    "project_lib_dir": self.config.get_optional_dir("lib"),
                     "project_libdeps_dir": join(
-                        get_project_libdeps_dir(), self.env_name
+                        self.config.get_optional_dir("libdeps"), self.env_name
                     ),
                 }
-            )  # yapf: disable
+            )
 
         for key, value in tpl_vars.items():
             if key.endswith(("_path", "_dir")):
@@ -109,7 +104,7 @@ class ProjectGenerator(object):
     def get_src_files(self):
         result = []
         with fs.cd(self.project_dir):
-            for root, _, files in os.walk(get_project_src_dir()):
+            for root, _, files in os.walk(self.config.get_optional_dir("src")):
                 for f in files:
                     result.append(relpath(join(root, f)))
         return result

platformio/managers/core.py

@@ -21,7 +21,7 @@ from platformio import __version__, exception, fs
 from platformio.compat import PY2, WINDOWS
 from platformio.managers.package import PackageManager
 from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
-from platformio.project.helpers import get_project_packages_dir
+from platformio.project.config import ProjectConfig
 
 CORE_PACKAGES = {
     "contrib-piohome": "^2.3.2",
@@ -40,8 +40,10 @@ PIOPLUS_AUTO_UPDATES_MAX = 100
 
 class CorePackageManager(PackageManager):
     def __init__(self):
+        config = ProjectConfig.get_instance()
+        packages_dir = config.get_optional_dir("packages")
         super(CorePackageManager, self).__init__(
-            get_project_packages_dir(),
+            packages_dir,
             [
                 "https://dl.bintray.com/platformio/dl-packages/manifest.json",
                 "http%s://dl.platformio.org/packages/manifest.json"

platformio/managers/lib.py

@@ -27,7 +27,7 @@ from platformio import app, exception, util
 from platformio.compat import glob_escape, string_types
 from platformio.managers.package import BasePkgManager
 from platformio.managers.platform import PlatformFactory, PlatformManager
-from platformio.project.helpers import get_project_global_lib_dir
+from platformio.project.config import ProjectConfig
 
 
 class LibraryManager(BasePkgManager):
@@ -35,9 +35,10 @@ class LibraryManager(BasePkgManager):
     FILE_CACHE_VALID = "30d"  # 1 month
 
     def __init__(self, package_dir=None):
-        if not package_dir:
+        self.config = ProjectConfig.get_instance()
-            package_dir = get_project_global_lib_dir()
+        super(LibraryManager, self).__init__(
-        super(LibraryManager, self).__init__(package_dir)
+            package_dir or self.config.get_optional_dir("globallib")
+        )
 
     @property
     def manifest_names(self):

platformio/managers/package.py

@@ -405,7 +405,7 @@ class PkgInstallerMixin(object):
                     self.parse_semver_version(manifest["version"], raise_exception=True)
                 ):
                     continue
-                elif not best or (
+                if not best or (
                     self.parse_semver_version(manifest["version"], raise_exception=True)
                     > self.parse_semver_version(best["version"], raise_exception=True)
                 ):

platformio/managers/platform.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=too-many-public-methods, too-many-instance-attributes
+
 import base64
 import os
 import re
@@ -32,12 +34,6 @@ from platformio.proc import (
     get_pythonexe_path,
 )
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (
-    get_project_boards_dir,
-    get_project_core_dir,
-    get_project_packages_dir,
-    get_project_platforms_dir,
-)
 
 try:
     from urllib.parse import quote
@@ -54,8 +50,9 @@ class PlatformManager(BasePkgManager):
                     "https" if app.get_setting("strict_ssl") else "http"
                 ),
             ]
+        self.config = ProjectConfig.get_instance()
         BasePkgManager.__init__(
-            self, package_dir or get_project_platforms_dir(), repositories
+            self, package_dir or self.config.get_optional_dir("platforms"), repositories
         )
 
     @property
@@ -164,7 +161,7 @@ class PlatformManager(BasePkgManager):
                     deppkgs[pkgname] = set()
                 deppkgs[pkgname].add(pkgmanifest["version"])
 
-        pm = PackageManager(get_project_packages_dir())
+        pm = PackageManager(self.config.get_optional_dir("packages"))
         for manifest in pm.get_installed():
             if manifest["name"] not in names:
                 continue
@@ -290,7 +287,7 @@ class PlatformPackagesMixin(object):
             version = opts.get("version", "")
             if name in without_packages:
                 continue
-            elif name in with_packages or not (
+            if name in with_packages or not (
                 skip_default_package or opts.get("optional", False)
             ):
                 if ":" in version:
@@ -385,8 +382,7 @@ class PlatformRunMixin(object):
         assert isinstance(variables, dict)
         assert isinstance(targets, list)
 
-        config = ProjectConfig.get_instance(variables["project_config"])
+        options = self.config.items(env=variables["pioenv"], as_dict=True)
-        options = config.items(env=variables["pioenv"], as_dict=True)
         if "framework" in options:
             # support PIO Core 3.0 dev/platforms
             options["pioframework"] = options["framework"]
@@ -421,7 +417,7 @@ class PlatformRunMixin(object):
             str(jobs),
             "--sconstruct",
             join(fs.get_source_dir(), "builder", "main.py"),
-        ]  # yapf: disable
+        ]
         args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
         # pylint: disable=protected-access
         args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0))
@@ -503,9 +499,7 @@ class PlatformRunMixin(object):
         click.echo(banner, err=True)
 
 
-class PlatformBase(  # pylint: disable=too-many-public-methods
+class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
-    PlatformPackagesMixin, PlatformRunMixin
-):
 
     PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__))
     _BOARDS_CACHE = {}
@@ -519,8 +513,10 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
         self._manifest = fs.load_json(manifest_path)
         self._custom_packages = None
 
-        self.pm = PackageManager(get_project_packages_dir(), self.package_repositories)
+        self.config = ProjectConfig.get_instance()
-
+        self.pm = PackageManager(
+            self.config.get_optional_dir("packages"), self.package_repositories
+        )
         # if self.engines and "platformio" in self.engines:
         #     if self.PIO_VERSION not in semantic_version.SimpleSpec(
         #             self.engines['platformio']):
@@ -619,8 +615,8 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
             self._BOARDS_CACHE[board_id] = config
 
         bdirs = [
-            get_project_boards_dir(),
+            self.config.get_optional_dir("boards"),
-            join(get_project_core_dir(), "boards"),
+            join(self.config.get_optional_dir("core"), "boards"),
             join(self.get_dir(), "boards"),
         ]
 

platformio/project/config.py

@@ -16,11 +16,12 @@ import glob
 import json
 import os
 import re
-from os.path import expanduser, getmtime, isfile
+from hashlib import sha1
 
 import click
 
 from platformio import exception
+from platformio.compat import WINDOWS, hashlib_encode_data
 from platformio.project.options import ProjectOptions
 
 try:
@@ -41,7 +42,7 @@ CONFIG_HEADER = """;PlatformIO Project Configuration File
 """
 
 
-class ProjectConfig(object):
+class ProjectConfigBase(object):
 
     INLINE_COMMENT_RE = re.compile(r"\s+;.*$")
     VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}")
@@ -49,7 +50,6 @@ class ProjectConfig(object):
     expand_interpolations = True
     warnings = []
 
-    _instances = {}
     _parser = None
     _parsed = []
 
@@ -66,33 +66,28 @@ class ProjectConfig(object):
             if not item or item.startswith((";", "#")):
                 continue
             if ";" in item:
-                item = ProjectConfig.INLINE_COMMENT_RE.sub("", item).strip()
+                item = ProjectConfigBase.INLINE_COMMENT_RE.sub("", item).strip()
             result.append(item)
         return result
 
     @staticmethod
-    def get_instance(path):
+    def get_default_path():
-        mtime = getmtime(path) if isfile(path) else 0
+        from platformio import app
-        instance = ProjectConfig._instances.get(path)
-        if instance and instance["mtime"] != mtime:
-            instance = None
-        if not instance:
-            instance = {"mtime": mtime, "config": ProjectConfig(path)}
-            ProjectConfig._instances[path] = instance
-        return instance["config"]
 
-    def __init__(self, path, parse_extra=True, expand_interpolations=True):
+        return app.get_session_var("custom_project_conf") or os.path.join(
+            os.getcwd(), "platformio.ini"
+        )
+
+    def __init__(self, path=None, parse_extra=True, expand_interpolations=True):
+        path = self.get_default_path() if path is None else path
         self.path = path
         self.expand_interpolations = expand_interpolations
         self.warnings = []
         self._parsed = []
         self._parser = ConfigParser.ConfigParser()
-        if isfile(path):
+        if path and os.path.isfile(path):
             self.read(path, parse_extra)
 
-    def __repr__(self):
-        return "<ProjectConfig %s>" % (self.path or "in-memory")
-
     def __getattr__(self, name):
         return getattr(self._parser, name)
 
@@ -111,7 +106,7 @@ class ProjectConfig(object):
         # load extra configs
         for pattern in self.get("platformio", "extra_configs", []):
             if pattern.startswith("~"):
-                pattern = expanduser(pattern)
+                pattern = os.path.expanduser(pattern)
             for item in glob.glob(pattern):
                 self.read(item)
 
@@ -165,7 +160,7 @@ class ProjectConfig(object):
                 unknown_conditions = [
                     ("%s.%s" % (scope, option)) not in ProjectOptions,
                     scope != "env" or not option.startswith(("custom_", "board_")),
-                ]  # yapf: disable
+                ]
                 if all(unknown_conditions):
                     self.warnings.append(
                         "Ignore unknown configuration option `%s` "
@@ -288,7 +283,7 @@ class ProjectConfig(object):
 
         # option is not specified by user
         if value is None:
-            return default
+            return default if default is not None else option_meta.default
 
         try:
             return self._cast_to(value, option_meta.type)
@@ -313,7 +308,7 @@ class ProjectConfig(object):
         return self.get("platformio", "default_envs", [])
 
     def validate(self, envs=None, silent=False):
-        if not isfile(self.path):
+        if not os.path.isfile(self.path):
             raise exception.NotPlatformIOProject(self.path)
         # check envs
         known = set(self.envs())
@@ -327,6 +322,93 @@ class ProjectConfig(object):
                 click.secho("Warning! %s" % warning, fg="yellow")
         return True
 
+
+class ProjectConfigDirsMixin(object):
+    def _get_core_dir(self, exists=False):
+        default = ProjectOptions["platformio.core_dir"].default
+        core_dir = self.get("platformio", "core_dir")
+        win_core_dir = None
+        if WINDOWS and core_dir == default:
+            win_core_dir = os.path.splitdrive(core_dir)[0] + "\\.platformio"
+            if os.path.isdir(win_core_dir):
+                core_dir = win_core_dir
+
+        if exists and not os.path.isdir(core_dir):
+            try:
+                os.makedirs(core_dir)
+            except OSError as e:
+                if win_core_dir:
+                    os.makedirs(win_core_dir)
+                    core_dir = win_core_dir
+                else:
+                    raise e
+
+        return core_dir
+
+    def get_optional_dir(self, name, exists=False):
+        if not ProjectOptions.get("platformio.%s_dir" % name):
+            raise ValueError("Unknown optional directory -> " + name)
+
+        if name == "core":
+            result = self._get_core_dir(exists)
+        else:
+            result = self.get("platformio", name + "_dir")
+
+        if result is None:
+            return None
+
+        project_dir = os.getcwd()
+
+        # patterns
+        if "$PROJECT_HASH" in result:
+            result = result.replace(
+                "$PROJECT_HASH",
+                "%s-%s"
+                % (
+                    os.path.basename(project_dir),
+                    sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
+                ),
+            )
+
+        if "$PROJECT_DIR" in result:
+            result = result.replace("$PROJECT_DIR", project_dir)
+        if "$PROJECT_CORE_DIR" in result:
+            result = result.replace("$PROJECT_CORE_DIR", self.get_optional_dir("core"))
+        if "$PROJECT_WORKSPACE_DIR" in result:
+            result = result.replace(
+                "$PROJECT_WORKSPACE_DIR", self.get_optional_dir("workspace")
+            )
+
+        if result.startswith("~"):
+            result = os.path.expanduser(result)
+
+        result = os.path.realpath(result)
+
+        if exists and not os.path.isdir(result):
+            os.makedirs(result)
+
+        return result
+
+
+class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin):
+
+    _instances = {}
+
+    @staticmethod
+    def get_instance(path=None):
+        path = ProjectConfig.get_default_path() if path is None else path
+        mtime = os.path.getmtime(path) if os.path.isfile(path) else 0
+        instance = ProjectConfig._instances.get(path)
+        if instance and instance["mtime"] != mtime:
+            instance = None
+        if not instance:
+            instance = {"mtime": mtime, "config": ProjectConfig(path)}
+            ProjectConfig._instances[path] = instance
+        return instance["config"]
+
+    def __repr__(self):
+        return "<ProjectConfig %s>" % (self.path or "in-memory")
+
     def to_json(self):
         result = {}
         for section in self.sections():

platformio/project/helpers.py

@@ -16,16 +16,7 @@ import json
 import os
 from hashlib import sha1
 from os import walk
-from os.path import (
+from os.path import dirname, expanduser, isdir, isfile, join
-    basename,
-    dirname,
-    expanduser,
-    isdir,
-    isfile,
-    join,
-    realpath,
-    splitdrive,
-)
 
 from click.testing import CliRunner
 
@@ -54,126 +45,32 @@ def find_project_dir_above(path):
     return None
 
 
-def get_project_optional_dir(name, default=None):
-    project_dir = get_project_dir()
-    config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
-    optional_dir = config.get("platformio", name)
-
-    if not optional_dir:
-        return default
-
-    if "$PROJECT_HASH" in optional_dir:
-        optional_dir = optional_dir.replace(
-            "$PROJECT_HASH",
-            "%s-%s"
-            % (
-                basename(project_dir),
-                sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
-            ),
-        )
-
-    if optional_dir.startswith("~"):
-        optional_dir = expanduser(optional_dir)
-
-    return realpath(optional_dir)
-
-
 def get_project_core_dir():
-    default = join(expanduser("~"), ".platformio")
+    """ Deprecated, use ProjectConfig.get_optional_dir("core") instead """
-    core_dir = get_project_optional_dir(
+    return ProjectConfig.get_instance(
-        "core_dir", get_project_optional_dir("home_dir", default)
+        join(get_project_dir(), "platformio.ini")
-    )
+    ).get_optional_dir("core", exists=True)
-    win_core_dir = None
-    if WINDOWS and core_dir == default:
-        win_core_dir = splitdrive(core_dir)[0] + "\\.platformio"
-        if isdir(win_core_dir):
-            core_dir = win_core_dir
-
-    if not isdir(core_dir):
-        try:
-            os.makedirs(core_dir)
-        except OSError as e:
-            if win_core_dir:
-                os.makedirs(win_core_dir)
-                core_dir = win_core_dir
-            else:
-                raise e
-
-    assert isdir(core_dir)
-    return core_dir
-
-
-def get_project_global_lib_dir():
-    return get_project_optional_dir(
-        "globallib_dir", join(get_project_core_dir(), "lib")
-    )
-
-
-def get_project_platforms_dir():
-    return get_project_optional_dir(
-        "platforms_dir", join(get_project_core_dir(), "platforms")
-    )
-
-
-def get_project_packages_dir():
-    return get_project_optional_dir(
-        "packages_dir", join(get_project_core_dir(), "packages")
-    )
 
 
 def get_project_cache_dir():
-    return get_project_optional_dir("cache_dir", join(get_project_core_dir(), ".cache"))
+    """ Deprecated, use ProjectConfig.get_optional_dir("cache") instead """
+    return ProjectConfig.get_instance(
+        join(get_project_dir(), "platformio.ini")
+    ).get_optional_dir("cache")
 
 
-def get_project_workspace_dir():
+def get_default_projects_dir():
-    return get_project_optional_dir("workspace_dir", join(get_project_dir(), ".pio"))
+    docs_dir = join(expanduser("~"), "Documents")
-
-
-def get_project_build_dir(force=False):
-    path = get_project_optional_dir(
-        "build_dir", join(get_project_workspace_dir(), "build")
-    )
     try:
-        if not isdir(path):
+        assert WINDOWS
-            os.makedirs(path)
+        import ctypes.wintypes  # pylint: disable=import-outside-toplevel
-    except Exception as e:  # pylint: disable=broad-except
-        if not force:
-            raise Exception(e)
-    return path
 
-
+        buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH)
-def get_project_libdeps_dir():
+        ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf)
-    return get_project_optional_dir(
+        docs_dir = buf.value
-        "libdeps_dir", join(get_project_workspace_dir(), "libdeps")
+    except:  # pylint: disable=bare-except
-    )
+        pass
-
+    return join(docs_dir, "PlatformIO", "Projects")
-
-def get_project_lib_dir():
-    return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib"))
-
-
-def get_project_include_dir():
-    return get_project_optional_dir("include_dir", join(get_project_dir(), "include"))
-
-
-def get_project_src_dir():
-    return get_project_optional_dir("src_dir", join(get_project_dir(), "src"))
-
-
-def get_project_test_dir():
-    return get_project_optional_dir("test_dir", join(get_project_dir(), "test"))
-
-
-def get_project_boards_dir():
-    return get_project_optional_dir("boards_dir", join(get_project_dir(), "boards"))
-
-
-def get_project_data_dir():
-    return get_project_optional_dir("data_dir", join(get_project_dir(), "data"))
-
-
-def get_project_shared_dir():
-    return get_project_optional_dir("shared_dir", join(get_project_dir(), "shared"))
 
 
 def compute_project_checksum(config):
@@ -185,7 +82,11 @@ def compute_project_checksum(config):
 
     # project file structure
     check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
-    for d in (get_project_include_dir(), get_project_src_dir(), get_project_lib_dir()):
+    for d in (
+        config.get_optional_dir("include"),
+        config.get_optional_dir("src"),
+        config.get_optional_dir("lib"),
+    ):
         if not isdir(d):
             continue
         chunks = []

platformio/project/options.py

@@ -14,13 +14,23 @@
 
 # pylint: disable=redefined-builtin, too-many-arguments
 
+import os
 from collections import OrderedDict, namedtuple
 
 import click
 
 ConfigOptionClass = namedtuple(
     "ConfigOption",
-    ["scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames"],
+    [
+        "scope",
+        "name",
+        "type",
+        "multiple",
+        "sysenvvar",
+        "buildenvvar",
+        "oldnames",
+        "default",
+    ],
 )
 
 
@@ -32,9 +42,10 @@ def ConfigOption(
     sysenvvar=None,
     buildenvvar=None,
     oldnames=None,
+    default=None,
 ):
     return ConfigOptionClass(
-        scope, name, type, multiple, sysenvvar, buildenvvar, oldnames
+        scope, name, type, multiple, sysenvvar, buildenvvar, oldnames, default
     )
 
 
@@ -63,40 +74,83 @@ ProjectOptions = OrderedDict(
             ConfigPlatformioOption(name="extra_configs", multiple=True),
             # Dirs
             ConfigPlatformioOption(
-                name="core_dir", oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR"
+                name="core_dir",
+                oldnames=["home_dir"],
+                sysenvvar="PLATFORMIO_CORE_DIR",
+                default=os.path.join(os.path.expanduser("~"), ".platformio"),
             ),
             ConfigPlatformioOption(
-                name="globallib_dir", sysenvvar="PLATFORMIO_GLOBALLIB_DIR"
+                name="globallib_dir",
+                sysenvvar="PLATFORMIO_GLOBALLIB_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", "lib"),
             ),
             ConfigPlatformioOption(
-                name="platforms_dir", sysenvvar="PLATFORMIO_PLATFORMS_DIR"
+                name="platforms_dir",
+                sysenvvar="PLATFORMIO_PLATFORMS_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", "platforms"),
             ),
             ConfigPlatformioOption(
-                name="packages_dir", sysenvvar="PLATFORMIO_PACKAGES_DIR"
+                name="packages_dir",
+                sysenvvar="PLATFORMIO_PACKAGES_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", "packages"),
+            ),
+            ConfigPlatformioOption(
+                name="cache_dir",
+                sysenvvar="PLATFORMIO_CACHE_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", ".cache"),
             ),
-            ConfigPlatformioOption(name="cache_dir", sysenvvar="PLATFORMIO_CACHE_DIR"),
             ConfigPlatformioOption(
                 name="build_cache_dir", sysenvvar="PLATFORMIO_BUILD_CACHE_DIR"
             ),
             ConfigPlatformioOption(
-                name="workspace_dir", sysenvvar="PLATFORMIO_WORKSPACE_DIR"
+                name="workspace_dir",
+                sysenvvar="PLATFORMIO_WORKSPACE_DIR",
+                default=os.path.join("$PROJECT_DIR", ".pio"),
             ),
-            ConfigPlatformioOption(name="build_dir", sysenvvar="PLATFORMIO_BUILD_DIR"),
             ConfigPlatformioOption(
-                name="libdeps_dir", sysenvvar="PLATFORMIO_LIBDEPS_DIR"
+                name="build_dir",
+                sysenvvar="PLATFORMIO_BUILD_DIR",
+                default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"),
             ),
-            ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"),
             ConfigPlatformioOption(
-                name="include_dir", sysenvvar="PLATFORMIO_INCLUDE_DIR"
+                name="libdeps_dir",
+                sysenvvar="PLATFORMIO_LIBDEPS_DIR",
+                default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"),
             ),
-            ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"),
-            ConfigPlatformioOption(name="test_dir", sysenvvar="PLATFORMIO_TEST_DIR"),
             ConfigPlatformioOption(
-                name="boards_dir", sysenvvar="PLATFORMIO_BOARDS_DIR"
+                name="lib_dir",
+                sysenvvar="PLATFORMIO_LIB_DIR",
+                default=os.path.join("$PROJECT_DIR", "lib"),
             ),
-            ConfigPlatformioOption(name="data_dir", sysenvvar="PLATFORMIO_DATA_DIR"),
             ConfigPlatformioOption(
-                name="shared_dir", sysenvvar="PLATFORMIO_SHARED_DIR"
+                name="include_dir",
+                sysenvvar="PLATFORMIO_INCLUDE_DIR",
+                default=os.path.join("$PROJECT_DIR", "include"),
+            ),
+            ConfigPlatformioOption(
+                name="src_dir",
+                sysenvvar="PLATFORMIO_SRC_DIR",
+                default=os.path.join("$PROJECT_DIR", "src"),
+            ),
+            ConfigPlatformioOption(
+                name="test_dir",
+                sysenvvar="PLATFORMIO_TEST_DIR",
+                default=os.path.join("$PROJECT_DIR", "test"),
+            ),
+            ConfigPlatformioOption(
+                name="boards_dir",
+                sysenvvar="PLATFORMIO_BOARDS_DIR",
+                default=os.path.join("$PROJECT_DIR", "boards"),
+            ),
+            ConfigPlatformioOption(
+                name="data_dir",
+                sysenvvar="PLATFORMIO_DATA_DIR",
+                default=os.path.join("$PROJECT_DIR", "data"),
+            ),
+            ConfigPlatformioOption(
+                name="shared_dir",
+                sysenvvar="PLATFORMIO_SHARED_DIR",
+                default=os.path.join("$PROJECT_DIR", "shared"),
             ),
             #
             # [env]

platformio/run/helpers.py

@@ -18,16 +18,14 @@ from os.path import isdir, isfile, join
 import click
 
 from platformio import fs
-from platformio.project.helpers import (
+from platformio.project.helpers import compute_project_checksum, get_project_dir
-    compute_project_checksum,
-    get_project_dir,
-    get_project_libdeps_dir,
-)
 
 
 def handle_legacy_libdeps(project_dir, config):
     legacy_libdeps_dir = join(project_dir, ".piolibdeps")
-    if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == get_project_libdeps_dir():
+    if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get_optional_dir(
+        "libdeps"
+    ):
         return
     if not config.has_section("env"):
         config.add_section("env")

platformio/run/processor.py

@@ -48,9 +48,11 @@ class EnvironmentProcessor(object):
         return variables
 
     def get_build_targets(self):
-        if self.targets:
+        return (
-            return [t for t in self.targets]
+            self.targets
-        return self.config.get("env:" + self.name, "targets", [])
+            if self.targets
+            else self.config.get("env:" + self.name, "targets", [])
+        )
 
     def process(self):
         if "platform" not in self.options:

platformio/test/native.py

@@ -14,9 +14,8 @@
 
 from os.path import join
 
-from platformio import fs, proc
+from platformio import proc
 from platformio.proc import LineBufferedAsyncPipe
-from platformio.project.helpers import get_project_build_dir
 from platformio.test.processor import TestProcessorBase
 
 
@@ -32,8 +31,7 @@ class NativeTestProcessor(TestProcessorBase):
         return self.run()
 
     def run(self):
-        with fs.cd(self.options["project_dir"]):
+        build_dir = self.options["project_config"].get_optional_dir("build")
-            build_dir = get_project_build_dir()
         result = proc.exec_command(
             [join(build_dir, self.env_name, "program")],
             stdout=LineBufferedAsyncPipe(self.on_run_out),

platformio/test/processor.py

@@ -20,7 +20,6 @@ from string import Template
 import click
 
 from platformio import exception
-from platformio.project.helpers import get_project_test_dir
 
 TRANSPORT_OPTIONS = {
     "arduino": {
@@ -104,7 +103,9 @@ class TestProcessorBase(object):
 
     def build_or_upload(self, target):
         if not self._outputcpp_generated:
-            self.generate_outputcpp(get_project_test_dir())
+            self.generate_outputcpp(
+                self.options["project_config"].get_optional_dir("test")
+            )
             self._outputcpp_generated = True
 
         if self.test_name != "*":
@@ -175,7 +176,7 @@ class TestProcessorBase(object):
                 "   $end;",
                 "}",
             ]
-        )  # yapf: disable
+        )
 
         def delete_tmptest_file(file_):
             try:

platformio/util.py

@@ -191,7 +191,7 @@ def get_logical_devices():
 
 def get_mdns_services():
     try:
-        import zeroconf
+        import zeroconf  # pylint: disable=import-outside-toplevel
     except ImportError:
         from site import addsitedir
         from platformio.managers.core import get_core_package_dir
@@ -199,7 +199,7 @@ def get_mdns_services():
         contrib_pysite_dir = get_core_package_dir("contrib-pysite")
         addsitedir(contrib_pysite_dir)
         sys.path.insert(0, contrib_pysite_dir)
-        import zeroconf
+        import zeroconf  # pylint: disable=import-outside-toplevel
 
     class mDNSListener(object):
         def __init__(self):
@@ -360,7 +360,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None):
             time.sleep(2 * total)
 
     raise exception.APIRequestError(
-        "Could not connect to PlatformIO API Service. " "Please try later."
+        "Could not connect to PlatformIO API Service. Please try later."
     )
 
 

tests/commands/test_check.py

@@ -164,7 +164,7 @@ def test_check_filter_sources(clirunner, check_dir):
     assert style == EXPECTED_STYLE
 
 
-def test_check_failed_if_no_source_files(clirunner, tmpdir):
+def test_check_no_source_files(clirunner, tmpdir):
     tmpdir.join("platformio.ini").write(DEFAULT_CONFIG)
     tmpdir.mkdir("src")
 
@@ -178,7 +178,7 @@ def test_check_failed_if_no_source_files(clirunner, tmpdir):
     assert style == 0
 
 
-def test_check_failed_if_bad_flag_passed(clirunner, check_dir):
+def test_check_bad_flag_passed(clirunner, check_dir):
     result = clirunner.invoke(
         cmd_check, ["--project-dir", str(check_dir), '"--flags=--UNKNOWN"']
     )

tests/test_examples.py

@@ -22,7 +22,6 @@ import pytest
 from platformio import util
 from platformio.managers.platform import PlatformFactory, PlatformManager
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import get_project_build_dir
 
 
 def pytest_generate_tests(metafunc):
@@ -71,11 +70,12 @@ def pytest_generate_tests(metafunc):
 @pytest.mark.examples
 def test_run(pioproject_dir):
     with util.cd(pioproject_dir):
-        build_dir = get_project_build_dir()
+        config = ProjectConfig()
+        build_dir = config.get_optional_dir("build")
         if isdir(build_dir):
             util.rmtree_(build_dir)
 
-        env_names = ProjectConfig(join(pioproject_dir, "platformio.ini")).envs()
+        env_names = config.envs()
         result = util.exec_command(
             ["platformio", "run", "-e", random.choice(env_names)]
         )

tests/test_projectconf.py

@@ -112,6 +112,15 @@ def test_warnings(config):
         config.validate(["non-existing-env"])
 
 
+def test_defaults(config):
+    assert config.get_optional_dir("core") == os.path.join(
+        os.path.expanduser("~"), ".platformio"
+    )
+    assert config.get_optional_dir("build_cache") == os.environ.get(
+        "PLATFORMIO_BUILD_CACHE_DIR"
+    )
+
+
 def test_sections(config):
     with pytest.raises(ConfigParser.NoSectionError):
         config.getraw("unknown_section", "unknown_option")