Fixed an issue when configuration file options partly ignored when `--project-conf` // Resolve #3034 (#3055)

* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` // Resolve #3034

* Py2 compatible makedirs

* Fix circle dependency

* Fix broken import in test examples

* Fix history

* Remove YAPF markers

* PyLint fix

* Fix invalid project conf path

* Move PIO Core to the root on Windows, issue with long CPPPATHs

* Respect global PLATFORMIO_BUILD_CACHE_DIR env var

* Fix Appveyor paths

* Minor changes

platformio/project/config.py

@@ -16,11 +16,12 @@ import glob
 import json
 import os
 import re
-from os.path import expanduser, getmtime, isfile
+from hashlib import sha1
 
 import click
 
 from platformio import exception
+from platformio.compat import WINDOWS, hashlib_encode_data
 from platformio.project.options import ProjectOptions
 
 try:
@@ -41,7 +42,7 @@ CONFIG_HEADER = """;PlatformIO Project Configuration File
 """
 
 
-class ProjectConfig(object):
+class ProjectConfigBase(object):
 
     INLINE_COMMENT_RE = re.compile(r"\s+;.*$")
     VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}")
@@ -49,7 +50,6 @@ class ProjectConfig(object):
     expand_interpolations = True
     warnings = []
 
-    _instances = {}
     _parser = None
     _parsed = []
 
@@ -66,33 +66,28 @@ class ProjectConfig(object):
             if not item or item.startswith((";", "#")):
                 continue
             if ";" in item:
-                item = ProjectConfig.INLINE_COMMENT_RE.sub("", item).strip()
+                item = ProjectConfigBase.INLINE_COMMENT_RE.sub("", item).strip()
             result.append(item)
         return result
 
     @staticmethod
-    def get_instance(path):
-        mtime = getmtime(path) if isfile(path) else 0
-        instance = ProjectConfig._instances.get(path)
-        if instance and instance["mtime"] != mtime:
-            instance = None
-        if not instance:
-            instance = {"mtime": mtime, "config": ProjectConfig(path)}
-            ProjectConfig._instances[path] = instance
-        return instance["config"]
+    def get_default_path():
+        from platformio import app
 
-    def __init__(self, path, parse_extra=True, expand_interpolations=True):
+        return app.get_session_var("custom_project_conf") or os.path.join(
+            os.getcwd(), "platformio.ini"
+        )
+
+    def __init__(self, path=None, parse_extra=True, expand_interpolations=True):
+        path = self.get_default_path() if path is None else path
         self.path = path
         self.expand_interpolations = expand_interpolations
         self.warnings = []
         self._parsed = []
         self._parser = ConfigParser.ConfigParser()
-        if isfile(path):
+        if path and os.path.isfile(path):
             self.read(path, parse_extra)
 
-    def __repr__(self):
-        return "<ProjectConfig %s>" % (self.path or "in-memory")
-
     def __getattr__(self, name):
         return getattr(self._parser, name)
 
@@ -111,7 +106,7 @@ class ProjectConfig(object):
         # load extra configs
         for pattern in self.get("platformio", "extra_configs", []):
             if pattern.startswith("~"):
-                pattern = expanduser(pattern)
+                pattern = os.path.expanduser(pattern)
             for item in glob.glob(pattern):
                 self.read(item)
 
@@ -165,7 +160,7 @@ class ProjectConfig(object):
                 unknown_conditions = [
                     ("%s.%s" % (scope, option)) not in ProjectOptions,
                     scope != "env" or not option.startswith(("custom_", "board_")),
-                ]  # yapf: disable
+                ]
                 if all(unknown_conditions):
                     self.warnings.append(
                         "Ignore unknown configuration option `%s` "
@@ -288,7 +283,7 @@ class ProjectConfig(object):
 
         # option is not specified by user
         if value is None:
-            return default
+            return default if default is not None else option_meta.default
 
         try:
             return self._cast_to(value, option_meta.type)
@@ -313,7 +308,7 @@ class ProjectConfig(object):
         return self.get("platformio", "default_envs", [])
 
     def validate(self, envs=None, silent=False):
-        if not isfile(self.path):
+        if not os.path.isfile(self.path):
             raise exception.NotPlatformIOProject(self.path)
         # check envs
         known = set(self.envs())
@@ -327,6 +322,93 @@ class ProjectConfig(object):
                 click.secho("Warning! %s" % warning, fg="yellow")
         return True
 
+
+class ProjectConfigDirsMixin(object):
+    def _get_core_dir(self, exists=False):
+        default = ProjectOptions["platformio.core_dir"].default
+        core_dir = self.get("platformio", "core_dir")
+        win_core_dir = None
+        if WINDOWS and core_dir == default:
+            win_core_dir = os.path.splitdrive(core_dir)[0] + "\\.platformio"
+            if os.path.isdir(win_core_dir):
+                core_dir = win_core_dir
+
+        if exists and not os.path.isdir(core_dir):
+            try:
+                os.makedirs(core_dir)
+            except OSError as e:
+                if win_core_dir:
+                    os.makedirs(win_core_dir)
+                    core_dir = win_core_dir
+                else:
+                    raise e
+
+        return core_dir
+
+    def get_optional_dir(self, name, exists=False):
+        if not ProjectOptions.get("platformio.%s_dir" % name):
+            raise ValueError("Unknown optional directory -> " + name)
+
+        if name == "core":
+            result = self._get_core_dir(exists)
+        else:
+            result = self.get("platformio", name + "_dir")
+
+        if result is None:
+            return None
+
+        project_dir = os.getcwd()
+
+        # patterns
+        if "$PROJECT_HASH" in result:
+            result = result.replace(
+                "$PROJECT_HASH",
+                "%s-%s"
+                % (
+                    os.path.basename(project_dir),
+                    sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
+                ),
+            )
+
+        if "$PROJECT_DIR" in result:
+            result = result.replace("$PROJECT_DIR", project_dir)
+        if "$PROJECT_CORE_DIR" in result:
+            result = result.replace("$PROJECT_CORE_DIR", self.get_optional_dir("core"))
+        if "$PROJECT_WORKSPACE_DIR" in result:
+            result = result.replace(
+                "$PROJECT_WORKSPACE_DIR", self.get_optional_dir("workspace")
+            )
+
+        if result.startswith("~"):
+            result = os.path.expanduser(result)
+
+        result = os.path.realpath(result)
+
+        if exists and not os.path.isdir(result):
+            os.makedirs(result)
+
+        return result
+
+
+class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin):
+
+    _instances = {}
+
+    @staticmethod
+    def get_instance(path=None):
+        path = ProjectConfig.get_default_path() if path is None else path
+        mtime = os.path.getmtime(path) if os.path.isfile(path) else 0
+        instance = ProjectConfig._instances.get(path)
+        if instance and instance["mtime"] != mtime:
+            instance = None
+        if not instance:
+            instance = {"mtime": mtime, "config": ProjectConfig(path)}
+            ProjectConfig._instances[path] = instance
+        return instance["config"]
+
+    def __repr__(self):
+        return "<ProjectConfig %s>" % (self.path or "in-memory")
+
     def to_json(self):
         result = {}
         for section in self.sections():

platformio/project/helpers.py

@@ -16,16 +16,7 @@ import json
 import os
 from hashlib import sha1
 from os import walk
-from os.path import (
-    basename,
-    dirname,
-    expanduser,
-    isdir,
-    isfile,
-    join,
-    realpath,
-    splitdrive,
-)
+from os.path import dirname, expanduser, isdir, isfile, join
 
 from click.testing import CliRunner
 
@@ -54,126 +45,32 @@ def find_project_dir_above(path):
     return None
 
 
-def get_project_optional_dir(name, default=None):
-    project_dir = get_project_dir()
-    config = ProjectConfig.get_instance(join(project_dir, "platformio.ini"))
-    optional_dir = config.get("platformio", name)
-
-    if not optional_dir:
-        return default
-
-    if "$PROJECT_HASH" in optional_dir:
-        optional_dir = optional_dir.replace(
-            "$PROJECT_HASH",
-            "%s-%s"
-            % (
-                basename(project_dir),
-                sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
-            ),
-        )
-
-    if optional_dir.startswith("~"):
-        optional_dir = expanduser(optional_dir)
-
-    return realpath(optional_dir)
-
-
 def get_project_core_dir():
-    default = join(expanduser("~"), ".platformio")
-    core_dir = get_project_optional_dir(
-        "core_dir", get_project_optional_dir("home_dir", default)
-    )
-    win_core_dir = None
-    if WINDOWS and core_dir == default:
-        win_core_dir = splitdrive(core_dir)[0] + "\\.platformio"
-        if isdir(win_core_dir):
-            core_dir = win_core_dir
-
-    if not isdir(core_dir):
-        try:
-            os.makedirs(core_dir)
-        except OSError as e:
-            if win_core_dir:
-                os.makedirs(win_core_dir)
-                core_dir = win_core_dir
-            else:
-                raise e
-
-    assert isdir(core_dir)
-    return core_dir
-
-
-def get_project_global_lib_dir():
-    return get_project_optional_dir(
-        "globallib_dir", join(get_project_core_dir(), "lib")
-    )
-
-
-def get_project_platforms_dir():
-    return get_project_optional_dir(
-        "platforms_dir", join(get_project_core_dir(), "platforms")
-    )
-
-
-def get_project_packages_dir():
-    return get_project_optional_dir(
-        "packages_dir", join(get_project_core_dir(), "packages")
-    )
+    """ Deprecated, use ProjectConfig.get_optional_dir("core") instead """
+    return ProjectConfig.get_instance(
+        join(get_project_dir(), "platformio.ini")
+    ).get_optional_dir("core", exists=True)
 
 
 def get_project_cache_dir():
-    return get_project_optional_dir("cache_dir", join(get_project_core_dir(), ".cache"))
+    """ Deprecated, use ProjectConfig.get_optional_dir("cache") instead """
+    return ProjectConfig.get_instance(
+        join(get_project_dir(), "platformio.ini")
+    ).get_optional_dir("cache")
 
 
-def get_project_workspace_dir():
-    return get_project_optional_dir("workspace_dir", join(get_project_dir(), ".pio"))
-
-
-def get_project_build_dir(force=False):
-    path = get_project_optional_dir(
-        "build_dir", join(get_project_workspace_dir(), "build")
-    )
+def get_default_projects_dir():
+    docs_dir = join(expanduser("~"), "Documents")
     try:
-        if not isdir(path):
-            os.makedirs(path)
-    except Exception as e:  # pylint: disable=broad-except
-        if not force:
-            raise Exception(e)
-    return path
+        assert WINDOWS
+        import ctypes.wintypes  # pylint: disable=import-outside-toplevel
 
-
-def get_project_libdeps_dir():
-    return get_project_optional_dir(
-        "libdeps_dir", join(get_project_workspace_dir(), "libdeps")
-    )
-
-
-def get_project_lib_dir():
-    return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib"))
-
-
-def get_project_include_dir():
-    return get_project_optional_dir("include_dir", join(get_project_dir(), "include"))
-
-
-def get_project_src_dir():
-    return get_project_optional_dir("src_dir", join(get_project_dir(), "src"))
-
-
-def get_project_test_dir():
-    return get_project_optional_dir("test_dir", join(get_project_dir(), "test"))
-
-
-def get_project_boards_dir():
-    return get_project_optional_dir("boards_dir", join(get_project_dir(), "boards"))
-
-
-def get_project_data_dir():
-    return get_project_optional_dir("data_dir", join(get_project_dir(), "data"))
-
-
-def get_project_shared_dir():
-    return get_project_optional_dir("shared_dir", join(get_project_dir(), "shared"))
+        buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH)
+        ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf)
+        docs_dir = buf.value
+    except:  # pylint: disable=bare-except
+        pass
+    return join(docs_dir, "PlatformIO", "Projects")
 
 
 def compute_project_checksum(config):
@@ -185,7 +82,11 @@ def compute_project_checksum(config):
 
     # project file structure
     check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
-    for d in (get_project_include_dir(), get_project_src_dir(), get_project_lib_dir()):
+    for d in (
+        config.get_optional_dir("include"),
+        config.get_optional_dir("src"),
+        config.get_optional_dir("lib"),
+    ):
         if not isdir(d):
             continue
         chunks = []

platformio/project/options.py

@@ -14,13 +14,23 @@
 
 # pylint: disable=redefined-builtin, too-many-arguments
 
+import os
 from collections import OrderedDict, namedtuple
 
 import click
 
 ConfigOptionClass = namedtuple(
     "ConfigOption",
-    ["scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames"],
+    [
+        "scope",
+        "name",
+        "type",
+        "multiple",
+        "sysenvvar",
+        "buildenvvar",
+        "oldnames",
+        "default",
+    ],
 )
 
 
@@ -32,9 +42,10 @@ def ConfigOption(
     sysenvvar=None,
     buildenvvar=None,
     oldnames=None,
+    default=None,
 ):
     return ConfigOptionClass(
-        scope, name, type, multiple, sysenvvar, buildenvvar, oldnames
+        scope, name, type, multiple, sysenvvar, buildenvvar, oldnames, default
     )
 
 
@@ -63,40 +74,83 @@ ProjectOptions = OrderedDict(
             ConfigPlatformioOption(name="extra_configs", multiple=True),
             # Dirs
             ConfigPlatformioOption(
-                name="core_dir", oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR"
+                name="core_dir",
+                oldnames=["home_dir"],
+                sysenvvar="PLATFORMIO_CORE_DIR",
+                default=os.path.join(os.path.expanduser("~"), ".platformio"),
             ),
             ConfigPlatformioOption(
-                name="globallib_dir", sysenvvar="PLATFORMIO_GLOBALLIB_DIR"
+                name="globallib_dir",
+                sysenvvar="PLATFORMIO_GLOBALLIB_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", "lib"),
             ),
             ConfigPlatformioOption(
-                name="platforms_dir", sysenvvar="PLATFORMIO_PLATFORMS_DIR"
+                name="platforms_dir",
+                sysenvvar="PLATFORMIO_PLATFORMS_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", "platforms"),
             ),
             ConfigPlatformioOption(
-                name="packages_dir", sysenvvar="PLATFORMIO_PACKAGES_DIR"
+                name="packages_dir",
+                sysenvvar="PLATFORMIO_PACKAGES_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", "packages"),
+            ),
+            ConfigPlatformioOption(
+                name="cache_dir",
+                sysenvvar="PLATFORMIO_CACHE_DIR",
+                default=os.path.join("$PROJECT_CORE_DIR", ".cache"),
             ),
-            ConfigPlatformioOption(name="cache_dir", sysenvvar="PLATFORMIO_CACHE_DIR"),
             ConfigPlatformioOption(
                 name="build_cache_dir", sysenvvar="PLATFORMIO_BUILD_CACHE_DIR"
             ),
             ConfigPlatformioOption(
-                name="workspace_dir", sysenvvar="PLATFORMIO_WORKSPACE_DIR"
+                name="workspace_dir",
+                sysenvvar="PLATFORMIO_WORKSPACE_DIR",
+                default=os.path.join("$PROJECT_DIR", ".pio"),
             ),
-            ConfigPlatformioOption(name="build_dir", sysenvvar="PLATFORMIO_BUILD_DIR"),
             ConfigPlatformioOption(
-                name="libdeps_dir", sysenvvar="PLATFORMIO_LIBDEPS_DIR"
+                name="build_dir",
+                sysenvvar="PLATFORMIO_BUILD_DIR",
+                default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"),
             ),
-            ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"),
             ConfigPlatformioOption(
-                name="include_dir", sysenvvar="PLATFORMIO_INCLUDE_DIR"
+                name="libdeps_dir",
+                sysenvvar="PLATFORMIO_LIBDEPS_DIR",
+                default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"),
             ),
-            ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"),
-            ConfigPlatformioOption(name="test_dir", sysenvvar="PLATFORMIO_TEST_DIR"),
             ConfigPlatformioOption(
-                name="boards_dir", sysenvvar="PLATFORMIO_BOARDS_DIR"
+                name="lib_dir",
+                sysenvvar="PLATFORMIO_LIB_DIR",
+                default=os.path.join("$PROJECT_DIR", "lib"),
             ),
-            ConfigPlatformioOption(name="data_dir", sysenvvar="PLATFORMIO_DATA_DIR"),
             ConfigPlatformioOption(
-                name="shared_dir", sysenvvar="PLATFORMIO_SHARED_DIR"
+                name="include_dir",
+                sysenvvar="PLATFORMIO_INCLUDE_DIR",
+                default=os.path.join("$PROJECT_DIR", "include"),
+            ),
+            ConfigPlatformioOption(
+                name="src_dir",
+                sysenvvar="PLATFORMIO_SRC_DIR",
+                default=os.path.join("$PROJECT_DIR", "src"),
+            ),
+            ConfigPlatformioOption(
+                name="test_dir",
+                sysenvvar="PLATFORMIO_TEST_DIR",
+                default=os.path.join("$PROJECT_DIR", "test"),
+            ),
+            ConfigPlatformioOption(
+                name="boards_dir",
+                sysenvvar="PLATFORMIO_BOARDS_DIR",
+                default=os.path.join("$PROJECT_DIR", "boards"),
+            ),
+            ConfigPlatformioOption(
+                name="data_dir",
+                sysenvvar="PLATFORMIO_DATA_DIR",
+                default=os.path.join("$PROJECT_DIR", "data"),
+            ),
+            ConfigPlatformioOption(
+                name="shared_dir",
+                sysenvvar="PLATFORMIO_SHARED_DIR",
+                default=os.path.join("$PROJECT_DIR", "shared"),
             ),
             #
             # [env]