Fixed an issue when configuration file options partly ignored when `--project-conf` // Resolve #3034 (#3055)

* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` // Resolve #3034

* Py2 compatible makedirs

* Fix circle dependency

* Fix broken import in test examples

* Fix history

* Remove YAPF markers

* PyLint fix

* Fix invalid project conf path

* Move PIO Core to the root on Windows, issue with long CPPPATHs

* Respect global PLATFORMIO_BUILD_CACHE_DIR env var

* Fix Appveyor paths

* Minor changes

platformio/managers/core.py

@@ -21,7 +21,7 @@ from platformio import __version__, exception, fs
 from platformio.compat import PY2, WINDOWS
 from platformio.managers.package import PackageManager
 from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
-from platformio.project.helpers import get_project_packages_dir
+from platformio.project.config import ProjectConfig
 
 CORE_PACKAGES = {
     "contrib-piohome": "^2.3.2",
@@ -40,8 +40,10 @@ PIOPLUS_AUTO_UPDATES_MAX = 100
 
 class CorePackageManager(PackageManager):
     def __init__(self):
+        config = ProjectConfig.get_instance()
+        packages_dir = config.get_optional_dir("packages")
         super(CorePackageManager, self).__init__(
-            get_project_packages_dir(),
+            packages_dir,
             [
                 "https://dl.bintray.com/platformio/dl-packages/manifest.json",
                 "http%s://dl.platformio.org/packages/manifest.json"

platformio/managers/lib.py

@@ -27,7 +27,7 @@ from platformio import app, exception, util
 from platformio.compat import glob_escape, string_types
 from platformio.managers.package import BasePkgManager
 from platformio.managers.platform import PlatformFactory, PlatformManager
-from platformio.project.helpers import get_project_global_lib_dir
+from platformio.project.config import ProjectConfig
 
 
 class LibraryManager(BasePkgManager):
@@ -35,9 +35,10 @@ class LibraryManager(BasePkgManager):
     FILE_CACHE_VALID = "30d"  # 1 month
 
     def __init__(self, package_dir=None):
-        if not package_dir:
-            package_dir = get_project_global_lib_dir()
-        super(LibraryManager, self).__init__(package_dir)
+        self.config = ProjectConfig.get_instance()
+        super(LibraryManager, self).__init__(
+            package_dir or self.config.get_optional_dir("globallib")
+        )
 
     @property
     def manifest_names(self):

platformio/managers/package.py

@@ -405,7 +405,7 @@ class PkgInstallerMixin(object):
                     self.parse_semver_version(manifest["version"], raise_exception=True)
                 ):
                     continue
-                elif not best or (
+                if not best or (
                     self.parse_semver_version(manifest["version"], raise_exception=True)
                     > self.parse_semver_version(best["version"], raise_exception=True)
                 ):

platformio/managers/platform.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=too-many-public-methods, too-many-instance-attributes
+
 import base64
 import os
 import re
@@ -32,12 +34,6 @@ from platformio.proc import (
     get_pythonexe_path,
 )
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (
-    get_project_boards_dir,
-    get_project_core_dir,
-    get_project_packages_dir,
-    get_project_platforms_dir,
-)
 
 try:
     from urllib.parse import quote
@@ -54,8 +50,9 @@ class PlatformManager(BasePkgManager):
                     "https" if app.get_setting("strict_ssl") else "http"
                 ),
             ]
+        self.config = ProjectConfig.get_instance()
         BasePkgManager.__init__(
-            self, package_dir or get_project_platforms_dir(), repositories
+            self, package_dir or self.config.get_optional_dir("platforms"), repositories
         )
 
     @property
@@ -164,7 +161,7 @@ class PlatformManager(BasePkgManager):
                     deppkgs[pkgname] = set()
                 deppkgs[pkgname].add(pkgmanifest["version"])
 
-        pm = PackageManager(get_project_packages_dir())
+        pm = PackageManager(self.config.get_optional_dir("packages"))
         for manifest in pm.get_installed():
             if manifest["name"] not in names:
                 continue
@@ -290,7 +287,7 @@ class PlatformPackagesMixin(object):
             version = opts.get("version", "")
             if name in without_packages:
                 continue
-            elif name in with_packages or not (
+            if name in with_packages or not (
                 skip_default_package or opts.get("optional", False)
             ):
                 if ":" in version:
@@ -385,8 +382,7 @@ class PlatformRunMixin(object):
         assert isinstance(variables, dict)
         assert isinstance(targets, list)
 
-        config = ProjectConfig.get_instance(variables["project_config"])
-        options = config.items(env=variables["pioenv"], as_dict=True)
+        options = self.config.items(env=variables["pioenv"], as_dict=True)
         if "framework" in options:
             # support PIO Core 3.0 dev/platforms
             options["pioframework"] = options["framework"]
@@ -421,7 +417,7 @@ class PlatformRunMixin(object):
             str(jobs),
             "--sconstruct",
             join(fs.get_source_dir(), "builder", "main.py"),
-        ]  # yapf: disable
+        ]
         args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
         # pylint: disable=protected-access
         args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0))
@@ -503,9 +499,7 @@ class PlatformRunMixin(object):
         click.echo(banner, err=True)
 
 
-class PlatformBase(  # pylint: disable=too-many-public-methods
-    PlatformPackagesMixin, PlatformRunMixin
-):
+class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
 
     PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__))
     _BOARDS_CACHE = {}
@@ -519,8 +513,10 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
         self._manifest = fs.load_json(manifest_path)
         self._custom_packages = None
 
-        self.pm = PackageManager(get_project_packages_dir(), self.package_repositories)
-
+        self.config = ProjectConfig.get_instance()
+        self.pm = PackageManager(
+            self.config.get_optional_dir("packages"), self.package_repositories
+        )
         # if self.engines and "platformio" in self.engines:
         #     if self.PIO_VERSION not in semantic_version.SimpleSpec(
         #             self.engines['platformio']):
@@ -619,8 +615,8 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
             self._BOARDS_CACHE[board_id] = config
 
         bdirs = [
-            get_project_boards_dir(),
-            join(get_project_core_dir(), "boards"),
+            self.config.get_optional_dir("boards"),
+            join(self.config.get_optional_dir("core"), "boards"),
             join(self.get_dir(), "boards"),
         ]