Add options to override default locations used by PlatformIO Core // Resolve #1615

HISTORY.rst

@@ -28,6 +28,7 @@ PlatformIO 4.0
 
   - Python 3 support (`issue #895 <https://github.com/platformio/platformio-core/issues/895>`_)
   - Added support for the latest Python "Click" package (CLI) (`issue #349 <https://github.com/platformio/platformio-core/issues/349>`_)
+  - Added options to override default locations used by PlatformIO Core (`core_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#core-dir>`__, `globallib_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#globallib-dir>`__, `platforms_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#platforms-dir>`__, `packages_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#packages-dir>`__, `cache_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#cache-dir>`__) (`issue #1615 <https://github.com/platformio/platformio-core/issues/1615>`_)
   - Removed line-buffering from `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__ command which was leading to omitting progress bar from upload tools (`issue #856 <https://github.com/platformio/platformio-core/issues/856>`_)
 
 * **Miscellaneous**

platformio/app.py

@@ -27,6 +27,7 @@ import requests
 from platformio import exception, lockfile, util
 from platformio.compat import PY2, WINDOWS
 from platformio.proc import is_ci
+from platformio.project.helpers import get_project_core_dir
 
 
 def projects_dir_validate(projects_dir):
@@ -90,7 +91,7 @@ class State(object):
         self.path = path
         self.lock = lock
         if not self.path:
-            self.path = join(util.get_home_dir(), "appstate.json")
+            self.path = join(get_project_core_dir(), "appstate.json")
         self._state = {}
         self._prev_state = {}
         self._lockfile = None
@@ -111,7 +112,7 @@ class State(object):
                 with codecs.open(self.path, "w", encoding="utf8") as fp:
                     json.dump(self._state, fp)
             except IOError:
-                raise exception.HomeDirPermissionsError(util.get_home_dir())
+                raise exception.HomeDirPermissionsError(get_project_core_dir())
         self._unlock_state_file()
 
     def _lock_state_file(self):

platformio/builder/main.py

@@ -32,11 +32,8 @@ from SCons.Script import Variables  # pylint: disable=import-error
 from platformio import util
 from platformio.compat import PY2, path_to_unicode
 from platformio.proc import get_pythonexe_path
+from platformio.project import helpers as project_helpers
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (
-    get_project_dir, get_projectbuild_dir, get_projectdata_dir,
-    get_projectinclude_dir, get_projectlib_dir, get_projectlibdeps_dir,
-    get_projectsrc_dir, get_projecttest_dir, get_projectworkspace_dir)
 
 AllowSubstExceptions(NameError)
 
@@ -108,23 +105,23 @@ DEFAULT_ENV_OPTIONS = dict(
     PIOVARIABLES=list(commonvars.keys()),
     ENV=environ,
     UNIX_TIME=int(time()),
-    PIOHOME_DIR=util.get_home_dir(),
+    PROJECT_DIR=project_helpers.get_project_dir(),
-    PROJECT_DIR=get_project_dir(),
+    PROJECTCORE_DIR=project_helpers.get_project_core_dir(),
-    PROJECTWORKSPACE_DIR=get_projectworkspace_dir(),
+    PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(),
-    PROJECTLIBDEPS_DIR=get_projectlibdeps_dir(),
+    PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(),
-    PROJECTINCLUDE_DIR=get_projectinclude_dir(),
+    PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(),
-    PROJECTSRC_DIR=get_projectsrc_dir(),
+    PROJECTSRC_DIR=project_helpers.get_project_src_dir(),
-    PROJECTTEST_DIR=get_projecttest_dir(),
+    PROJECTTEST_DIR=project_helpers.get_project_test_dir(),
-    PROJECTDATA_DIR=get_projectdata_dir(),
+    PROJECTDATA_DIR=project_helpers.get_project_data_dir(),
-    PROJECTBUILD_DIR=get_projectbuild_dir(),
+    PROJECTBUILD_DIR=project_helpers.get_project_build_dir(),
     BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"),
     BUILDSRC_DIR=join("$BUILD_DIR", "src"),
     BUILDTEST_DIR=join("$BUILD_DIR", "test"),
     LIBPATH=["$BUILD_DIR"],
     LIBSOURCE_DIRS=[
-        get_projectlib_dir(),
+        project_helpers.get_project_lib_dir(),
         join("$PROJECTLIBDEPS_DIR", "$PIOENV"),
-        join("$PIOHOME_DIR", "lib")
+        project_helpers.get_project_global_lib_dir()
     ],
     PROGNAME="program",
     PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),

platformio/commands/init.py

@@ -26,8 +26,8 @@ from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.managers.platform import PlatformManager
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import (
-    get_projectinclude_dir, get_projectlib_dir, get_projectsrc_dir,
+    get_project_include_dir, get_project_lib_dir, get_project_src_dir,
-    get_projecttest_dir, is_platformio_project)
+    get_project_test_dir, is_platformio_project)
 
 
 def validate_boards(ctx, param, value):  # pylint: disable=W0613
@@ -155,10 +155,10 @@ def init_base_project(project_dir):
     ProjectConfig(join(project_dir, "platformio.ini")).save()
     with util.cd(project_dir):
         dir_to_readme = [
-            (get_projectsrc_dir(), None),
+            (get_project_src_dir(), None),
-            (get_projectinclude_dir(), init_include_readme),
+            (get_project_include_dir(), init_include_readme),
-            (get_projectlib_dir(), init_lib_readme),
+            (get_project_lib_dir(), init_lib_readme),
-            (get_projecttest_dir(), init_test_readme),
+            (get_project_test_dir(), init_test_readme),
         ]
         for (path, cb) in dir_to_readme:
             if isdir(path):

platformio/commands/lib.py

@@ -27,7 +27,8 @@ from platformio.managers.lib import (LibraryManager, get_builtin_libs,
 from platformio.proc import is_ci
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import (
-    get_project_dir, get_projectlibdeps_dir, is_platformio_project)
+    get_project_dir, get_project_global_lib_dir, get_project_libdeps_dir,
+    is_platformio_project)
 
 try:
     from urllib.parse import quote
@@ -73,12 +74,12 @@ def cli(ctx, **options):
         return
     storage_dirs = list(options['storage_dir'])
     if options['global']:
-        storage_dirs.append(join(util.get_home_dir(), "lib"))
+        storage_dirs.append(get_project_global_lib_dir())
     if not storage_dirs:
         if is_platformio_project():
             storage_dirs = [get_project_dir()]
         elif is_ci():
-            storage_dirs = [join(util.get_home_dir(), "lib")]
+            storage_dirs = [get_project_global_lib_dir()]
             click.secho(
                 "Warning! Global library storage is used automatically. "
                 "Please use `platformio lib --global %s` command to remove "
@@ -87,7 +88,7 @@ def cli(ctx, **options):
 
     if not storage_dirs:
         raise exception.NotGlobalLibDir(get_project_dir(),
-                                        join(util.get_home_dir(), "lib"),
+                                        get_project_global_lib_dir(),
                                         ctx.invoked_subcommand)
 
     ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options['environment']
@@ -99,7 +100,7 @@ def cli(ctx, **options):
             ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
             continue
         with util.cd(storage_dir):
-            libdeps_dir = get_projectlibdeps_dir()
+            libdeps_dir = get_project_libdeps_dir()
         config = ProjectConfig.get_instance(
             join(storage_dir, "platformio.ini"))
         config.validate(options['environment'])

platformio/commands/run.py

@@ -28,8 +28,8 @@ from platformio.commands.platform import \
 from platformio.managers.platform import PlatformFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import (
-    calculate_project_hash, find_project_dir_above, get_project_dir,
+    calculate_project_hash, find_project_dir_above, get_project_build_dir,
-    get_projectbuild_dir, get_projectlibdeps_dir)
+    get_project_dir, get_project_libdeps_dir)
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
 
@@ -71,12 +71,12 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf,
         # clean obsolete build dir
         if not disable_auto_clean:
             try:
-                _clean_build_dir(get_projectbuild_dir())
+                _clean_build_dir(get_project_build_dir())
             except:  # pylint: disable=bare-except
                 click.secho(
                     "Can not remove temporary directory `%s`. Please remove "
                     "it manually to avoid build issues" %
-                    get_projectbuild_dir(force=True),
+                    get_project_build_dir(force=True),
                     fg="yellow")
 
         config = ProjectConfig.get_instance(
@@ -252,7 +252,7 @@ class EnvironmentProcessor(object):
 def _handle_legacy_libdeps(project_dir, config):
     legacy_libdeps_dir = join(project_dir, ".piolibdeps")
     if (not isdir(legacy_libdeps_dir)
-            or legacy_libdeps_dir == get_projectlibdeps_dir()):
+            or legacy_libdeps_dir == get_project_libdeps_dir()):
         return
     if not config.has_section("env"):
         config.add_section("env")
@@ -273,7 +273,7 @@ def _handle_legacy_libdeps(project_dir, config):
 def _autoinstall_libdeps(ctx, envname, libraries, verbose=False):
     if not libraries:
         return
-    libdeps_dir = join(get_projectlibdeps_dir(), envname)
+    libdeps_dir = join(get_project_libdeps_dir(), envname)
     ctx.meta.update({
         CTX_META_STORAGE_DIRS_KEY: [libdeps_dir],
         CTX_META_STORAGE_LIBDEPS_KEY: {

platformio/ide/projectgenerator.py

@@ -26,7 +26,7 @@ from platformio.commands.run import cli as cmd_run
 from platformio.compat import PY2, WINDOWS, get_file_contents
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import (
-    get_projectlib_dir, get_projectlibdeps_dir, get_projectsrc_dir)
+    get_project_lib_dir, get_project_libdeps_dir, get_project_src_dir)
 
 
 class ProjectGenerator(object):
@@ -91,7 +91,7 @@ class ProjectGenerator(object):
     def get_src_files(self):
         result = []
         with util.cd(self.project_dir):
-            for root, _, files in os.walk(get_projectsrc_dir()):
+            for root, _, files in os.walk(get_project_src_dir()):
                 for f in files:
                     result.append(relpath(join(root, f)))
         return result
@@ -142,10 +142,10 @@ class ProjectGenerator(object):
                 "src_files": self.get_src_files(),
                 "user_home_dir": abspath(expanduser("~")),
                 "project_dir": self.project_dir,
-                "project_src_dir": get_projectsrc_dir(),
+                "project_src_dir": get_project_src_dir(),
-                "project_lib_dir": get_projectlib_dir(),
+                "project_lib_dir": get_project_lib_dir(),
                 "project_libdeps_dir": join(
-                    get_projectlibdeps_dir(), self.env_name),
+                    get_project_libdeps_dir(), self.env_name),
                 "systype": util.get_systype(),
                 "platformio_path": self._fix_os_path(
                     sys.argv[0] if isfile(sys.argv[0])

platformio/maintenance.py

@@ -35,6 +35,7 @@ from platformio.managers.core import update_core_packages
 from platformio.managers.lib import LibraryManager
 from platformio.managers.platform import PlatformFactory, PlatformManager
 from platformio.proc import is_ci, is_container
+from platformio.project.helpers import get_project_core_dir
 
 
 def on_platformio_start(ctx, force, caller):
@@ -113,7 +114,7 @@ class Upgrader(object):
     @staticmethod
     def _upgrade_to_3_0_0(ctx):
         # convert custom board configuration
-        boards_dir = join(util.get_home_dir(), "boards")
+        boards_dir = join(get_project_core_dir(), "boards")
         if isdir(boards_dir):
             for item in os.listdir(boards_dir):
                 if not item.endswith(".json"):

platformio/managers/core.py

@@ -24,6 +24,7 @@ from platformio import __version__, exception, util
 from platformio.compat import PY2, WINDOWS
 from platformio.managers.package import PackageManager
 from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
+from platformio.project.helpers import get_project_packages_dir
 
 CORE_PACKAGES = {
     "contrib-piohome": "^2.1.0",
@@ -42,12 +43,11 @@ PIOPLUS_AUTO_UPDATES_MAX = 100
 class CorePackageManager(PackageManager):
 
     def __init__(self):
-        super(CorePackageManager, self).__init__(
+        super(CorePackageManager, self).__init__(get_project_packages_dir(), [
-            join(util.get_home_dir(), "packages"), [
+            "https://dl.bintray.com/platformio/dl-packages/manifest.json",
-                "https://dl.bintray.com/platformio/dl-packages/manifest.json",
+            "http%s://dl.platformio.org/packages/manifest.json" %
-                "http%s://dl.platformio.org/packages/manifest.json" %
+            ("" if sys.version_info < (2, 7, 9) else "s")
-                ("" if sys.version_info < (2, 7, 9) else "s")
+        ])
-            ])
 
     def install(  # pylint: disable=keyword-arg-before-vararg
             self,

platformio/managers/lib.py

@@ -26,13 +26,14 @@ from platformio import app, commands, exception, util
 from platformio.compat import glob_escape, string_types
 from platformio.managers.package import BasePkgManager
 from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.project.helpers import get_project_global_lib_dir
 
 
 class LibraryManager(BasePkgManager):
 
     def __init__(self, package_dir=None):
         if not package_dir:
-            package_dir = join(util.get_home_dir(), "lib")
+            package_dir = get_project_global_lib_dir()
         super(LibraryManager, self).__init__(package_dir)
 
     @property

platformio/managers/platform.py

@@ -29,7 +29,9 @@ from platformio.managers.core import get_core_package_dir
 from platformio.managers.package import BasePkgManager, PackageManager
 from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv,
                              exec_command, get_pythonexe_path)
-from platformio.project.helpers import get_projectboards_dir
+from platformio.project.helpers import (
+    get_project_boards_dir, get_project_core_dir, get_project_packages_dir,
+    get_project_platforms_dir)
 
 try:
     from urllib.parse import quote
@@ -48,9 +50,8 @@ class PlatformManager(BasePkgManager):
                 "{0}://dl.platformio.org/platforms/manifest.json".format(
                     "https" if app.get_setting("enable_ssl") else "http")
             ]
-        BasePkgManager.__init__(
+        BasePkgManager.__init__(self, package_dir
-            self, package_dir or join(util.get_home_dir(), "platforms"),
+                                or get_project_platforms_dir(), repositories)
-            repositories)
 
     @property
     def manifest_names(self):
@@ -156,7 +157,7 @@ class PlatformManager(BasePkgManager):
                     deppkgs[pkgname] = set()
                 deppkgs[pkgname].add(pkgmanifest['version'])
 
-        pm = PackageManager(join(util.get_home_dir(), "packages"))
+        pm = PackageManager(get_project_packages_dir())
         for manifest in pm.get_installed():
             if manifest['name'] not in names:
                 continue
@@ -481,8 +482,8 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
         self.manifest_path = manifest_path
         self._manifest = util.load_json(manifest_path)
 
-        self.pm = PackageManager(
+        self.pm = PackageManager(get_project_packages_dir(),
-            join(util.get_home_dir(), "packages"), self.package_repositories)
+                                 self.package_repositories)
 
         self.silent = False
         self.verbose = False
@@ -579,8 +580,8 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
             self._BOARDS_CACHE[board_id] = config
 
         bdirs = [
-            get_projectboards_dir(),
+            get_project_boards_dir(),
-            join(util.get_home_dir(), "boards"),
+            join(get_project_core_dir(), "boards"),
             join(self.get_dir(), "boards"),
         ]
 

platformio/project/config.py

@@ -45,16 +45,20 @@ KNOWN_PLATFORMIO_OPTIONS = [
     "extra_configs",
 
     # Dirs
-    "home_dir",
+    "core_dir",
+    "globallib_dir",
+    "platforms_dir",
+    "packages_dir",
+    "cache_dir",
     "workspace_dir",
-    "lib_dir",
+    "build_dir",
     "libdeps_dir",
+    "lib_dir",
     "include_dir",
     "src_dir",
-    "build_dir",
-    "data_dir",
     "test_dir",
-    "boards_dir"
+    "boards_dir",
+    "data_dir"
 ]
 
 KNOWN_ENV_OPTIONS = [

platformio/project/helpers.py

@@ -15,7 +15,8 @@
 import os
 from hashlib import sha1
 from os import walk
-from os.path import abspath, dirname, expanduser, isdir, isfile, join
+from os.path import (abspath, dirname, expanduser, isdir, isfile, join,
+                     splitdrive)
 
 from platformio import __version__
 from platformio.compat import PY2, WINDOWS
@@ -76,14 +77,57 @@ def get_project_optional_dir(name, default=None):
     return paths
 
 
-def get_projectworkspace_dir():
+def get_project_core_dir():
+    core_dir = get_project_optional_dir(
+        "core_dir",
+        get_project_optional_dir("home_dir",
+                                 join(expanduser("~"), ".platformio")))
+    win_core_dir = None
+    if WINDOWS:
+        win_core_dir = splitdrive(core_dir)[0] + "\\.platformio"
+        if isdir(win_core_dir):
+            core_dir = win_core_dir
+
+    if not isdir(core_dir):
+        try:
+            os.makedirs(core_dir)
+        except:  # pylint: disable=bare-except
+            if win_core_dir:
+                os.makedirs(win_core_dir)
+                core_dir = win_core_dir
+
+    assert isdir(core_dir)
+    return core_dir
+
+
+def get_project_global_lib_dir():
+    return get_project_optional_dir("globallib_dir",
+                                    join(get_project_core_dir(), "lib"))
+
+
+def get_project_platforms_dir():
+    return get_project_optional_dir("platforms_dir",
+                                    join(get_project_core_dir(), "platforms"))
+
+
+def get_project_packages_dir():
+    return get_project_optional_dir("packages_dir",
+                                    join(get_project_core_dir(), "packages"))
+
+
+def get_project_cache_dir():
+    return get_project_optional_dir("cache_dir",
+                                    join(get_project_core_dir(), ".cache"))
+
+
+def get_project_workspace_dir():
     return get_project_optional_dir("workspace_dir",
                                     join(get_project_dir(), ".pio"))
 
 
-def get_projectbuild_dir(force=False):
+def get_project_build_dir(force=False):
     path = get_project_optional_dir("build_dir",
-                                    join(get_projectworkspace_dir(), "build"))
+                                    join(get_project_workspace_dir(), "build"))
     try:
         if not isdir(path):
             os.makedirs(path)
@@ -93,35 +137,35 @@ def get_projectbuild_dir(force=False):
     return path
 
 
-def get_projectlibdeps_dir():
+def get_project_libdeps_dir():
     return get_project_optional_dir(
-        "libdeps_dir", join(get_projectworkspace_dir(), "libdeps"))
+        "libdeps_dir", join(get_project_workspace_dir(), "libdeps"))
 
 
-def get_projectlib_dir():
+def get_project_lib_dir():
     return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib"))
 
 
-def get_projectsrc_dir():
+def get_project_include_dir():
-    return get_project_optional_dir("src_dir", join(get_project_dir(), "src"))
-
-
-def get_projectinclude_dir():
     return get_project_optional_dir("include_dir",
                                     join(get_project_dir(), "include"))
 
 
-def get_projecttest_dir():
+def get_project_src_dir():
+    return get_project_optional_dir("src_dir", join(get_project_dir(), "src"))
+
+
+def get_project_test_dir():
     return get_project_optional_dir("test_dir", join(get_project_dir(),
                                                      "test"))
 
 
-def get_projectboards_dir():
+def get_project_boards_dir():
     return get_project_optional_dir("boards_dir",
                                     join(get_project_dir(), "boards"))
 
 
-def get_projectdata_dir():
+def get_project_data_dir():
     return get_project_optional_dir("data_dir", join(get_project_dir(),
                                                      "data"))
 
@@ -129,7 +173,7 @@ def get_projectdata_dir():
 def calculate_project_hash():
     check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
     chunks = [__version__]
-    for d in (get_projectsrc_dir(), get_projectlib_dir()):
+    for d in (get_project_src_dir(), get_project_lib_dir()):
         if not isdir(d):
             continue
         for root, _, files in walk(d):

platformio/util.py

@@ -25,8 +25,7 @@ import sys
 import time
 from functools import wraps
 from glob import glob
-from os.path import (abspath, basename, dirname, expanduser, isdir, isfile,
+from os.path import abspath, basename, dirname, isdir, isfile, join
-                     join, splitdrive)
 from shutil import rmtree
 
 import click
@@ -38,11 +37,25 @@ from platformio.compat import PY2, WINDOWS, get_file_contents, path_to_unicode
 from platformio.proc import LineBufferedAsyncPipe as AsyncPipe
 from platformio.proc import exec_command, is_ci, where_is_program
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (
+from platformio.project.helpers import \
-    get_project_dir, get_project_optional_dir, get_projectboards_dir,
+    get_project_boards_dir as get_projectboards_dir
-    get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir,
+from platformio.project.helpers import \
-    get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir,
+    get_project_build_dir as get_projectbuild_dir
-    is_platformio_project)
+from platformio.project.helpers import get_project_cache_dir as get_cache_dir
+from platformio.project.helpers import get_project_core_dir as get_home_dir
+from platformio.project.helpers import \
+    get_project_data_dir as get_projectdata_dir
+from platformio.project.helpers import get_project_dir
+from platformio.project.helpers import \
+    get_project_lib_dir as get_projectlib_dir
+from platformio.project.helpers import \
+    get_project_libdeps_dir as get_projectlibdeps_dir
+from platformio.project.helpers import get_project_optional_dir
+from platformio.project.helpers import \
+    get_project_src_dir as get_projectsrc_dir
+from platformio.project.helpers import \
+    get_project_test_dir as get_projecttest_dir
+from platformio.project.helpers import is_platformio_project
 
 
 class cd(object):
@@ -135,32 +148,6 @@ def pioversion_to_intstr():
     return [int(i) for i in vermatch.group(1).split(".")[:3]]
 
 
-def get_home_dir():
-    home_dir = get_project_optional_dir("home_dir",
-                                        join(expanduser("~"), ".platformio"))
-    win_home_dir = None
-    if WINDOWS:
-        win_home_dir = splitdrive(home_dir)[0] + "\\.platformio"
-        if isdir(win_home_dir):
-            home_dir = win_home_dir
-
-    if not isdir(home_dir):
-        try:
-            os.makedirs(home_dir)
-        except:  # pylint: disable=bare-except
-            if win_home_dir:
-                os.makedirs(win_home_dir)
-                home_dir = win_home_dir
-
-    assert isdir(home_dir)
-    return home_dir
-
-
-def get_cache_dir():
-    return get_project_optional_dir("cache_dir", join(get_home_dir(),
-                                                      ".cache"))
-
-
 def get_source_dir():
     curpath = abspath(__file__)
     if not isfile(curpath):