diff --git a/HISTORY.rst b/HISTORY.rst index 650412d3..379224a7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,6 +28,7 @@ PlatformIO 4.0 - Python 3 support (`issue #895 `_) - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) + - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) * **Miscellaneous** diff --git a/docs b/docs index 82316b22..dde6b061 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 82316b22ff38d4e62dd38dd8e465afa38bd062b5 +Subproject commit dde6b061a61872e35a920775c20a64063b445065 diff --git a/platformio/app.py b/platformio/app.py index b8e3b691..798fecab 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -27,6 +27,7 @@ import requests from platformio import exception, lockfile, util from platformio.compat import PY2, WINDOWS from platformio.proc import is_ci +from platformio.project.helpers import get_project_core_dir def projects_dir_validate(projects_dir): @@ -90,7 +91,7 @@ class State(object): self.path = path self.lock = lock if not self.path: - self.path = join(util.get_home_dir(), "appstate.json") + self.path = join(get_project_core_dir(), "appstate.json") self._state = {} self._prev_state = {} self._lockfile = None @@ -111,7 +112,7 @@ class State(object): with codecs.open(self.path, "w", encoding="utf8") as fp: json.dump(self._state, fp) except IOError: - raise exception.HomeDirPermissionsError(util.get_home_dir()) + raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() def _lock_state_file(self): diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 4604fc9c..8358d52e 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -32,11 +32,8 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import util from platformio.compat import PY2, path_to_unicode from platformio.proc import get_pythonexe_path +from platformio.project import helpers as project_helpers from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_dir, get_projectbuild_dir, get_projectdata_dir, - get_projectinclude_dir, get_projectlib_dir, get_projectlibdeps_dir, - get_projectsrc_dir, get_projecttest_dir, get_projectworkspace_dir) AllowSubstExceptions(NameError) @@ -108,23 +105,23 @@ DEFAULT_ENV_OPTIONS = dict( PIOVARIABLES=list(commonvars.keys()), ENV=environ, UNIX_TIME=int(time()), - PIOHOME_DIR=util.get_home_dir(), - PROJECT_DIR=get_project_dir(), - PROJECTWORKSPACE_DIR=get_projectworkspace_dir(), - PROJECTLIBDEPS_DIR=get_projectlibdeps_dir(), - PROJECTINCLUDE_DIR=get_projectinclude_dir(), - PROJECTSRC_DIR=get_projectsrc_dir(), - PROJECTTEST_DIR=get_projecttest_dir(), - PROJECTDATA_DIR=get_projectdata_dir(), - PROJECTBUILD_DIR=get_projectbuild_dir(), + PROJECT_DIR=project_helpers.get_project_dir(), + PROJECTCORE_DIR=project_helpers.get_project_core_dir(), + PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(), + PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(), + PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(), + PROJECTSRC_DIR=project_helpers.get_project_src_dir(), + PROJECTTEST_DIR=project_helpers.get_project_test_dir(), + PROJECTDATA_DIR=project_helpers.get_project_data_dir(), + PROJECTBUILD_DIR=project_helpers.get_project_build_dir(), BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"), BUILDSRC_DIR=join("$BUILD_DIR", "src"), BUILDTEST_DIR=join("$BUILD_DIR", "test"), LIBPATH=["$BUILD_DIR"], LIBSOURCE_DIRS=[ - get_projectlib_dir(), + project_helpers.get_project_lib_dir(), join("$PROJECTLIBDEPS_DIR", "$PIOENV"), - join("$PIOHOME_DIR", "lib") + project_helpers.get_project_global_lib_dir() ], PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), diff --git a/platformio/commands/init.py b/platformio/commands/init.py index b8f293d2..c002cc74 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -26,8 +26,8 @@ from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig from platformio.project.helpers import ( - get_projectinclude_dir, get_projectlib_dir, get_projectsrc_dir, - get_projecttest_dir, is_platformio_project) + get_project_include_dir, get_project_lib_dir, get_project_src_dir, + get_project_test_dir, is_platformio_project) def validate_boards(ctx, param, value): # pylint: disable=W0613 @@ -155,10 +155,10 @@ def init_base_project(project_dir): ProjectConfig(join(project_dir, "platformio.ini")).save() with util.cd(project_dir): dir_to_readme = [ - (get_projectsrc_dir(), None), - (get_projectinclude_dir(), init_include_readme), - (get_projectlib_dir(), init_lib_readme), - (get_projecttest_dir(), init_test_readme), + (get_project_src_dir(), None), + (get_project_include_dir(), init_include_readme), + (get_project_lib_dir(), init_lib_readme), + (get_project_test_dir(), init_test_readme), ] for (path, cb) in dir_to_readme: if isdir(path): diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index bf3458b1..6ad3252b 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -27,7 +27,8 @@ from platformio.managers.lib import (LibraryManager, get_builtin_libs, from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import ( - get_project_dir, get_projectlibdeps_dir, is_platformio_project) + get_project_dir, get_project_global_lib_dir, get_project_libdeps_dir, + is_platformio_project) try: from urllib.parse import quote @@ -73,12 +74,12 @@ def cli(ctx, **options): return storage_dirs = list(options['storage_dir']) if options['global']: - storage_dirs.append(join(util.get_home_dir(), "lib")) + storage_dirs.append(get_project_global_lib_dir()) if not storage_dirs: if is_platformio_project(): storage_dirs = [get_project_dir()] elif is_ci(): - storage_dirs = [join(util.get_home_dir(), "lib")] + storage_dirs = [get_project_global_lib_dir()] click.secho( "Warning! Global library storage is used automatically. " "Please use `platformio lib --global %s` command to remove " @@ -87,7 +88,7 @@ def cli(ctx, **options): if not storage_dirs: raise exception.NotGlobalLibDir(get_project_dir(), - join(util.get_home_dir(), "lib"), + get_project_global_lib_dir(), ctx.invoked_subcommand) ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options['environment'] @@ -99,7 +100,7 @@ def cli(ctx, **options): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue with util.cd(storage_dir): - libdeps_dir = get_projectlibdeps_dir() + libdeps_dir = get_project_libdeps_dir() config = ProjectConfig.get_instance( join(storage_dir, "platformio.ini")) config.validate(options['environment']) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 27a1a93b..5ee46c8f 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -28,8 +28,8 @@ from platformio.commands.platform import \ from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.helpers import ( - calculate_project_hash, find_project_dir_above, get_project_dir, - get_projectbuild_dir, get_projectlibdeps_dir) + calculate_project_hash, find_project_dir_above, get_project_build_dir, + get_project_dir, get_project_libdeps_dir) # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @@ -71,12 +71,12 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, # clean obsolete build dir if not disable_auto_clean: try: - _clean_build_dir(get_projectbuild_dir()) + _clean_build_dir(get_project_build_dir()) except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " "it manually to avoid build issues" % - get_projectbuild_dir(force=True), + get_project_build_dir(force=True), fg="yellow") config = ProjectConfig.get_instance( @@ -252,7 +252,7 @@ class EnvironmentProcessor(object): def _handle_legacy_libdeps(project_dir, config): legacy_libdeps_dir = join(project_dir, ".piolibdeps") if (not isdir(legacy_libdeps_dir) - or legacy_libdeps_dir == get_projectlibdeps_dir()): + or legacy_libdeps_dir == get_project_libdeps_dir()): return if not config.has_section("env"): config.add_section("env") @@ -273,7 +273,7 @@ def _handle_legacy_libdeps(project_dir, config): def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): if not libraries: return - libdeps_dir = join(get_projectlibdeps_dir(), envname) + libdeps_dir = join(get_project_libdeps_dir(), envname) ctx.meta.update({ CTX_META_STORAGE_DIRS_KEY: [libdeps_dir], CTX_META_STORAGE_LIBDEPS_KEY: { diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 58b87cf2..9898dd88 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -26,7 +26,7 @@ from platformio.commands.run import cli as cmd_run from platformio.compat import PY2, WINDOWS, get_file_contents from platformio.project.config import ProjectConfig from platformio.project.helpers import ( - get_projectlib_dir, get_projectlibdeps_dir, get_projectsrc_dir) + get_project_lib_dir, get_project_libdeps_dir, get_project_src_dir) class ProjectGenerator(object): @@ -91,7 +91,7 @@ class ProjectGenerator(object): def get_src_files(self): result = [] with util.cd(self.project_dir): - for root, _, files in os.walk(get_projectsrc_dir()): + for root, _, files in os.walk(get_project_src_dir()): for f in files: result.append(relpath(join(root, f))) return result @@ -142,10 +142,10 @@ class ProjectGenerator(object): "src_files": self.get_src_files(), "user_home_dir": abspath(expanduser("~")), "project_dir": self.project_dir, - "project_src_dir": get_projectsrc_dir(), - "project_lib_dir": get_projectlib_dir(), + "project_src_dir": get_project_src_dir(), + "project_lib_dir": get_project_lib_dir(), "project_libdeps_dir": join( - get_projectlibdeps_dir(), self.env_name), + get_project_libdeps_dir(), self.env_name), "systype": util.get_systype(), "platformio_path": self._fix_os_path( sys.argv[0] if isfile(sys.argv[0]) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 23fdb6c2..ea7c75f2 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -35,6 +35,7 @@ from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager from platformio.managers.platform import PlatformFactory, PlatformManager from platformio.proc import is_ci, is_container +from platformio.project.helpers import get_project_core_dir def on_platformio_start(ctx, force, caller): @@ -113,7 +114,7 @@ class Upgrader(object): @staticmethod def _upgrade_to_3_0_0(ctx): # convert custom board configuration - boards_dir = join(util.get_home_dir(), "boards") + boards_dir = join(get_project_core_dir(), "boards") if isdir(boards_dir): for item in os.listdir(boards_dir): if not item.endswith(".json"): diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 700a928d..bda6c795 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,6 +24,7 @@ from platformio import __version__, exception, util from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path +from platformio.project.helpers import get_project_packages_dir CORE_PACKAGES = { "contrib-piohome": "^2.1.0", @@ -42,12 +43,11 @@ PIOPLUS_AUTO_UPDATES_MAX = 100 class CorePackageManager(PackageManager): def __init__(self): - super(CorePackageManager, self).__init__( - join(util.get_home_dir(), "packages"), [ - "https://dl.bintray.com/platformio/dl-packages/manifest.json", - "http%s://dl.platformio.org/packages/manifest.json" % - ("" if sys.version_info < (2, 7, 9) else "s") - ]) + super(CorePackageManager, self).__init__(get_project_packages_dir(), [ + "https://dl.bintray.com/platformio/dl-packages/manifest.json", + "http%s://dl.platformio.org/packages/manifest.json" % + ("" if sys.version_info < (2, 7, 9) else "s") + ]) def install( # pylint: disable=keyword-arg-before-vararg self, diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index ca143b58..e26a6cb8 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -26,13 +26,14 @@ from platformio import app, commands, exception, util from platformio.compat import glob_escape, string_types from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.project.helpers import get_project_global_lib_dir class LibraryManager(BasePkgManager): def __init__(self, package_dir=None): if not package_dir: - package_dir = join(util.get_home_dir(), "lib") + package_dir = get_project_global_lib_dir() super(LibraryManager, self).__init__(package_dir) @property diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index b5537937..56625efe 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -29,7 +29,9 @@ from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, exec_command, get_pythonexe_path) -from platformio.project.helpers import get_projectboards_dir +from platformio.project.helpers import ( + get_project_boards_dir, get_project_core_dir, get_project_packages_dir, + get_project_platforms_dir) try: from urllib.parse import quote @@ -48,9 +50,8 @@ class PlatformManager(BasePkgManager): "{0}://dl.platformio.org/platforms/manifest.json".format( "https" if app.get_setting("enable_ssl") else "http") ] - BasePkgManager.__init__( - self, package_dir or join(util.get_home_dir(), "platforms"), - repositories) + BasePkgManager.__init__(self, package_dir + or get_project_platforms_dir(), repositories) @property def manifest_names(self): @@ -156,7 +157,7 @@ class PlatformManager(BasePkgManager): deppkgs[pkgname] = set() deppkgs[pkgname].add(pkgmanifest['version']) - pm = PackageManager(join(util.get_home_dir(), "packages")) + pm = PackageManager(get_project_packages_dir()) for manifest in pm.get_installed(): if manifest['name'] not in names: continue @@ -481,8 +482,8 @@ class PlatformBase( # pylint: disable=too-many-public-methods self.manifest_path = manifest_path self._manifest = util.load_json(manifest_path) - self.pm = PackageManager( - join(util.get_home_dir(), "packages"), self.package_repositories) + self.pm = PackageManager(get_project_packages_dir(), + self.package_repositories) self.silent = False self.verbose = False @@ -579,8 +580,8 @@ class PlatformBase( # pylint: disable=too-many-public-methods self._BOARDS_CACHE[board_id] = config bdirs = [ - get_projectboards_dir(), - join(util.get_home_dir(), "boards"), + get_project_boards_dir(), + join(get_project_core_dir(), "boards"), join(self.get_dir(), "boards"), ] diff --git a/platformio/project/config.py b/platformio/project/config.py index 27557d56..7b5baac0 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -45,16 +45,20 @@ KNOWN_PLATFORMIO_OPTIONS = [ "extra_configs", # Dirs - "home_dir", + "core_dir", + "globallib_dir", + "platforms_dir", + "packages_dir", + "cache_dir", "workspace_dir", - "lib_dir", + "build_dir", "libdeps_dir", + "lib_dir", "include_dir", "src_dir", - "build_dir", - "data_dir", "test_dir", - "boards_dir" + "boards_dir", + "data_dir" ] KNOWN_ENV_OPTIONS = [ diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index f215a511..a2acce4d 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -15,7 +15,8 @@ import os from hashlib import sha1 from os import walk -from os.path import abspath, dirname, expanduser, isdir, isfile, join +from os.path import (abspath, dirname, expanduser, isdir, isfile, join, + splitdrive) from platformio import __version__ from platformio.compat import PY2, WINDOWS @@ -76,14 +77,57 @@ def get_project_optional_dir(name, default=None): return paths -def get_projectworkspace_dir(): +def get_project_core_dir(): + core_dir = get_project_optional_dir( + "core_dir", + get_project_optional_dir("home_dir", + join(expanduser("~"), ".platformio"))) + win_core_dir = None + if WINDOWS: + win_core_dir = splitdrive(core_dir)[0] + "\\.platformio" + if isdir(win_core_dir): + core_dir = win_core_dir + + if not isdir(core_dir): + try: + os.makedirs(core_dir) + except: # pylint: disable=bare-except + if win_core_dir: + os.makedirs(win_core_dir) + core_dir = win_core_dir + + assert isdir(core_dir) + return core_dir + + +def get_project_global_lib_dir(): + return get_project_optional_dir("globallib_dir", + join(get_project_core_dir(), "lib")) + + +def get_project_platforms_dir(): + return get_project_optional_dir("platforms_dir", + join(get_project_core_dir(), "platforms")) + + +def get_project_packages_dir(): + return get_project_optional_dir("packages_dir", + join(get_project_core_dir(), "packages")) + + +def get_project_cache_dir(): + return get_project_optional_dir("cache_dir", + join(get_project_core_dir(), ".cache")) + + +def get_project_workspace_dir(): return get_project_optional_dir("workspace_dir", join(get_project_dir(), ".pio")) -def get_projectbuild_dir(force=False): +def get_project_build_dir(force=False): path = get_project_optional_dir("build_dir", - join(get_projectworkspace_dir(), "build")) + join(get_project_workspace_dir(), "build")) try: if not isdir(path): os.makedirs(path) @@ -93,35 +137,35 @@ def get_projectbuild_dir(force=False): return path -def get_projectlibdeps_dir(): +def get_project_libdeps_dir(): return get_project_optional_dir( - "libdeps_dir", join(get_projectworkspace_dir(), "libdeps")) + "libdeps_dir", join(get_project_workspace_dir(), "libdeps")) -def get_projectlib_dir(): +def get_project_lib_dir(): return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) -def get_projectsrc_dir(): - return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) - - -def get_projectinclude_dir(): +def get_project_include_dir(): return get_project_optional_dir("include_dir", join(get_project_dir(), "include")) -def get_projecttest_dir(): +def get_project_src_dir(): + return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) + + +def get_project_test_dir(): return get_project_optional_dir("test_dir", join(get_project_dir(), "test")) -def get_projectboards_dir(): +def get_project_boards_dir(): return get_project_optional_dir("boards_dir", join(get_project_dir(), "boards")) -def get_projectdata_dir(): +def get_project_data_dir(): return get_project_optional_dir("data_dir", join(get_project_dir(), "data")) @@ -129,7 +173,7 @@ def get_projectdata_dir(): def calculate_project_hash(): check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") chunks = [__version__] - for d in (get_projectsrc_dir(), get_projectlib_dir()): + for d in (get_project_src_dir(), get_project_lib_dir()): if not isdir(d): continue for root, _, files in walk(d): diff --git a/platformio/util.py b/platformio/util.py index 304d3a14..edfd8e60 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -25,8 +25,7 @@ import sys import time from functools import wraps from glob import glob -from os.path import (abspath, basename, dirname, expanduser, isdir, isfile, - join, splitdrive) +from os.path import abspath, basename, dirname, isdir, isfile, join from shutil import rmtree import click @@ -38,11 +37,25 @@ from platformio.compat import PY2, WINDOWS, get_file_contents, path_to_unicode from platformio.proc import LineBufferedAsyncPipe as AsyncPipe from platformio.proc import exec_command, is_ci, where_is_program from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_dir, get_project_optional_dir, get_projectboards_dir, - get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir, - get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir, - is_platformio_project) +from platformio.project.helpers import \ + get_project_boards_dir as get_projectboards_dir +from platformio.project.helpers import \ + get_project_build_dir as get_projectbuild_dir +from platformio.project.helpers import get_project_cache_dir as get_cache_dir +from platformio.project.helpers import get_project_core_dir as get_home_dir +from platformio.project.helpers import \ + get_project_data_dir as get_projectdata_dir +from platformio.project.helpers import get_project_dir +from platformio.project.helpers import \ + get_project_lib_dir as get_projectlib_dir +from platformio.project.helpers import \ + get_project_libdeps_dir as get_projectlibdeps_dir +from platformio.project.helpers import get_project_optional_dir +from platformio.project.helpers import \ + get_project_src_dir as get_projectsrc_dir +from platformio.project.helpers import \ + get_project_test_dir as get_projecttest_dir +from platformio.project.helpers import is_platformio_project class cd(object): @@ -135,32 +148,6 @@ def pioversion_to_intstr(): return [int(i) for i in vermatch.group(1).split(".")[:3]] -def get_home_dir(): - home_dir = get_project_optional_dir("home_dir", - join(expanduser("~"), ".platformio")) - win_home_dir = None - if WINDOWS: - win_home_dir = splitdrive(home_dir)[0] + "\\.platformio" - if isdir(win_home_dir): - home_dir = win_home_dir - - if not isdir(home_dir): - try: - os.makedirs(home_dir) - except: # pylint: disable=bare-except - if win_home_dir: - os.makedirs(win_home_dir) - home_dir = win_home_dir - - assert isdir(home_dir) - return home_dir - - -def get_cache_dir(): - return get_project_optional_dir("cache_dir", join(get_home_dir(), - ".cache")) - - def get_source_dir(): curpath = abspath(__file__) if not isfile(curpath):