diff --git a/HISTORY.rst b/HISTORY.rst index 9801729e..bcb6d008 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,8 +11,10 @@ PlatformIO Core 5 5.2.3 (2021-??-??) ~~~~~~~~~~~~~~~~~~ +- Improved PlatformIO directory interpolation (``${platformio.***_dir}``) in `"platformio.ini" `__ configuration file (`issue #3934 `_) - Fixed an issue when the "$PROJECT_DIR" gets the full path to "platformio.ini", not the directory name (`issue #4086 `_) + 5.2.2 (2021-10-20) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 66f67cb3..2f0abf6a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 66f67cb3357265ab52a0567b67337ca7fda6c3e4 +Subproject commit 2f0abf6aba38c9310b08fd5200ed36eeacbfdb5a diff --git a/platformio/app.py b/platformio/app.py index e630dd3c..845d7e81 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -21,17 +21,17 @@ import os import platform import socket import uuid -from os.path import dirname, isdir, isfile, join, realpath from platformio import __version__, exception, fs, proc from platformio.compat import IS_WINDOWS, hashlib_encode_data from platformio.package.lockfile import LockFile -from platformio.project.helpers import get_default_projects_dir, get_project_core_dir +from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_default_projects_dir def projects_dir_validate(projects_dir): - assert isdir(projects_dir) - return realpath(projects_dir) + assert os.path.isdir(projects_dir) + return os.path.realpath(projects_dir) DEFAULT_SETTINGS = { @@ -91,7 +91,10 @@ class State(object): self.path = path self.lock = lock if not self.path: - self.path = join(get_project_core_dir(), "appstate.json") + core_dir = ProjectConfig.get_instance().get("platformio", "core_dir") + if not os.path.isdir(core_dir): + os.makedirs(core_dir) + self.path = os.path.join(core_dir, "appstate.json") self._storage = {} self._lockfile = None self.modified = False @@ -99,7 +102,7 @@ class State(object): def __enter__(self): try: self._lock_state_file() - if isfile(self.path): + if os.path.isfile(self.path): self._storage = fs.load_json(self.path) assert isinstance(self._storage, dict) except ( @@ -117,7 +120,7 @@ class State(object): with open(self.path, mode="w", encoding="utf8") as fp: fp.write(json.dumps(self._storage)) except IOError: - raise exception.HomeDirPermissionsError(get_project_core_dir()) + raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) self._unlock_state_file() def _lock_state_file(self): @@ -127,7 +130,7 @@ class State(object): try: self._lockfile.acquire() except IOError: - raise exception.HomeDirPermissionsError(dirname(self.path)) + raise exception.HomeDirPermissionsError(os.path.dirname(self.path)) def _unlock_state_file(self): if hasattr(self, "_lockfile") and self._lockfile: diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 7f719cca..055eaf56 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -109,22 +109,22 @@ env.Replace( config = env.GetProjectConfig() env.Replace( PROJECT_DIR=get_project_dir(), - PROJECT_CORE_DIR=config.get_optional_dir("core"), - PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"), - PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"), - PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"), - PROJECT_INCLUDE_DIR=config.get_optional_dir("include"), - PROJECT_SRC_DIR=config.get_optional_dir("src"), - PROJECTSRC_DIR=config.get_optional_dir("src"), # legacy for dev/platform - PROJECT_TEST_DIR=config.get_optional_dir("test"), - PROJECT_DATA_DIR=config.get_optional_dir("data"), - PROJECTDATA_DIR=config.get_optional_dir("data"), # legacy for dev/platform - PROJECT_BUILD_DIR=config.get_optional_dir("build"), - BUILD_CACHE_DIR=config.get_optional_dir("build_cache"), + PROJECT_CORE_DIR=config.get("platformio", "core_dir"), + PROJECT_PACKAGES_DIR=config.get("platformio", "packages_dir"), + PROJECT_WORKSPACE_DIR=config.get("platformio", "workspace_dir"), + PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"), + PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"), + PROJECT_SRC_DIR=config.get("platformio", "src_dir"), + PROJECTSRC_DIR=config.get("platformio", "src_dir"), # legacy for dev/platform + PROJECT_TEST_DIR=config.get("platformio", "test_dir"), + PROJECT_DATA_DIR=config.get("platformio", "data_dir"), + PROJECTDATA_DIR=config.get("platformio", "data_dir"), # legacy for dev/platform + PROJECT_BUILD_DIR=config.get("platformio", "build_dir"), + BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"), LIBSOURCE_DIRS=[ - config.get_optional_dir("lib"), + config.get("platformio", "lib_dir"), os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), - config.get_optional_dir("globallib"), + config.get("platformio", "globallib_dir"), ], ) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 9d278b20..3903f879 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -63,7 +63,7 @@ def _dump_includes(env): # include Unity framework if there are tests in project includes["unity"] = [] auto_install_unity = False - test_dir = env.GetProjectConfig().get_optional_dir("test") + test_dir = env.GetProjectConfig().get("platformio", "test_dir") if os.path.isdir(test_dir) and os.listdir(test_dir) != ["README"]: auto_install_unity = True unity_dir = get_core_package_dir( diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 87673f99..82ab7566 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -106,8 +106,8 @@ def cli( ) default_patterns = [ - config.get_optional_dir("src"), - config.get_optional_dir("include"), + config.get("platformio", "src_dir"), + config.get("platformio", "include_dir"), ] tool_options = dict( verbose=verbose, diff --git a/platformio/commands/check/tools/clangtidy.py b/platformio/commands/check/tools/clangtidy.py index 3206cdba..d6f99058 100644 --- a/platformio/commands/check/tools/clangtidy.py +++ b/platformio/commands/check/tools/clangtidy.py @@ -77,7 +77,7 @@ class ClangtidyCheckTool(CheckToolBase): includes = [] for inc in self.cpp_includes: if self.options.get("skip_packages") and inc.lower().startswith( - self.config.get_optional_dir("packages").lower() + self.config.get("platformio", "packages_dir").lower() ): continue includes.append(inc) diff --git a/platformio/commands/check/tools/cppcheck.py b/platformio/commands/check/tools/cppcheck.py index bc06799a..99512d52 100644 --- a/platformio/commands/check/tools/cppcheck.py +++ b/platformio/commands/check/tools/cppcheck.py @@ -84,7 +84,7 @@ class CppcheckCheckTool(CheckToolBase): if ( args.get("file", "") .lower() - .startswith(self.config.get_optional_dir("packages").lower()) + .startswith(self.config.get("platformio", "packages_dir").lower()) ): if args["id"] in breaking_defect_ids: if self.options.get("verbose"): @@ -201,7 +201,7 @@ class CppcheckCheckTool(CheckToolBase): result = [] for inc in self.cpp_includes: if self.options.get("skip_packages") and inc.lower().startswith( - self.config.get_optional_dir("packages").lower() + self.config.get("platformio", "packages_dir").lower() ): continue result.append(inc) diff --git a/platformio/commands/check/tools/pvsstudio.py b/platformio/commands/check/tools/pvsstudio.py index 36298d25..a872c0af 100644 --- a/platformio/commands/check/tools/pvsstudio.py +++ b/platformio/commands/check/tools/pvsstudio.py @@ -43,7 +43,7 @@ class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-at with open(self._tmp_cfg_file, mode="w", encoding="utf8") as fp: fp.write( "exclude-path = " - + self.config.get_optional_dir("packages").replace("\\", "/") + + self.config.get("platformio", "packages_dir").replace("\\", "/") ) with open(self._tmp_cmd_file, mode="w", encoding="utf8") as fp: diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index 3c0ce465..b567f06b 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -14,16 +14,15 @@ from __future__ import absolute_import -from os.path import join +import os from platformio import __version__, app, fs, util -from platformio.project.helpers import get_project_core_dir, is_platformio_project +from platformio.project.config import ProjectConfig +from platformio.project.helpers import is_platformio_project class AppRPC: - APPSTATE_PATH = join(get_project_core_dir(), "homestate.json") - IGNORE_STORAGE_KEYS = [ "cid", "coreVersion", @@ -34,9 +33,16 @@ class AppRPC: "projectsDir", ] + @staticmethod + def get_state_path(): + core_dir = ProjectConfig.get_instance().get("platformio", "core_dir") + if not os.path.isdir(core_dir): + os.makedirs(core_dir) + return os.path.join(core_dir, "homestate.json") + @staticmethod def load_state(): - with app.State(AppRPC.APPSTATE_PATH, lock=True) as state: + with app.State(AppRPC.get_state_path(), lock=True) as state: storage = state.get("storage", {}) # base data @@ -72,7 +78,7 @@ class AppRPC: @staticmethod def save_state(state): - with app.State(AppRPC.APPSTATE_PATH, lock=True) as s: + with app.State(AppRPC.get_state_path(), lock=True) as s: s.clear() s.update(state) storage = s.get("storage", {}) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 4c48a5d9..535bc9be 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -81,7 +81,7 @@ class ProjectRPC: data["description"] = config.get("platformio", "description") data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", [])) - libdeps_dir = config.get_optional_dir("libdeps") + libdeps_dir = config.get("platformio", "libdeps_dir") for section in config.sections(): if not section.startswith("env:"): continue @@ -253,7 +253,7 @@ class ProjectRPC: with fs.cd(project_dir): config = ProjectConfig() - src_dir = config.get_optional_dir("src") + src_dir = config.get("platformio", "src_dir") main_path = os.path.join( src_dir, "main.%s" % ("cpp" if is_cpp_project else "c") ) @@ -308,7 +308,7 @@ class ProjectRPC: ) with fs.cd(project_dir): config = ProjectConfig() - src_dir = config.get_optional_dir("src") + src_dir = config.get("platformio", "src_dir") if os.path.isdir(src_dir): fs.rmtree(src_dir) shutil.copytree(arduino_project_dir, src_dir, symlinks=True) diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index dac03816..7b6c7bb7 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -43,7 +43,7 @@ CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps" def get_project_global_lib_dir(): - return ProjectConfig.get_instance().get_optional_dir("globallib") + return ProjectConfig.get_instance().get("platformio", "globallib_dir") @click.group(short_help="Library manager") @@ -111,7 +111,7 @@ def cli(ctx, **options): os.path.join(storage_dir, "platformio.ini") ) config.validate(options["environment"], silent=in_silence) - libdeps_dir = config.get_optional_dir("libdeps") + libdeps_dir = config.get("platformio", "libdeps_dir") for env in config.envs(): if options["environment"] and env not in options["environment"]: continue diff --git a/platformio/commands/project.py b/platformio/commands/project.py index d45d6dcd..c79979fd 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -226,10 +226,10 @@ def init_base_project(project_dir): config = ProjectConfig() config.save() dir_to_readme = [ - (config.get_optional_dir("src"), None), - (config.get_optional_dir("include"), init_include_readme), - (config.get_optional_dir("lib"), init_lib_readme), - (config.get_optional_dir("test"), init_test_readme), + (config.get("platformio", "src_dir"), None), + (config.get("platformio", "include_dir"), init_include_readme), + (config.get("platformio", "lib_dir"), init_lib_readme), + (config.get("platformio", "test_dir"), init_test_readme), ] for (path, cb) in dir_to_readme: if os.path.isdir(path): diff --git a/platformio/commands/remote/client/agent_service.py b/platformio/commands/remote/client/agent_service.py index 5918d205..01af44da 100644 --- a/platformio/commands/remote/client/agent_service.py +++ b/platformio/commands/remote/client/agent_service.py @@ -13,7 +13,6 @@ # limitations under the License. import os -from os.path import getatime, getmtime, isdir, isfile, join from twisted.logger import LogLevel # pylint: disable=import-error from twisted.spread import pb # pylint: disable=import-error @@ -25,15 +24,16 @@ from platformio.commands.remote.ac.serial import SerialPortAsyncCmd from platformio.commands.remote.client.base import RemoteClientBase from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError -from platformio.project.helpers import get_project_core_dir class RemoteAgentService(RemoteClientBase): def __init__(self, name, share, working_dir=None): RemoteClientBase.__init__(self) self.log_level = LogLevel.info - self.working_dir = working_dir or join(get_project_core_dir(), "remote") - if not isdir(self.working_dir): + self.working_dir = working_dir or os.path.join( + ProjectConfig.get_instance().get("platformio", "core_dir"), "remote" + ) + if not os.path.isdir(self.working_dir): os.makedirs(self.working_dir) if name: self.name = str(name)[:50] @@ -138,14 +138,14 @@ class RemoteAgentService(RemoteClientBase): self, command, options ): assert options and "project_id" in options - project_dir = join(self.working_dir, "projects", options["project_id"]) - origin_pio_ini = join(project_dir, "platformio.ini") - back_pio_ini = join(project_dir, "platformio.ini.bak") + project_dir = os.path.join(self.working_dir, "projects", options["project_id"]) + origin_pio_ini = os.path.join(project_dir, "platformio.ini") + back_pio_ini = os.path.join(project_dir, "platformio.ini.bak") # remove insecure project options try: conf = ProjectConfig(origin_pio_ini) - if isfile(back_pio_ini): + if os.path.isfile(back_pio_ini): os.remove(back_pio_ini) os.rename(origin_pio_ini, back_pio_ini) # cleanup @@ -159,7 +159,10 @@ class RemoteAgentService(RemoteClientBase): conf.save(origin_pio_ini) # restore A/M times - os.utime(origin_pio_ini, (getatime(back_pio_ini), getmtime(back_pio_ini))) + os.utime( + origin_pio_ini, + (os.path.getatime(back_pio_ini), os.path.getmtime(back_pio_ini)), + ) except NotPlatformIOProjectError as e: raise pb.Error(str(e)) @@ -194,8 +197,8 @@ class RemoteAgentService(RemoteClientBase): paused_acs.append(ac) def _cb_on_end(): - if isfile(back_pio_ini): - if isfile(origin_pio_ini): + if os.path.isfile(back_pio_ini): + if os.path.isfile(origin_pio_ini): os.remove(origin_pio_ini) os.rename(back_pio_ini, origin_pio_ini) for ac in paused_acs: diff --git a/platformio/commands/remote/client/run_or_test.py b/platformio/commands/remote/client/run_or_test.py index 10a9b008..cdc1465d 100644 --- a/platformio/commands/remote/client/run_or_test.py +++ b/platformio/commands/remote/client/run_or_test.py @@ -69,8 +69,8 @@ class RunOrTestClient(AsyncClientBase): os.path.join(self.options["project_dir"], "platformio.ini") ) psync.add_item(cfg.path, "platformio.ini") - psync.add_item(cfg.get_optional_dir("shared"), "shared") - psync.add_item(cfg.get_optional_dir("boards"), "boards") + psync.add_item(cfg.get("platformio", "shared_dir"), "shared") + psync.add_item(cfg.get("platformio", "boards_dir"), "boards") if self.options["force_remote"]: self._add_project_source_items(cfg, psync) @@ -78,26 +78,26 @@ class RunOrTestClient(AsyncClientBase): self._add_project_binary_items(cfg, psync) if self.command == "test": - psync.add_item(cfg.get_optional_dir("test"), "test") + psync.add_item(cfg.get("platformio", "test_dir"), "test") def _add_project_source_items(self, cfg, psync): - psync.add_item(cfg.get_optional_dir("lib"), "lib") + psync.add_item(cfg.get("platformio", "lib_dir"), "lib") psync.add_item( - cfg.get_optional_dir("include"), + cfg.get("platformio", "include_dir"), "include", cb_filter=self._cb_tarfile_filter, ) psync.add_item( - cfg.get_optional_dir("src"), "src", cb_filter=self._cb_tarfile_filter + cfg.get("platformio", "src_dir"), "src", cb_filter=self._cb_tarfile_filter ) if set(["buildfs", "uploadfs", "uploadfsota"]) & set( self.options.get("target", []) ): - psync.add_item(cfg.get_optional_dir("data"), "data") + psync.add_item(cfg.get("platformio", "data_dir"), "data") @staticmethod def _add_project_binary_items(cfg, psync): - build_dir = cfg.get_optional_dir("build") + build_dir = cfg.get("platformio", "build_dir") for env_name in os.listdir(build_dir): env_dir = os.path.join(build_dir, env_name) if not os.path.isdir(env_dir): diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 5684f119..85fddd72 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -101,7 +101,7 @@ def cli( # clean obsolete build dir if not disable_auto_clean: - build_dir = config.get_optional_dir("build") + build_dir = config.get("platformio", "build_dir") try: clean_build_dir(build_dir, config) except: # pylint: disable=bare-except diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index c976d935..925b5584 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -23,8 +23,8 @@ from platformio.project.helpers import compute_project_checksum, get_project_dir def handle_legacy_libdeps(project_dir, config): legacy_libdeps_dir = join(project_dir, ".piolibdeps") - if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get_optional_dir( - "libdeps" + if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get( + "platformio", "libdeps_dir" ): return if not config.has_section("env"): diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index 63d1727c..c207d0ae 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -64,7 +64,7 @@ def system_info(json_output): } data["core_dir"] = { "title": "PlatformIO Core Directory", - "value": project_config.get_optional_dir("core"), + "value": project_config.get("platformio", "core_dir"), } data["platformio_exe"] = { "title": "PlatformIO Core Executable", @@ -88,7 +88,7 @@ def system_info(json_output): "title": "Tools & Toolchains", "value": len( ToolPackageManager( - project_config.get_optional_dir("packages") + project_config.get("platformio", "packages_dir") ).get_installed() ), } diff --git a/platformio/commands/test/helpers.py b/platformio/commands/test/helpers.py index ce72360f..e490ea7c 100644 --- a/platformio/commands/test/helpers.py +++ b/platformio/commands/test/helpers.py @@ -18,7 +18,7 @@ from platformio import exception def get_test_names(config): - test_dir = config.get_optional_dir("test") + test_dir = config.get("platformio", "test_dir") if not os.path.isdir(test_dir): raise exception.TestDirNotExists(test_dir) names = [] diff --git a/platformio/commands/test/native.py b/platformio/commands/test/native.py index a73c03fe..3c30e97b 100644 --- a/platformio/commands/test/native.py +++ b/platformio/commands/test/native.py @@ -31,7 +31,7 @@ class NativeTestProcessor(TestProcessorBase): return self.run() def run(self): - build_dir = self.options["project_config"].get_optional_dir("build") + build_dir = self.options["project_config"].get("platformio", "build_dir") result = proc.exec_command( [join(build_dir, self.env_name, "program")], stdout=LineBufferedAsyncPipe(self.on_run_out), diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 3e9a519d..0e1e367f 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -124,7 +124,7 @@ class TestProcessorBase(object): def build_or_upload(self, target): if not self._output_file_generated: self.generate_output_file( - self.options["project_config"].get_optional_dir("test") + self.options["project_config"].get("platformio", "test_dir") ) self._output_file_generated = True diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index de726bc9..bcafa8fd 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -20,7 +20,7 @@ from platformio.compat import string_types from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.helpers import reveal_debug_port from platformio.project.config import ProjectConfig -from platformio.project.helpers import get_project_core_dir, load_project_ide_data +from platformio.project.helpers import load_project_ide_data from platformio.project.options import ProjectOptions @@ -208,7 +208,7 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes def reveal_patterns(self, source, recursive=True): program_path = self.program_path or "" patterns = { - "PLATFORMIO_CORE_DIR": get_project_core_dir(), + "PLATFORMIO_CORE_DIR": self.project_config.get("platformio", "core_dir"), "PYTHONEXE": proc.get_pythonexe_path(), "PROJECT_DIR": os.getcwd(), "PROG_PATH": program_path, diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 56930b9a..f6112196 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -79,11 +79,11 @@ class ProjectGenerator(object): tpl_vars.update( { "src_files": self.get_src_files(), - "project_src_dir": self.config.get_optional_dir("src"), - "project_lib_dir": self.config.get_optional_dir("lib"), - "project_test_dir": self.config.get_optional_dir("test"), + "project_src_dir": self.config.get("platformio", "src_dir"), + "project_lib_dir": self.config.get("platformio", "lib_dir"), + "project_test_dir": self.config.get("platformio", "test_dir"), "project_libdeps_dir": os.path.join( - self.config.get_optional_dir("libdeps"), self.env_name + self.config.get("platformio", "libdeps_dir"), self.env_name ), } ) @@ -103,7 +103,7 @@ class ProjectGenerator(object): def get_src_files(self): result = [] with fs.cd(self.project_dir): - for root, _, files in os.walk(self.config.get_optional_dir("src")): + for root, _, files in os.walk(self.config.get("platformio", "src_dir")): for f in files: result.append(os.path.relpath(os.path.join(root, f))) return result diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index 9df294ee..c0edf6aa 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -40,7 +40,7 @@ % end % % def _get_lib_dirs(envname): -% env_libdeps_dir = os.path.join(config.get_optional_dir("libdeps"), envname) +% env_libdeps_dir = os.path.join(config.get("platformio", "libdeps_dir"), envname) % env_lib_extra_dirs = config.get("env:" + envname, "lib_extra_dirs", []) % return _fix_lib_dirs([env_libdeps_dir] + env_lib_extra_dirs) % end diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index efe8a361..ecce6a22 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -31,7 +31,7 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an self.config = ProjectConfig.get_instance() super(PlatformPackageManager, self).__init__( PackageType.PLATFORM, - package_dir or self.config.get_optional_dir("platforms"), + package_dir or self.config.get("platformio", "platforms_dir"), ) @property diff --git a/platformio/package/manager/tool.py b/platformio/package/manager/tool.py index 60aededd..70a76377 100644 --- a/platformio/package/manager/tool.py +++ b/platformio/package/manager/tool.py @@ -20,7 +20,7 @@ from platformio.project.config import ProjectConfig class ToolPackageManager(BasePackageManager): # pylint: disable=too-many-ancestors def __init__(self, package_dir=None): if not package_dir: - package_dir = ProjectConfig.get_instance().get_optional_dir("packages") + package_dir = ProjectConfig.get_instance().get("platformio", "packages_dir") super(ToolPackageManager, self).__init__(PackageType.TOOL, package_dir) @property diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 1d7911b0..d5ce103d 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -45,7 +45,7 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub self._custom_packages = None self.config = ProjectConfig.get_instance() - self.pm = ToolPackageManager(self.config.get_optional_dir("packages")) + self.pm = ToolPackageManager(self.config.get("platformio", "packages_dir")) @property def name(self): @@ -145,8 +145,8 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub self._BOARDS_CACHE[board_id] = config bdirs = [ - self.config.get_optional_dir("boards"), - os.path.join(self.config.get_optional_dir("core"), "boards"), + self.config.get("platformio", "boards_dir"), + os.path.join(self.config.get("platformio", "core_dir"), "boards"), os.path.join(self.get_dir(), "boards"), ] diff --git a/platformio/project/config.py b/platformio/project/config.py index 187cb993..30c8a9ae 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -13,7 +13,6 @@ # limitations under the License. import glob -import hashlib import json import os import re @@ -21,7 +20,7 @@ import re import click from platformio import fs -from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types +from platformio.compat import string_types from platformio.project import exception from platformio.project.options import ProjectOptions @@ -311,8 +310,11 @@ class ProjectConfigBase(object): if not option_meta: return value + if option_meta.validate: + value = option_meta.validate(value) if option_meta.multiple: value = self.parse_multi_values(value or []) + try: return self.cast_to(value, option_meta.type) except click.BadParameter as e: @@ -353,74 +355,7 @@ class ProjectConfigBase(object): return True -class ProjectConfigDirsMixin(object): - def _get_core_dir(self, exists=False): - default = ProjectOptions["platformio.core_dir"].default - core_dir = self.get("platformio", "core_dir") - win_core_dir = None - if IS_WINDOWS and core_dir == default: - win_core_dir = os.path.splitdrive(core_dir)[0] + "\\.platformio" - if os.path.isdir(win_core_dir): - core_dir = win_core_dir - - if exists and not os.path.isdir(core_dir): - try: - os.makedirs(core_dir) - except OSError as e: - if win_core_dir: - os.makedirs(win_core_dir) - core_dir = win_core_dir - else: - raise e - - return core_dir - - def get_optional_dir(self, name, exists=False): - if not ProjectOptions.get("platformio.%s_dir" % name): - raise ValueError("Unknown optional directory -> " + name) - - if name == "core": - result = self._get_core_dir(exists) - else: - result = self.get("platformio", name + "_dir") - - if result is None: - return None - - project_dir = os.getcwd() - - # patterns - if "$PROJECT_HASH" in result: - result = result.replace( - "$PROJECT_HASH", - "%s-%s" - % ( - os.path.basename(project_dir), - hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], - ), - ) - - if "$PROJECT_DIR" in result: - result = result.replace("$PROJECT_DIR", project_dir) - if "$PROJECT_CORE_DIR" in result: - result = result.replace("$PROJECT_CORE_DIR", self.get_optional_dir("core")) - if "$PROJECT_WORKSPACE_DIR" in result: - result = result.replace( - "$PROJECT_WORKSPACE_DIR", self.get_optional_dir("workspace") - ) - - if result.startswith("~"): - result = fs.expanduser(result) - - result = os.path.realpath(result) - - if exists and not os.path.isdir(result): - os.makedirs(result) - - return result - - -class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): +class ProjectConfig(ProjectConfigBase): _instances = {} diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 655e9685..100c1a75 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -15,8 +15,6 @@ import json import os from hashlib import sha1 -from os import walk -from os.path import dirname, isdir, isfile, join from click.testing import CliRunner @@ -32,65 +30,50 @@ def get_project_dir(): def is_platformio_project(project_dir=None): if not project_dir: project_dir = get_project_dir() - return isfile(join(project_dir, "platformio.ini")) + return os.path.isfile(os.path.join(project_dir, "platformio.ini")) def find_project_dir_above(path): - if isfile(path): - path = dirname(path) + if os.path.isfile(path): + path = os.path.dirname(path) if is_platformio_project(path): return path - if isdir(dirname(path)): - return find_project_dir_above(dirname(path)) + if os.path.isdir(os.path.dirname(path)): + return find_project_dir_above(os.path.dirname(path)) return None -def get_project_core_dir(): - """Deprecated, use ProjectConfig.get_optional_dir("core") instead""" - return ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini") - ).get_optional_dir("core", exists=True) - - def get_project_cache_dir(): - """Deprecated, use ProjectConfig.get_optional_dir("cache") instead""" - return ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini") - ).get_optional_dir("cache") + """Deprecated, use ProjectConfig.get("platformio", "cache_dir") instead""" + return ProjectConfig.get_instance().get("platformio", "cache_dir") def get_project_global_lib_dir(): """ - Deprecated, use ProjectConfig.get_optional_dir("globallib") instead + Deprecated, use ProjectConfig.get("platformio", "globallib_dir") instead "platformio-node-helpers" depends on it """ - return ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini") - ).get_optional_dir("globallib") + return ProjectConfig.get_instance().get("platformio", "globallib_dir") def get_project_lib_dir(): """ - Deprecated, use ProjectConfig.get_optional_dir("lib") instead + Deprecated, use ProjectConfig.get("platformio", "lib_dir") instead "platformio-node-helpers" depends on it """ - return ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini") - ).get_optional_dir("lib") + return ProjectConfig.get_instance().get("platformio", "lib_dir") def get_project_libdeps_dir(): """ - Deprecated, use ProjectConfig.get_optional_dir("libdeps") instead + Deprecated, use ProjectConfig.get("platformio", "libdeps_dir") instead "platformio-node-helpers" depends on it """ - return ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini") - ).get_optional_dir("libdeps") + return ProjectConfig.get_instance().get("platformio", "libdeps_dir") def get_default_projects_dir(): - docs_dir = join(fs.expanduser("~"), "Documents") + docs_dir = os.path.join(fs.expanduser("~"), "Documents") try: assert IS_WINDOWS import ctypes.wintypes # pylint: disable=import-outside-toplevel @@ -100,7 +83,7 @@ def get_default_projects_dir(): docs_dir = buf.value except: # pylint: disable=bare-except pass - return join(docs_dir, "PlatformIO", "Projects") + return os.path.join(docs_dir, "PlatformIO", "Projects") def compute_project_checksum(config): @@ -113,16 +96,16 @@ def compute_project_checksum(config): # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") for d in ( - config.get_optional_dir("include"), - config.get_optional_dir("src"), - config.get_optional_dir("lib"), + config.get("platformio", "include_dir"), + config.get("platformio", "src_dir"), + config.get("platformio", "lib_dir"), ): - if not isdir(d): + if not os.path.isdir(d): continue chunks = [] - for root, _, files in walk(d): + for root, _, files in os.walk(d): for f in files: - path = join(root, f) + path = os.path.join(root, f) if path.endswith(check_suffixes): chunks.append(path) if not chunks: @@ -171,8 +154,8 @@ def _load_project_ide_data(project_dir, env_names): def _load_cached_project_ide_data(project_dir, env_names): build_dir = ProjectConfig.get_instance( - join(project_dir, "platformio.ini") - ).get_optional_dir("build") + os.path.join(project_dir, "platformio.ini") + ).get("platformio", "build_dir") result = {} for name in env_names: if not os.path.isfile(os.path.join(build_dir, name, "idedata.json")): diff --git a/platformio/project/options.py b/platformio/project/options.py index 3c9db96c..f94a74c6 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -14,12 +14,14 @@ # pylint: disable=redefined-builtin, too-many-arguments +import hashlib import os from collections import OrderedDict import click from platformio import fs +from platformio.compat import IS_WINDOWS, hashlib_encode_data class ConfigOption(object): # pylint: disable=too-many-instance-attributes @@ -35,6 +37,7 @@ class ConfigOption(object): # pylint: disable=too-many-instance-attributes buildenvvar=None, oldnames=None, default=None, + validate=None, ): self.scope = scope self.group = group @@ -46,6 +49,7 @@ class ConfigOption(object): # pylint: disable=too-many-instance-attributes self.buildenvvar = buildenvvar self.oldnames = oldnames self.default = default + self.validate = validate def as_dict(self): result = dict( @@ -60,13 +64,11 @@ class ConfigOption(object): # pylint: disable=too-many-instance-attributes ) if isinstance(self.type, click.ParamType): result["type"] = self.type.name - if isinstance(self.type, (click.IntRange, click.FloatRange)): result["min"] = self.type.min result["max"] = self.type.max if isinstance(self.type, click.Choice): result["choices"] = self.type.choices - return result @@ -78,6 +80,50 @@ def ConfigEnvOption(*args, **kwargs): return ConfigOption("env", *args, **kwargs) +def calculate_path_hash(path): + return "%s-%s" % ( + os.path.basename(path), + hashlib.sha1(hashlib_encode_data(path)).hexdigest()[:10], + ) + + +def expand_dir_templates(path): + project_dir = os.getcwd() + tpls = { + "$PROJECT_DIR": lambda: project_dir, + "$PROJECT_HASH": lambda: calculate_path_hash(project_dir), + } + done = False + while not done: + done = True + for tpl, cb in tpls.items(): + if tpl not in path: + continue + path = path.replace(tpl, cb()) + done = False + return path + + +def validate_dir(path): + if not path: + return path + if path.startswith("~"): + path = fs.expanduser(path) + if "$" in path: + path = expand_dir_templates(path) + return os.path.realpath(path) + + +def validate_core_dir(path): + default = ProjectOptions["platformio.core_dir"].default + win_core_dir = None + if IS_WINDOWS and path == default: + win_core_dir = os.path.splitdrive(path)[0] + "\\.platformio" + if os.path.isdir(win_core_dir): + path = win_core_dir + return validate_dir(path) + + ProjectOptions = OrderedDict( [ ("%s.%s" % (option.scope, option.name), option) @@ -121,6 +167,7 @@ ProjectOptions = OrderedDict( oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR", default=os.path.join(fs.expanduser("~"), ".platformio"), + validate=validate_core_dir, ), ConfigPlatformioOption( group="directory", @@ -130,7 +177,8 @@ ProjectOptions = OrderedDict( "Finder (LDF) looks for global libraries" ), sysenvvar="PLATFORMIO_GLOBALLIB_DIR", - default=os.path.join("$PROJECT_CORE_DIR", "lib"), + default=os.path.join("${platformio.core_dir}", "lib"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -140,7 +188,8 @@ ProjectOptions = OrderedDict( "platforms" ), sysenvvar="PLATFORMIO_PLATFORMS_DIR", - default=os.path.join("$PROJECT_CORE_DIR", "platforms"), + default=os.path.join("${platformio.core_dir}", "platforms"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -149,7 +198,8 @@ ProjectOptions = OrderedDict( "A location where PlatformIO Core keeps installed packages" ), sysenvvar="PLATFORMIO_PACKAGES_DIR", - default=os.path.join("$PROJECT_CORE_DIR", "packages"), + default=os.path.join("${platformio.core_dir}", "packages"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -160,7 +210,8 @@ ProjectOptions = OrderedDict( "other service information)" ), sysenvvar="PLATFORMIO_CACHE_DIR", - default=os.path.join("$PROJECT_CORE_DIR", ".cache"), + default=os.path.join("${platformio.core_dir}", ".cache"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -171,6 +222,7 @@ ProjectOptions = OrderedDict( "build environments" ), sysenvvar="PLATFORMIO_BUILD_CACHE_DIR", + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -182,6 +234,7 @@ ProjectOptions = OrderedDict( ), sysenvvar="PLATFORMIO_WORKSPACE_DIR", default=os.path.join("$PROJECT_DIR", ".pio"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -192,7 +245,8 @@ ProjectOptions = OrderedDict( "and other cached information" ), sysenvvar="PLATFORMIO_BUILD_DIR", - default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"), + default=os.path.join("${platformio.workspace_dir}", "build"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -202,7 +256,8 @@ ProjectOptions = OrderedDict( "dependencies declared via `lib_deps` option" ), sysenvvar="PLATFORMIO_LIBDEPS_DIR", - default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"), + default=os.path.join("${platformio.workspace_dir}", "libdeps"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -213,6 +268,7 @@ ProjectOptions = OrderedDict( ), sysenvvar="PLATFORMIO_INCLUDE_DIR", default=os.path.join("$PROJECT_DIR", "include"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -223,6 +279,7 @@ ProjectOptions = OrderedDict( ), sysenvvar="PLATFORMIO_SRC_DIR", default=os.path.join("$PROJECT_DIR", "src"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -230,6 +287,7 @@ ProjectOptions = OrderedDict( description="A storage for the custom/private project libraries", sysenvvar="PLATFORMIO_LIB_DIR", default=os.path.join("$PROJECT_DIR", "lib"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -240,6 +298,7 @@ ProjectOptions = OrderedDict( ), sysenvvar="PLATFORMIO_DATA_DIR", default=os.path.join("$PROJECT_DIR", "data"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -250,6 +309,7 @@ ProjectOptions = OrderedDict( ), sysenvvar="PLATFORMIO_TEST_DIR", default=os.path.join("$PROJECT_DIR", "test"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -257,6 +317,7 @@ ProjectOptions = OrderedDict( description="A global storage for custom board manifests", sysenvvar="PLATFORMIO_BOARDS_DIR", default=os.path.join("$PROJECT_DIR", "boards"), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", @@ -267,6 +328,7 @@ ProjectOptions = OrderedDict( ), sysenvvar="PLATFORMIO_SHARED_DIR", default=os.path.join("$PROJECT_DIR", "shared"), + validate=validate_dir, ), # # [env] diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 19d4a98e..5bd118e7 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -486,5 +486,5 @@ def test_update_without_metadata(isolated_pio_core, tmpdir_factory): # update lm = LibraryPackageManager(str(storage_dir)) new_pkg = lm.update(pkg, silent=True) - assert len(lm.get_installed()) == 3 + assert len(lm.get_installed()) == 4 assert new_pkg.metadata.spec.owner == "ottowinter" diff --git a/tests/test_examples.py b/tests/test_examples.py index 9e2b7cc0..0be516eb 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -67,7 +67,7 @@ def pytest_generate_tests(metafunc): def test_run(pioproject_dir): with fs.cd(pioproject_dir): config = ProjectConfig() - build_dir = config.get_optional_dir("build") + build_dir = config.get("platformio", "build_dir") if os.path.isdir(build_dir): fs.rmtree(build_dir) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 99c71ec0..ca859218 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -18,12 +18,15 @@ import os import pytest +from platformio import fs from platformio.project.config import ConfigParser, ProjectConfig from platformio.project.exception import InvalidProjectConfError, UnknownEnvNamesError +from platformio.project.options import calculate_path_hash BASE_CONFIG = """ [platformio] env_default = base, extra_2 +build_dir = ~/tmp/pio-$PROJECT_HASH extra_configs = extra_envs.ini extra_debug.ini @@ -83,17 +86,23 @@ lib_install = 574 build_flags = ${custom.debug_flags} ${custom.extra_flags} lib_ignore = ${env.lib_ignore}, Lib3 upload_port = /dev/extra_2/port +debug_server = ${custom.debug_server} """ EXTRA_DEBUG_CONFIG = """ # Override original "custom.debug_flags" [custom] debug_flags = -D DEBUG=1 +debug_server = + ${platformio.packages_dir}/tool-openocd/openocd + --help [env:extra_2] build_flags = -Og """ +DEFAULT_CORE_DIR = os.path.join(fs.expanduser("~"), ".platformio") + @pytest.fixture(scope="module") def config(tmpdir_factory): @@ -124,7 +133,7 @@ def test_warnings(config): def test_defaults(config): - assert config.get_optional_dir("core") == os.path.join( + assert config.get("platformio", "core_dir") == os.path.join( os.path.expanduser("~"), ".platformio" ) assert config.get("strict_ldf", "lib_deps", ["Empty"]) == ["Empty"] @@ -272,6 +281,14 @@ def test_getraw_value(config): assert config.getraw("env", "monitor_speed") == "9600" assert config.getraw("env:test_extends", "monitor_speed") == "115200" + # dir options + packages_dir = os.path.join(DEFAULT_CORE_DIR, "packages") + assert config.get("platformio", "packages_dir") == packages_dir + assert ( + config.getraw("custom", "debug_server") + == f"\n{packages_dir}/tool-openocd/openocd\n--help" + ) + def test_get_value(config): assert config.get("custom", "debug_flags") == "-D DEBUG=1" @@ -293,6 +310,15 @@ def test_get_value(config): "-D CUSTOM_DEBUG_FLAG", ] + # dir options + assert config.get("platformio", "packages_dir") == os.path.join( + DEFAULT_CORE_DIR, "packages" + ) + assert config.get("env:extra_2", "debug_server") == [ + os.path.join(DEFAULT_CORE_DIR, "packages", "tool-openocd", "openocd"), + "--help", + ] + def test_items(config): assert config.items("custom") == [ @@ -300,6 +326,11 @@ def test_items(config): ("lib_flags", "-lc -lm"), ("extra_flags", ""), ("lib_ignore", "LibIgnoreCustom"), + ( + "debug_server", + "\n%s/tool-openocd/openocd\n--help" + % os.path.join(DEFAULT_CORE_DIR, "packages"), + ), ] assert config.items(env="base") == [ ("build_flags", ["-D DEBUG=1"]), @@ -326,6 +357,13 @@ def test_items(config): ("build_flags", ["-Og"]), ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), ("upload_port", "/dev/extra_2/port"), + ( + "debug_server", + [ + "%s/tool-openocd/openocd" % os.path.join(DEFAULT_CORE_DIR, "packages"), + "--help", + ], + ), ("monitor_speed", 9600), ("custom_monitor_speed", "115200"), ("lib_deps", ["Lib1", "Lib2"]), @@ -426,6 +464,11 @@ def test_dump(tmpdir_factory): ( "platformio", [ + ( + "build_dir", + "%s-%s" + % (fs.expanduser("~/tmp/pio"), calculate_path_hash(os.getcwd())), + ), ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), ("default_envs", ["base", "extra_2"]), ],