From d2abac9b18ff47cd2ab32312995ba52e78cfd62a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Sep 2019 14:13:53 +0300 Subject: [PATCH] Fixed an issue when configuration file options partly ignored when ``--project-conf`` // Resolve #3034 (#3055) * Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` // Resolve #3034 * Py2 compatible makedirs * Fix circle dependency * Fix broken import in test examples * Fix history * Remove YAPF markers * PyLint fix * Fix invalid project conf path * Move PIO Core to the root on Windows, issue with long CPPPATHs * Respect global PLATFORMIO_BUILD_CACHE_DIR env var * Fix Appveyor paths * Minor changes --- .appveyor.yml | 5 +- .pylintrc | 3 +- HISTORY.rst | 1 + platformio/__main__.py | 2 +- platformio/app.py | 31 +++-- platformio/builder/main.py | 57 +++++---- platformio/builder/tools/pioide.py | 2 +- platformio/builder/tools/piolib.py | 15 ++- platformio/builder/tools/piomisc.py | 14 +-- platformio/builder/tools/platformio.py | 12 +- platformio/check/tools/cppcheck.py | 3 +- platformio/commands/check.py | 22 ++-- platformio/commands/debug.py | 10 +- platformio/commands/device.py | 10 +- platformio/commands/init.py | 19 ++- platformio/commands/lib.py | 18 ++- platformio/commands/platform.py | 2 +- platformio/commands/run.py | 18 +-- platformio/commands/test.py | 15 ++- platformio/commands/upgrade.py | 2 +- platformio/debug/client.py | 10 +- platformio/debug/server.py | 2 +- platformio/exception.py | 2 +- platformio/home/rpc/handlers/project.py | 27 +++-- platformio/ide/projectgenerator.py | 19 ++- platformio/managers/core.py | 6 +- platformio/managers/lib.py | 9 +- platformio/managers/package.py | 2 +- platformio/managers/platform.py | 34 +++--- platformio/project/config.py | 126 ++++++++++++++++---- platformio/project/helpers.py | 147 ++++-------------------- platformio/project/options.py | 88 +++++++++++--- platformio/run/helpers.py | 10 +- platformio/run/processor.py | 8 +- platformio/test/native.py | 6 +- platformio/test/processor.py | 7 +- platformio/util.py | 6 +- tests/commands/test_check.py | 4 +- tests/test_examples.py | 6 +- tests/test_projectconf.py | 9 ++ 40 files changed, 411 insertions(+), 378 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c9703e56..12d10edf 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,14 +6,15 @@ platform: environment: matrix: - TOXENV: "py27" - PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P2_{build} + PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P2_{build} - TOXENV: "py36" - PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P3_{build} + PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P3_{build} install: - cmd: git submodule update --init --recursive - cmd: SET PATH=C:\MinGW\bin;%PATH% + - cmd: SET PLATFORMIO_CORE_DIR=C:\.pio - cmd: pip install --force-reinstall tox test_script: diff --git a/.pylintrc b/.pylintrc index e4203273..67ce4ef7 100644 --- a/.pylintrc +++ b/.pylintrc @@ -11,4 +11,5 @@ disable= too-few-public-methods, useless-object-inheritance, useless-import-alias, - fixme + fixme, + bad-option-value diff --git a/HISTORY.rst b/HISTORY.rst index d381bdd3..213c54f5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,7 @@ PlatformIO Core 4.0 * Added ``--no-ansi`` flag for `PIO Core `__ to disable ANSI control characters * Fixed an issue with project generator for `CLion IDE `__ when 2 environments were used (`issue #2824 `_) * Fixed default PIO Unified Debugger configuration for `J-Link probe `__ +* Fixed an issue when configuration file options partly ignored when using custom ``--project-conf`` (`issue #3034 `_) 4.0.3 (2019-08-30) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__main__.py b/platformio/__main__.py index dbf66524..043befbd 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -70,7 +70,7 @@ def configure(): # https://urllib3.readthedocs.org # /en/latest/security.html#insecureplatformwarning try: - import urllib3 + import urllib3 # pylint: disable=import-outside-toplevel urllib3.disable_warnings() except (AttributeError, ImportError): diff --git a/platformio/app.py b/platformio/app.py index 2858b37a..42dac9c0 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -17,7 +17,7 @@ import hashlib import os import uuid from os import environ, getenv, listdir, remove -from os.path import abspath, dirname, expanduser, isdir, isfile, join +from os.path import abspath, dirname, isdir, isfile, join from time import time import requests @@ -25,21 +25,11 @@ import requests from platformio import exception, fs, lockfile from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data from platformio.proc import is_ci -from platformio.project.helpers import get_project_cache_dir, get_project_core_dir - - -def get_default_projects_dir(): - docs_dir = join(expanduser("~"), "Documents") - try: - assert WINDOWS - import ctypes.wintypes - - buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) - ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf) - docs_dir = buf.value - except: # pylint: disable=bare-except - pass - return join(docs_dir, "PlatformIO", "Projects") +from platformio.project.helpers import ( + get_default_projects_dir, + get_project_cache_dir, + get_project_core_dir, +) def projects_dir_validate(projects_dir): @@ -88,7 +78,12 @@ DEFAULT_SETTINGS = { }, } -SESSION_VARS = {"command_ctx": None, "force_option": False, "caller_id": None} +SESSION_VARS = { + "command_ctx": None, + "force_option": False, + "caller_id": None, + "custom_project_conf": None, +} class State(object): @@ -415,6 +410,6 @@ def get_cid(): uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) cid = str(cid) - if WINDOWS or os.getuid() > 0: # yapf: disable pylint: disable=no-member + if WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) return cid diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 52adb5a3..0fef773b 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -31,7 +31,7 @@ from platformio import fs from platformio.compat import PY2, dump_json_to_unicode from platformio.managers.platform import PlatformBase from platformio.proc import get_pythonexe_path -from platformio.project import helpers as project_helpers +from platformio.project.helpers import get_project_dir AllowSubstExceptions(NameError) @@ -44,7 +44,7 @@ clivars.AddVariables( ("PIOENV",), ("PIOTEST_RUNNING_NAME",), ("UPLOAD_PORT",), -) # yapf: disable +) DEFAULT_ENV_OPTIONS = dict( tools=[ @@ -67,26 +67,10 @@ DEFAULT_ENV_OPTIONS = dict( # Propagating External Environment ENV=environ, UNIX_TIME=int(time()), - PROJECT_DIR=project_helpers.get_project_dir(), - PROJECTCORE_DIR=project_helpers.get_project_core_dir(), - PROJECTPACKAGES_DIR=project_helpers.get_project_packages_dir(), - PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(), - PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(), - PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(), - PROJECTSRC_DIR=project_helpers.get_project_src_dir(), - PROJECTTEST_DIR=project_helpers.get_project_test_dir(), - PROJECTDATA_DIR=project_helpers.get_project_data_dir(), - PROJECTBUILD_DIR=project_helpers.get_project_build_dir(), - BUILDCACHE_DIR=project_helpers.get_project_optional_dir("build_cache_dir"), - BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"), - BUILDSRC_DIR=join("$BUILD_DIR", "src"), - BUILDTEST_DIR=join("$BUILD_DIR", "test"), + BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"), + BUILD_SRC_DIR=join("$BUILD_DIR", "src"), + BUILD_TEST_DIR=join("$BUILD_DIR", "test"), LIBPATH=["$BUILD_DIR"], - LIBSOURCE_DIRS=[ - project_helpers.get_project_lib_dir(), - join("$PROJECTLIBDEPS_DIR", "$PIOENV"), - project_helpers.get_project_global_lib_dir(), - ], PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), PYTHONEXE=get_pythonexe_path(), @@ -110,10 +94,33 @@ env.Replace( } ) -if env.subst("$BUILDCACHE_DIR"): - if not isdir(env.subst("$BUILDCACHE_DIR")): - makedirs(env.subst("$BUILDCACHE_DIR")) - env.CacheDir("$BUILDCACHE_DIR") +# Setup project optional directories +config = env.GetProjectConfig() +env.Replace( + PROJECT_DIR=get_project_dir(), + PROJECT_CORE_DIR=config.get_optional_dir("core"), + PROJECT_PACKAGES_DIR=config.get_optional_dir("packages"), + PROJECT_WORKSPACE_DIR=config.get_optional_dir("workspace"), + PROJECT_LIBDEPS_DIR=config.get_optional_dir("libdeps"), + PROJECT_INCLUDE_DIR=config.get_optional_dir("include"), + PROJECT_SRC_DIR=config.get_optional_dir("src"), + PROJECTSRC_DIR=config.get_optional_dir("src"), # legacy for dev/platform + PROJECT_TEST_DIR=config.get_optional_dir("test"), + PROJECT_DATA_DIR=config.get_optional_dir("data"), + PROJECTDATA_DIR=config.get_optional_dir("data"), # legacy for dev/platform + PROJECT_BUILD_DIR=config.get_optional_dir("build"), + BUILD_CACHE_DIR=config.get_optional_dir("build_cache"), + LIBSOURCE_DIRS=[ + config.get_optional_dir("lib"), + join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), + config.get_optional_dir("globallib"), + ], +) + +if env.subst("$BUILD_CACHE_DIR"): + if not isdir(env.subst("$BUILD_CACHE_DIR")): + makedirs(env.subst("$BUILD_CACHE_DIR")) + env.CacheDir("$BUILD_CACHE_DIR") if int(ARGUMENTS.get("ISATTY", 0)): # pylint: disable=protected-access diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 39d388a4..606a3bb0 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -54,7 +54,7 @@ def _dump_includes(env, projenv): if unity_dir: includes.append(unity_dir) - includes.extend([env.subst("$PROJECTINCLUDE_DIR"), env.subst("$PROJECTSRC_DIR")]) + includes.extend([env.subst("$PROJECT_INCLUDE_DIR"), env.subst("$PROJECT_SRC_DIR")]) # remove duplicates result = [] diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index e67b9f3b..5ca4b2a2 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -601,8 +601,7 @@ class MbedLibBuilder(LibBuilderBase): mbed_config_path = join(self.env.subst(p), "mbed_config.h") if isfile(mbed_config_path): break - else: - mbed_config_path = None + mbed_config_path = None if not mbed_config_path: return None @@ -821,16 +820,16 @@ class ProjectAsLibBuilder(LibBuilderBase): @property def include_dir(self): - include_dir = self.env.subst("$PROJECTINCLUDE_DIR") + include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") return include_dir if isdir(include_dir) else None @property def src_dir(self): - return self.env.subst("$PROJECTSRC_DIR") + return self.env.subst("$PROJECT_SRC_DIR") def get_include_dirs(self): include_dirs = [] - project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR") + project_include_dir = self.env.subst("$PROJECT_INCLUDE_DIR") if isdir(project_include_dir): include_dirs.append(project_include_dir) for include_dir in LibBuilderBase.get_include_dirs(self): @@ -845,9 +844,9 @@ class ProjectAsLibBuilder(LibBuilderBase): if "__test" in COMMAND_LINE_TARGETS: items.extend( [ - join("$PROJECTTEST_DIR", item) + join("$PROJECT_TEST_DIR", item) for item in self.env.MatchSourceFiles( - "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER" + "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) ] ) @@ -896,7 +895,7 @@ class ProjectAsLibBuilder(LibBuilderBase): not_found_uri.append(uri) did_install = False - lm = LibraryManager(self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) + lm = LibraryManager(self.env.subst(join("$PROJECT_LIBDEPS_DIR", "$PIOENV"))) for uri in not_found_uri: try: lm.install(uri) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 78295ea3..a1b564b9 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -117,7 +117,7 @@ class InoToCPPConverter(object): stropen = True newlines.append(line[:-1]) continue - elif stropen: + if stropen: newlines[len(newlines) - 1] += line[:-1] continue elif stropen and line.endswith(('",', '";')): @@ -199,7 +199,7 @@ class InoToCPPConverter(object): def ConvertInoToCpp(env): - src_dir = glob_escape(env.subst("$PROJECTSRC_DIR")) + src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR")) ino_nodes = env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde")) if not ino_nodes: return @@ -256,7 +256,7 @@ def GetActualLDScript(env): if f == "-T": script_in_next = True continue - elif script_in_next: + if script_in_next: script_in_next = False raw_script = f elif f.startswith("-Wl,-T"): @@ -309,7 +309,7 @@ def PioClean(env, clean_dir): env.Exit(0) -def ProcessDebug(env): +def ConfigureDebugTarget(env): if not env.subst("$PIODEBUGFLAGS"): env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"]) env.Append( @@ -322,7 +322,7 @@ def ProcessDebug(env): env.Append(BUILD_UNFLAGS=unflags) -def ProcessTest(env): +def ConfigureTestTarget(env): env.Append( CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], CPPPATH=[join("$BUILD_DIR", "UnityTestLib")], @@ -361,7 +361,7 @@ def generate(env): env.AddMethod(GetActualLDScript) env.AddMethod(VerboseAction) env.AddMethod(PioClean) - env.AddMethod(ProcessDebug) - env.AddMethod(ProcessTest) + env.AddMethod(ConfigureDebugTarget) + env.AddMethod(ConfigureTestTarget) env.AddMethod(GetExtraScripts) return env diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index a33dafb3..e197b0fb 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -67,15 +67,17 @@ def _build_project_deps(env): is_test = "__test" in COMMAND_LINE_TARGETS if is_test: projenv.BuildSources( - "$BUILDTEST_DIR", "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER" + "$BUILD_TEST_DIR", "$PROJECT_TEST_DIR", "$PIOTEST_SRC_FILTER" ) if not is_test or env.GetProjectOption("test_build_project_src", False): - projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", env.get("SRC_FILTER")) + projenv.BuildSources( + "$BUILD_SRC_DIR", "$PROJECT_SRC_DIR", env.get("SRC_FILTER") + ) if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS: sys.stderr.write( "Error: Nothing to build. Please put your source code files " - "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR") + "to '%s' folder\n" % env.subst("$PROJECT_SRC_DIR") ) env.Exit(1) @@ -102,7 +104,7 @@ def BuildProgram(env): env.Replace(AS="$CC", ASCOM="$ASPPCOM") if "debug" in COMMAND_LINE_TARGETS or env.GetProjectOption("build_type") == "debug": - env.ProcessDebug() + env.ConfigureDebugTarget() # process extra flags from board if "BOARD" in env and "build.extra_flags" in env.BoardConfig(): @@ -121,7 +123,7 @@ def BuildProgram(env): env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) if "__test" in COMMAND_LINE_TARGETS: - env.ProcessTest() + env.ConfigureTestTarget() # build project with dependencies _build_project_deps(env) diff --git a/platformio/check/tools/cppcheck.py b/platformio/check/tools/cppcheck.py index f26e7824..d0f1d6eb 100644 --- a/platformio/check/tools/cppcheck.py +++ b/platformio/check/tools/cppcheck.py @@ -19,7 +19,6 @@ from tempfile import NamedTemporaryFile from platformio.check.defect import DefectItem from platformio.check.tools.base import CheckToolBase from platformio.managers.core import get_core_package_dir -from platformio.project.helpers import get_project_core_dir class CppcheckCheckTool(CheckToolBase): @@ -110,7 +109,7 @@ class CppcheckCheckTool(CheckToolBase): cmd.append("--file-list=%s" % self._generate_src_file()) cmd.append("--includes-file=%s" % self._generate_inc_file()) - core_dir = get_project_core_dir() + core_dir = self.config.get_optional_dir("core") cmd.append("--suppress=*:%s*" % core_dir) cmd.append("--suppress=unmatchedSuppression:%s*" % core_dir) diff --git a/platformio/commands/check.py b/platformio/commands/check.py index 1c46640f..392e618d 100644 --- a/platformio/commands/check.py +++ b/platformio/commands/check.py @@ -17,23 +17,18 @@ import os from collections import Counter -from os.path import basename, dirname, isfile, join +from os.path import basename, dirname, isfile from time import time import click from tabulate import tabulate -from platformio import exception, fs, util +from platformio import app, exception, fs, util from platformio.check.defect import DefectItem from platformio.check.tools import CheckToolFactory from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - find_project_dir_above, - get_project_dir, - get_project_include_dir, - get_project_src_dir, -) +from platformio.project.helpers import find_project_dir_above, get_project_dir @click.command("check", short_help="Run a static analysis tool on code") @@ -72,15 +67,15 @@ def cli( verbose, json_output, ): + app.set_session_var("custom_project_conf", project_conf) + # find project directory on upper level if isfile(project_dir): project_dir = find_project_dir_above(project_dir) results = [] with fs.cd(project_dir): - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini") - ) + config = ProjectConfig.get_instance(project_conf) config.validate(environment) default_envs = config.default_envs() @@ -103,7 +98,10 @@ def cli( default_filter = [ "+<%s/>" % basename(d) - for d in (get_project_src_dir(), get_project_include_dir()) + for d in ( + config.get_optional_dir("src"), + config.get_optional_dir("include"), + ) ] tool_options = dict( diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 470fdd97..21b0bfd8 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -17,11 +17,11 @@ import os import signal -from os.path import isfile, join +from os.path import isfile import click -from platformio import exception, fs, proc, util +from platformio import app, exception, fs, proc, util from platformio.debug import helpers from platformio.managers.core import inject_contrib_pysite from platformio.project.config import ProjectConfig @@ -54,6 +54,8 @@ from platformio.project.helpers import is_platformio_project, load_project_ide_d @click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) @click.pass_context def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unprocessed): + app.set_session_var("custom_project_conf", project_conf) + # use env variables from Eclipse or CLion for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"): if is_platformio_project(project_dir): @@ -62,9 +64,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro project_dir = os.getenv(sysenv) with fs.cd(project_dir): - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini") - ) + config = ProjectConfig.get_instance(project_conf) config.validate(envs=[environment] if environment else None) env_name = environment or helpers.get_default_debug_env(config) diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 351ba641..45f94ae7 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -15,12 +15,11 @@ import sys from fnmatch import fnmatch from os import getcwd -from os.path import join import click from serial.tools import miniterm -from platformio import exception, util +from platformio import exception, fs, util from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig @@ -175,7 +174,8 @@ def device_list( # pylint: disable=too-many-branches def device_monitor(**kwargs): # pylint: disable=too-many-branches env_options = {} try: - env_options = get_project_options(kwargs["project_dir"], kwargs["environment"]) + with fs.cd(kwargs["project_dir"]): + env_options = get_project_options(kwargs["environment"]) for k in ("port", "speed", "rts", "dtr"): k2 = "monitor_%s" % k if k == "speed": @@ -225,8 +225,8 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches raise exception.MinitermException(e) -def get_project_options(project_dir, environment=None): - config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) +def get_project_options(environment=None): + config = ProjectConfig.get_instance() config.validate(envs=[environment] if environment else None) if not environment: default_envs = config.default_envs() diff --git a/platformio/commands/init.py b/platformio/commands/init.py index 03a08fab..ebf8a3f6 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -24,13 +24,7 @@ from platformio.commands.platform import platform_install as cli_platform_instal from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_include_dir, - get_project_lib_dir, - get_project_src_dir, - get_project_test_dir, - is_platformio_project, -) +from platformio.project.helpers import is_platformio_project def validate_boards(ctx, param, value): # pylint: disable=W0613 @@ -133,13 +127,14 @@ def cli( def init_base_project(project_dir): - ProjectConfig(join(project_dir, "platformio.ini")).save() with fs.cd(project_dir): + config = ProjectConfig() + config.save() dir_to_readme = [ - (get_project_src_dir(), None), - (get_project_include_dir(), init_include_readme), - (get_project_lib_dir(), init_lib_readme), - (get_project_test_dir(), init_test_readme), + (config.get_optional_dir("src"), None), + (config.get_optional_dir("include"), init_include_readme), + (config.get_optional_dir("lib"), init_lib_readme), + (config.get_optional_dir("test"), init_test_readme), ] for (path, cb) in dir_to_readme: if isdir(path): diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 6244da99..83e6cff4 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -21,18 +21,13 @@ import click import semantic_version from tabulate import tabulate -from platformio import exception, fs, util +from platformio import exception, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib from platformio.proc import is_ci from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_dir, - get_project_global_lib_dir, - get_project_libdeps_dir, - is_platformio_project, -) +from platformio.project.helpers import get_project_dir, is_platformio_project try: from urllib.parse import quote @@ -45,6 +40,10 @@ CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs" CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps" +def get_project_global_lib_dir(): + return ProjectConfig.get_instance().get_optional_dir("globallib") + + @click.group(short_help="Library Manager") @click.option( "-d", @@ -105,10 +104,9 @@ def cli(ctx, **options): if not is_platformio_project(storage_dir): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue - with fs.cd(storage_dir): - libdeps_dir = get_project_libdeps_dir() config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini")) config.validate(options["environment"], silent=in_silence) + libdeps_dir = config.get_optional_dir("libdeps") for env in config.envs(): if options["environment"] and env not in options["environment"]: continue @@ -336,7 +334,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): click.secho(" *", fg="green") click.secho("For example: DS*, PCA*, DHT* and etc.\n", fg="yellow") click.echo( - "For more examples and advanced search syntax, " "please use documentation:" + "For more examples and advanced search syntax, please use documentation:" ) click.secho( "https://docs.platformio.org/page/userguide/lib/cmd_search.html\n", diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 30afd2c9..968f978b 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -331,7 +331,7 @@ def platform_uninstall(platforms): for platform in platforms: if pm.uninstall(platform): click.secho( - "The platform '%s' has been successfully " "uninstalled!" % platform, + "The platform '%s' has been successfully uninstalled!" % platform, fg="green", ) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index b9cf3459..e378c528 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -14,16 +14,16 @@ from multiprocessing import cpu_count from os import getcwd -from os.path import isfile, join +from os.path import isfile from time import time import click from tabulate import tabulate -from platformio import exception, fs, util +from platformio import app, exception, fs, util from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.project.config import ProjectConfig -from platformio.project.helpers import find_project_dir_above, get_project_build_dir +from platformio.project.helpers import find_project_dir_above from platformio.run.helpers import clean_build_dir, handle_legacy_libdeps from platformio.run.processor import EnvironmentProcessor from platformio.test.processor import CTX_META_TEST_IS_RUNNING @@ -81,6 +81,8 @@ def cli( verbose, disable_auto_clean, ): + app.set_session_var("custom_project_conf", project_conf) + # find project directory on upper level if isfile(project_dir): project_dir = find_project_dir_above(project_dir) @@ -88,20 +90,18 @@ def cli( is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta with fs.cd(project_dir): - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini") - ) + config = ProjectConfig.get_instance(project_conf) config.validate(environment) # clean obsolete build dir if not disable_auto_clean: + build_dir = config.get_optional_dir("build") try: - clean_build_dir(get_project_build_dir(), config) + clean_build_dir(build_dir, config) except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " - "it manually to avoid build issues" - % get_project_build_dir(force=True), + "it manually to avoid build issues" % build_dir, fg="yellow", ) diff --git a/platformio/commands/test.py b/platformio/commands/test.py index b46c6e42..49b188df 100644 --- a/platformio/commands/test.py +++ b/platformio/commands/test.py @@ -22,9 +22,8 @@ from time import time import click from tabulate import tabulate -from platformio import exception, fs, util +from platformio import app, exception, fs, util from platformio.project.config import ProjectConfig -from platformio.project.helpers import get_project_test_dir from platformio.test.embedded import EmbeddedTestProcessor from platformio.test.native import NativeTestProcessor @@ -97,17 +96,17 @@ def cli( # pylint: disable=redefined-builtin monitor_dtr, verbose, ): + app.set_session_var("custom_project_conf", project_conf) + with fs.cd(project_dir): - test_dir = get_project_test_dir() + config = ProjectConfig.get_instance(project_conf) + config.validate(envs=environment) + + test_dir = config.get_optional_dir("test") if not isdir(test_dir): raise exception.TestDirNotExists(test_dir) test_names = get_test_names(test_dir) - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini") - ) - config.validate(envs=environment) - click.echo("Verbose mode can be enabled via `-v, --verbose` option") click.secho("Collected %d items" % len(test_names), bold=True) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 8c0d3496..a933ab7f 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -92,7 +92,7 @@ WARNING! Don't use `sudo` for the rest PlatformIO commands. def get_pip_package(to_develop): if not to_develop: return "platformio" - dl_url = "https://github.com/platformio/" "platformio-core/archive/develop.zip" + dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip" cache_dir = get_project_cache_dir() if not os.path.isdir(cache_dir): os.makedirs(cache_dir) diff --git a/platformio/debug/client.py b/platformio/debug/client.py index 63b07c5e..eca409e5 100644 --- a/platformio/debug/client.py +++ b/platformio/debug/client.py @@ -91,7 +91,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes self.project_dir, "-l", "10", - ] # yapf: disable + ] args.extend(self.args) if not gdb_path: raise exception.DebugInvalidOptions("GDB client is not configured") @@ -139,14 +139,14 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes " echo Warning! Undefined pio_reset_target command\\n", " mon reset", "end", - ] + commands # yapf: disable + ] + commands if not any("define pio_reset_halt_target" in cmd for cmd in commands): commands = [ "define pio_reset_halt_target", " echo Warning! Undefined pio_reset_halt_target command\\n", " mon reset halt", "end", - ] + commands # yapf: disable + ] + commands if not any("define pio_restart_target" in cmd for cmd in commands): commands += [ "define pio_restart_target", @@ -154,7 +154,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes " $INIT_BREAK", " %s" % ("continue" if patterns["INIT_BREAK"] else "next"), "end", - ] # yapf: disable + ] banner = [ "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n", @@ -243,7 +243,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes % self.debug_options["init_break"] ) self.console_log( - "PlatformIO: More configuration options -> " "http://bit.ly/pio-debug" + "PlatformIO: More configuration options -> http://bit.ly/pio-debug" ) self.transport.write( b"0-exec-continue\n" if helpers.is_mi_mode(self.args) else b"continue\n" diff --git a/platformio/debug/server.py b/platformio/debug/server.py index fbce1268..49d730a0 100644 --- a/platformio/debug/server.py +++ b/platformio/debug/server.py @@ -64,7 +64,7 @@ class DebugServer(BaseProcess): self._debug_port = ":3333" openocd_pipe_allowed = all( [not self.debug_options["port"], "openocd" in server_executable] - ) # yapf: disable + ) if openocd_pipe_allowed: args = [] if server["cwd"]: diff --git a/platformio/exception.py b/platformio/exception.py index c9f3f509..6e5910f8 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -148,7 +148,7 @@ class FDSizeMismatch(PlatformIOPackageException): class FDSHASumMismatch(PlatformIOPackageException): MESSAGE = ( - "The 'sha1' sum '{0}' of downloaded file '{1}' " "is not equal to remote '{2}'" + "The 'sha1' sum '{0}' of downloaded file '{1}' is not equal to remote '{2}'" ) diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 4a0acfb5..097f7e45 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -28,20 +28,16 @@ from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_libdeps_dir, - get_project_src_dir, - is_platformio_project, -) +from platformio.project.helpers import is_platformio_project class ProjectRPC(object): @staticmethod def _get_projects(project_dirs=None): - def _get_project_data(project_dir): + def _get_project_data(): data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} - config = ProjectConfig(join(project_dir, "platformio.ini")) - libdeps_dir = get_project_libdeps_dir() + config = ProjectConfig() + libdeps_dir = config.get_optional_dir("libdeps") data["libExtraDirs"].extend(config.get("platformio", "lib_extra_dirs", [])) @@ -76,7 +72,7 @@ class ProjectRPC(object): boards = [] try: with fs.cd(project_dir): - data = _get_project_data(project_dir) + data = _get_project_data() except exception.PlatformIOProjectException: continue @@ -178,9 +174,10 @@ class ProjectRPC(object): "", "void loop() {", " // put your main code here, to run repeatedly:", - "}" "", + "}", + "", ] - ) # yapf: disable + ) elif framework == "mbed": main_content = "\n".join( [ @@ -196,11 +193,12 @@ class ProjectRPC(object): "}", "", ] - ) # yapf: disable + ) if not main_content: return project_dir with fs.cd(project_dir): - src_dir = get_project_src_dir() + config = ProjectConfig() + src_dir = config.get_optional_dir("src") main_path = join(src_dir, "main.cpp") if isfile(main_path): return project_dir @@ -258,7 +256,8 @@ class ProjectRPC(object): @staticmethod def _finalize_arduino_import(_, project_dir, arduino_project_dir): with fs.cd(project_dir): - src_dir = get_project_src_dir() + config = ProjectConfig() + src_dir = config.get_optional_dir("src") if isdir(src_dir): fs.rmtree(src_dir) shutil.copytree(arduino_project_dir, src_dir) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 75252866..defeaafa 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -23,12 +23,7 @@ from platformio import fs, util from platformio.compat import get_file_contents from platformio.proc import where_is_program from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_lib_dir, - get_project_libdeps_dir, - get_project_src_dir, - load_project_ide_data, -) +from platformio.project.helpers import load_project_ide_data class ProjectGenerator(object): @@ -76,7 +71,7 @@ class ProjectGenerator(object): else where_is_program("platformio"), "env_path": os.getenv("PATH"), "env_pathsep": os.pathsep, - } # yapf: disable + } # default env configuration tpl_vars.update(self.config.items(env=self.env_name, as_dict=True)) @@ -87,13 +82,13 @@ class ProjectGenerator(object): tpl_vars.update( { "src_files": self.get_src_files(), - "project_src_dir": get_project_src_dir(), - "project_lib_dir": get_project_lib_dir(), + "project_src_dir": self.config.get_optional_dir("src"), + "project_lib_dir": self.config.get_optional_dir("lib"), "project_libdeps_dir": join( - get_project_libdeps_dir(), self.env_name + self.config.get_optional_dir("libdeps"), self.env_name ), } - ) # yapf: disable + ) for key, value in tpl_vars.items(): if key.endswith(("_path", "_dir")): @@ -109,7 +104,7 @@ class ProjectGenerator(object): def get_src_files(self): result = [] with fs.cd(self.project_dir): - for root, _, files in os.walk(get_project_src_dir()): + for root, _, files in os.walk(self.config.get_optional_dir("src")): for f in files: result.append(relpath(join(root, f))) return result diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 9d1487d6..ee59e36b 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,7 +21,7 @@ from platformio import __version__, exception, fs from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path -from platformio.project.helpers import get_project_packages_dir +from platformio.project.config import ProjectConfig CORE_PACKAGES = { "contrib-piohome": "^2.3.2", @@ -40,8 +40,10 @@ PIOPLUS_AUTO_UPDATES_MAX = 100 class CorePackageManager(PackageManager): def __init__(self): + config = ProjectConfig.get_instance() + packages_dir = config.get_optional_dir("packages") super(CorePackageManager, self).__init__( - get_project_packages_dir(), + packages_dir, [ "https://dl.bintray.com/platformio/dl-packages/manifest.json", "http%s://dl.platformio.org/packages/manifest.json" diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 943126d7..85cf4b29 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -27,7 +27,7 @@ from platformio import app, exception, util from platformio.compat import glob_escape, string_types from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager -from platformio.project.helpers import get_project_global_lib_dir +from platformio.project.config import ProjectConfig class LibraryManager(BasePkgManager): @@ -35,9 +35,10 @@ class LibraryManager(BasePkgManager): FILE_CACHE_VALID = "30d" # 1 month def __init__(self, package_dir=None): - if not package_dir: - package_dir = get_project_global_lib_dir() - super(LibraryManager, self).__init__(package_dir) + self.config = ProjectConfig.get_instance() + super(LibraryManager, self).__init__( + package_dir or self.config.get_optional_dir("globallib") + ) @property def manifest_names(self): diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 01cf5254..f940767b 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -405,7 +405,7 @@ class PkgInstallerMixin(object): self.parse_semver_version(manifest["version"], raise_exception=True) ): continue - elif not best or ( + if not best or ( self.parse_semver_version(manifest["version"], raise_exception=True) > self.parse_semver_version(best["version"], raise_exception=True) ): diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 124a4245..2536386a 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-public-methods, too-many-instance-attributes + import base64 import os import re @@ -32,12 +34,6 @@ from platformio.proc import ( get_pythonexe_path, ) from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_boards_dir, - get_project_core_dir, - get_project_packages_dir, - get_project_platforms_dir, -) try: from urllib.parse import quote @@ -54,8 +50,9 @@ class PlatformManager(BasePkgManager): "https" if app.get_setting("strict_ssl") else "http" ), ] + self.config = ProjectConfig.get_instance() BasePkgManager.__init__( - self, package_dir or get_project_platforms_dir(), repositories + self, package_dir or self.config.get_optional_dir("platforms"), repositories ) @property @@ -164,7 +161,7 @@ class PlatformManager(BasePkgManager): deppkgs[pkgname] = set() deppkgs[pkgname].add(pkgmanifest["version"]) - pm = PackageManager(get_project_packages_dir()) + pm = PackageManager(self.config.get_optional_dir("packages")) for manifest in pm.get_installed(): if manifest["name"] not in names: continue @@ -290,7 +287,7 @@ class PlatformPackagesMixin(object): version = opts.get("version", "") if name in without_packages: continue - elif name in with_packages or not ( + if name in with_packages or not ( skip_default_package or opts.get("optional", False) ): if ":" in version: @@ -385,8 +382,7 @@ class PlatformRunMixin(object): assert isinstance(variables, dict) assert isinstance(targets, list) - config = ProjectConfig.get_instance(variables["project_config"]) - options = config.items(env=variables["pioenv"], as_dict=True) + options = self.config.items(env=variables["pioenv"], as_dict=True) if "framework" in options: # support PIO Core 3.0 dev/platforms options["pioframework"] = options["framework"] @@ -421,7 +417,7 @@ class PlatformRunMixin(object): str(jobs), "--sconstruct", join(fs.get_source_dir(), "builder", "main.py"), - ] # yapf: disable + ] args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) # pylint: disable=protected-access args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) @@ -503,9 +499,7 @@ class PlatformRunMixin(object): click.echo(banner, err=True) -class PlatformBase( # pylint: disable=too-many-public-methods - PlatformPackagesMixin, PlatformRunMixin -): +class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) _BOARDS_CACHE = {} @@ -519,8 +513,10 @@ class PlatformBase( # pylint: disable=too-many-public-methods self._manifest = fs.load_json(manifest_path) self._custom_packages = None - self.pm = PackageManager(get_project_packages_dir(), self.package_repositories) - + self.config = ProjectConfig.get_instance() + self.pm = PackageManager( + self.config.get_optional_dir("packages"), self.package_repositories + ) # if self.engines and "platformio" in self.engines: # if self.PIO_VERSION not in semantic_version.SimpleSpec( # self.engines['platformio']): @@ -619,8 +615,8 @@ class PlatformBase( # pylint: disable=too-many-public-methods self._BOARDS_CACHE[board_id] = config bdirs = [ - get_project_boards_dir(), - join(get_project_core_dir(), "boards"), + self.config.get_optional_dir("boards"), + join(self.config.get_optional_dir("core"), "boards"), join(self.get_dir(), "boards"), ] diff --git a/platformio/project/config.py b/platformio/project/config.py index 28e4c955..d0abf25c 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -16,11 +16,12 @@ import glob import json import os import re -from os.path import expanduser, getmtime, isfile +from hashlib import sha1 import click from platformio import exception +from platformio.compat import WINDOWS, hashlib_encode_data from platformio.project.options import ProjectOptions try: @@ -41,7 +42,7 @@ CONFIG_HEADER = """;PlatformIO Project Configuration File """ -class ProjectConfig(object): +class ProjectConfigBase(object): INLINE_COMMENT_RE = re.compile(r"\s+;.*$") VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") @@ -49,7 +50,6 @@ class ProjectConfig(object): expand_interpolations = True warnings = [] - _instances = {} _parser = None _parsed = [] @@ -66,33 +66,28 @@ class ProjectConfig(object): if not item or item.startswith((";", "#")): continue if ";" in item: - item = ProjectConfig.INLINE_COMMENT_RE.sub("", item).strip() + item = ProjectConfigBase.INLINE_COMMENT_RE.sub("", item).strip() result.append(item) return result @staticmethod - def get_instance(path): - mtime = getmtime(path) if isfile(path) else 0 - instance = ProjectConfig._instances.get(path) - if instance and instance["mtime"] != mtime: - instance = None - if not instance: - instance = {"mtime": mtime, "config": ProjectConfig(path)} - ProjectConfig._instances[path] = instance - return instance["config"] + def get_default_path(): + from platformio import app - def __init__(self, path, parse_extra=True, expand_interpolations=True): + return app.get_session_var("custom_project_conf") or os.path.join( + os.getcwd(), "platformio.ini" + ) + + def __init__(self, path=None, parse_extra=True, expand_interpolations=True): + path = self.get_default_path() if path is None else path self.path = path self.expand_interpolations = expand_interpolations self.warnings = [] self._parsed = [] self._parser = ConfigParser.ConfigParser() - if isfile(path): + if path and os.path.isfile(path): self.read(path, parse_extra) - def __repr__(self): - return "" % (self.path or "in-memory") - def __getattr__(self, name): return getattr(self._parser, name) @@ -111,7 +106,7 @@ class ProjectConfig(object): # load extra configs for pattern in self.get("platformio", "extra_configs", []): if pattern.startswith("~"): - pattern = expanduser(pattern) + pattern = os.path.expanduser(pattern) for item in glob.glob(pattern): self.read(item) @@ -165,7 +160,7 @@ class ProjectConfig(object): unknown_conditions = [ ("%s.%s" % (scope, option)) not in ProjectOptions, scope != "env" or not option.startswith(("custom_", "board_")), - ] # yapf: disable + ] if all(unknown_conditions): self.warnings.append( "Ignore unknown configuration option `%s` " @@ -288,7 +283,7 @@ class ProjectConfig(object): # option is not specified by user if value is None: - return default + return default if default is not None else option_meta.default try: return self._cast_to(value, option_meta.type) @@ -313,7 +308,7 @@ class ProjectConfig(object): return self.get("platformio", "default_envs", []) def validate(self, envs=None, silent=False): - if not isfile(self.path): + if not os.path.isfile(self.path): raise exception.NotPlatformIOProject(self.path) # check envs known = set(self.envs()) @@ -327,6 +322,93 @@ class ProjectConfig(object): click.secho("Warning! %s" % warning, fg="yellow") return True + +class ProjectConfigDirsMixin(object): + def _get_core_dir(self, exists=False): + default = ProjectOptions["platformio.core_dir"].default + core_dir = self.get("platformio", "core_dir") + win_core_dir = None + if WINDOWS and core_dir == default: + win_core_dir = os.path.splitdrive(core_dir)[0] + "\\.platformio" + if os.path.isdir(win_core_dir): + core_dir = win_core_dir + + if exists and not os.path.isdir(core_dir): + try: + os.makedirs(core_dir) + except OSError as e: + if win_core_dir: + os.makedirs(win_core_dir) + core_dir = win_core_dir + else: + raise e + + return core_dir + + def get_optional_dir(self, name, exists=False): + if not ProjectOptions.get("platformio.%s_dir" % name): + raise ValueError("Unknown optional directory -> " + name) + + if name == "core": + result = self._get_core_dir(exists) + else: + result = self.get("platformio", name + "_dir") + + if result is None: + return None + + project_dir = os.getcwd() + + # patterns + if "$PROJECT_HASH" in result: + result = result.replace( + "$PROJECT_HASH", + "%s-%s" + % ( + os.path.basename(project_dir), + sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], + ), + ) + + if "$PROJECT_DIR" in result: + result = result.replace("$PROJECT_DIR", project_dir) + if "$PROJECT_CORE_DIR" in result: + result = result.replace("$PROJECT_CORE_DIR", self.get_optional_dir("core")) + if "$PROJECT_WORKSPACE_DIR" in result: + result = result.replace( + "$PROJECT_WORKSPACE_DIR", self.get_optional_dir("workspace") + ) + + if result.startswith("~"): + result = os.path.expanduser(result) + + result = os.path.realpath(result) + + if exists and not os.path.isdir(result): + os.makedirs(result) + + return result + + +class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): + + _instances = {} + + @staticmethod + def get_instance(path=None): + path = ProjectConfig.get_default_path() if path is None else path + mtime = os.path.getmtime(path) if os.path.isfile(path) else 0 + instance = ProjectConfig._instances.get(path) + if instance and instance["mtime"] != mtime: + instance = None + if not instance: + instance = {"mtime": mtime, "config": ProjectConfig(path)} + ProjectConfig._instances[path] = instance + return instance["config"] + + def __repr__(self): + return "" % (self.path or "in-memory") + def to_json(self): result = {} for section in self.sections(): diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index f1e6702e..e28c6643 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -16,16 +16,7 @@ import json import os from hashlib import sha1 from os import walk -from os.path import ( - basename, - dirname, - expanduser, - isdir, - isfile, - join, - realpath, - splitdrive, -) +from os.path import dirname, expanduser, isdir, isfile, join from click.testing import CliRunner @@ -54,126 +45,32 @@ def find_project_dir_above(path): return None -def get_project_optional_dir(name, default=None): - project_dir = get_project_dir() - config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) - optional_dir = config.get("platformio", name) - - if not optional_dir: - return default - - if "$PROJECT_HASH" in optional_dir: - optional_dir = optional_dir.replace( - "$PROJECT_HASH", - "%s-%s" - % ( - basename(project_dir), - sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], - ), - ) - - if optional_dir.startswith("~"): - optional_dir = expanduser(optional_dir) - - return realpath(optional_dir) - - def get_project_core_dir(): - default = join(expanduser("~"), ".platformio") - core_dir = get_project_optional_dir( - "core_dir", get_project_optional_dir("home_dir", default) - ) - win_core_dir = None - if WINDOWS and core_dir == default: - win_core_dir = splitdrive(core_dir)[0] + "\\.platformio" - if isdir(win_core_dir): - core_dir = win_core_dir - - if not isdir(core_dir): - try: - os.makedirs(core_dir) - except OSError as e: - if win_core_dir: - os.makedirs(win_core_dir) - core_dir = win_core_dir - else: - raise e - - assert isdir(core_dir) - return core_dir - - -def get_project_global_lib_dir(): - return get_project_optional_dir( - "globallib_dir", join(get_project_core_dir(), "lib") - ) - - -def get_project_platforms_dir(): - return get_project_optional_dir( - "platforms_dir", join(get_project_core_dir(), "platforms") - ) - - -def get_project_packages_dir(): - return get_project_optional_dir( - "packages_dir", join(get_project_core_dir(), "packages") - ) + """ Deprecated, use ProjectConfig.get_optional_dir("core") instead """ + return ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini") + ).get_optional_dir("core", exists=True) def get_project_cache_dir(): - return get_project_optional_dir("cache_dir", join(get_project_core_dir(), ".cache")) + """ Deprecated, use ProjectConfig.get_optional_dir("cache") instead """ + return ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini") + ).get_optional_dir("cache") -def get_project_workspace_dir(): - return get_project_optional_dir("workspace_dir", join(get_project_dir(), ".pio")) - - -def get_project_build_dir(force=False): - path = get_project_optional_dir( - "build_dir", join(get_project_workspace_dir(), "build") - ) +def get_default_projects_dir(): + docs_dir = join(expanduser("~"), "Documents") try: - if not isdir(path): - os.makedirs(path) - except Exception as e: # pylint: disable=broad-except - if not force: - raise Exception(e) - return path + assert WINDOWS + import ctypes.wintypes # pylint: disable=import-outside-toplevel - -def get_project_libdeps_dir(): - return get_project_optional_dir( - "libdeps_dir", join(get_project_workspace_dir(), "libdeps") - ) - - -def get_project_lib_dir(): - return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) - - -def get_project_include_dir(): - return get_project_optional_dir("include_dir", join(get_project_dir(), "include")) - - -def get_project_src_dir(): - return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) - - -def get_project_test_dir(): - return get_project_optional_dir("test_dir", join(get_project_dir(), "test")) - - -def get_project_boards_dir(): - return get_project_optional_dir("boards_dir", join(get_project_dir(), "boards")) - - -def get_project_data_dir(): - return get_project_optional_dir("data_dir", join(get_project_dir(), "data")) - - -def get_project_shared_dir(): - return get_project_optional_dir("shared_dir", join(get_project_dir(), "shared")) + buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) + ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf) + docs_dir = buf.value + except: # pylint: disable=bare-except + pass + return join(docs_dir, "PlatformIO", "Projects") def compute_project_checksum(config): @@ -185,7 +82,11 @@ def compute_project_checksum(config): # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") - for d in (get_project_include_dir(), get_project_src_dir(), get_project_lib_dir()): + for d in ( + config.get_optional_dir("include"), + config.get_optional_dir("src"), + config.get_optional_dir("lib"), + ): if not isdir(d): continue chunks = [] diff --git a/platformio/project/options.py b/platformio/project/options.py index 0dc66343..f7294c40 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -14,13 +14,23 @@ # pylint: disable=redefined-builtin, too-many-arguments +import os from collections import OrderedDict, namedtuple import click ConfigOptionClass = namedtuple( "ConfigOption", - ["scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames"], + [ + "scope", + "name", + "type", + "multiple", + "sysenvvar", + "buildenvvar", + "oldnames", + "default", + ], ) @@ -32,9 +42,10 @@ def ConfigOption( sysenvvar=None, buildenvvar=None, oldnames=None, + default=None, ): return ConfigOptionClass( - scope, name, type, multiple, sysenvvar, buildenvvar, oldnames + scope, name, type, multiple, sysenvvar, buildenvvar, oldnames, default ) @@ -63,40 +74,83 @@ ProjectOptions = OrderedDict( ConfigPlatformioOption(name="extra_configs", multiple=True), # Dirs ConfigPlatformioOption( - name="core_dir", oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR" + name="core_dir", + oldnames=["home_dir"], + sysenvvar="PLATFORMIO_CORE_DIR", + default=os.path.join(os.path.expanduser("~"), ".platformio"), ), ConfigPlatformioOption( - name="globallib_dir", sysenvvar="PLATFORMIO_GLOBALLIB_DIR" + name="globallib_dir", + sysenvvar="PLATFORMIO_GLOBALLIB_DIR", + default=os.path.join("$PROJECT_CORE_DIR", "lib"), ), ConfigPlatformioOption( - name="platforms_dir", sysenvvar="PLATFORMIO_PLATFORMS_DIR" + name="platforms_dir", + sysenvvar="PLATFORMIO_PLATFORMS_DIR", + default=os.path.join("$PROJECT_CORE_DIR", "platforms"), ), ConfigPlatformioOption( - name="packages_dir", sysenvvar="PLATFORMIO_PACKAGES_DIR" + name="packages_dir", + sysenvvar="PLATFORMIO_PACKAGES_DIR", + default=os.path.join("$PROJECT_CORE_DIR", "packages"), + ), + ConfigPlatformioOption( + name="cache_dir", + sysenvvar="PLATFORMIO_CACHE_DIR", + default=os.path.join("$PROJECT_CORE_DIR", ".cache"), ), - ConfigPlatformioOption(name="cache_dir", sysenvvar="PLATFORMIO_CACHE_DIR"), ConfigPlatformioOption( name="build_cache_dir", sysenvvar="PLATFORMIO_BUILD_CACHE_DIR" ), ConfigPlatformioOption( - name="workspace_dir", sysenvvar="PLATFORMIO_WORKSPACE_DIR" + name="workspace_dir", + sysenvvar="PLATFORMIO_WORKSPACE_DIR", + default=os.path.join("$PROJECT_DIR", ".pio"), ), - ConfigPlatformioOption(name="build_dir", sysenvvar="PLATFORMIO_BUILD_DIR"), ConfigPlatformioOption( - name="libdeps_dir", sysenvvar="PLATFORMIO_LIBDEPS_DIR" + name="build_dir", + sysenvvar="PLATFORMIO_BUILD_DIR", + default=os.path.join("$PROJECT_WORKSPACE_DIR", "build"), ), - ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"), ConfigPlatformioOption( - name="include_dir", sysenvvar="PLATFORMIO_INCLUDE_DIR" + name="libdeps_dir", + sysenvvar="PLATFORMIO_LIBDEPS_DIR", + default=os.path.join("$PROJECT_WORKSPACE_DIR", "libdeps"), ), - ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"), - ConfigPlatformioOption(name="test_dir", sysenvvar="PLATFORMIO_TEST_DIR"), ConfigPlatformioOption( - name="boards_dir", sysenvvar="PLATFORMIO_BOARDS_DIR" + name="lib_dir", + sysenvvar="PLATFORMIO_LIB_DIR", + default=os.path.join("$PROJECT_DIR", "lib"), ), - ConfigPlatformioOption(name="data_dir", sysenvvar="PLATFORMIO_DATA_DIR"), ConfigPlatformioOption( - name="shared_dir", sysenvvar="PLATFORMIO_SHARED_DIR" + name="include_dir", + sysenvvar="PLATFORMIO_INCLUDE_DIR", + default=os.path.join("$PROJECT_DIR", "include"), + ), + ConfigPlatformioOption( + name="src_dir", + sysenvvar="PLATFORMIO_SRC_DIR", + default=os.path.join("$PROJECT_DIR", "src"), + ), + ConfigPlatformioOption( + name="test_dir", + sysenvvar="PLATFORMIO_TEST_DIR", + default=os.path.join("$PROJECT_DIR", "test"), + ), + ConfigPlatformioOption( + name="boards_dir", + sysenvvar="PLATFORMIO_BOARDS_DIR", + default=os.path.join("$PROJECT_DIR", "boards"), + ), + ConfigPlatformioOption( + name="data_dir", + sysenvvar="PLATFORMIO_DATA_DIR", + default=os.path.join("$PROJECT_DIR", "data"), + ), + ConfigPlatformioOption( + name="shared_dir", + sysenvvar="PLATFORMIO_SHARED_DIR", + default=os.path.join("$PROJECT_DIR", "shared"), ), # # [env] diff --git a/platformio/run/helpers.py b/platformio/run/helpers.py index d9481433..a4c79e83 100644 --- a/platformio/run/helpers.py +++ b/platformio/run/helpers.py @@ -18,16 +18,14 @@ from os.path import isdir, isfile, join import click from platformio import fs -from platformio.project.helpers import ( - compute_project_checksum, - get_project_dir, - get_project_libdeps_dir, -) +from platformio.project.helpers import compute_project_checksum, get_project_dir def handle_legacy_libdeps(project_dir, config): legacy_libdeps_dir = join(project_dir, ".piolibdeps") - if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == get_project_libdeps_dir(): + if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get_optional_dir( + "libdeps" + ): return if not config.has_section("env"): config.add_section("env") diff --git a/platformio/run/processor.py b/platformio/run/processor.py index 219444d8..490ae497 100644 --- a/platformio/run/processor.py +++ b/platformio/run/processor.py @@ -48,9 +48,11 @@ class EnvironmentProcessor(object): return variables def get_build_targets(self): - if self.targets: - return [t for t in self.targets] - return self.config.get("env:" + self.name, "targets", []) + return ( + self.targets + if self.targets + else self.config.get("env:" + self.name, "targets", []) + ) def process(self): if "platform" not in self.options: diff --git a/platformio/test/native.py b/platformio/test/native.py index 73029e98..f028e4f3 100644 --- a/platformio/test/native.py +++ b/platformio/test/native.py @@ -14,9 +14,8 @@ from os.path import join -from platformio import fs, proc +from platformio import proc from platformio.proc import LineBufferedAsyncPipe -from platformio.project.helpers import get_project_build_dir from platformio.test.processor import TestProcessorBase @@ -32,8 +31,7 @@ class NativeTestProcessor(TestProcessorBase): return self.run() def run(self): - with fs.cd(self.options["project_dir"]): - build_dir = get_project_build_dir() + build_dir = self.options["project_config"].get_optional_dir("build") result = proc.exec_command( [join(build_dir, self.env_name, "program")], stdout=LineBufferedAsyncPipe(self.on_run_out), diff --git a/platformio/test/processor.py b/platformio/test/processor.py index b04a2789..e228a8c0 100644 --- a/platformio/test/processor.py +++ b/platformio/test/processor.py @@ -20,7 +20,6 @@ from string import Template import click from platformio import exception -from platformio.project.helpers import get_project_test_dir TRANSPORT_OPTIONS = { "arduino": { @@ -104,7 +103,9 @@ class TestProcessorBase(object): def build_or_upload(self, target): if not self._outputcpp_generated: - self.generate_outputcpp(get_project_test_dir()) + self.generate_outputcpp( + self.options["project_config"].get_optional_dir("test") + ) self._outputcpp_generated = True if self.test_name != "*": @@ -175,7 +176,7 @@ class TestProcessorBase(object): " $end;", "}", ] - ) # yapf: disable + ) def delete_tmptest_file(file_): try: diff --git a/platformio/util.py b/platformio/util.py index bfd4abfb..cff14277 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -191,7 +191,7 @@ def get_logical_devices(): def get_mdns_services(): try: - import zeroconf + import zeroconf # pylint: disable=import-outside-toplevel except ImportError: from site import addsitedir from platformio.managers.core import get_core_package_dir @@ -199,7 +199,7 @@ def get_mdns_services(): contrib_pysite_dir = get_core_package_dir("contrib-pysite") addsitedir(contrib_pysite_dir) sys.path.insert(0, contrib_pysite_dir) - import zeroconf + import zeroconf # pylint: disable=import-outside-toplevel class mDNSListener(object): def __init__(self): @@ -360,7 +360,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): time.sleep(2 * total) raise exception.APIRequestError( - "Could not connect to PlatformIO API Service. " "Please try later." + "Could not connect to PlatformIO API Service. Please try later." ) diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index c509a1f9..e4bb6db9 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -164,7 +164,7 @@ def test_check_filter_sources(clirunner, check_dir): assert style == EXPECTED_STYLE -def test_check_failed_if_no_source_files(clirunner, tmpdir): +def test_check_no_source_files(clirunner, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src") @@ -178,7 +178,7 @@ def test_check_failed_if_no_source_files(clirunner, tmpdir): assert style == 0 -def test_check_failed_if_bad_flag_passed(clirunner, check_dir): +def test_check_bad_flag_passed(clirunner, check_dir): result = clirunner.invoke( cmd_check, ["--project-dir", str(check_dir), '"--flags=--UNKNOWN"'] ) diff --git a/tests/test_examples.py b/tests/test_examples.py index 013ecb26..eac51469 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -22,7 +22,6 @@ import pytest from platformio import util from platformio.managers.platform import PlatformFactory, PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import get_project_build_dir def pytest_generate_tests(metafunc): @@ -71,11 +70,12 @@ def pytest_generate_tests(metafunc): @pytest.mark.examples def test_run(pioproject_dir): with util.cd(pioproject_dir): - build_dir = get_project_build_dir() + config = ProjectConfig() + build_dir = config.get_optional_dir("build") if isdir(build_dir): util.rmtree_(build_dir) - env_names = ProjectConfig(join(pioproject_dir, "platformio.ini")).envs() + env_names = config.envs() result = util.exec_command( ["platformio", "run", "-e", random.choice(env_names)] ) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index a5127dd0..2ec24521 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -112,6 +112,15 @@ def test_warnings(config): config.validate(["non-existing-env"]) +def test_defaults(config): + assert config.get_optional_dir("core") == os.path.join( + os.path.expanduser("~"), ".platformio" + ) + assert config.get_optional_dir("build_cache") == os.environ.get( + "PLATFORMIO_BUILD_CACHE_DIR" + ) + + def test_sections(config): with pytest.raises(ConfigParser.NoSectionError): config.getraw("unknown_section", "unknown_option")