From 0ce23438368e4d9fa323649c0ad7f9027ce26938 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 17:08:00 +0300 Subject: [PATCH] Do not pass project settings as SCons arguments // Resolve #1637 --- platformio/builder/main.py | 98 ++----- platformio/builder/tools/pioide.py | 5 +- platformio/builder/tools/piolib.py | 26 +- platformio/builder/tools/piomisc.py | 2 +- platformio/builder/tools/pioplatform.py | 53 ++-- platformio/builder/tools/pioproject.py | 49 ++++ platformio/commands/run.py | 352 ------------------------ platformio/commands/run/__init__.py | 15 + platformio/commands/run/command.py | 119 ++++++++ platformio/commands/run/helpers.py | 127 +++++++++ platformio/commands/run/processor.py | 115 ++++++++ platformio/managers/platform.py | 15 +- platformio/project/config.py | 20 +- platformio/project/options.py | 4 +- 14 files changed, 513 insertions(+), 487 deletions(-) create mode 100644 platformio/builder/tools/pioproject.py delete mode 100644 platformio/commands/run.py create mode 100644 platformio/commands/run/__init__.py create mode 100644 platformio/commands/run/command.py create mode 100644 platformio/commands/run/helpers.py create mode 100644 platformio/commands/run/processor.py diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 8358d52e..225af059 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -16,7 +16,7 @@ import base64 import json import sys from os import environ -from os.path import expanduser, join +from os.path import join from time import time from SCons.Script import ARGUMENTS # pylint: disable=import-error @@ -33,76 +33,29 @@ from platformio import util from platformio.compat import PY2, path_to_unicode from platformio.proc import get_pythonexe_path from platformio.project import helpers as project_helpers -from platformio.project.config import ProjectConfig AllowSubstExceptions(NameError) -# allow common variables from INI file -commonvars = Variables(None) -commonvars.AddVariables( +# append CLI arguments to build environment +clivars = Variables(None) +clivars.AddVariables( ("PLATFORM_MANIFEST",), ("BUILD_SCRIPT",), - ("EXTRA_SCRIPTS",), + ("PROJECT_CONFIG",), ("PIOENV",), ("PIOTEST",), - ("PIOPLATFORM",), - ("PIOFRAMEWORK",), - - # build options - ("BUILD_FLAGS",), - ("SRC_BUILD_FLAGS",), - ("BUILD_UNFLAGS",), - ("SRC_FILTER",), - - # library options - ("LIB_LDF_MODE",), - ("LIB_COMPAT_MODE",), - ("LIB_DEPS",), - ("LIB_IGNORE",), - ("LIB_EXTRA_DIRS",), - ("LIB_ARCHIVE",), - - # board options - ("BOARD",), - # deprecated options, use board_{object.path} instead - ("BOARD_MCU",), - ("BOARD_F_CPU",), - ("BOARD_F_FLASH",), - ("BOARD_FLASH_MODE",), - # end of deprecated options - - # upload options - ("UPLOAD_PORT",), - ("UPLOAD_PROTOCOL",), - ("UPLOAD_SPEED",), - ("UPLOAD_FLAGS",), - ("UPLOAD_RESETMETHOD",), - - # test options - ("TEST_BUILD_PROJECT_SRC",), - - # debug options - ("DEBUG_TOOL",), - ("DEBUG_SVD_PATH",), - + ("UPLOAD_PORT",) ) # yapf: disable -MULTILINE_VARS = [ - "EXTRA_SCRIPTS", "PIOFRAMEWORK", "BUILD_FLAGS", "SRC_BUILD_FLAGS", - "BUILD_UNFLAGS", "UPLOAD_FLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE", - "LIB_EXTRA_DIRS" -] - DEFAULT_ENV_OPTIONS = dict( tools=[ "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform", - "piowinhooks", "piolib", "pioupload", "piomisc", "pioide" - ], # yapf: disable + "pioproject", "piowinhooks", "piolib", "pioupload", "piomisc", "pioide" + ], toolpath=[join(util.get_source_dir(), "builder", "tools")], - variables=commonvars, + variables=clivars, # Propagating External Environment - PIOVARIABLES=list(commonvars.keys()), ENV=environ, UNIX_TIME=int(time()), PROJECT_DIR=project_helpers.get_project_dir(), @@ -136,38 +89,21 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)): env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) -# decode common variables -for k in list(commonvars.keys()): - if k in env: - env[k] = base64.b64decode(env[k]) - if isinstance(env[k], bytes): - env[k] = env[k].decode() - if k in MULTILINE_VARS: - env[k] = ProjectConfig.parse_multi_values(env[k]) - if env.GetOption('clean'): env.PioClean(env.subst("$BUILD_DIR")) env.Exit(0) elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): print("Verbose mode can be enabled via `-v, --verbose` option") -# Handle custom variables from system environment -for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPTS", - "UPLOAD_PORT", "UPLOAD_FLAGS", "LIB_EXTRA_DIRS"): - k = "PLATFORMIO_%s" % var - if k not in environ: - continue - if var in ("UPLOAD_PORT", ): - env[var] = environ.get(k) - continue - env.Append(**{var: ProjectConfig.parse_multi_values(environ.get(k))}) +# Load variables from CLI +for key in list(clivars.keys()): + if key in env: + env[key] = base64.b64decode(env[key]) + if isinstance(env[key], bytes): + env[key] = env[key].decode() -env.Prepend(LIBSOURCE_DIRS=env.get("LIB_EXTRA_DIRS", [])) -env['LIBSOURCE_DIRS'] = [ - expanduser(d) if d.startswith("~") else d for d in env['LIBSOURCE_DIRS'] -] - -env.LoadPioPlatform(commonvars) +env.LoadProjectOptions() +env.LoadPioPlatform() env.SConscriptChdir(0) env.SConsignFile( diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 12ebefba..4c42ee88 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -113,7 +113,7 @@ def _dump_defines(env): def _get_svd_path(env): - svd_path = env.subst("$DEBUG_SVD_PATH") + svd_path = env.GetProjectOption("debug_svd_path") if svd_path: return abspath(svd_path) @@ -139,8 +139,7 @@ def DumpIDEData(env, projenv): LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS" data = { - "libsource_dirs": - [env.subst(l) for l in env.get("LIBSOURCE_DIRS", [])], + "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()], "defines": _dump_defines(env), "includes": diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index b63dd0f9..dba7d108 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -23,8 +23,8 @@ import os import re import sys from glob import glob -from os.path import (basename, commonprefix, dirname, isdir, isfile, join, - realpath, sep) +from os.path import (basename, commonprefix, dirname, expanduser, isdir, + isfile, join, realpath, sep) import SCons.Scanner # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error @@ -207,17 +207,18 @@ class LibBuilderBase(object): @property def lib_archive(self): - return self.env.get("LIB_ARCHIVE", "") != "false" + return self.env.GetProjectOption("lib_archive", True) @property def lib_ldf_mode(self): return self.validate_ldf_mode( - self.env.get("LIB_LDF_MODE", self.LDF_MODE_DEFAULT)) + self.env.GetProjectOption("lib_ldf_mode", self.LDF_MODE_DEFAULT)) @property def lib_compat_mode(self): return self.validate_compat_mode( - self.env.get("LIB_COMPAT_MODE", self.COMPAT_MODE_DEFAULT)) + self.env.GetProjectOption("lib_compat_mode", + self.COMPAT_MODE_DEFAULT)) @property def depbuilders(self): @@ -867,7 +868,7 @@ class ProjectAsLibBuilder(LibBuilderBase): pass def process_dependencies(self): # pylint: disable=too-many-branches - uris = self.env.get("LIB_DEPS", []) + uris = self.env.GetProjectOption("lib_deps", []) if not uris: return storage_dirs = [] @@ -907,6 +908,14 @@ class ProjectAsLibBuilder(LibBuilderBase): return result +def GetLibSourceDirs(env): + items = env.GetProjectOption("lib_extra_dirs", []) + items.extend(env['LIBSOURCE_DIRS']) + return [ + expanduser(item) if item.startswith("~") else item for item in items + ] + + def GetLibBuilders(env): # pylint: disable=too-many-branches if "__PIO_LIB_BUILDERS" in DefaultEnvironment(): @@ -920,7 +929,7 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches def _check_lib_builder(lb): compat_mode = lb.lib_compat_mode - if lb.name in env.get("LIB_IGNORE", []): + if lb.name in env.GetProjectOption("lib_ignore", []): if verbose: sys.stderr.write("Ignored library %s\n" % lb.path) return None @@ -939,7 +948,7 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches return True found_incompat = False - for libs_dir in env['LIBSOURCE_DIRS']: + for libs_dir in env.GetLibSourceDirs(): libs_dir = env.subst(libs_dir) if not isdir(libs_dir): continue @@ -1038,6 +1047,7 @@ def exists(_): def generate(env): + env.AddMethod(GetLibSourceDirs) env.AddMethod(GetLibBuilders) env.AddMethod(ConfigureProjectLibBuilder) return env diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index bfc41a76..3d434955 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -322,7 +322,7 @@ def ProcessTest(env): def GetExtraScripts(env, scope): items = [] - for item in env.get("EXTRA_SCRIPTS", []): + for item in env.GetProjectOption("extra_scripts", []): if scope == "post" and ":" not in item: items.append(item) elif item.startswith("%s:" % scope): diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index cc61a709..144f1f4b 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -14,7 +14,6 @@ from __future__ import absolute_import -import base64 import sys from os.path import isdir, isfile, join @@ -23,6 +22,7 @@ from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import exception, util from platformio.compat import WINDOWS from platformio.managers.platform import PlatformFactory +from platformio.project.config import ProjectOptions # pylint: disable=too-many-branches, too-many-locals @@ -33,10 +33,10 @@ def initPioPlatform(name): def PioPlatform(env): - variables = {} - for name in env['PIOVARIABLES']: - if name in env: - variables[name.lower()] = env[name] + variables = env.GetProjectOptions(as_dict=True) + if "framework" in variables: + # support PIO Core 3.0 dev/platforms + variables['pioframework'] = variables['framework'] p = initPioPlatform(env['PLATFORM_MANIFEST']) p.configure_default_packages(variables, COMMAND_LINE_TARGETS) return p @@ -63,7 +63,7 @@ def GetFrameworkScript(env, framework): return script_path -def LoadPioPlatform(env, variables): +def LoadPioPlatform(env): p = env.PioPlatform() installed_packages = p.get_installed_packages() @@ -92,36 +92,25 @@ def LoadPioPlatform(env, variables): env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")]) if "BOARD" not in env: - # handle _MCU and _F_CPU variables for AVR native - for key, value in variables.UnknownVariables().items(): - if not key.startswith("BOARD_"): - continue - value = base64.b64decode(value) - if isinstance(value, bytes): - value = value.decode() - env.Replace(**{key.upper().replace("BUILD.", ""): value}) return - # update board manifest with a custom data + # update board manifest with overridden data from INI config board_config = env.BoardConfig() - for key, value in variables.UnknownVariables().items(): - if not key.startswith("BOARD_"): - continue - value = base64.b64decode(value) - if isinstance(value, bytes): - value = value.decode() - board_config.update(key.lower()[6:], value) + for option, value in env.GetProjectOptions(): + if option.startswith("board_"): + board_config.update(option.lower()[6:], value) - # update default environment variables - for key in list(variables.keys()): - if key in env or \ - not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]): + # load default variables from board config + for option_meta in ProjectOptions.values(): + if not option_meta.buildenvvar or option_meta.buildenvvar in env: continue - _opt, _val = key.lower().split("_", 1) - if _opt == "board": - _opt = "build" - if _val in board_config.get(_opt): - env.Replace(**{key: board_config.get("%s.%s" % (_opt, _val))}) + data_path = (option_meta.name[6:] + if option_meta.name.startswith("board_") else + option_meta.name.replace("_", ".")) + try: + env[option_meta.buildenvvar] = board_config.get(data_path) + except KeyError: + pass if "build.ldscript" in board_config: env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript")) @@ -165,7 +154,7 @@ def PrintConfiguration(env): data = [ "CURRENT(%s)" % board_config.get_debug_tool_name( - env.subst("$DEBUG_TOOL")) + env.GetProjectOption("debug_tool")) ] onboard = [] external = [] diff --git a/platformio/builder/tools/pioproject.py b/platformio/builder/tools/pioproject.py new file mode 100644 index 00000000..5797755d --- /dev/null +++ b/platformio/builder/tools/pioproject.py @@ -0,0 +1,49 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +from platformio.project.config import ProjectConfig, ProjectOptions + + +def GetProjectConfig(env): + return ProjectConfig.get_instance(env['PROJECT_CONFIG']) + + +def GetProjectOptions(env, as_dict=False): + return env.GetProjectConfig().items(env=env['PIOENV'], as_dict=as_dict) + + +def GetProjectOption(env, option, default=None): + return env.GetProjectConfig().get("env:" + env['PIOENV'], option, default) + + +def LoadProjectOptions(env): + for option, value in env.GetProjectOptions(): + option_meta = ProjectOptions.get("env." + option) + if not option_meta or not option_meta.buildenvvar: + continue + env[option_meta.buildenvvar] = value + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(GetProjectConfig) + env.AddMethod(GetProjectOptions) + env.AddMethod(GetProjectOption) + env.AddMethod(LoadProjectOptions) + return env diff --git a/platformio/commands/run.py b/platformio/commands/run.py deleted file mode 100644 index 41fca7e0..00000000 --- a/platformio/commands/run.py +++ /dev/null @@ -1,352 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os import getcwd, makedirs -from os.path import getmtime, isdir, isfile, join -from time import time - -import click - -from platformio import exception, telemetry, util -from platformio.commands.device import device_monitor as cmd_device_monitor -from platformio.commands.lib import (CTX_META_STORAGE_DIRS_KEY, - CTX_META_STORAGE_LIBDEPS_KEY) -from platformio.commands.lib import lib_install as cmd_lib_install -from platformio.commands.platform import \ - platform_install as cmd_platform_install -from platformio.managers.platform import PlatformFactory -from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - calculate_project_hash, find_project_dir_above, get_project_build_dir, - get_project_dir, get_project_libdeps_dir) - -# pylint: disable=too-many-arguments,too-many-locals,too-many-branches - - -@click.command("run", short_help="Process project environments") -@click.option("-e", "--environment", multiple=True) -@click.option("-t", "--target", multiple=True) -@click.option("--upload-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=True, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option( - "-c", - "--project-conf", - type=click.Path( - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) -@click.option("-s", "--silent", is_flag=True) -@click.option("-v", "--verbose", is_flag=True) -@click.option("--disable-auto-clean", is_flag=True) -@click.pass_context -def cli(ctx, environment, target, upload_port, project_dir, project_conf, - silent, verbose, disable_auto_clean): - # find project directory on upper level - if isfile(project_dir): - project_dir = find_project_dir_above(project_dir) - - with util.cd(project_dir): - # clean obsolete build dir - if not disable_auto_clean: - try: - _clean_build_dir(get_project_build_dir()) - except: # pylint: disable=bare-except - click.secho( - "Can not remove temporary directory `%s`. Please remove " - "it manually to avoid build issues" % - get_project_build_dir(force=True), - fg="yellow") - - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini")) - config.validate(environment) - - _handle_legacy_libdeps(project_dir, config) - - results = [] - start_time = time() - default_envs = config.default_envs() - for envname in config.envs(): - skipenv = any([ - environment and envname not in environment, not environment - and default_envs and envname not in default_envs - ]) - if skipenv: - results.append((envname, None)) - continue - - if not silent and any( - status is not None for (_, status) in results): - click.echo() - - options = config.items(env=envname, as_dict=True) - if "piotest" not in options and "piotest" in ctx.meta: - options['piotest'] = ctx.meta['piotest'] - - ep = EnvironmentProcessor(ctx, envname, options, target, - upload_port, silent, verbose) - result = (envname, ep.process()) - results.append(result) - if result[1] and "monitor" in ep.get_build_targets() and \ - "nobuild" not in ep.get_build_targets(): - ctx.invoke( - cmd_device_monitor, - environment=environment[0] if environment else None) - - found_error = any(status is False for (_, status) in results) - - if (found_error or not silent) and len(results) > 1: - click.echo() - print_summary(results, start_time) - - if found_error: - raise exception.ReturnErrorCode(1) - return True - - -class EnvironmentProcessor(object): - - DEFAULT_DUMP_OPTIONS = ("platform", "framework", "board") - - IGNORE_BUILD_OPTIONS = [ - "test_transport", "test_filter", "test_ignore", "test_port", - "test_speed", "debug_port", "debug_init_cmds", "debug_extra_cmds", - "debug_server", "debug_init_break", "debug_load_cmd", - "debug_load_mode", "monitor_port", "monitor_speed", "monitor_rts", - "monitor_dtr" - ] - - REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"} - - def __init__( - self, # pylint: disable=R0913 - cmd_ctx, - name, - options, - targets, - upload_port, - silent, - verbose): - self.cmd_ctx = cmd_ctx - self.name = name - self.options = options - self.targets = targets - self.upload_port = upload_port - self.silent = silent - self.verbose = verbose - - def process(self): - terminal_width, _ = click.get_terminal_size() - start_time = time() - env_dump = [] - - for k, v in self.options.items(): - self.options[k] = self.options[k].strip() - if self.verbose or k in self.DEFAULT_DUMP_OPTIONS: - env_dump.append("%s: %s" % (k, ", ".join( - ProjectConfig.parse_multi_values(v)))) - - if not self.silent: - click.echo("Processing %s (%s)" % (click.style( - self.name, fg="cyan", bold=True), "; ".join(env_dump))) - click.secho("-" * terminal_width, bold=True) - - result = self._run() - is_error = result['returncode'] != 0 - - if self.silent and not is_error: - return True - - if is_error or "piotest_processor" not in self.cmd_ctx.meta: - print_header( - "[%s] Took %.2f seconds" % ( - (click.style("ERROR", fg="red", bold=True) if is_error else - click.style("SUCCESS", fg="green", bold=True)), - time() - start_time), - is_error=is_error) - - return not is_error - - def get_build_variables(self): - variables = {"pioenv": self.name} - if self.upload_port: - variables['upload_port'] = self.upload_port - for k, v in self.options.items(): - if k in self.REMAPED_OPTIONS: - k = self.REMAPED_OPTIONS[k] - if k in self.IGNORE_BUILD_OPTIONS: - continue - if k == "targets" or (k == "upload_port" and self.upload_port): - continue - variables[k] = v - return variables - - def get_build_targets(self): - targets = [] - if self.targets: - targets = [t for t in self.targets] - elif "targets" in self.options: - targets = self.options['targets'].split(", ") - return targets - - def _run(self): - if "platform" not in self.options: - raise exception.UndefinedEnvPlatform(self.name) - - build_vars = self.get_build_variables() - build_targets = self.get_build_targets() - - telemetry.on_run_environment(self.options, build_targets) - - # skip monitor target, we call it above - if "monitor" in build_targets: - build_targets.remove("monitor") - if "nobuild" not in build_targets: - # install dependent libraries - if "lib_install" in self.options: - _autoinstall_libdeps(self.cmd_ctx, self.name, [ - int(d.strip()) - for d in self.options['lib_install'].split(",") - if d.strip() - ], self.verbose) - if "lib_deps" in self.options: - _autoinstall_libdeps( - self.cmd_ctx, self.name, - ProjectConfig.parse_multi_values(self.options['lib_deps']), - self.verbose) - - try: - p = PlatformFactory.newPlatform(self.options['platform']) - except exception.UnknownPlatform: - self.cmd_ctx.invoke( - cmd_platform_install, - platforms=[self.options['platform']], - skip_default_package=True) - p = PlatformFactory.newPlatform(self.options['platform']) - - return p.run(build_vars, build_targets, self.silent, self.verbose) - - -def _handle_legacy_libdeps(project_dir, config): - legacy_libdeps_dir = join(project_dir, ".piolibdeps") - if (not isdir(legacy_libdeps_dir) - or legacy_libdeps_dir == get_project_libdeps_dir()): - return - if not config.has_section("env"): - config.add_section("env") - lib_extra_dirs = [] - if config.has_option("env", "lib_extra_dirs"): - lib_extra_dirs = config.getlist("env", "lib_extra_dirs") - lib_extra_dirs.append(legacy_libdeps_dir) - config.set("env", "lib_extra_dirs", lib_extra_dirs) - click.secho( - "DEPRECATED! A legacy library storage `{0}` has been found in a " - "project. \nPlease declare project dependencies in `platformio.ini`" - " file using `lib_deps` option and remove `{0}` folder." - "\nMore details -> http://docs.platformio.org/page/projectconf/" - "section_env_library.html#lib-deps".format(legacy_libdeps_dir), - fg="yellow") - - -def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): - if not libraries: - return - libdeps_dir = join(get_project_libdeps_dir(), envname) - ctx.meta.update({ - CTX_META_STORAGE_DIRS_KEY: [libdeps_dir], - CTX_META_STORAGE_LIBDEPS_KEY: { - libdeps_dir: libraries - } - }) - try: - ctx.invoke(cmd_lib_install, silent=not verbose) - except exception.InternetIsOffline as e: - click.secho(str(e), fg="yellow") - - -def _clean_build_dir(build_dir): - # remove legacy ".pioenvs" folder - legacy_build_dir = join(get_project_dir(), ".pioenvs") - if isdir(legacy_build_dir) and legacy_build_dir != build_dir: - util.rmtree_(legacy_build_dir) - - structhash_file = join(build_dir, "structure.hash") - proj_hash = calculate_project_hash() - - # if project's config is modified - if (isdir(build_dir) and getmtime( - join(get_project_dir(), "platformio.ini")) > getmtime(build_dir)): - util.rmtree_(build_dir) - - # check project structure - if isdir(build_dir) and isfile(structhash_file): - with open(structhash_file) as f: - if f.read() == proj_hash: - return - util.rmtree_(build_dir) - - if not isdir(build_dir): - makedirs(build_dir) - - with open(structhash_file, "w") as f: - f.write(proj_hash) - - -def print_header(label, is_error=False, fg=None): - terminal_width, _ = click.get_terminal_size() - width = len(click.unstyle(label)) - half_line = "=" * int((terminal_width - width - 2) / 2) - click.secho( - "%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) - - -def print_summary(results, start_time): - print_header("[%s]" % click.style("SUMMARY")) - - envname_max_len = 0 - for (envname, _) in results: - if len(envname) > envname_max_len: - envname_max_len = len(envname) - - successed = True - for (envname, status) in results: - status_str = click.style("SUCCESS", fg="green") - if status is False: - successed = False - status_str = click.style("ERROR", fg="red") - elif status is None: - status_str = click.style("SKIP", fg="yellow") - - format_str = ( - "Environment {0:<" + str(envname_max_len + 9) + "}\t[{1}]") - click.echo( - format_str.format(click.style(envname, fg="cyan"), status_str), - err=status is False) - - print_header( - "[%s] Took %.2f seconds" % ( - (click.style("SUCCESS", fg="green", bold=True) if successed else - click.style("ERROR", fg="red", bold=True)), time() - start_time), - is_error=not successed) diff --git a/platformio/commands/run/__init__.py b/platformio/commands/run/__init__.py new file mode 100644 index 00000000..05d4d370 --- /dev/null +++ b/platformio/commands/run/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.run.command import cli diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py new file mode 100644 index 00000000..1913314b --- /dev/null +++ b/platformio/commands/run/command.py @@ -0,0 +1,119 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import getcwd +from os.path import isfile, join +from time import time + +import click + +from platformio import exception, util +from platformio.commands.device import device_monitor as cmd_device_monitor +from platformio.commands.run.helpers import ( + _clean_build_dir, _handle_legacy_libdeps, print_summary) +from platformio.commands.run.processor import EnvironmentProcessor +from platformio.project.config import ProjectConfig +from platformio.project.helpers import (find_project_dir_above, + get_project_build_dir) + +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches + + +@click.command("run", short_help="Process project environments") +@click.option("-e", "--environment", multiple=True) +@click.option("-t", "--target", multiple=True) +@click.option("--upload-port") +@click.option( + "-d", + "--project-dir", + default=getcwd, + type=click.Path( + exists=True, + file_okay=True, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) +@click.option("-s", "--silent", is_flag=True) +@click.option("-v", "--verbose", is_flag=True) +@click.option("--disable-auto-clean", is_flag=True) +@click.pass_context +def cli(ctx, environment, target, upload_port, project_dir, project_conf, + silent, verbose, disable_auto_clean): + # find project directory on upper level + if isfile(project_dir): + project_dir = find_project_dir_above(project_dir) + + with util.cd(project_dir): + # clean obsolete build dir + if not disable_auto_clean: + try: + _clean_build_dir(get_project_build_dir()) + except: # pylint: disable=bare-except + click.secho( + "Can not remove temporary directory `%s`. Please remove " + "it manually to avoid build issues" % + get_project_build_dir(force=True), + fg="yellow") + + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(environment) + + _handle_legacy_libdeps(project_dir, config) + + results = [] + start_time = time() + default_envs = config.default_envs() + for envname in config.envs(): + skipenv = any([ + environment and envname not in environment, not environment + and default_envs and envname not in default_envs + ]) + if skipenv: + results.append((envname, None)) + continue + + if not silent and any( + status is not None for (_, status) in results): + click.echo() + + ep = EnvironmentProcessor(ctx, envname, config, target, + upload_port, silent, verbose) + result = (envname, ep.process()) + results.append(result) + + if result[1] and "monitor" in ep.get_build_targets() and \ + "nobuild" not in ep.get_build_targets(): + ctx.invoke( + cmd_device_monitor, + environment=environment[0] if environment else None) + + found_error = any(status is False for (_, status) in results) + + if (found_error or not silent) and len(results) > 1: + click.echo() + print_summary(results, start_time) + + if found_error: + raise exception.ReturnErrorCode(1) + return True diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py new file mode 100644 index 00000000..fd1ac7e3 --- /dev/null +++ b/platformio/commands/run/helpers.py @@ -0,0 +1,127 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import makedirs +from os.path import getmtime, isdir, isfile, join +from time import time + +import click + +from platformio import exception, util +from platformio.commands.lib import (CTX_META_STORAGE_DIRS_KEY, + CTX_META_STORAGE_LIBDEPS_KEY) +from platformio.commands.lib import lib_install as cmd_lib_install +from platformio.project.helpers import ( + calculate_project_hash, get_project_dir, get_project_libdeps_dir) + + +def _handle_legacy_libdeps(project_dir, config): + legacy_libdeps_dir = join(project_dir, ".piolibdeps") + if (not isdir(legacy_libdeps_dir) + or legacy_libdeps_dir == get_project_libdeps_dir()): + return + if not config.has_section("env"): + config.add_section("env") + lib_extra_dirs = config.get("env", "lib_extra_dirs", []) + lib_extra_dirs.append(legacy_libdeps_dir) + config.set("env", "lib_extra_dirs", lib_extra_dirs) + click.secho( + "DEPRECATED! A legacy library storage `{0}` has been found in a " + "project. \nPlease declare project dependencies in `platformio.ini`" + " file using `lib_deps` option and remove `{0}` folder." + "\nMore details -> http://docs.platformio.org/page/projectconf/" + "section_env_library.html#lib-deps".format(legacy_libdeps_dir), + fg="yellow") + + +def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): + if not libraries: + return + libdeps_dir = join(get_project_libdeps_dir(), envname) + ctx.meta.update({ + CTX_META_STORAGE_DIRS_KEY: [libdeps_dir], + CTX_META_STORAGE_LIBDEPS_KEY: { + libdeps_dir: libraries + } + }) + try: + ctx.invoke(cmd_lib_install, silent=not verbose) + except exception.InternetIsOffline as e: + click.secho(str(e), fg="yellow") + + +def _clean_build_dir(build_dir): + # remove legacy ".pioenvs" folder + legacy_build_dir = join(get_project_dir(), ".pioenvs") + if isdir(legacy_build_dir) and legacy_build_dir != build_dir: + util.rmtree_(legacy_build_dir) + + structhash_file = join(build_dir, "structure.hash") + proj_hash = calculate_project_hash() + + # if project's config is modified + if (isdir(build_dir) and getmtime( + join(get_project_dir(), "platformio.ini")) > getmtime(build_dir)): + util.rmtree_(build_dir) + + # check project structure + if isdir(build_dir) and isfile(structhash_file): + with open(structhash_file) as f: + if f.read() == proj_hash: + return + util.rmtree_(build_dir) + + if not isdir(build_dir): + makedirs(build_dir) + + with open(structhash_file, "w") as f: + f.write(proj_hash) + + +def print_header(label, is_error=False, fg=None): + terminal_width, _ = click.get_terminal_size() + width = len(click.unstyle(label)) + half_line = "=" * int((terminal_width - width - 2) / 2) + click.secho( + "%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) + + +def print_summary(results, start_time): + print_header("[%s]" % click.style("SUMMARY")) + + succeeded_nums = 0 + failed_nums = 0 + envname_max_len = max( + [len(click.style(envname, fg="cyan")) for (envname, _) in results]) + for (envname, status) in results: + if status is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif status is None: + status_str = click.style("IGNORED", fg="yellow") + else: + succeeded_nums += 1 + status_str = click.style("SUCCESS", fg="green") + + format_str = "Environment {0:<%d}\t[{1}]" % envname_max_len + click.echo( + format_str.format(click.style(envname, fg="cyan"), status_str), + err=status is False) + + print_header( + "%s%d succeeded in %.2f seconds" % + ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums, + time() - start_time), + is_error=failed_nums, + fg="red" if failed_nums else "green") diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py new file mode 100644 index 00000000..6d57f257 --- /dev/null +++ b/platformio/commands/run/processor.py @@ -0,0 +1,115 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import time + +import click + +from platformio import exception, telemetry +from platformio.commands.platform import \ + platform_install as cmd_platform_install +from platformio.commands.run.helpers import _autoinstall_libdeps, print_header +from platformio.managers.platform import PlatformFactory + +# pylint: disable=too-many-instance-attributes + + +class EnvironmentProcessor(object): + + DEFAULT_PRINT_OPTIONS = ("platform", "framework", "board") + + def __init__( # pylint: disable=too-many-arguments + self, cmd_ctx, name, config, targets, upload_port, silent, + verbose): + self.cmd_ctx = cmd_ctx + self.name = name + self.config = config + self.targets = targets + self.upload_port = upload_port + self.silent = silent + self.verbose = verbose + self.options = config.items(env=name, as_dict=True) + + def process(self): + terminal_width, _ = click.get_terminal_size() + start_time = time() + env_dump = [] + + for k, v in self.options.items(): + if self.verbose or k in self.DEFAULT_PRINT_OPTIONS: + env_dump.append( + "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)) + + if not self.silent: + click.echo("Processing %s (%s)" % (click.style( + self.name, fg="cyan", bold=True), "; ".join(env_dump))) + click.secho("-" * terminal_width, bold=True) + + result = self._run_platform() + is_error = result['returncode'] != 0 + + if self.silent and not is_error: + return True + + if is_error or "piotest_processor" not in self.cmd_ctx.meta: + print_header( + "[%s] Took %.2f seconds" % ( + (click.style("ERROR", fg="red", bold=True) if is_error else + click.style("SUCCESS", fg="green", bold=True)), + time() - start_time), + is_error=is_error) + + return not is_error + + def get_build_variables(self): + variables = {"pioenv": self.name, "project_config": self.config.path} + if "piotest" in self.cmd_ctx.meta: + variables['piotest'] = self.cmd_ctx.meta['piotest'] + if self.upload_port: + # override upload port with a custom from CLI + variables['upload_port'] = self.upload_port + return variables + + def get_build_targets(self): + if self.targets: + return [t for t in self.targets] + return self.config.get("env:" + self.name, "targets", []) + + def _run_platform(self): + if "platform" not in self.options: + raise exception.UndefinedEnvPlatform(self.name) + + build_vars = self.get_build_variables() + build_targets = self.get_build_targets() + + telemetry.on_run_environment(self.options, build_targets) + + # skip monitor target, we call it above + if "monitor" in build_targets: + build_targets.remove("monitor") + if "nobuild" not in build_targets and "lib_deps" in self.options: + _autoinstall_libdeps( + self.cmd_ctx, self.name, + self.config.get("env:" + self.name, "lib_deps"), self.verbose) + + try: + p = PlatformFactory.newPlatform(self.options['platform']) + except exception.UnknownPlatform: + self.cmd_ctx.invoke( + cmd_platform_install, + platforms=[self.options['platform']], + skip_default_package=True) + p = PlatformFactory.newPlatform(self.options['platform']) + + return p.run(build_vars, build_targets, self.silent, self.verbose) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8a5de2ef..81982219 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -29,6 +29,7 @@ from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, exec_command, get_pythonexe_path) +from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_project_boards_dir, get_project_core_dir, get_project_packages_dir, get_project_platforms_dir) @@ -358,7 +359,12 @@ class PlatformRunMixin(object): assert isinstance(variables, dict) assert isinstance(targets, list) - self.configure_default_packages(variables, targets) + config = ProjectConfig.get_instance(variables['project_config']) + options = config.items(env=variables['pioenv'], as_dict=True) + if "framework" in options: + # support PIO Core 3.0 dev/platforms + options['pioframework'] = options['framework'] + self.configure_default_packages(options, targets) self.install_packages(silent=True) self.silent = silent @@ -611,12 +617,9 @@ class PlatformBase( # pylint: disable=too-many-public-methods def get_package_type(self, name): return self.packages[name].get("type") - def configure_default_packages(self, variables, targets): + def configure_default_packages(self, options, targets): # enable used frameworks - frameworks = variables.get("pioframework", []) - if not isinstance(frameworks, list): - frameworks = frameworks.split(", ") - for framework in frameworks: + for framework in options.get("framework", []): if not self.frameworks: continue framework = framework.lower().strip() diff --git a/platformio/project/config.py b/platformio/project/config.py index a1b3b428..decbab72 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -174,7 +174,7 @@ class ProjectConfig(object): return self.getraw(section, option) def get(self, section, option, default=None): - value = default + value = None try: value = self.getraw(section, option) except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): @@ -185,7 +185,7 @@ class ProjectConfig(object): option_meta = ProjectOptions.get( "%s.%s" % (section.split(":", 1)[0], option)) if not option_meta: - return value + return default if value and option_meta.multiple: value = self.parse_multi_values(value) @@ -203,6 +203,22 @@ class ProjectConfig(object): elif envvar_value and not value: value = envvar_value + # option is not specified by user + if value is None: + return default + + # cast types + if not isinstance(value, (list, tuple)): + value = [value] + for i, v in enumerate(value): + if option_meta.type == bool: + value[i] = v in ("1", "true", "yes") + elif option_meta.type == int: + value[i] = int(v) + elif option_meta.type == float: + value[i] = float(v) + value = value if option_meta.multiple else value[0] + return value def envs(self): diff --git a/platformio/project/options.py b/platformio/project/options.py index 7c68a884..a5dd5615 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -151,7 +151,7 @@ ProjectOptions = OrderedDict([ # Monitor ConfigEnvOption(name="monitor_port"), ConfigEnvOption( - name="monitor_speed", oldnames=["monitor_baud"], type=int), + name="monitor_speed", oldnames=["monitor_baud"]), ConfigEnvOption(name="monitor_rts"), ConfigEnvOption(name="monitor_dtr"), ConfigEnvOption(name="monitor_flags", multiple=True), @@ -168,7 +168,7 @@ ProjectOptions = OrderedDict([ sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"), ConfigEnvOption(name="lib_ldf_mode"), ConfigEnvOption(name="lib_compat_mode"), - ConfigEnvOption(name="lib_archive", type=bool), # FIXME: B + ConfigEnvOption(name="lib_archive", type=bool), # Test ConfigEnvOption(name="test_filter", multiple=True),