Pre/Post extra scripting for advanced control of PIO Build System // Resolve #891

HISTORY.rst

@@ -7,6 +7,8 @@ PlatformIO 3.0
 3.4.1 (2017-??-??)
 ~~~~~~~~~~~~~~~~~~
 
+* Pre/Post extra scripting for advanced control of PIO Build System
+  (`issue #891 <https://github.com/platformio/platformio-core/issues/891>`_)
 * Added ``monitor_*`` options to white-list for `Project Configuration File "platformio.ini" <http://docs.platformio.org/page/projectconf.html>`__
   (`issue #982 <https://github.com/platformio/platformio-core/issues/982>`_)
 * Use a root of library when filtering source code using

platformio/builder/main.py

@@ -32,7 +32,7 @@ commonvars = Variables(None)
 commonvars.AddVariables(
     ("PLATFORM_MANIFEST",),
     ("BUILD_SCRIPT",),
-    ("EXTRA_SCRIPT",),
+    ("EXTRA_SCRIPTS",),
     ("PIOENV",),
     ("PIOTEST",),
     ("PIOPLATFORM",),
@@ -67,6 +67,11 @@ commonvars.AddVariables(
 
 )  # yapf: disable
 
+MULTILINE_VARS = [
+    "EXTRA_SCRIPTS", "PIOFRAMEWORK", "BUILD_FLAGS", "SRC_BUILD_FLAGS",
+    "BUILD_UNFLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE", "LIB_EXTRA_DIRS"
+]
+
 DEFAULT_ENV_OPTIONS = dict(
     tools=[
         "ar", "as", "gcc", "g++", "gnulink", "platformio", "pioplatform",
@@ -110,8 +115,8 @@ env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS)
 for k in commonvars.keys():
     if k in env:
         env[k] = base64.b64decode(env[k])
-        if "\n" in env[k]:
+        if k in MULTILINE_VARS:
-            env[k] = [v.strip() for v in env[k].split("\n") if v.strip()]
+            env[k] = util.parse_conf_multi_values(env[k])
 
 if env.GetOption('clean'):
     env.PioClean(env.subst("$BUILD_DIR"))
@@ -120,38 +125,30 @@ elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     print "Verbose mode can be enabled via `-v, --verbose` option"
 
 # Handle custom variables from system environment
-for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPT",
+for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPTS",
             "UPLOAD_PORT", "UPLOAD_FLAGS", "LIB_EXTRA_DIRS"):
     k = "PLATFORMIO_%s" % var
     if k not in environ:
         continue
-    if var in ("UPLOAD_PORT", "EXTRA_SCRIPT") or not env.get(var):
+    if var in ("UPLOAD_PORT", ):
         env[var] = environ.get(k)
-    elif isinstance(env[var], list):
-        env.Append(**{var: environ.get(k)})
-    else:
-        env[var] = "%s%s%s" % (environ.get(k), ", "
-                               if var == "LIB_EXTRA_DIRS" else " ", env[var])
-
-# Parse comma separated items
-for opt in ("PIOFRAMEWORK", "LIB_DEPS", "LIB_IGNORE", "LIB_EXTRA_DIRS"):
-    if opt not in env or isinstance(env[opt], list):
         continue
-    env[opt] = [l.strip() for l in env[opt].split(", ") if l.strip()]
+    env.Append(**{var: util.parse_conf_multi_values(environ.get(k))})
 
 # Configure extra library source directories for LDF
 if util.get_project_optional_dir("lib_extra_dirs"):
-    items = util.get_project_optional_dir("lib_extra_dirs")
+    env.Prepend(LIBSOURCE_DIRS=util.parse_conf_multi_values(
-    env.Prepend(LIBSOURCE_DIRS=[
+        util.get_project_optional_dir("lib_extra_dirs")))
-        l.strip() for l in items.split("\n" if "\n" in items else ", ")
-        if l.strip()
-    ])
 env.Prepend(LIBSOURCE_DIRS=env.get("LIB_EXTRA_DIRS", []))
 
 env.LoadPioPlatform(commonvars)
 
 env.SConscriptChdir(0)
 env.SConsignFile(join("$PROJECTPIOENVS_DIR", ".sconsign.dblite"))
+
+for item in env.GetPreExtraScripts():
+    env.SConscript(item, exports="env")
+
 env.SConscript("$BUILD_SCRIPT")
 
 AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS + ["size"]))
@@ -160,8 +157,8 @@ AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS + ["size"]))
 if "UPLOAD_FLAGS" in env:
     env.Append(UPLOADERFLAGS=["$UPLOAD_FLAGS"])
 
-if env.get("EXTRA_SCRIPT"):
+for item in env.GetPostExtraScripts():
-    env.SConscript(env.get("EXTRA_SCRIPT"), exports="env")
+    env.SConscript(item, exports="env")
 
 if "envdump" in COMMAND_LINE_TARGETS:
     print env.Dump()

platformio/builder/tools/piomisc.py

@@ -297,6 +297,20 @@ def ProcessTest(env):
         duplicate=False)
 
 
+def GetPreExtraScripts(env):
+    return [
+        item[4:] for item in env.get("EXTRA_SCRIPTS", [])
+        if item.startswith("pre:")
+    ]
+
+
+def GetPostExtraScripts(env):
+    return [
+        item[5:] if item.startswith("post:") else item
+        for item in env.get("EXTRA_SCRIPTS", []) if not item.startswith("pre:")
+    ]
+
+
 def exists(_):
     return True
 
@@ -309,4 +323,6 @@ def generate(env):
     env.AddMethod(PioClean)
     env.AddMethod(ProcessDebug)
     env.AddMethod(ProcessTest)
+    env.AddMethod(GetPreExtraScripts)
+    env.AddMethod(GetPostExtraScripts)
     return env

platformio/commands/run.py

@@ -77,10 +77,8 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
 
         env_default = None
         if config.has_option("platformio", "env_default"):
-            env_default = [
+            env_default = util.parse_conf_multi_values(
-                e.strip()
+                config.get("platformio", "env_default"))
-                for e in config.get("platformio", "env_default").split(", ")
-            ]
 
         results = []
         start_time = time()
@@ -130,7 +128,7 @@ class EnvironmentProcessor(object):
     KNOWN_OPTIONS = ("platform", "framework", "board", "board_mcu",
                      "board_f_cpu", "board_f_flash", "board_flash_mode",
                      "build_flags", "src_build_flags", "build_unflags",
-                     "src_filter", "extra_script", "targets", "upload_port",
+                     "src_filter", "extra_scripts", "targets", "upload_port",
                      "upload_protocol", "upload_speed", "upload_flags",
                      "upload_resetmethod", "lib_deps", "lib_ignore",
                      "lib_extra_dirs", "lib_ldf_mode", "lib_compat_mode",
@@ -149,7 +147,11 @@ class EnvironmentProcessor(object):
 
     REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"}
 
-    RENAMED_OPTIONS = {"lib_use": "lib_deps", "lib_force": "lib_deps"}
+    RENAMED_OPTIONS = {
+        "lib_use": "lib_deps",
+        "lib_force": "lib_deps",
+        "extra_script": "extra_scripts"
+    }
 
     RENAMED_PLATFORMS = {"espressif": "espressif8266"}
 
@@ -178,13 +180,13 @@ class EnvironmentProcessor(object):
             self.options[k] = self.options[k].strip()
 
         if not self.silent:
-            click.echo("[%s] Processing %s (%s)" %
+            click.echo(
-                       (datetime.now().strftime("%c"),
+                "[%s] Processing %s (%s)" %
-                        click.style(self.name, fg="cyan", bold=True),
+                (datetime.now().strftime("%c"),
-                        "; ".join([
+                 click.style(self.name, fg="cyan", bold=True), "; ".join([
-                            "%s: %s" % (k, v.replace("\n", ", "))
+                     "%s: %s" % (k, ", ".join(util.parse_conf_multi_values(v)))
-                            for k, v in self.options.items()
+                     for k, v in self.options.items()
-                        ])))
+                 ])))
             click.secho("-" * terminal_width, bold=True)
 
         self.options = self._validate_options(self.options)
@@ -276,12 +278,10 @@ class EnvironmentProcessor(object):
                     if d.strip()
                 ], self.verbose)
             if "lib_deps" in self.options:
-                _autoinstall_libdeps(self.cmd_ctx, [
+                _autoinstall_libdeps(
-                    d.strip()
+                    self.cmd_ctx,
-                    for d in self.options['lib_deps'].split(
+                    util.parse_conf_multi_values(self.options['lib_deps']),
-                        "\n" if "\n" in self.options['lib_deps'] else ", ")
+                    self.verbose)
-                    if d.strip()
-                ], self.verbose)
 
         try:
             p = PlatformFactory.newPlatform(self.options['platform'])
@@ -296,6 +296,8 @@ class EnvironmentProcessor(object):
 
 
 def _autoinstall_libdeps(ctx, libraries, verbose=False):
+    if not libraries:
+        return
     storage_dir = util.get_projectlibdeps_dir()
     ctx.obj = LibraryManager(storage_dir)
     if verbose:

platformio/managers/core.py

@@ -41,8 +41,7 @@ class CorePackageManager(PackageManager):
                 ("" if sys.version_info < (2, 7, 9) else "s")
             ])
 
-    def install(self, name, requirements=None, *args,
+    def install(self, name, requirements=None, *args, **kwargs):  # pylint: disable=arguments-differ
-                **kwargs):  # pylint: disable=arguments-differ
         PackageManager.install(self, name, requirements, *args, **kwargs)
         self.cleanup_packages()
         return self.get_package_dir(name, requirements)

platformio/util.py

@@ -327,6 +327,15 @@ def load_project_config(path=None):
     return cp
 
 
+def parse_conf_multi_values(items):
+    if not items:
+        return []
+    return [
+        item.strip() for item in items.split("\n" if "\n" in items else ", ")
+        if item.strip()
+    ]
+
+
 def change_filemtime(path, time):
     os.utime(path, (time, time))