Include external configuration files with "extra_configs" option // Resolve #1590

.pylintrc

@@ -20,4 +20,4 @@ confidence=
 # --disable=W"
 # disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
 
-disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens,useless-object-inheritance,useless-import-alias
+disable=fixme,locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens,useless-object-inheritance,useless-import-alias

HISTORY.rst

@@ -7,8 +7,10 @@ PlatformIO 4.0
 4.0.0 (2019-??-??)
 ~~~~~~~~~~~~~~~~~~
 
-* Added Python 3.5+ support
+* Python 3 support
   (`issue #895 <https://github.com/platformio/platformio-core/issues/895>`_)
+* Include external configuration files with "extra_configs" option
+  (`issue #1590 <https://github.com/platformio/platformio-core/issues/1590>`_)
 
 PlatformIO 3.0
 --------------

platformio/commands/run.py

@@ -130,7 +130,7 @@ class EnvironmentProcessor(object):
     KNOWN_PLATFORMIO_OPTIONS = [
         "description", "env_default", "home_dir", "lib_dir", "libdeps_dir",
         "include_dir", "src_dir", "build_dir", "data_dir", "test_dir",
-        "boards_dir", "lib_extra_dirs"
+        "boards_dir", "lib_extra_dirs", "extra_configs"
     ]
 
     KNOWN_ENV_OPTIONS = [

platformio/exception.py

@@ -207,7 +207,7 @@ class InvalidLibConfURL(PlatformioException):
 
 class InvalidProjectConf(PlatformioException):
 
-    MESSAGE = "Invalid `platformio.ini`, project configuration file: '{0}'"
+    MESSAGE = ("Invalid '{0}' (project configuration file): '{1}'")
 
 
 class BuildScriptNotFound(PlatformioException):

platformio/project/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/project/config.py

@@ -0,0 +1,106 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import os
+import re
+
+import click
+
+from platformio import exception
+
+try:
+    import ConfigParser as ConfigParser
+except ImportError:
+    import configparser as ConfigParser
+
+
+class ProjectConfig(object):
+
+    VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}")
+
+    _parser = None
+    _parsed = []
+
+    @staticmethod
+    def parse_multi_values(items):
+        result = []
+        if not items:
+            return result
+        inline_comment_re = re.compile(r"\s+;.*$")
+        for item in items.split("\n" if "\n" in items else ", "):
+            item = item.strip()
+            # comment
+            if not item or item.startswith((";", "#")):
+                continue
+            if ";" in item:
+                item = inline_comment_re.sub("", item).strip()
+            result.append(item)
+        return result
+
+    def __init__(self, path):
+        self.path = path
+        self._parsed = []
+        self._parser = ConfigParser.ConfigParser()
+        self.read(path)
+
+    def read(self, path):
+        if path in self._parsed:
+            return
+        self._parsed.append(path)
+        try:
+            self._parser.read(path)
+        except ConfigParser.Error as e:
+            raise exception.InvalidProjectConf(path, str(e))
+
+        # load extra configs
+        if (not self._parser.has_section("platformio")
+                or not self._parser.has_option("platformio", "extra_configs")):
+            return
+        extra_configs = self.parse_multi_values(
+            self.get("platformio", "extra_configs"))
+        for pattern in extra_configs:
+            for item in glob.glob(pattern):
+                self.read(item)
+
+    def __getattr__(self, name):
+        return getattr(self._parser, name)
+
+    def items(self, section):
+        items = []
+        for option in self._parser.options(section):
+            items.append((option, self._parser.get(section, option)))
+        return items
+
+    def get(self, section, option):
+        try:
+            value = self._parser.get(section, option)
+        except ConfigParser.Error as e:
+            raise exception.InvalidProjectConf(self.path, str(e))
+        if "${" not in value or "}" not in value:
+            return value
+        return self.VARTPL_RE.sub(self._re_sub_handler, value)
+
+    def _re_sub_handler(self, match):
+        section, option = match.group(1), match.group(2)
+        if section in ("env",
+                       "sysenv") and not self._parser.has_section(section):
+            if section == "env":
+                click.secho(
+                    "Warning! Access to system environment variable via "
+                    "`${{env.{0}}}` is deprecated. Please use "
+                    "`${{sysenv.{0}}}` instead".format(option),
+                    fg="yellow")
+            return os.getenv(option)
+        return self._parser.get(section, option)

platformio/util.py

@@ -33,52 +33,17 @@ import click
 import requests
 
 from platformio import __apiurl__, __version__, exception
+from platformio.project.config import ProjectConfig
 
 # pylint: disable=too-many-ancestors
 
 PY2 = sys.version_info[0] == 2
 if PY2:
-    import ConfigParser as ConfigParser
     string_types = basestring  # pylint: disable=undefined-variable
 else:
-    import configparser as ConfigParser
     string_types = str
 
 
-class ProjectConfig(ConfigParser.ConfigParser):
-
-    VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}")
-
-    def items(self, section, **_):  # pylint: disable=arguments-differ
-        items = []
-        for option in ConfigParser.ConfigParser.options(self, section):
-            items.append((option, self.get(section, option)))
-        return items
-
-    def get(  # pylint: disable=arguments-differ
-            self, section, option, **kwargs):
-        try:
-            value = ConfigParser.ConfigParser.get(self, section, option,
-                                                  **kwargs)
-        except ConfigParser.Error as e:
-            raise exception.InvalidProjectConf(str(e))
-        if "${" not in value or "}" not in value:
-            return value
-        return self.VARTPL_RE.sub(self._re_sub_handler, value)
-
-    def _re_sub_handler(self, match):
-        section, option = match.group(1), match.group(2)
-        if section in ("env", "sysenv") and not self.has_section(section):
-            if section == "env":
-                click.secho(
-                    "Warning! Access to system environment variable via "
-                    "`${{env.{0}}}` is deprecated. Please use "
-                    "`${{sysenv.{0}}}` instead".format(option),
-                    fg="yellow")
-            return os.getenv(option)
-        return self.get(section, option)
-
-
 class AsyncPipe(Thread):
 
     def __init__(self, outcallback=None):
@@ -347,34 +312,17 @@ def get_projectdata_dir():
                                                      "data"))
 
 
-def load_project_config(path=None):
+def load_project_config(path=None):  # FIXME:
     if not path or isdir(path):
         path = join(path or get_project_dir(), "platformio.ini")
     if not isfile(path):
         raise exception.NotPlatformIOProject(
             dirname(path) if path.endswith("platformio.ini") else path)
-    cp = ProjectConfig()
-    try:
-        cp.read(path)
-    except ConfigParser.Error as e:
-        raise exception.InvalidProjectConf(str(e))
-    return cp
+    return ProjectConfig(path)
 
 
-def parse_conf_multi_values(items):
-    result = []
-    if not items:
-        return result
-    inline_comment_re = re.compile(r"\s+;.*$")
-    for item in items.split("\n" if "\n" in items else ", "):
-        item = item.strip()
-        # comment
-        if not item or item.startswith((";", "#")):
-            continue
-        if ";" in item:
-            item = inline_comment_re.sub("", item).strip()
-        result.append(item)
-    return result
+def parse_conf_multi_values(items):  # FIXME:
+    return ProjectConfig.parse_multi_values(items)
 
 
 def change_filemtime(path, mtime):

tests/test_projectconf.py

@@ -0,0 +1,74 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.project.config import ProjectConfig
+
+BASE_CONFIG = """
+[platformio]
+extra_configs =
+  extra_envs.ini
+  extra_debug.ini
+
+[common]
+debug_flags = -D RELEASE
+lib_flags = -lc -lm
+
+[env:esp-wrover-kit]
+platform = espressif32
+framework = espidf
+board = esp-wrover-kit
+build_flags = ${common.debug_flags}
+"""
+
+EXTRA_ENVS_CONFIG = """
+[env:esp32dev]
+platform = espressif32
+framework = espidf
+board = esp32dev
+build_flags = ${common.lib_flags} ${common.debug_flags}
+
+[env:lolin32]
+platform = espressif32
+framework = espidf
+board = lolin32
+build_flags = ${common.debug_flags}
+"""
+
+EXTRA_DEBUG_CONFIG = """
+# Override base "common.debug_flags"
+[common]
+debug_flags = -D DEBUG=1
+
+[env:lolin32]
+build_flags = -Og
+"""
+
+
+def test_parser(tmpdir):
+    tmpdir.join("platformio.ini").write(BASE_CONFIG)
+    tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG)
+    tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG)
+
+    config = None
+    with tmpdir.as_cwd():
+        config = ProjectConfig(tmpdir.join("platformio.ini").strpath)
+    assert config
+
+    assert config.sections() == [
+        "platformio", "common", "env:esp-wrover-kit", "env:esp32dev",
+        "env:lolin32"
+    ]
+    assert config.get("common", "debug_flags") == "-D DEBUG=1"
+    assert config.get("env:esp32dev", "build_flags") == "-lc -lm -D DEBUG=1"
+    assert config.get("env:lolin32", "build_flags") == "-Og"