Implement "extends" for project configuration // Resolve #2953

HISTORY.rst

@@ -3,12 +3,13 @@ Release Notes
 
 .. _release_notes_4_0:
 
-PlatformIO 4.0
+PlatformIO Core 4.0
---------------
+-------------------
 
-4.0.4 (2019-??-??)
+4.1.0 (2019-??-??)
 ~~~~~~~~~~~~~~~~~~
 
+* Extend project environment configuration in "platformio.ini" with other sections using a new `extends <http://docs.platformio.org/page/projectconf/section_env_advanced.html#extends>`__ option (`issue #2953 <https://github.com/platformio/platformio-core/issues/2953>`_)
 * Fixed an issue with project generator for `CLion IDE <http://docs.platformio.org/page/ide/clion.html>`__ when 2 environments were used (`issue #2824 <https://github.com/platformio/platformio-core/issues/2824>`_)
 
 4.0.3 (2019-08-30)
@@ -109,8 +110,8 @@ PlatformIO 4.0
   - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules <http://docs.platformio.org/page/faq.html#platformio-udev-rules>`__ (`issue #2442 <https://github.com/platformio/platformio-core/issues/2442>`_)
   - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 <https://github.com/platformio/platformio-core/issues/2559>`_)
 
-PlatformIO 3.0
+PlatformIO Core 3.0
---------------
+-------------------
 
 3.6.7 (2019-04-23)
 ~~~~~~~~~~~~~~~~~~
@@ -710,8 +711,8 @@ PlatformIO 3.0
   (`issue #742 <https://github.com/platformio/platformio-core/issues/742>`_)
 * Stopped supporting Python 2.6
 
-PlatformIO 2.0
+PlatformIO Core 2.0
---------------
+--------------------
 
 2.11.2 (2016-08-02)
 ~~~~~~~~~~~~~~~~~~~
@@ -1496,8 +1497,8 @@ PlatformIO 2.0
 * Fixed bug with creating copies of source files
   (`issue #177 <https://github.com/platformio/platformio-core/issues/177>`_)
 
-PlatformIO 1.0
+PlatformIO Core 1.0
---------------
+-------------------
 
 1.5.0 (2015-05-15)
 ~~~~~~~~~~~~~~~~~~
@@ -1687,8 +1688,8 @@ PlatformIO 1.0
   error (`issue #81 <https://github.com/platformio/platformio-core/issues/81>`_)
 * Several bug fixes, increased stability and performance improvements
 
-PlatformIO 0.0
+PlatformIO Core 0.0
---------------
+-------------------
 
 0.10.2 (2015-01-06)
 ~~~~~~~~~~~~~~~~~~~

platformio/project/config.py

@@ -90,6 +90,9 @@ class ProjectConfig(object):
         if isfile(path):
             self.read(path, parse_extra)
 
+    def __repr__(self):
+        return "<ProjectConfig %s>" % (self.path or "in-memory")
+
     def __getattr__(self, name):
         return getattr(self._parser, name)
 
@@ -163,34 +166,46 @@ class ProjectConfig(object):
                         "in section [%s]" % (option, section))
         return True
 
+    def walk_options(self, root_section):
+        extends_queue = (["env", root_section] if
+                         root_section.startswith("env:") else [root_section])
+        extends_done = []
+        while extends_queue:
+            section = extends_queue.pop()
+            extends_done.append(section)
+            if not self._parser.has_section(section):
+                continue
+            for option in self._parser.options(section):
+                yield (section, option)
+            if self._parser.has_option(section, "extends"):
+                extends_queue.extend(
+                    self.parse_multi_values(
+                        self._parser.get(section, "extends"))[::-1])
+
     def options(self, section=None, env=None):
+        result = []
         assert section or env
         if not section:
             section = "env:" + env
-        options = self._parser.options(section)
 
-        # handle global options from [env]
+        for _, option in self.walk_options(section):
-        if ((env or section.startswith("env:"))
+            if option not in result:
-                and self._parser.has_section("env")):
+                result.append(option)
-            for option in self._parser.options("env"):
-                if option not in options:
-                    options.append(option)
 
         # handle system environment variables
         scope = section.split(":", 1)[0]
         for option_meta in ProjectOptions.values():
-            if option_meta.scope != scope or option_meta.name in options:
+            if option_meta.scope != scope or option_meta.name in result:
                 continue
             if option_meta.sysenvvar and option_meta.sysenvvar in os.environ:
-                options.append(option_meta.name)
+                result.append(option_meta.name)
 
-        return options
+        return result
 
     def has_option(self, section, option):
         if self._parser.has_option(section, option):
             return True
-        return (section.startswith("env:") and self._parser.has_section("env")
+        return option in self.options(section)
-                and self._parser.has_option("env", option))
 
     def items(self, section=None, env=None, as_dict=False):
         assert section or env
@@ -215,12 +230,16 @@ class ProjectConfig(object):
         if not self.expand_interpolations:
             return self._parser.get(section, option)
 
-        try:
+        value = None
+        found = False
+        for sec, opt in self.walk_options(section):
+            if opt == option:
+                value = self._parser.get(sec, option)
+                found = True
+                break
+
+        if not found:
             value = self._parser.get(section, option)
-        except ConfigParser.NoOptionError as e:
-            if not section.startswith("env:"):
-                raise e
-            value = self._parser.get("env", option)
 
         if "${" not in value or "}" not in value:
             return value
@@ -267,13 +286,13 @@ class ProjectConfig(object):
             return default
 
         try:
-            return self._covert_value(value, option_meta.type)
+            return self._cast_to(value, option_meta.type)
         except click.BadParameter as e:
             raise exception.ProjectOptionValueError(e.format_message(), option,
                                                     section)
 
     @staticmethod
-    def _covert_value(value, to_type):
+    def _cast_to(value, to_type):
         items = value
         if not isinstance(value, (list, tuple)):
             items = [value]

platformio/project/options.py

@@ -90,6 +90,7 @@ ProjectOptions = OrderedDict([
         #
         # [env]
         #
+        ConfigEnvOption(name="extends", multiple=True),
 
         # Generic
         ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"),

tests/test_projectconf.py

@@ -34,6 +34,17 @@ lib_deps =
     Lib2
 lib_ignore = ${custom.lib_ignore}
 
+[strict_ldf]
+lib_ldf_mode = chain+
+lib_compat_mode = strict
+
+[monitor_custom]
+monitor_speed = 9600
+
+[strict_settings]
+extends = strict_ldf, monitor_custom
+build_flags = -D RELEASE
+
 [custom]
 debug_flags = -D RELEASE
 lib_flags = -lc -lm
@@ -43,6 +54,10 @@ lib_ignore = LibIgnoreCustom
 [env:base]
 build_flags = ${custom.debug_flags} ${custom.extra_flags}
 targets =
+
+[env:test_extends]
+extends = strict_settings
+
 """
 
 EXTRA_ENVS_CONFIG = """
@@ -66,52 +81,70 @@ build_flags = -Og
 """
 
 
-def test_real_config(tmpdir):
+@pytest.fixture(scope="module")
+def config(tmpdir_factory):
+    tmpdir = tmpdir_factory.mktemp("project")
     tmpdir.join("platformio.ini").write(BASE_CONFIG)
     tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG)
     tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG)
-
-    config = None
     with tmpdir.as_cwd():
-        config = ProjectConfig(tmpdir.join("platformio.ini").strpath)
+        return ProjectConfig(tmpdir.join("platformio.ini").strpath)
-    assert config
-    assert len(config.warnings) == 2
-    assert "lib_install" in config.warnings[1]
 
-    config.validate(["extra_2", "base"], silent=True)
+
-    with pytest.raises(UnknownEnvNames):
+def test_empty_config():
-        config.validate(["non-existing-env"])
+    config = ProjectConfig("/non/existing/platformio.ini")
 
     # unknown section
     with pytest.raises(ConfigParser.NoSectionError):
         config.getraw("unknown_section", "unknown_option")
-    # unknown option
-    with pytest.raises(ConfigParser.NoOptionError):
-        config.getraw("custom", "unknown_option")
-    # unknown option even if exists in [env]
-    with pytest.raises(ConfigParser.NoOptionError):
-        config.getraw("platformio", "monitor_speed")
 
-    # sections
+    assert config.sections() == []
+    assert config.get("section", "option") is None
+    assert config.get("section", "option", 13) == 13
+
+
+def test_warnings(config):
+    config.validate(["extra_2", "base"], silent=True)
+    assert len(config.warnings) == 2
+    assert "lib_install" in config.warnings[1]
+
+    with pytest.raises(UnknownEnvNames):
+        config.validate(["non-existing-env"])
+
+
+def test_sections(config):
+    with pytest.raises(ConfigParser.NoSectionError):
+        config.getraw("unknown_section", "unknown_option")
+
     assert config.sections() == [
-        "platformio", "env", "custom", "env:base", "env:extra_1", "env:extra_2"
+        "platformio", "env", "strict_ldf", "monitor_custom", "strict_settings",
+        "custom", "env:base", "env:test_extends", "env:extra_1", "env:extra_2"
     ]
 
-    # envs
+
-    assert config.envs() == ["base", "extra_1", "extra_2"]
+def test_envs(config):
+    assert config.envs() == ["base", "test_extends", "extra_1", "extra_2"]
     assert config.default_envs() == ["base", "extra_2"]
 
-    # options
+
+def test_options(config):
     assert config.options(env="base") == [
         "build_flags", "targets", "monitor_speed", "lib_deps", "lib_ignore"
     ]
+    assert config.options(env="test_extends") == [
+        "extends", "build_flags", "lib_ldf_mode", "lib_compat_mode",
+        "monitor_speed", "lib_deps", "lib_ignore"
+    ]
 
-    # has_option
+
+def test_has_option(config):
     assert config.has_option("env:base", "monitor_speed")
     assert not config.has_option("custom", "monitor_speed")
     assert not config.has_option("env:extra_1", "lib_install")
+    assert config.has_option("env:test_extends", "lib_compat_mode")
 
-    # sysenv
+
+def test_sysenv_options(config):
     assert config.get("custom", "extra_flags") is None
     assert config.get("env:base", "build_flags") == ["-D DEBUG=1"]
     assert config.get("env:base", "upload_port") is None
@@ -126,72 +159,83 @@ def test_real_config(tmpdir):
     assert config.get("env:base", "upload_port") == "/dev/sysenv/port"
     assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port"
 
-    # getraw
+    # env var as option
-    assert config.getraw("env:base", "targets") == ""
+    assert config.options(env="test_extends") == [
-    assert config.getraw("env:extra_1", "lib_deps") == "574"
+        "extends", "build_flags", "lib_ldf_mode", "lib_compat_mode",
-    assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1"
+        "monitor_speed", "lib_deps", "lib_ignore", "upload_port"
-
-    # get
-    assert config.get("custom", "debug_flags") == "-D DEBUG=1"
-    assert config.get("env:extra_1", "build_flags") == [
-        "-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2"
     ]
-    assert config.get("env:extra_2", "build_flags") == [
-        "-Og", "-DSYSENVDEPS1 -DSYSENVDEPS2"]
-    assert config.get("env:extra_2", "monitor_speed") == "115200"
-    assert config.get("env:base", "build_flags") == ([
-        "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2"
-    ])
 
-    # items
+    # sysenv
-    assert config.items("custom") == [
+    os.environ["PLATFORMIO_HOME_DIR"] = "/custom/core/dir"
-        ("debug_flags", "-D DEBUG=1"),
+    assert config.get("platformio", "core_dir") == "/custom/core/dir"
-        ("lib_flags", "-lc -lm"),
-        ("extra_flags", "-L /usr/local/lib"),
-        ("lib_ignore", "LibIgnoreCustom")
-    ]  # yapf: disable
-    assert config.items(env="base") == [
-        ("build_flags", [
-            "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2"]),
-        ("targets", []),
-        ("monitor_speed", "115200"),
-        ("lib_deps", ["Lib1", "Lib2"]),
-        ("lib_ignore", ["LibIgnoreCustom"]),
-        ("upload_port", "/dev/sysenv/port")
-    ]  # yapf: disable
-    assert config.items(env="extra_1") == [
-        ("build_flags", ["-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2"]),
-        ("lib_deps", ["574"]),
-        ("monitor_speed", "115200"),
-        ("lib_ignore", ["LibIgnoreCustom"]),
-        ("upload_port", "/dev/sysenv/port")
-    ]  # yapf: disable
-    assert config.items(env="extra_2") == [
-        ("build_flags", ["-Og", "-DSYSENVDEPS1 -DSYSENVDEPS2"]),
-        ("lib_ignore", ["LibIgnoreCustom", "Lib3"]),
-        ("upload_port", "/dev/extra_2/port"),
-        ("monitor_speed", "115200"),
-        ("lib_deps", ["Lib1", "Lib2"])
-    ]  # yapf: disable
 
     # cleanup system environment variables
     del os.environ["PLATFORMIO_BUILD_FLAGS"]
     del os.environ["PLATFORMIO_UPLOAD_PORT"]
     del os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"]
-
-
-def test_empty_config():
-    config = ProjectConfig("/non/existing/platformio.ini")
-
-    # unknown section
-    with pytest.raises(ConfigParser.NoSectionError):
-        config.getraw("unknown_section", "unknown_option")
-
-    assert config.sections() == []
-    assert config.get("section", "option") is None
-    assert config.get("section", "option", 13) == 13
-
-    # sysenv
-    os.environ["PLATFORMIO_HOME_DIR"] = "/custom/core/dir"
-    assert config.get("platformio", "core_dir") == "/custom/core/dir"
     del os.environ["PLATFORMIO_HOME_DIR"]
+
+
+def test_getraw_value(config):
+    # unknown option
+    with pytest.raises(ConfigParser.NoOptionError):
+        config.getraw("custom", "unknown_option")
+    # unknown option even if exists in [env]
+    with pytest.raises(ConfigParser.NoOptionError):
+        config.getraw("platformio", "monitor_speed")
+
+    # known
+    assert config.getraw("env:base", "targets") == ""
+    assert config.getraw("env:extra_1", "lib_deps") == "574"
+    assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1"
+
+    # extended
+    assert config.getraw("env:test_extends", "lib_ldf_mode") == "chain+"
+    assert config.getraw("env", "monitor_speed") == "115200"
+    assert config.getraw("env:test_extends", "monitor_speed") == "9600"
+
+
+def test_get_value(config):
+    assert config.get("custom", "debug_flags") == "-D DEBUG=1"
+    assert config.get("env:extra_1", "build_flags") == ["-lc -lm -D DEBUG=1"]
+    assert config.get("env:extra_2", "build_flags") == ["-Og"]
+    assert config.get("env:extra_2", "monitor_speed") == "115200"
+    assert config.get("env:base", "build_flags") == ["-D DEBUG=1"]
+
+
+def test_items(config):
+    assert config.items("custom") == [
+        ("debug_flags", "-D DEBUG=1"),
+        ("lib_flags", "-lc -lm"),
+        ("extra_flags", None),
+        ("lib_ignore", "LibIgnoreCustom")
+    ]  # yapf: disable
+    assert config.items(env="base") == [
+        ("build_flags", ["-D DEBUG=1"]),
+        ("targets", []),
+        ("monitor_speed", "115200"),
+        ("lib_deps", ["Lib1", "Lib2"]),
+        ("lib_ignore", ["LibIgnoreCustom"]),
+    ]  # yapf: disable
+    assert config.items(env="extra_1") == [
+        ("build_flags", ["-lc -lm -D DEBUG=1"]),
+        ("lib_deps", ["574"]),
+        ("monitor_speed", "115200"),
+        ("lib_ignore", ["LibIgnoreCustom"]),
+    ]  # yapf: disable
+    assert config.items(env="extra_2") == [
+        ("build_flags", ["-Og"]),
+        ("lib_ignore", ["LibIgnoreCustom", "Lib3"]),
+        ("upload_port", "/dev/extra_2/port"),
+        ("monitor_speed", "115200"),
+        ("lib_deps", ["Lib1", "Lib2"])
+    ]  # yapf: disable
+    assert config.items(env="test_extends") == [
+        ("extends", ["strict_settings"]),
+        ("build_flags", ["-D RELEASE"]),
+        ("lib_ldf_mode", "chain+"),
+        ("lib_compat_mode", "strict"),
+        ("monitor_speed", "9600"),
+        ("lib_deps", ["Lib1", "Lib2"]),
+        ("lib_ignore", ["LibIgnoreCustom"])
+    ]  # yapf: disable