Implement "extends" for project configuration // Resolve #2953

2019-08-31 23:39:41 +03:00
parent bdce78ba6f
commit fe237f15aa
5 changed files with 180 additions and 115 deletions
--- a/HISTORY.rst
+++ b/HISTORY.rst
@ -3,12 +3,13 @@ Release Notes

 .. _release_notes_4_0:

-PlatformIO 4.0
--------------
+PlatformIO Core 4.0
+-------------------

-4.0.4 (2019-??-??)
+4.1.0 (2019-??-??)
 ~~~~~~~~~~~~~~~~~~

+* Extend project environment configuration in "platformio.ini" with other sections using a new `extends <http://docs.platformio.org/page/projectconf/section_env_advanced.html#extends>`__ option (`issue #2953 <https://github.com/platformio/platformio-core/issues/2953>`_)
 * Fixed an issue with project generator for `CLion IDE <http://docs.platformio.org/page/ide/clion.html>`__ when 2 environments were used (`issue #2824 <https://github.com/platformio/platformio-core/issues/2824>`_)

 4.0.3 (2019-08-30)
@ -109,8 +110,8 @@ PlatformIO 4.0
  - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules <http://docs.platformio.org/page/faq.html#platformio-udev-rules>`__ (`issue #2442 <https://github.com/platformio/platformio-core/issues/2442>`_)
  - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 <https://github.com/platformio/platformio-core/issues/2559>`_)

-PlatformIO 3.0
--------------
+PlatformIO Core 3.0
+-------------------

 3.6.7 (2019-04-23)
 ~~~~~~~~~~~~~~~~~~
@ -710,8 +711,8 @@ PlatformIO 3.0
  (`issue #742 <https://github.com/platformio/platformio-core/issues/742>`_)
 * Stopped supporting Python 2.6

-PlatformIO 2.0
--------------
+PlatformIO Core 2.0
+--------------------

 2.11.2 (2016-08-02)
 ~~~~~~~~~~~~~~~~~~~
@ -1496,8 +1497,8 @@ PlatformIO 2.0
 * Fixed bug with creating copies of source files
  (`issue #177 <https://github.com/platformio/platformio-core/issues/177>`_)

-PlatformIO 1.0
--------------
+PlatformIO Core 1.0
+-------------------

 1.5.0 (2015-05-15)
 ~~~~~~~~~~~~~~~~~~
@ -1687,8 +1688,8 @@ PlatformIO 1.0
  error (`issue #81 <https://github.com/platformio/platformio-core/issues/81>`_)
 * Several bug fixes, increased stability and performance improvements

-PlatformIO 0.0
--------------
+PlatformIO Core 0.0
+-------------------

 0.10.2 (2015-01-06)
 ~~~~~~~~~~~~~~~~~~~
--- a/2
+++ b/2
--- a/platformio/project/config.py
+++ b/platformio/project/config.py
@ -90,6 +90,9 @@ class ProjectConfig(object):
        if isfile(path):
            self.read(path, parse_extra)

+    def __repr__(self):
+        return "<ProjectConfig %s>" % (self.path or "in-memory")
+
    def __getattr__(self, name):
        return getattr(self._parser, name)

@ -163,34 +166,46 @@ class ProjectConfig(object):
                        "in section [%s]" % (option, section))
        return True

+    def walk_options(self, root_section):
+        extends_queue = (["env", root_section] if
+                         root_section.startswith("env:") else [root_section])
+        extends_done = []
+        while extends_queue:
+            section = extends_queue.pop()
+            extends_done.append(section)
+            if not self._parser.has_section(section):
+                continue
+            for option in self._parser.options(section):
+                yield (section, option)
+            if self._parser.has_option(section, "extends"):
+                extends_queue.extend(
+                    self.parse_multi_values(
+                        self._parser.get(section, "extends"))[::-1])
+
    def options(self, section=None, env=None):
+        result = []
        assert section or env
        if not section:
            section = "env:" + env
-        options = self._parser.options(section)

-        # handle global options from [env]
-        if ((env or section.startswith("env:"))
-                and self._parser.has_section("env")):
-            for option in self._parser.options("env"):
-                if option not in options:
-                    options.append(option)
+        for _, option in self.walk_options(section):
+            if option not in result:
+                result.append(option)

        # handle system environment variables
        scope = section.split(":", 1)[0]
        for option_meta in ProjectOptions.values():
-            if option_meta.scope != scope or option_meta.name in options:
+            if option_meta.scope != scope or option_meta.name in result:
                continue
            if option_meta.sysenvvar and option_meta.sysenvvar in os.environ:
-                options.append(option_meta.name)
+                result.append(option_meta.name)

-        return options
+        return result

    def has_option(self, section, option):
        if self._parser.has_option(section, option):
            return True
-        return (section.startswith("env:") and self._parser.has_section("env")
-                and self._parser.has_option("env", option))
+        return option in self.options(section)

    def items(self, section=None, env=None, as_dict=False):
        assert section or env
@ -215,12 +230,16 @@ class ProjectConfig(object):
        if not self.expand_interpolations:
            return self._parser.get(section, option)

-        try:
+        value = None
+        found = False
+        for sec, opt in self.walk_options(section):
+            if opt == option:
+                value = self._parser.get(sec, option)
+                found = True
+                break
+
+        if not found:
            value = self._parser.get(section, option)
-        except ConfigParser.NoOptionError as e:
-            if not section.startswith("env:"):
-                raise e
-            value = self._parser.get("env", option)

        if "${" not in value or "}" not in value:
            return value
@ -267,13 +286,13 @@ class ProjectConfig(object):
            return default

        try:
-            return self._covert_value(value, option_meta.type)
+            return self._cast_to(value, option_meta.type)
        except click.BadParameter as e:
            raise exception.ProjectOptionValueError(e.format_message(), option,
                                                    section)

    @staticmethod
-    def _covert_value(value, to_type):
+    def _cast_to(value, to_type):
        items = value
        if not isinstance(value, (list, tuple)):
            items = [value]
--- a/platformio/project/options.py
+++ b/platformio/project/options.py
@ -90,6 +90,7 @@ ProjectOptions = OrderedDict([
        #
        # [env]
        #
+        ConfigEnvOption(name="extends", multiple=True),

        # Generic
        ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"),
--- a/tests/test_projectconf.py
+++ b/tests/test_projectconf.py
@ -34,6 +34,17 @@ lib_deps =
    Lib2
 lib_ignore = ${custom.lib_ignore}

+[strict_ldf]
+lib_ldf_mode = chain+
+lib_compat_mode = strict
+
+[monitor_custom]
+monitor_speed = 9600
+
+[strict_settings]
+extends = strict_ldf, monitor_custom
+build_flags = -D RELEASE
+
 [custom]
 debug_flags = -D RELEASE
 lib_flags = -lc -lm
@ -43,6 +54,10 @@ lib_ignore = LibIgnoreCustom
 [env:base]
 build_flags = ${custom.debug_flags} ${custom.extra_flags}
 targets =
+
+[env:test_extends]
+extends = strict_settings
+
 """

 EXTRA_ENVS_CONFIG = """
@ -66,52 +81,70 @@ build_flags = -Og
 """


-def test_real_config(tmpdir):
+@pytest.fixture(scope="module")
+def config(tmpdir_factory):
+    tmpdir = tmpdir_factory.mktemp("project")
    tmpdir.join("platformio.ini").write(BASE_CONFIG)
    tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG)
    tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG)
-
-    config = None
    with tmpdir.as_cwd():
-        config = ProjectConfig(tmpdir.join("platformio.ini").strpath)
-    assert config
-    assert len(config.warnings) == 2
-    assert "lib_install" in config.warnings[1]
+        return ProjectConfig(tmpdir.join("platformio.ini").strpath)

-    config.validate(["extra_2", "base"], silent=True)
-    with pytest.raises(UnknownEnvNames):
-        config.validate(["non-existing-env"])
+
+def test_empty_config():
+    config = ProjectConfig("/non/existing/platformio.ini")

    # unknown section
    with pytest.raises(ConfigParser.NoSectionError):
        config.getraw("unknown_section", "unknown_option")
-    # unknown option
-    with pytest.raises(ConfigParser.NoOptionError):
-        config.getraw("custom", "unknown_option")
-    # unknown option even if exists in [env]
-    with pytest.raises(ConfigParser.NoOptionError):
-        config.getraw("platformio", "monitor_speed")

-    # sections
+    assert config.sections() == []
+    assert config.get("section", "option") is None
+    assert config.get("section", "option", 13) == 13
+
+
+def test_warnings(config):
+    config.validate(["extra_2", "base"], silent=True)
+    assert len(config.warnings) == 2
+    assert "lib_install" in config.warnings[1]
+
+    with pytest.raises(UnknownEnvNames):
+        config.validate(["non-existing-env"])
+
+
+def test_sections(config):
+    with pytest.raises(ConfigParser.NoSectionError):
+        config.getraw("unknown_section", "unknown_option")
+
    assert config.sections() == [
-        "platformio", "env", "custom", "env:base", "env:extra_1", "env:extra_2"
+        "platformio", "env", "strict_ldf", "monitor_custom", "strict_settings",
+        "custom", "env:base", "env:test_extends", "env:extra_1", "env:extra_2"
    ]

-    # envs
-    assert config.envs() == ["base", "extra_1", "extra_2"]
+
+def test_envs(config):
+    assert config.envs() == ["base", "test_extends", "extra_1", "extra_2"]
    assert config.default_envs() == ["base", "extra_2"]

-    # options
+
+def test_options(config):
    assert config.options(env="base") == [
        "build_flags", "targets", "monitor_speed", "lib_deps", "lib_ignore"
    ]
+    assert config.options(env="test_extends") == [
+        "extends", "build_flags", "lib_ldf_mode", "lib_compat_mode",
+        "monitor_speed", "lib_deps", "lib_ignore"
+    ]

-    # has_option
+
+def test_has_option(config):
    assert config.has_option("env:base", "monitor_speed")
    assert not config.has_option("custom", "monitor_speed")
    assert not config.has_option("env:extra_1", "lib_install")
+    assert config.has_option("env:test_extends", "lib_compat_mode")

-    # sysenv
+
+def test_sysenv_options(config):
    assert config.get("custom", "extra_flags") is None
    assert config.get("env:base", "build_flags") == ["-D DEBUG=1"]
    assert config.get("env:base", "upload_port") is None
@ -126,72 +159,83 @@ def test_real_config(tmpdir):
    assert config.get("env:base", "upload_port") == "/dev/sysenv/port"
    assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port"

-    # getraw
-    assert config.getraw("env:base", "targets") == ""
-    assert config.getraw("env:extra_1", "lib_deps") == "574"
-    assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1"
-
-    # get
-    assert config.get("custom", "debug_flags") == "-D DEBUG=1"
-    assert config.get("env:extra_1", "build_flags") == [
-        "-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2"
+    # env var as option
+    assert config.options(env="test_extends") == [
+        "extends", "build_flags", "lib_ldf_mode", "lib_compat_mode",
+        "monitor_speed", "lib_deps", "lib_ignore", "upload_port"
    ]
-    assert config.get("env:extra_2", "build_flags") == [
-        "-Og", "-DSYSENVDEPS1 -DSYSENVDEPS2"]
-    assert config.get("env:extra_2", "monitor_speed") == "115200"
-    assert config.get("env:base", "build_flags") == ([
-        "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2"
-    ])

-    # items
-    assert config.items("custom") == [
-        ("debug_flags", "-D DEBUG=1"),
-        ("lib_flags", "-lc -lm"),
-        ("extra_flags", "-L /usr/local/lib"),
-        ("lib_ignore", "LibIgnoreCustom")
-    ]  # yapf: disable
-    assert config.items(env="base") == [
-        ("build_flags", [
-            "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2"]),
-        ("targets", []),
-        ("monitor_speed", "115200"),
-        ("lib_deps", ["Lib1", "Lib2"]),
-        ("lib_ignore", ["LibIgnoreCustom"]),
-        ("upload_port", "/dev/sysenv/port")
-    ]  # yapf: disable
-    assert config.items(env="extra_1") == [
-        ("build_flags", ["-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2"]),
-        ("lib_deps", ["574"]),
-        ("monitor_speed", "115200"),
-        ("lib_ignore", ["LibIgnoreCustom"]),
-        ("upload_port", "/dev/sysenv/port")
-    ]  # yapf: disable
-    assert config.items(env="extra_2") == [
-        ("build_flags", ["-Og", "-DSYSENVDEPS1 -DSYSENVDEPS2"]),
-        ("lib_ignore", ["LibIgnoreCustom", "Lib3"]),
-        ("upload_port", "/dev/extra_2/port"),
-        ("monitor_speed", "115200"),
-        ("lib_deps", ["Lib1", "Lib2"])
-    ]  # yapf: disable
+    # sysenv
+    os.environ["PLATFORMIO_HOME_DIR"] = "/custom/core/dir"
+    assert config.get("platformio", "core_dir") == "/custom/core/dir"

    # cleanup system environment variables
    del os.environ["PLATFORMIO_BUILD_FLAGS"]
    del os.environ["PLATFORMIO_UPLOAD_PORT"]
    del os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"]
-
-
-def test_empty_config():
-    config = ProjectConfig("/non/existing/platformio.ini")
-
-    # unknown section
-    with pytest.raises(ConfigParser.NoSectionError):
-        config.getraw("unknown_section", "unknown_option")
-
-    assert config.sections() == []
-    assert config.get("section", "option") is None
-    assert config.get("section", "option", 13) == 13
-
-    # sysenv
-    os.environ["PLATFORMIO_HOME_DIR"] = "/custom/core/dir"
-    assert config.get("platformio", "core_dir") == "/custom/core/dir"
    del os.environ["PLATFORMIO_HOME_DIR"]
+
+
+def test_getraw_value(config):
+    # unknown option
+    with pytest.raises(ConfigParser.NoOptionError):
+        config.getraw("custom", "unknown_option")
+    # unknown option even if exists in [env]
+    with pytest.raises(ConfigParser.NoOptionError):
+        config.getraw("platformio", "monitor_speed")
+
+    # known
+    assert config.getraw("env:base", "targets") == ""
+    assert config.getraw("env:extra_1", "lib_deps") == "574"
+    assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1"
+
+    # extended
+    assert config.getraw("env:test_extends", "lib_ldf_mode") == "chain+"
+    assert config.getraw("env", "monitor_speed") == "115200"
+    assert config.getraw("env:test_extends", "monitor_speed") == "9600"
+
+
+def test_get_value(config):
+    assert config.get("custom", "debug_flags") == "-D DEBUG=1"
+    assert config.get("env:extra_1", "build_flags") == ["-lc -lm -D DEBUG=1"]
+    assert config.get("env:extra_2", "build_flags") == ["-Og"]
+    assert config.get("env:extra_2", "monitor_speed") == "115200"
+    assert config.get("env:base", "build_flags") == ["-D DEBUG=1"]
+
+
+def test_items(config):
+    assert config.items("custom") == [
+        ("debug_flags", "-D DEBUG=1"),
+        ("lib_flags", "-lc -lm"),
+        ("extra_flags", None),
+        ("lib_ignore", "LibIgnoreCustom")
+    ]  # yapf: disable
+    assert config.items(env="base") == [
+        ("build_flags", ["-D DEBUG=1"]),
+        ("targets", []),
+        ("monitor_speed", "115200"),
+        ("lib_deps", ["Lib1", "Lib2"]),
+        ("lib_ignore", ["LibIgnoreCustom"]),
+    ]  # yapf: disable
+    assert config.items(env="extra_1") == [
+        ("build_flags", ["-lc -lm -D DEBUG=1"]),
+        ("lib_deps", ["574"]),
+        ("monitor_speed", "115200"),
+        ("lib_ignore", ["LibIgnoreCustom"]),
+    ]  # yapf: disable
+    assert config.items(env="extra_2") == [
+        ("build_flags", ["-Og"]),
+        ("lib_ignore", ["LibIgnoreCustom", "Lib3"]),
+        ("upload_port", "/dev/extra_2/port"),
+        ("monitor_speed", "115200"),
+        ("lib_deps", ["Lib1", "Lib2"])
+    ]  # yapf: disable
+    assert config.items(env="test_extends") == [
+        ("extends", ["strict_settings"]),
+        ("build_flags", ["-D RELEASE"]),
+        ("lib_ldf_mode", "chain+"),
+        ("lib_compat_mode", "strict"),
+        ("monitor_speed", "9600"),
+        ("lib_deps", ["Lib1", "Lib2"]),
+        ("lib_ignore", ["LibIgnoreCustom"])
+    ]  # yapf: disable