Merge branch 'release/v4.2.1'

HISTORY.rst

@@ -1,10 +1,21 @@
 Release Notes
 =============
 
-.. _release_notes_4_0:
+.. _release_notes_4:
+
+PlatformIO Core 4
+-----------------
+
+4.2.1 (2020-02-17)
+~~~~~~~~~~~~~~~~~~
+
+* Improved VSCode template with special ``forceInclude`` field for direct includes via ``-include`` flag (`issue #3379 <https://github.com/platformio/platformio-core/issues/3379>`_)
+* Improved support of PIO Home on card-sized PC (Raspberry Pi, etc.) (`issue #3313 <https://github.com/platformio/platformio-core/issues/3313>`_)
+* Froze "marshmallow" dependency to 2.X for Python 2 (`issue #3380 <https://github.com/platformio/platformio-core/issues/3380>`_)
+* Fixed "TypeError: unsupported operand type(s)" when system environment variable is used by project configuration parser (`issue #3377 <https://github.com/platformio/platformio-core/issues/3377>`_)
+* Fixed an issue when Library Dependency Finder (LDF) ignores custom "libLDFMode" and "libCompatMode" options in `library.json <http://docs.platformio.org/page/librarymanager/config.html>`__
+* Fixed an issue when generating of compilation database "compile_commands.json" does not work with Python 2.7 (`issue #3378 <https://github.com/platformio/platformio-core/issues/3378>`_)
 
-PlatformIO Core 4.0
--------------------
 
 4.2.0 (2020-02-12)
 ~~~~~~~~~~~~~~~~~~
@@ -182,8 +193,8 @@ PlatformIO Core 4.0
   - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules <http://docs.platformio.org/page/faq.html#platformio-udev-rules>`__ (`issue #2442 <https://github.com/platformio/platformio-core/issues/2442>`_)
   - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 <https://github.com/platformio/platformio-core/issues/2559>`_)
 
-PlatformIO Core 3.0
+PlatformIO Core 3
--------------------
+-----------------
 
 3.6.7 (2019-04-23)
 ~~~~~~~~~~~~~~~~~~
@@ -783,8 +794,8 @@ PlatformIO Core 3.0
   (`issue #742 <https://github.com/platformio/platformio-core/issues/742>`_)
 * Stopped supporting Python 2.6
 
-PlatformIO Core 2.0
+PlatformIO Core 2
---------------------
+-----------------
 
 2.11.2 (2016-08-02)
 ~~~~~~~~~~~~~~~~~~~
@@ -1569,8 +1580,8 @@ PlatformIO Core 2.0
 * Fixed bug with creating copies of source files
   (`issue #177 <https://github.com/platformio/platformio-core/issues/177>`_)
 
-PlatformIO Core 1.0
+PlatformIO Core 1
--------------------
+-----------------
 
 1.5.0 (2015-05-15)
 ~~~~~~~~~~~~~~~~~~
@@ -1760,8 +1771,8 @@ PlatformIO Core 1.0
   error (`issue #81 <https://github.com/platformio/platformio-core/issues/81>`_)
 * Several bug fixes, increased stability and performance improvements
 
-PlatformIO Core 0.0
+PlatformIO Core Preview
--------------------
+-----------------------
 
 0.10.2 (2015-01-06)
 ~~~~~~~~~~~~~~~~~~~

platformio/__init__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = (4, 2, 0)
+VERSION = (4, 2, 1)
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/builder/tools/compilation_db.py

@@ -25,6 +25,7 @@ import os
 import SCons
 
 from platformio.builder.tools.platformio import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT
+from platformio.proc import where_is_program
 
 # Implements the ability for SCons to emit a compilation database for the MongoDB project. See
 # http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation
@@ -49,7 +50,7 @@ class __CompilationDbNode(SCons.Node.Python.Value):
         self.Decider(changed_since_last_build_node)
 
 
-def changed_since_last_build_node(child, target, prev_ni, node):
+def changed_since_last_build_node(*args, **kwargs):
     """ Dummy decider to force always building"""
     return True
 
@@ -145,7 +146,6 @@ def ScanCompilationDb(node, env, path):
 
 
 def generate(env, **kwargs):
-
     static_obj, shared_obj = SCons.Tool.createObjBuilders(env)
 
     env["COMPILATIONDB_COMSTR"] = kwargs.get(
@@ -195,6 +195,14 @@ def generate(env, **kwargs):
     )
 
     def CompilationDatabase(env, target):
+        # Resolve absolute path of toolchain
+        for cmd in ("CC", "CXX", "AS"):
+            if cmd not in env:
+                continue
+            env[cmd] = where_is_program(
+                env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")
+            )
+
         result = env.__COMPILATIONDB_Database(target=target, source=[])
 
         env.AlwaysBuild(result)

platformio/builder/tools/piolib.py

@@ -727,22 +727,16 @@ class PlatformIOLibBuilder(LibBuilderBase):
     @property
     def lib_ldf_mode(self):
         return self.validate_ldf_mode(
-            self.env.GetProjectOption(
+            self._manifest.get("build", {}).get(
-                "lib_ldf_mode",
+                "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self)
-                self._manifest.get("build", {}).get(
-                    "libLDFMode", LibBuilderBase.lib_ldf_mode.fget(self)
-                ),
             )
         )
 
     @property
     def lib_compat_mode(self):
         return self.validate_compat_mode(
-            self.env.GetProjectOption(
+            self._manifest.get("build", {}).get(
-                "lib_compat_mode",
+                "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self)
-                self._manifest.get("build", {}).get(
-                    "libCompatMode", LibBuilderBase.lib_compat_mode.fget(self)
-                ),
             )
         )
 

platformio/commands/debug/server.py

@@ -58,7 +58,7 @@ class DebugServer(BaseProcess):
                 "\nCould not launch Debug Server '%s'. Please check that it "
                 "is installed and is included in a system PATH\n\n"
                 "See documentation or contact contact@platformio.org:\n"
-                "http://docs.platformio.org/page/plus/debugging.html\n"
+                "https://docs.platformio.org/page/plus/debugging.html\n"
                 % server_executable
             )
 

platformio/commands/home/command.py

@@ -22,7 +22,11 @@ import click
 
 from platformio import exception
 from platformio.compat import WINDOWS
-from platformio.managers.core import get_core_package_dir, inject_contrib_pysite
+from platformio.managers.core import (
+    build_contrib_pysite_deps,
+    get_core_package_dir,
+    inject_contrib_pysite,
+)
 
 
 @click.command("home", short_help="PIO Home")
@@ -50,7 +54,13 @@ def cli(port, host, no_open, shutdown_timeout):
 
     # import contrib modules
     inject_contrib_pysite()
-    from autobahn.twisted.resource import WebSocketResource
+
+    try:
+        from autobahn.twisted.resource import WebSocketResource
+    except:  # pylint: disable=bare-except
+        build_contrib_pysite_deps(get_core_package_dir("contrib-pysite"))
+        from autobahn.twisted.resource import WebSocketResource
+
     from twisted.internet import reactor
     from twisted.web import server
 

platformio/commands/run/helpers.py

@@ -36,7 +36,7 @@ def handle_legacy_libdeps(project_dir, config):
         "DEPRECATED! A legacy library storage `{0}` has been found in a "
         "project. \nPlease declare project dependencies in `platformio.ini`"
         " file using `lib_deps` option and remove `{0}` folder."
-        "\nMore details -> http://docs.platformio.org/page/projectconf/"
+        "\nMore details -> https://docs.platformio.org/page/projectconf/"
         "section_env_library.html#lib-deps".format(legacy_libdeps_dir),
         fg="yellow",
     )

platformio/exception.py

@@ -301,6 +301,6 @@ class TestDirNotExists(PlatformioException):
         "A test folder '{0}' does not exist.\nPlease create 'test' "
         "directory in project's root and put a test set.\n"
         "More details about Unit "
-        "Testing: http://docs.platformio.org/page/plus/"
+        "Testing: https://docs.platformio.org/page/plus/"
         "unit-testing.html"
     )

platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl

@@ -1,11 +1,8 @@
-{
+% import os
-    "configurations": [
-        {
-            "name": "!!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags"
-        },
-        {
 % import platform
-% from os.path import commonprefix, dirname, isdir
+% import re
+%
+% import click
 %
 % systype = platform.system().lower()
 %
@@ -17,38 +14,79 @@
 %   return " " in flag and systype == "windows"
 % end
 %
-% def _split_flags(flags):
+% def split_args(args_string):
+%   return click.parser.split_arg_string(to_unix_path(args_string))
+% end
+%
+% def filter_args(args, allowed, ignore=None):
+%   if not allowed:
+%     return []
+%   end
+%
+%   ignore = ignore or []
 %   result = []
 %   i = 0
-%   flags = flags.strip()
+%   length = len(args)
-%   while i < len(flags):
+%     while(i < length):
-%       current_arg = []
+%      if any(args[i].startswith(f) for f in allowed) and not any(
-%       while i < len(flags) and flags[i] != " ":
+%        args[i].startswith(f) for f in ignore):
-%         if flags[i] == '"':
+%        result.append(args[i])
-%           quotes_idx = flags.find('"', i + 1)
+%        if i + 1 < length and not args[i + 1].startswith("-"):
-%           current_arg.extend(flags[i + 1:quotes_idx])
+%          i += 1
-%           i = quotes_idx + 1
+%          result.append(args[i])
-%         else:
+%        end
-%           current_arg.append(flags[i])
-%           i = i + 1
-%         end
 %       end
-%       arg = "".join(current_arg)
+%      i += 1
-%       if arg.strip():
+%    end
-%         result.append(arg.strip())
+%    return result
-%       end
+% end
-%       i = i + 1
+%
+% def _find_abs_path(inc, inc_paths):
+%   for path in inc_paths:
+%     if os.path.isfile(os.path.join(path, inc)):
+%       return os.path.join(path, inc)
+%     end
+%   end
+%   return inc
+% end
+%
+% def _find_forced_includes(flags, inc_paths):
+%   result = []
+%   for f in flags:
+%     inc = ""
+%     if f.startswith("-include") and f.split("-include")[1].strip():
+%       inc = f.split("-include")[1].strip()
+%     elif not f.startswith("-"):
+%       inc = f
+%     end
+%     if inc:
+%       result.append(_find_abs_path(inc, inc_paths))
+%     end
 %   end
 %   return result
 % end
 %
 % cleaned_includes = []
 % for include in includes:
-%   if "toolchain-" not in dirname(commonprefix([include, cc_path])) and isdir(include):
+%   if "toolchain-" not in os.path.dirname(os.path.commonprefix(
+%       [include, cc_path])) and os.path.isdir(include):
 %     cleaned_includes.append(include)
 %   end
 % end
 %
+% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
+% cc_stds = STD_RE.findall(cc_flags)
+% cxx_stds = STD_RE.findall(cxx_flags)
+% cc_m_flags = split_args(cc_flags)
+% forced_includes = _find_forced_includes(
+%   filter_args(cc_m_flags, ["-include"]), cleaned_includes)
+%
+{
+    "configurations": [
+        {
+            "name": "!!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags"
+        },
+        {
 % if systype == "windows":
             "name": "Win32",
 % elif systype == "darwin":
@@ -79,21 +117,26 @@
                 ""
             ],
             "intelliSenseMode": "clang-x64",
-% import re
-% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
-% cc_stds = STD_RE.findall(cc_flags)
-% cxx_stds = STD_RE.findall(cxx_flags)
-%
 % if cc_stds:
             "cStandard": "c{{ cc_stds[-1] }}",
 % end
 % if cxx_stds:
             "cppStandard": "c++{{ cxx_stds[-1] }}",
+% end
+% if forced_includes:
+            "forcedInclude": [
+% for include in forced_includes:
+                "{{ include }}",
+% end
+                ""
+            ],
 % end
             "compilerPath": "{{ cc_path }}",
             "compilerArgs": [
-% for flag in [ '"%s"' % _escape(f) if _escape_required(f) else f for f in _split_flags(
+% for flag in [
-%       cc_flags) if f.startswith(("-m", "-i", "@"))]:
+%     '"%s"' % _escape(f) if _escape_required(f) else f
+%     for f in filter_args(cc_m_flags, ["-m", "-i", "@"], ["-include"])
+% ]:
                 "{{ flag }}",
 % end
                 ""

platformio/managers/core.py

@@ -12,12 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import subprocess
 import sys
 from os.path import dirname, join
 
-from platformio import __version__, exception, fs
+from platformio import __version__, exception, fs, util
 from platformio.compat import PY2, WINDOWS
 from platformio.managers.package import PackageManager
 from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
@@ -25,7 +26,7 @@ from platformio.project.config import ProjectConfig
 
 CORE_PACKAGES = {
     "contrib-piohome": "~3.1.0",
-    "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]),
+    "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
     "tool-pioplus": "^2.6.1",
     "tool-unity": "~1.20500.0",
     "tool-scons": "~2.20501.7" if PY2 else "~3.30102.0",
@@ -113,6 +114,79 @@ def inject_contrib_pysite():
     sys.path.insert(0, contrib_pysite_dir)
 
 
+def build_contrib_pysite_deps(target_dir):
+    if os.path.isdir(target_dir):
+        util.rmtree_(target_dir)
+    os.makedirs(target_dir)
+    with open(os.path.join(target_dir, "package.json"), "w") as fp:
+        json.dump(
+            dict(
+                name="contrib-pysite",
+                version="2.%d%d.0" % (sys.version_info.major, sys.version_info.minor),
+                system=util.get_systype(),
+            ),
+            fp,
+        )
+
+    pythonexe = get_pythonexe_path()
+    for dep in get_contrib_pysite_deps():
+        subprocess.call(
+            [
+                pythonexe,
+                "-m",
+                "pip",
+                "install",
+                "--no-cache-dir",
+                "--no-compile",
+                "-t",
+                target_dir,
+                dep,
+            ]
+        )
+    return True
+
+
+def get_contrib_pysite_deps():
+    sys_type = util.get_systype()
+    py_version = "%d%d" % (sys.version_info.major, sys.version_info.minor)
+
+    twisted_version = "19.7.0"
+    result = [
+        "twisted == %s" % twisted_version,
+        "autobahn == 19.10.1",
+        "json-rpc == 1.12.1",
+    ]
+
+    # twisted[tls], see setup.py for %twisted_version%
+    result.extend(
+        ["pyopenssl >= 16.0.0", "service_identity >= 18.1.0", "idna >= 0.6, != 2.3"]
+    )
+
+    # zeroconf
+    if sys.version_info.major < 3:
+        result.append(
+            "https://github.com/ivankravets/python-zeroconf/" "archive/pio-py27.zip"
+        )
+    else:
+        result.append("zeroconf == 0.23.0")
+
+    if "windows" in sys_type:
+        result.append("pypiwin32 == 223")
+        # workaround for twisted wheels
+        twisted_wheel = (
+            "https://download.lfd.uci.edu/pythonlibs/g5apjq5m/Twisted-"
+            "%s-cp%s-cp%sm-win%s.whl"
+            % (
+                twisted_version,
+                py_version,
+                py_version,
+                "_amd64" if "amd64" in sys_type else "32",
+            )
+        )
+        result[0] = twisted_wheel
+    return result
+
+
 def pioplus_call(args, **kwargs):
     if WINDOWS and sys.version_info < (2, 7, 6):
         raise exception.PlatformioException(

platformio/package/exception.py

@@ -41,5 +41,5 @@ class ManifestValidationError(ManifestException):
     def __str__(self):
         return (
             "Invalid manifest fields: %s. \nPlease check specification -> "
-            "http://docs.platformio.org/page/librarymanager/config.html" % self.messages
+            "htts://docs.platformio.org/page/librarymanager/config.html" % self.messages
         )

platformio/project/config.py

@@ -273,7 +273,9 @@ class ProjectConfigBase(object):
                     if envvar_value:
                         break
             if envvar_value and option_meta.multiple:
-                value += ("" if value == MISSING else "\n") + envvar_value
+                if value == MISSING:
+                    value = ""
+                value += ("\n" if value else "") + envvar_value
             elif envvar_value and value == MISSING:
                 value = envvar_value
 

setup.py

@@ -23,6 +23,8 @@ from platformio import (
     __url__,
     __version__,
 )
+from platformio.compat import PY2
+
 
 install_requires = [
     "bottle<0.13",
@@ -33,9 +35,10 @@ install_requires = [
     "semantic_version>=2.8.1,<3",
     "tabulate>=0.8.3,<1",
     "pyelftools>=0.25,<1",
-    "marshmallow>=2.20.5",
+    "marshmallow%s" % (">=2,<3" if PY2 else ">=2"),
 ]
 
+
 setup(
     name=__title__,
     version=__version__,

tests/test_projectconf.py

@@ -185,6 +185,7 @@ def test_sysenv_options(config):
     assert config.get("env:base", "upload_port") is None
     assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port"
     os.environ["PLATFORMIO_BUILD_FLAGS"] = "-DSYSENVDEPS1 -DSYSENVDEPS2"
+    os.environ["PLATFORMIO_BUILD_UNFLAGS"] = "-DREMOVE_MACRO"
     os.environ["PLATFORMIO_UPLOAD_PORT"] = "/dev/sysenv/port"
     os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] = "-L /usr/local/lib"
     assert config.get("custom", "extra_flags") == "-L /usr/local/lib"
@@ -194,6 +195,7 @@ def test_sysenv_options(config):
     ]
     assert config.get("env:base", "upload_port") == "/dev/sysenv/port"
     assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port"
+    assert config.get("env:base", "build_unflags") == ["-DREMOVE_MACRO"]
 
     # env var as option
     assert config.options(env="test_extends") == [
@@ -206,6 +208,7 @@ def test_sysenv_options(config):
         "lib_deps",
         "lib_ignore",
         "custom_builtin_option",
+        "build_unflags",
         "upload_port",
     ]
 
@@ -215,6 +218,7 @@ def test_sysenv_options(config):
 
     # cleanup system environment variables
     del os.environ["PLATFORMIO_BUILD_FLAGS"]
+    del os.environ["PLATFORMIO_BUILD_UNFLAGS"]
     del os.environ["PLATFORMIO_UPLOAD_PORT"]
     del os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"]
     del os.environ["PLATFORMIO_HOME_DIR"]