From 326eb4a681c2bc5c7048d8537cd0871d4eab0ef4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 14 Mar 2019 20:12:18 +0200 Subject: [PATCH 1/9] Fix "FileExistsError" when "platformio ci" command is used in pair with "--keep-build-dir" option --- HISTORY.rst | 5 +++++ platformio/commands/ci.py | 4 ++-- tests/commands/test_ci.py | 32 +++++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 84ee3562..32e2aa8d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,11 @@ Release Notes PlatformIO 3.0 -------------- +3.6.6 (2019-??-??) +~~~~~~~~~~~~~~~~~~ + +* Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option + 3.6.5 (2019-03-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 0b5b3e4e..13224fac 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -55,7 +55,6 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument "--build-dir", default=mkdtemp, type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, @@ -134,7 +133,8 @@ def _copy_contents(dst_dir, contents): if dst_dir_name == "src" and len(items['dirs']) == 1: copytree(list(items['dirs']).pop(), dst_dir, symlinks=True) else: - makedirs(dst_dir) + if not isdir(dst_dir): + makedirs(dst_dir) for d in items['dirs']: copytree(d, join(dst_dir, basename(d)), symlinks=True) diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index ec4077ba..67ff4420 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import join +from os.path import isfile, join from platformio.commands.ci import cli as cmd_ci from platformio.commands.lib import cli as cmd_lib @@ -32,6 +32,36 @@ def test_ci_boards(clirunner, validate_cliresult): validate_cliresult(result) +def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): + build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) + result = clirunner.invoke(cmd_ci, [ + join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", + "--build-dir", build_dir + ]) + validate_cliresult(result) + assert not isfile(join(build_dir, "platformio.ini")) + + +def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): + build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) + result = clirunner.invoke(cmd_ci, [ + join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", + "--build-dir", build_dir, "--keep-build-dir" + ]) + validate_cliresult(result) + assert isfile(join(build_dir, "platformio.ini")) + + # 2nd attempt + result = clirunner.invoke(cmd_ci, [ + join("examples", "wiring-blink", "src", "main.cpp"), "-b", "metro", + "--build-dir", build_dir, "--keep-build-dir" + ]) + validate_cliresult(result) + + assert "board: uno" in result.output + assert "board: metro" in result.output + + def test_ci_project_conf(clirunner, validate_cliresult): project_dir = join("examples", "wiring-blink") result = clirunner.invoke(cmd_ci, [ From 86e464110174796e8ba6362224012d5424f8ab74 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 14 Mar 2019 20:16:15 +0200 Subject: [PATCH 2/9] Bump version to 3.6.6a1 --- docs | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 768ccfd2..7349afe0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 768ccfd2b4a3e508addb48f7f6d1fdebad5d946c +Subproject commit 7349afe05572deec8cdf28378d8339519af9615d diff --git a/platformio/__init__.py b/platformio/__init__.py index d0f05a30..93a24702 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, 5) +VERSION = (3, 6, "6a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From d7e2d05f60fb2c10770eb6d0c364d84c3d5cc5d4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Mar 2019 18:49:20 +0200 Subject: [PATCH 3/9] Fix error with conflicting declaration of a prototype (Arduino sketch preprocessor) --- HISTORY.rst | 1 + platformio/builder/tools/piomisc.py | 13 ++++++++++--- tests/ino2cpp/basic/basic.ino | 8 ++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 32e2aa8d..f59d36aa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,7 @@ PlatformIO 3.0 3.6.6 (2019-??-??) ~~~~~~~~~~~~~~~~~~ +* Fixed error with conflicting declaration of a prototype (Arduino sketch preprocessor) * Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option 3.6.5 (2019-03-07) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index be030f39..c2173bd2 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -33,10 +33,10 @@ class InoToCPPConverter(object): PROTOTYPE_RE = re.compile( r"""^( (?:template\<.*\>\s*)? # template - ([a-z_\d\&]+\*?\s+){1,2} # return type + ([a-z_\d\&]+\*?\s+){1,2} # return type ([a-z_\d]+\s*) # name of prototype \([a-z_,\.\*\&\[\]\s\d]*\) # arguments - )\s*\{ # must end with { + )\s*(\{|;) # must end with `{` or `;` """, re.X | re.M | re.I) DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I) PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)" @@ -162,7 +162,14 @@ class InoToCPPConverter(object): if not prototypes: return contents - prototype_names = set([m.group(3).strip() for m in prototypes]) + # skip already declared prototypes + declared = set( + m.group(1).strip() for m in prototypes if m.group(4) == ";") + prototypes = [ + m for m in prototypes if m.group(1).strip() not in declared + ] + + prototype_names = set(m.group(3).strip() for m in prototypes) split_pos = prototypes[0].start() match_ptrs = re.search( self.PROTOPTRS_TPLRE % ("|".join(prototype_names)), diff --git a/tests/ino2cpp/basic/basic.ino b/tests/ino2cpp/basic/basic.ino index 0c5d7511..fcec8edd 100644 --- a/tests/ino2cpp/basic/basic.ino +++ b/tests/ino2cpp/basic/basic.ino @@ -49,4 +49,12 @@ void fooCallback(){ } +extern "C" { +void some_extern(const char *fmt, ...); +}; + +void some_extern(const char *fmt, ...) { + +} + // юнікод From 9c1cc97776230c0dac2877b3967a6a30252a22be Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Mar 2019 21:16:02 +0200 Subject: [PATCH 4/9] Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode // Resolve #2243 --- HISTORY.rst | 4 +++- platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f59d36aa..9e9e5a44 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,7 +7,9 @@ PlatformIO 3.0 3.6.6 (2019-??-??) ~~~~~~~~~~~~~~~~~~ -* Fixed error with conflicting declaration of a prototype (Arduino sketch preprocessor) +* Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode + (`issue #2243 `_) +* Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor) * Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option 3.6.5 (2019-03-07) diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index 5f0343de..75d77ffd 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -1,7 +1,9 @@ { - "!!! WARNING !!!": "PLEASE DO NOT MODIFY THIS FILE! USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags", "configurations": [ { + "name": "!!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags" + }, + { % import platform % from os.path import commonprefix, dirname % From 06fe557a2069b3ebe1aac6602b54ef48ab7a3d73 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Mar 2019 21:24:43 +0200 Subject: [PATCH 5/9] Fix "SameFileError" when CI is used in pair with --keep-build-dir // Resolve #2227 --- platformio/commands/ci.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 13224fac..8fb1bf89 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -145,7 +145,10 @@ def _copy_contents(dst_dir, contents): dst_dir = join(dst_dir, mkdtemp(dir=dst_dir)) for f in items['files']: - copyfile(f, join(dst_dir, basename(f))) + dst_file = join(dst_dir, basename(f)) + if f == dst_file: + continue + copyfile(f, dst_file) def _exclude_contents(dst_dir, patterns): From e33b0fe291bcd3e5d67a831f2390f802fc53e716 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Mar 2019 14:24:43 +0200 Subject: [PATCH 6/9] Fix an issue when PlatformIO Build System does not pick up "mbed_lib.json" files from libraries // Resolve #2164 --- HISTORY.rst | 2 ++ platformio/builder/tools/piolib.py | 57 +++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9e9e5a44..3dccab4c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ PlatformIO 3.0 * Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode (`issue #2243 `_) +* Fixed an issue when PlatformIO Build System does not pick up "mbed_lib.json" files from libraries + (`issue #2164 `_) * Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor) * Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 17cefb0c..c6d1f45f 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -70,6 +70,8 @@ class LibBuilderFactory(object): # check source files for root, _, files in os.walk(path, followlinks=True): + if "mbed_lib.json" in files: + return ["mbed"] for fname in files: if not env.IsFileWithExt( fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT): @@ -260,7 +262,6 @@ class LibBuilderBase(object): def process_extra_options(self): with util.cd(self.path): - self.env.ProcessUnFlags(self.build_unflags) self.env.ProcessFlags(self.build_flags) if self.extra_script: self.env.SConscriptChdir(1) @@ -270,6 +271,7 @@ class LibBuilderBase(object): "env": self.env, "pio_lib_builder": self }) + self.env.ProcessUnFlags(self.build_unflags) def process_dependencies(self): if not self.dependencies: @@ -588,6 +590,59 @@ class MbedLibBuilder(LibBuilderBase): def is_frameworks_compatible(self, frameworks): return util.items_in_list(frameworks, ["mbed"]) + @property + def build_flags(self): + return self._mbed_lib_json_to_build_flags() + + def _mbed_lib_json_to_build_flags(self): # pylint: disable=too-many-locals + json_files = [ + join(root, "mbed_lib.json") + for root, _, files in os.walk(self.path) + if "mbed_lib.json" in files + ] + if not json_files: + return None + + build_flags = [] + cppdefines = str(self.env.Flatten(self.env.subst("$CPPDEFINES"))) + for p in json_files: + manifest = util.load_json(p) + + # default macros + build_flags.extend(["-D" + m for m in manifest.get("macros", [])]) + + macros = {} + # configuration items + for key, options in manifest.get("config", {}).items(): + if "value" not in options: + continue + macros[key] = dict( + name=options.get("macro_name"), value=options.get("value")) + # overrode items per target + for target, options in manifest.get("target_overrides", + {}).items(): + if target != "*" and "TARGET_" + target not in cppdefines: + continue + build_flags.extend( + ["-D" + m for m in options.get("target.macros_add", [])]) + for key, value in options.items(): + if not key.startswith("target.") and key in macros: + macros[key]['value'] = value + for key, macro in macros.items(): + name = macro['name'] + value = macro['value'] + if not name: + name = key + if "." not in name: + name = manifest.get("name") + "." + name + name = re.sub(r"[^a-z\d]+", "_", name, flags=re.I).upper() + name = "MBED_CONF_" + name + if isinstance(value, bool): + value = 1 if value else 0 + build_flags.append("-D%s=%s" % (name, value)) + + return build_flags + class PlatformIOLibBuilder(LibBuilderBase): From 00b162608ec8c3ab0b34cca791ac0e8da27bca56 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Mar 2019 17:44:01 +0200 Subject: [PATCH 7/9] Fix an issue with incorrect order of project "include" and "src" paths in ``CPPPATH`` // Resolve #1914 --- HISTORY.rst | 2 ++ platformio/builder/tools/piolib.py | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3dccab4c..f78ece9f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,8 @@ PlatformIO 3.0 (`issue #2164 `_) * Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor) * Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option +* Fixed an issue with incorrect order of project "include" and "src" paths in ``CPPPATH`` + (`issue #1914 `_) 3.6.5 (2019-03-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index c6d1f45f..635f5920 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -177,10 +177,11 @@ class LibBuilderBase(object): if isdir(join(self.path, "src")) else self.path) def get_include_dirs(self): - items = [self.src_dir] + items = [] include_dir = self.include_dir if include_dir and include_dir not in items: items.append(include_dir) + items.append(self.src_dir) return items @property @@ -756,10 +757,11 @@ class ProjectAsLibBuilder(LibBuilderBase): return self.env.subst("$PROJECTSRC_DIR") def get_include_dirs(self): - include_dirs = LibBuilderBase.get_include_dirs(self) + include_dirs = [] project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR") if isdir(project_include_dir): include_dirs.append(project_include_dir) + include_dirs.extend(LibBuilderBase.get_include_dirs(self)) return include_dirs def get_search_files(self): @@ -827,8 +829,9 @@ class ProjectAsLibBuilder(LibBuilderBase): def build(self): self._is_built = True # do not build Project now + result = LibBuilderBase.build(self) self.env.PrependUnique(CPPPATH=self.get_include_dirs()) - return LibBuilderBase.build(self) + return result def GetLibBuilders(env): # pylint: disable=too-many-branches From 760499c095aa2a15e051d24b642744ea26f20552 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Mar 2019 23:54:53 +0200 Subject: [PATCH 8/9] Append configuration settings from "mbed_lib.json" to "mbed_config.h" // Resolve #2164 --- platformio/builder/tools/piolib.py | 136 ++++++++++++++++++++--------- 1 file changed, 94 insertions(+), 42 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 635f5920..c628f50c 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=no-member, no-self-use, unused-argument +# pylint: disable=no-member, no-self-use, unused-argument, too-many-lines # pylint: disable=too-many-instance-attributes, too-many-public-methods from __future__ import absolute_import @@ -591,58 +591,110 @@ class MbedLibBuilder(LibBuilderBase): def is_frameworks_compatible(self, frameworks): return util.items_in_list(frameworks, ["mbed"]) - @property - def build_flags(self): - return self._mbed_lib_json_to_build_flags() + def process_extra_options(self): + self._process_mbed_lib_confs() + return super(MbedLibBuilder, self).process_extra_options() - def _mbed_lib_json_to_build_flags(self): # pylint: disable=too-many-locals - json_files = [ + def _process_mbed_lib_confs(self): + mbed_lib_paths = [ join(root, "mbed_lib.json") for root, _, files in os.walk(self.path) if "mbed_lib.json" in files ] - if not json_files: + if not mbed_lib_paths: return None - build_flags = [] + mbed_config_path = None + for p in self.env.get("CPPPATH"): + mbed_config_path = join(self.env.subst(p), "mbed_config.h") + if isfile(mbed_config_path): + break + else: + mbed_config_path = None + if not mbed_config_path: + return None + + macros = {} + for mbed_lib_path in mbed_lib_paths: + macros.update(self._mbed_lib_conf_parse_macros(mbed_lib_path)) + + self._mbed_conf_append_macros(mbed_config_path, macros) + return True + + @staticmethod + def _mbed_normalize_macro(macro): + name = macro + value = None + if "=" in macro: + name, value = macro.split("=", 1) + return dict(name=name, value=value) + + def _mbed_lib_conf_parse_macros(self, mbed_lib_path): + macros = {} cppdefines = str(self.env.Flatten(self.env.subst("$CPPDEFINES"))) - for p in json_files: - manifest = util.load_json(p) + manifest = util.load_json(mbed_lib_path) - # default macros - build_flags.extend(["-D" + m for m in manifest.get("macros", [])]) + # default macros + for macro in manifest.get("macros", []): + macro = self._mbed_normalize_macro(macro) + macros[macro['name']] = macro - macros = {} - # configuration items - for key, options in manifest.get("config", {}).items(): - if "value" not in options: + # configuration items + for key, options in manifest.get("config", {}).items(): + if "value" not in options: + continue + macros[key] = dict( + name=options.get("macro_name"), value=options.get("value")) + + # overrode items per target + for target, options in manifest.get("target_overrides", {}).items(): + if target != "*" and "TARGET_" + target not in cppdefines: + continue + for macro in options.get("target.macros_add", []): + macro = self._mbed_normalize_macro(macro) + macros[macro['name']] = macro + for key, value in options.items(): + if not key.startswith("target.") and key in macros: + macros[key]['value'] = value + + # normalize macro names + for key, macro in macros.items(): + if not macro['name']: + macro['name'] = key + if "." not in macro['name']: + macro['name'] = "%s.%s" % (manifest.get("name"), + macro['name']) + macro['name'] = re.sub( + r"[^a-z\d]+", "_", macro['name'], flags=re.I).upper() + macro['name'] = "MBED_CONF_" + macro['name'] + if isinstance(macro['value'], bool): + macro['value'] = 1 if macro['value'] else 0 + + return {macro["name"]: macro["value"] for macro in macros.values()} + + def _mbed_conf_append_macros(self, mbed_config_path, macros): + lines = [] + with open(mbed_config_path) as fp: + for line in fp.readlines(): + line = line.strip() + if line == "#endif": + lines.append( + "// PlatformIO Library Dependency Finder (LDF)") + lines.extend([ + "#define %s %s" % (name, + value if value is not None else "") + for name, value in macros.items() + ]) + lines.append("") + if not line.startswith("#define"): + lines.append(line) continue - macros[key] = dict( - name=options.get("macro_name"), value=options.get("value")) - # overrode items per target - for target, options in manifest.get("target_overrides", - {}).items(): - if target != "*" and "TARGET_" + target not in cppdefines: - continue - build_flags.extend( - ["-D" + m for m in options.get("target.macros_add", [])]) - for key, value in options.items(): - if not key.startswith("target.") and key in macros: - macros[key]['value'] = value - for key, macro in macros.items(): - name = macro['name'] - value = macro['value'] - if not name: - name = key - if "." not in name: - name = manifest.get("name") + "." + name - name = re.sub(r"[^a-z\d]+", "_", name, flags=re.I).upper() - name = "MBED_CONF_" + name - if isinstance(value, bool): - value = 1 if value else 0 - build_flags.append("-D%s=%s" % (name, value)) - - return build_flags + tokens = line.split() + if len(tokens) < 2 or tokens[1] not in macros: + lines.append(line) + lines.append("") + with open(mbed_config_path, "w") as fp: + fp.write("\n".join(lines)) class PlatformIOLibBuilder(LibBuilderBase): From 8cd35fb5379c99624e24c1d8de26f9b8c181d347 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 29 Mar 2019 12:43:16 +0200 Subject: [PATCH 9/9] Bump version to 3.6.6 --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f78ece9f..f244ded7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,7 +4,7 @@ Release Notes PlatformIO 3.0 -------------- -3.6.6 (2019-??-??) +3.6.6 (2019-03-29) ~~~~~~~~~~~~~~~~~~ * Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode diff --git a/docs b/docs index 7349afe0..3a6c69a1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7349afe05572deec8cdf28378d8339519af9615d +Subproject commit 3a6c69a1ae227479321de75184aa8678104fc37f diff --git a/platformio/__init__.py b/platformio/__init__.py index 93a24702..6b913ff6 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "6a1") +VERSION = (3, 6, 6) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"