From 091901912397dc653a64659ff70b69263d029903 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Nov 2021 23:19:22 +0200 Subject: [PATCH 01/56] Bump version to 5.2.4a1 --- HISTORY.rst | 3 +++ docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ff5da7e2..168fc737 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,9 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +5.2.4 (2021-??-??) +~~~~~~~~~~~~~~~~~~ + 5.2.3 (2021-11-05) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index fed771ae..277e3337 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit fed771ae8d8cb6acfe3b19c2e3afd411537ca6f5 +Subproject commit 277e33379b59fa244b1b925488953cde2004486d diff --git a/platformio/__init__.py b/platformio/__init__.py index 61793cfd..7ecfc27c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, 3) +VERSION = (5, 2, "4a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 831a2582ed3e78958852991dc2a66a0d25319dd8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 Nov 2021 19:31:49 +0200 Subject: [PATCH 02/56] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 277e3337..5a7258e0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 277e33379b59fa244b1b925488953cde2004486d +Subproject commit 5a7258e0f6890265c8fa709a74fff7fd4d0bec9d From d3049a8d62febc5bbddcf7f68b228bece0250f5d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 Nov 2021 20:08:18 +0200 Subject: [PATCH 03/56] Fix test --- tests/package/test_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 5bd118e7..a42110e9 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -214,8 +214,8 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory): # test conflicted names lm = LibraryPackageManager(str(tmpdir_factory.mktemp("conflicted-storage"))) - lm.install("4@2.6.1", silent=True) - lm.install("5357@2.6.1", silent=True) + lm.install("z3t0/IRremote@2.6.1", silent=True) + lm.install("mbed-yuhki50/IRremote", silent=True) assert len(lm.get_installed()) == 2 # Tools From 2786bfbeb8b85896f2d369ca3c38cf9f31722090 Mon Sep 17 00:00:00 2001 From: Pedro Barreto Date: Tue, 9 Nov 2021 20:45:12 +0000 Subject: [PATCH 04/56] Escape spaces in CLion CMakeListsPrivate template - FIXES #4085 (#4105) This fix adds spaces to the regex substitutions on CMakeListsPrivate.txt add_definitions. Fixes #4102 --- platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl index c0edf6aa..a7a04ec2 100644 --- a/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl @@ -75,7 +75,7 @@ set(CMAKE_CXX_STANDARD {{ cxx_stds[-1] }}) if (CMAKE_BUILD_TYPE MATCHES "{{ env_name }}") % for define in defines: - add_definitions(-D{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}) + add_definitions(-D{{!re.sub(r"([\"\(\)\ #])", r"\\\1", define)}}) % end % for include in filter_includes(includes): @@ -99,7 +99,7 @@ endif() % for env, data in ide_data.items(): if (CMAKE_BUILD_TYPE MATCHES "{{ env }}") % for define in data["defines"]: - add_definitions(-D{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}) + add_definitions(-D{{!re.sub(r"([\"\(\)\ #])", r"\\\1", define)}}) % end % for include in filter_includes(data["includes"]): From 7d78e4a60aeac6195d839a6004ca78c64e64f6bb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Nov 2021 22:49:00 +0200 Subject: [PATCH 05/56] Fixed an issue with the CLion project generator when a macro contains a space // Resolve #4102 --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 168fc737..fceec618 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,8 @@ PlatformIO Core 5 5.2.4 (2021-??-??) ~~~~~~~~~~~~~~~~~~ +- Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 `_) + 5.2.3 (2021-11-05) ~~~~~~~~~~~~~~~~~~ From 001f075a4966c01a842c7260475dde7c60b78b13 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Nov 2021 22:49:21 +0200 Subject: [PATCH 06/56] Bump version to 5.2.4a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 7ecfc27c..dbae072d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "4a1") +VERSION = (5, 2, "4a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 4687665ff3706f02fae8cd1ea220ae502a4f6ec9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 12 Nov 2021 15:17:25 +0200 Subject: [PATCH 07/56] Improved support for projects located on a network share // Resolve #3417 , Resolve #3926 , Resolve #4102 --- HISTORY.rst | 1 + platformio/app.py | 2 +- platformio/builder/main.py | 25 ++++++++++--------- platformio/builder/tools/pioide.py | 12 ++++----- platformio/builder/tools/piolib.py | 10 ++++---- platformio/builder/tools/piomisc.py | 2 +- platformio/builder/tools/platformio.py | 4 +-- platformio/commands/check/defect.py | 2 +- platformio/commands/check/tools/base.py | 6 ++--- platformio/commands/ci.py | 4 +-- .../commands/home/rpc/handlers/project.py | 2 +- platformio/debug/process/gdb.py | 2 +- platformio/fs.py | 25 ++++++++++++++++--- platformio/package/lockfile.py | 2 +- platformio/package/manager/base.py | 4 +-- platformio/project/generator.py | 6 +++-- platformio/project/helpers.py | 2 +- platformio/project/options.py | 2 +- 18 files changed, 67 insertions(+), 46 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fceec618..144744df 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ PlatformIO Core 5 5.2.4 (2021-??-??) ~~~~~~~~~~~~~~~~~~ +- Improved support for projects located on a network share (`issue #3417 `_, `issue #3926 `_, `issue #4099 `_) - Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 `_) 5.2.3 (2021-11-05) diff --git a/platformio/app.py b/platformio/app.py index d6930a47..374f4014 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -31,7 +31,7 @@ from platformio.project.helpers import get_default_projects_dir def projects_dir_validate(projects_dir): assert os.path.isdir(projects_dir) - return os.path.realpath(projects_dir) + return os.path.abspath(projects_dir) DEFAULT_SETTINGS = { diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 055eaf56..ce79301b 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -128,15 +128,20 @@ env.Replace( ], ) -if ( - compat.IS_WINDOWS - and sys.version_info >= (3, 8) - and env["PROJECT_DIR"].startswith("\\\\") -): +if int(ARGUMENTS.get("ISATTY", 0)): + # pylint: disable=protected-access + click._compat.isatty = lambda stream: True + +if compat.IS_WINDOWS and sys.version_info >= (3, 8) and os.getcwd().startswith("\\\\"): + click.secho("!!! WARNING !!!\t\t" * 3, fg="red") click.secho( - "There is a known issue with Python 3.8+ and mapped network drives on " - "Windows.\nSee a solution at:\n" - "https://github.com/platformio/platformio-core/issues/3417", + "Your project is located on a mapped network drive but the " + "current command-line shell does not support the UNC paths.", + fg="yellow", + ) + click.secho( + "Please move your project to a physical drive or check this workaround: " + "https://bit.ly/3kuU5mP\n", fg="yellow", ) @@ -145,10 +150,6 @@ if env.subst("$BUILD_CACHE_DIR"): os.makedirs(env.subst("$BUILD_CACHE_DIR")) env.CacheDir("$BUILD_CACHE_DIR") -if int(ARGUMENTS.get("ISATTY", 0)): - # pylint: disable=protected-access - click._compat.isatty = lambda stream: True - is_clean_all = "cleanall" in COMMAND_LINE_TARGETS if env.GetOption("clean") or is_clean_all: env.PioClean(is_clean_all) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 3903f879..5a6c2851 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -32,14 +32,14 @@ def _dump_includes(env): env.subst("$PROJECT_SRC_DIR"), ] includes["build"].extend( - [os.path.realpath(env.subst(item)) for item in env.get("CPPPATH", [])] + [os.path.abspath(env.subst(item)) for item in env.get("CPPPATH", [])] ) # installed libs includes["compatlib"] = [] for lb in env.GetLibBuilders(): includes["compatlib"].extend( - [os.path.realpath(inc) for inc in lb.get_include_dirs()] + [os.path.abspath(inc) for inc in lb.get_include_dirs()] ) # includes from toolchains @@ -56,9 +56,7 @@ def _dump_includes(env): os.path.join(toolchain_dir, "*", "include*"), ] for g in toolchain_incglobs: - includes["toolchain"].extend( - [os.path.realpath(inc) for inc in glob.glob(g)] - ) + includes["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)]) # include Unity framework if there are tests in project includes["unity"] = [] @@ -132,7 +130,7 @@ def _dump_defines(env): def _get_svd_path(env): svd_path = env.GetProjectOption("debug_svd_path") if svd_path: - return os.path.realpath(svd_path) + return os.path.abspath(svd_path) if "BOARD" not in env: return None @@ -147,7 +145,7 @@ def _get_svd_path(env): # default file from ./platform/misc/svd folder p = env.PioPlatform() if os.path.isfile(os.path.join(p.get_dir(), "misc", "svd", svd_path)): - return os.path.realpath(os.path.join(p.get_dir(), "misc", "svd", svd_path)) + return os.path.abspath(os.path.join(p.get_dir(), "misc", "svd", svd_path)) return None diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 588238b8..bd6f35f4 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -125,7 +125,7 @@ class LibBuilderBase(object): def __init__(self, env, path, manifest=None, verbose=False): self.env = env.Clone() self.envorigin = env.Clone() - self.path = os.path.realpath(env.subst(path)) + self.path = os.path.abspath(env.subst(path)) self.verbose = verbose try: @@ -290,7 +290,7 @@ class LibBuilderBase(object): if self.extra_script: self.env.SConscriptChdir(1) self.env.SConscript( - os.path.realpath(self.extra_script), + os.path.abspath(self.extra_script), exports={"env": self.env, "pio_lib_builder": self}, ) self.env.ProcessUnFlags(self.build_unflags) @@ -750,14 +750,14 @@ class PlatformIOLibBuilder(LibBuilderBase): def include_dir(self): if "includeDir" in self._manifest.get("build", {}): with fs.cd(self.path): - return os.path.realpath(self._manifest.get("build").get("includeDir")) + return os.path.abspath(self._manifest.get("build").get("includeDir")) return LibBuilderBase.include_dir.fget(self) # pylint: disable=no-member @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): with fs.cd(self.path): - return os.path.realpath(self._manifest.get("build").get("srcDir")) + return os.path.abspath(self._manifest.get("build").get("srcDir")) return LibBuilderBase.src_dir.fget(self) # pylint: disable=no-member @property @@ -1024,7 +1024,7 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches found_incompat = False for storage_dir in env.GetLibSourceDirs(): - storage_dir = os.path.realpath(storage_dir) + storage_dir = os.path.abspath(storage_dir) if not os.path.isdir(storage_dir): continue for item in sorted(os.listdir(storage_dir)): diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 3551c62b..af703185 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -376,7 +376,7 @@ def GetExtraScripts(env, scope): if not items: return items with fs.cd(env.subst("$PROJECT_DIR")): - return [os.path.realpath(item) for item in items] + return [os.path.abspath(item) for item in items] def exists(_): diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index a9355c49..61274dc7 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -207,12 +207,12 @@ def ParseFlagsExtended(env, flags): # pylint: disable=too-many-branches for k in ("CPPPATH", "LIBPATH"): for i, p in enumerate(result.get(k, [])): if os.path.isdir(p): - result[k][i] = os.path.realpath(p) + result[k][i] = os.path.abspath(p) # fix relative path for "-include" for i, f in enumerate(result.get("CCFLAGS", [])): if isinstance(f, tuple) and f[0] == "-include": - result["CCFLAGS"][i] = (f[0], env.File(os.path.realpath(f[1].get_path()))) + result["CCFLAGS"][i] = (f[0], env.File(os.path.abspath(f[1].get_path()))) return result diff --git a/platformio/commands/check/defect.py b/platformio/commands/check/defect.py index f337b077..5e907d3e 100644 --- a/platformio/commands/check/defect.py +++ b/platformio/commands/check/defect.py @@ -86,7 +86,7 @@ class DefectItem(object): "severity": self.SEVERITY_LABELS[self.severity], "category": self.category, "message": self.message, - "file": os.path.realpath(self.file), + "file": os.path.abspath(self.file), "line": self.line, "column": self.column, "callstack": self.callstack, diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index da38c97e..d5328d1a 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -201,11 +201,11 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes def _add_file(path): if path.endswith(header_extensions): - result["headers"].append(os.path.realpath(path)) + result["headers"].append(os.path.abspath(path)) elif path.endswith(c_extension): - result["c"].append(os.path.realpath(path)) + result["c"].append(os.path.abspath(path)) elif path.endswith(cpp_extensions): - result["c++"].append(os.path.realpath(path)) + result["c++"].append(os.path.abspath(path)) for pattern in patterns: for item in glob.glob(pattern, recursive=True): diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 58d8fef7..050baa65 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -33,7 +33,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument for i, p in enumerate(value): if p.startswith("~"): value[i] = fs.expanduser(p) - value[i] = os.path.realpath(value[i]) + value[i] = os.path.abspath(value[i]) if not glob.glob(value[i], recursive=True): invalid_path = p break @@ -162,7 +162,7 @@ def _exclude_contents(dst_dir, patterns): for p in patterns: contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True) for path in contents: - path = os.path.realpath(path) + path = os.path.abspath(path) if os.path.isdir(path): fs.rmtree(path) elif os.path.isfile(path): diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 1c033f51..8418fa60 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -93,7 +93,7 @@ class ProjectRPC: # skip non existing folders and resolve full path for key in ("envLibdepsDirs", "libExtraDirs"): data[key] = [ - fs.expanduser(d) if d.startswith("~") else os.path.realpath(d) + fs.expanduser(d) if d.startswith("~") else os.path.abspath(d) for d in data[key] if os.path.isdir(d) ] diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index ed8483bb..1ff395f5 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -61,7 +61,7 @@ class GDBClientProcess(DebugClientProcess): def _get_data_dir(gdb_path): if "msp430" in gdb_path: return None - gdb_data_dir = os.path.realpath( + gdb_data_dir = os.path.abspath( os.path.join(os.path.dirname(gdb_path), "..", "share", "gdb") ) return gdb_data_dir if os.path.isdir(gdb_data_dir) else None diff --git a/platformio/fs.py b/platformio/fs.py index 258fe530..6acfc726 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -24,7 +24,7 @@ import sys import click -from platformio import exception +from platformio import exception, proc from platformio.compat import IS_WINDOWS @@ -41,7 +41,7 @@ class cd(object): def get_source_dir(): - curpath = os.path.realpath(__file__) + curpath = os.path.abspath(__file__) if not os.path.isfile(curpath): for p in sys.path: if os.path.isfile(os.path.join(p, __file__)): @@ -119,7 +119,7 @@ def ensure_udev_rules(): if not any(os.path.isfile(p) for p in installed_rules): raise exception.MissedUdevRules - origin_path = os.path.realpath( + origin_path = os.path.abspath( os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules") ) if not os.path.isfile(origin_path): @@ -181,6 +181,25 @@ def to_unix_path(path): return re.sub(r"[\\]+", "/", path) +def normalize_path(path): + path = os.path.abspath(path) + if not IS_WINDOWS or not path.startswith("\\\\"): + return path + try: + result = proc.exec_command(["net", "use"]) + if result["returncode"] != 0: + return path + share_re = re.compile(r"\s([A-Z]\:)\s+(\\\\[^\s]+)") + for line in result["out"].split("\n"): + share = share_re.search(line) + if not share: + continue + path = path.replace(share.group(2), share.group(1)) + except OSError: + pass + return path + + def expanduser(path): """ Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE diff --git a/platformio/package/lockfile.py b/platformio/package/lockfile.py index b04cd428..a24f59e7 100644 --- a/platformio/package/lockfile.py +++ b/platformio/package/lockfile.py @@ -48,7 +48,7 @@ class LockFile(object): def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY): self.timeout = timeout self.delay = delay - self._lock_path = os.path.realpath(path) + ".lock" + self._lock_path = os.path.abspath(path) + ".lock" self._fp = None def _lock(self): diff --git a/platformio/package/manager/base.py b/platformio/package/manager/base.py index 3664a72e..16409d7c 100644 --- a/platformio/package/manager/base.py +++ b/platformio/package/manager/base.py @@ -252,9 +252,9 @@ class BasePackageManager( # pylint: disable=too-many-public-methods # external "URL" mismatch if spec.external: # local folder mismatch - if os.path.realpath(spec.url) == os.path.realpath(pkg.path) or ( + if os.path.abspath(spec.url) == os.path.abspath(pkg.path) or ( spec.url.startswith("file://") - and os.path.realpath(pkg.path) == os.path.realpath(spec.url[7:]) + and os.path.abspath(pkg.path) == os.path.abspath(spec.url[7:]) ): return True if spec.url != pkg.metadata.spec.url: diff --git a/platformio/project/generator.py b/platformio/project/generator.py index adf54e10..31d3620b 100644 --- a/platformio/project/generator.py +++ b/platformio/project/generator.py @@ -82,7 +82,7 @@ class ProjectGenerator(object): "project_dir": self.project_dir, "original_env_name": self.original_env_name, "env_name": self.env_name, - "user_home_dir": os.path.realpath(fs.expanduser("~")), + "user_home_dir": os.path.abspath(fs.expanduser("~")), "platformio_path": sys.argv[0] if os.path.isfile(sys.argv[0]) else where_is_program("platformio"), @@ -125,7 +125,9 @@ class ProjectGenerator(object): with fs.cd(self.project_dir): for root, _, files in os.walk(self.config.get("platformio", "src_dir")): for f in files: - result.append(os.path.relpath(os.path.join(root, f))) + result.append( + os.path.relpath(os.path.join(os.path.realpath(root), f)) + ) return result def get_tpls(self): diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 4cf66470..2736cae8 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig def get_project_dir(): - return os.getcwd() + return fs.normalize_path(os.getcwd()) def is_platformio_project(project_dir=None): diff --git a/platformio/project/options.py b/platformio/project/options.py index d47cf461..d9103161 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -114,7 +114,7 @@ def validate_dir(path): path = fs.expanduser(path) if "$" in path: path = expand_dir_templates(path) - return os.path.realpath(path) + return fs.normalize_path(path) def validate_core_dir(path): From 3022cb6955e4f099a15c6b5ece53a48e9af436f4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 12 Nov 2021 15:17:55 +0200 Subject: [PATCH 08/56] Bump version to 5.2.4a3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index dbae072d..7c86004d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "4a2") +VERSION = (5, 2, "4a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 5e2c5c793f4a7009471c98c610b6d0b0bf9562b1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 15 Nov 2021 11:28:57 +0200 Subject: [PATCH 09/56] SPDX License List v3.15 --- platformio/package/manifest/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 416dccfd..fcd84296 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -253,7 +253,7 @@ class ManifestSchema(BaseSchema): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.14" + version = "3.15" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" "v%s/json/licenses.json" % version From 5cb5c9713ed8c9c254736293a87b6478f6a49e08 Mon Sep 17 00:00:00 2001 From: valeros Date: Mon, 15 Nov 2021 19:22:41 +0200 Subject: [PATCH 10/56] Wrap the path to PlatformIO core in the NetBeans project template This fixes a possible issue when the path to PlatformIO contains a whitespace Resolve #4096 --- HISTORY.rst | 1 + .../project/tpls/netbeans/nbproject/configurations.xml.tpl | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 144744df..f4159d22 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,7 @@ PlatformIO Core 5 - Improved support for projects located on a network share (`issue #3417 `_, `issue #3926 `_, `issue #4099 `_) - Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 `_) +- Fixed an issue with the NetBeans project generator when the path to PlatformIO contains a space (`issue #4096 `_) 5.2.3 (2021-11-05) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl b/platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl index b0b7e5c7..099e0e41 100644 --- a/platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl +++ b/platformio/project/tpls/netbeans/nbproject/configurations.xml.tpl @@ -30,8 +30,8 @@ . - {{platformio_path}} -f -c netbeans run - {{platformio_path}} -f -c netbeans run --target clean + "{{platformio_path}}" -f -c netbeans run + "{{platformio_path}}" -f -c netbeans run --target clean % cleaned_includes = filter_includes(includes) From 0343548f6ee41e8343ae64aa9a60ff94392e84e1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 18 Nov 2021 13:14:55 +0200 Subject: [PATCH 11/56] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 5a7258e0..7b3abdb6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5a7258e0f6890265c8fa709a74fff7fd4d0bec9d +Subproject commit 7b3abdb69625fcac959bbaa6a7aae0174afc16af From 1800c29b445d5c8f0bd1ec7dec94fabdc5718e12 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 18 Nov 2021 13:17:26 +0200 Subject: [PATCH 12/56] Upgraded build engine to the SCons 4.3 --- HISTORY.rst | 1 + platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index f4159d22..a5ea55aa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ PlatformIO Core 5 ~~~~~~~~~~~~~~~~~~ - Improved support for projects located on a network share (`issue #3417 `_, `issue #3926 `_, `issue #4099 `_) +- Upgraded build engine to the SCons 4.3 (`release notes `__) - Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 `_) - Fixed an issue with the NetBeans project generator when the path to PlatformIO contains a space (`issue #4096 `_) diff --git a/platformio/__init__.py b/platformio/__init__.py index 7c86004d..2aae84dc 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -50,7 +50,7 @@ __core_packages__ = { "contrib-piohome": "~3.4.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~4.40200.0", + "tool-scons": "~4.40300.0", "tool-cppcheck": "~1.260.0", "tool-clangtidy": "~1.120001.0", "tool-pvs-studio": "~7.14.0", From 507df1f5071da08b399f2cdfe27b774d20c072a7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 18 Nov 2021 13:31:49 +0200 Subject: [PATCH 13/56] Extend platform manifest test with a package owner --- platformio/package/manifest/parser.py | 11 ++++++++--- tests/package/test_manifest.py | 7 ++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index f3d35559..3598d516 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -658,9 +658,14 @@ class PlatformJsonManifestParser(BaseManifestParser): @staticmethod def _parse_dependencies(raw): - return [ - dict(name=name, version=opts.get("version")) for name, opts in raw.items() - ] + result = [] + for name, opts in raw.items(): + item = {"name": name} + for k in ("owner", "version"): + if k in opts: + item[k] = opts[k] + result.append(item) + return result class PackageJsonManifestParser(BaseManifestParser): diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 2cc64ddb..ff6803eb 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -576,6 +576,7 @@ def test_platform_json_schema(): "packages": { "toolchain-atmelavr": { "type": "toolchain", + "owner": "platformio", "version": "~1.50400.0" }, "framework-arduinoavr": { @@ -623,7 +624,11 @@ def test_platform_json_schema(): "dependencies": [ {"name": "framework-arduinoavr", "version": "~4.2.0"}, {"name": "tool-avrdude", "version": "~1.60300.0"}, - {"name": "toolchain-atmelavr", "version": "~1.50400.0"}, + { + "name": "toolchain-atmelavr", + "owner": "platformio", + "version": "~1.50400.0", + }, ], }, ) From 68243aa95b4ebd6ec37c3e4f3bd8bebcd0edb749 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 18 Nov 2021 17:55:35 +0200 Subject: [PATCH 14/56] Added support for a new "headers" field in "library.json" --- HISTORY.rst | 1 + docs | 2 +- platformio/package/manifest/parser.py | 17 +++++++++++++---- platformio/package/manifest/schema.py | 7 +++++++ tests/package/test_manifest.py | 7 +++++++ 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a5ea55aa..68e1f024 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ PlatformIO Core 5 5.2.4 (2021-??-??) ~~~~~~~~~~~~~~~~~~ +- Added support for a new ``headers`` field in `library.json `__ (declare a list of header files that can be included in a project source files using ``#include <...>`` directive) - Improved support for projects located on a network share (`issue #3417 `_, `issue #3926 `_, `issue #4099 `_) - Upgraded build engine to the SCons 4.3 (`release notes `__) - Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 `_) diff --git a/docs b/docs index 7b3abdb6..1e2679b0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7b3abdb69625fcac959bbaa6a7aae0174afc16af +Subproject commit 1e2679b0ef3b373d2d324cd89e7ac6a2a14dbfe7 diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 3598d516..78c4da1e 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -167,7 +167,7 @@ class BaseManifestParser(object): return self._data @staticmethod - def str_to_list(value, sep=",", lowercase=True): + def str_to_list(value, sep=",", lowercase=False): if isinstance(value, string_types): value = value.split(sep) assert isinstance(value, list) @@ -323,8 +323,10 @@ class LibraryJsonManifestParser(BaseManifestParser): # normalize Union[str, list] fields for k in ("keywords", "platforms", "frameworks"): if k in data: - data[k] = self.str_to_list(data[k], sep=",") + data[k] = self.str_to_list(data[k], sep=",", lowercase=True) + if "headers" in data: + data["headers"] = self.str_to_list(data["headers"], sep=",") if "authors" in data: data["authors"] = self._parse_authors(data["authors"]) if "platforms" in data: @@ -430,7 +432,9 @@ class ModuleJsonManifestParser(BaseManifestParser): if "dependencies" in data: data["dependencies"] = self._parse_dependencies(data["dependencies"]) if "keywords" in data: - data["keywords"] = self.str_to_list(data["keywords"], sep=",") + data["keywords"] = self.str_to_list( + data["keywords"], sep=",", lowercase=True + ) return data def _parse_authors(self, raw): @@ -480,6 +484,9 @@ class LibraryPropertiesManifestParser(BaseManifestParser): export=self._parse_export(), ) ) + if "includes" in data: + data["headers"] = self.str_to_list(data["includes"], ",") + del data["includes"] if "author" in data: data["authors"] = self._parse_authors(data) for key in ("author", "maintainer"): @@ -674,7 +681,9 @@ class PackageJsonManifestParser(BaseManifestParser): def parse(self, contents): data = json.loads(contents) if "keywords" in data: - data["keywords"] = self.str_to_list(data["keywords"], sep=",") + data["keywords"] = self.str_to_list( + data["keywords"], sep=",", lowercase=True + ) data = self._parse_system(data) data = self._parse_homepage(data) data = self._parse_repository(data) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index fcd84296..cedb84ad 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -209,6 +209,13 @@ class ManifestSchema(BaseSchema): ] ) ) + headers = StrictListField( + fields.Str( + validate=[ + validate.Length(min=1, max=255), + ] + ) + ) # platform.json specific title = fields.Str(validate=validate.Length(min=1, max=100)) diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index ff6803eb..18c6a355 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -29,6 +29,7 @@ def test_library_json_parser(): { "name": "TestPackage", "keywords": "kw1, KW2, kw3", + "headers": "include1.h, Include2.hpp", "platforms": ["atmelavr", "espressif"], "repository": { "type": "git", @@ -62,6 +63,7 @@ def test_library_json_parser(): }, "export": {"exclude": [".gitignore", "tests"], "include": ["mylib"]}, "keywords": ["kw1", "kw2", "kw3"], + "headers": ["include1.h", "Include2.hpp"], "homepage": "http://old.url.format", "build": {"flags": ["-DHELLO"]}, "dependencies": [ @@ -76,6 +78,7 @@ def test_library_json_parser(): contents = """ { "keywords": ["sound", "audio", "music", "SD", "card", "playback"], + "headers": ["include 1.h", "include Space.hpp"], "frameworks": "arduino", "platforms": "atmelavr", "export": { @@ -94,6 +97,7 @@ def test_library_json_parser(): raw_data, { "keywords": ["sound", "audio", "music", "sd", "card", "playback"], + "headers": ["include 1.h", "include Space.hpp"], "frameworks": ["arduino"], "export": {"exclude": ["audio_samples"]}, "platforms": ["atmelavr"], @@ -205,6 +209,7 @@ sentence=This is Arduino library customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third ignore_empty_field= +includes=Arduino.h, Arduino Space.hpp """ raw_data = parser.LibraryPropertiesManifestParser(contents).as_dict() raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) @@ -225,6 +230,7 @@ ignore_empty_field= {"name": "Maintainer Author", "maintainer": True}, ], "keywords": ["uncategorized"], + "headers": ["Arduino.h", "Arduino Space.hpp"], "customField": "Custom Value", "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third", "dependencies": [ @@ -530,6 +536,7 @@ includes=MozziGuts.h }, "platforms": ["*"], "frameworks": ["arduino"], + "headers": ["MozziGuts.h"], "export": { "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] }, From 9e0ded958ca4fe6a4dca9b15c128bd22d3603981 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 18 Nov 2021 17:56:18 +0200 Subject: [PATCH 15/56] Bump version to 5.2.4a4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 2aae84dc..c915aafe 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "4a3") +VERSION = (5, 2, "4a4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 8dc68a01fd663a68ad87b6616aa18299d96ae829 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Nov 2021 22:08:10 +0200 Subject: [PATCH 16/56] Do not print empty errors --- platformio/commands/home/rpc/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index c10fa176..eb5bd6a5 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -92,6 +92,6 @@ class WebSocketJSONRPCServer(WebSocketEndpoint): async def _handle_rpc(self, websocket, data): # pylint: disable=no-member response = await self.factory.manager.get_response_for_payload(data) - if response.error: + if response.error and response.error.data: click.secho("Error: %s" % response.error.data, fg="red", err=True) await websocket.send_text(self.factory.manager.serialize(response.body)) From 3f71067b677f8a7ca7f03a79ec3c031ecf107f9b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Nov 2021 22:08:57 +0200 Subject: [PATCH 17/56] Update zeroconf deps to 0.37.* --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 79bc7092..2b81a839 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ minimal_requirements = [ ] if not PY2: - minimal_requirements.append("zeroconf==0.36.*") + minimal_requirements.append("zeroconf==0.37.*") home_requirements = [ "aiofiles==0.7.*", From df2f1d10fd1e4f2b1834efb5fef19bc7b2933548 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 25 Nov 2021 22:19:01 +0200 Subject: [PATCH 18/56] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 1e2679b0..61b67f11 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1e2679b0ef3b373d2d324cd89e7ac6a2a14dbfe7 +Subproject commit 61b67f11788e257ac105cbb210aff22525c1d20b From 7219c9f806475cc7385e68c6d7a7a06443a0e4ab Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 25 Nov 2021 22:19:47 +0200 Subject: [PATCH 19/56] Ignore duplicated manifest values --- platformio/package/manifest/parser.py | 21 +++++++++++++-------- tests/package/test_manifest.py | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 78c4da1e..f317b0a3 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -167,7 +167,7 @@ class BaseManifestParser(object): return self._data @staticmethod - def str_to_list(value, sep=",", lowercase=False): + def str_to_list(value, sep=",", lowercase=False, unique=False): if isinstance(value, string_types): value = value.split(sep) assert isinstance(value, list) @@ -178,6 +178,8 @@ class BaseManifestParser(object): continue if lowercase: item = item.lower() + if unique and item in result: + continue result.append(item) return result @@ -323,10 +325,12 @@ class LibraryJsonManifestParser(BaseManifestParser): # normalize Union[str, list] fields for k in ("keywords", "platforms", "frameworks"): if k in data: - data[k] = self.str_to_list(data[k], sep=",", lowercase=True) + data[k] = self.str_to_list( + data[k], sep=",", lowercase=True, unique=True + ) if "headers" in data: - data["headers"] = self.str_to_list(data["headers"], sep=",") + data["headers"] = self.str_to_list(data["headers"], sep=",", unique=True) if "authors" in data: data["authors"] = self._parse_authors(data["authors"]) if "platforms" in data: @@ -370,7 +374,8 @@ class LibraryJsonManifestParser(BaseManifestParser): for item in raw: if item == "espressif": item = "espressif8266" - result.append(item) + if item not in result: + result.append(item) return result @staticmethod @@ -433,7 +438,7 @@ class ModuleJsonManifestParser(BaseManifestParser): data["dependencies"] = self._parse_dependencies(data["dependencies"]) if "keywords" in data: data["keywords"] = self.str_to_list( - data["keywords"], sep=",", lowercase=True + data["keywords"], sep=",", lowercase=True, unique=True ) return data @@ -485,7 +490,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): ) ) if "includes" in data: - data["headers"] = self.str_to_list(data["includes"], ",") + data["headers"] = self.str_to_list(data["includes"], sep=",", unique=True) del data["includes"] if "author" in data: data["authors"] = self._parse_authors(data) @@ -650,7 +655,7 @@ class PlatformJsonManifestParser(BaseManifestParser): def parse(self, contents): data = json.loads(contents) if "keywords" in data: - data["keywords"] = self.str_to_list(data["keywords"], sep=",") + data["keywords"] = self.str_to_list(data["keywords"], sep=",", unique=True) if "frameworks" in data: data["frameworks"] = self._parse_frameworks(data["frameworks"]) if "packages" in data: @@ -682,7 +687,7 @@ class PackageJsonManifestParser(BaseManifestParser): data = json.loads(contents) if "keywords" in data: data["keywords"] = self.str_to_list( - data["keywords"], sep=",", lowercase=True + data["keywords"], sep=",", lowercase=True, unique=True ) data = self._parse_system(data) data = self._parse_homepage(data) diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 18c6a355..e75868e1 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -28,7 +28,7 @@ def test_library_json_parser(): contents = """ { "name": "TestPackage", - "keywords": "kw1, KW2, kw3", + "keywords": "kw1, KW2, kw3, KW2", "headers": "include1.h, Include2.hpp", "platforms": ["atmelavr", "espressif"], "repository": { From f63d899c4220e6394392037e20c251c0b0a1c818 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 25 Nov 2021 22:35:44 +0200 Subject: [PATCH 20/56] Ignore duplicated manifest values --- platformio/package/manifest/parser.py | 2 +- tests/package/test_manifest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index f317b0a3..4be71944 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -557,7 +557,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): continue if arch == "*": return ["*"] - if arch in platforms_map: + if arch in platforms_map and platforms_map[arch] not in result: result.append(platforms_map[arch]) return result diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index e75868e1..94dd29f1 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -440,7 +440,7 @@ sentence=A library for monochrome TFTs and OLEDs paragraph=Supported display controller: SSD1306, SSD1309, SSD1322, SSD1325 category=Display url=https://github.com/olikraus/u8glib -architectures=avr,sam +architectures=avr,sam,samd depends=First Library (=2.0.0), Second Library (>=1.2.0), Third """ raw_data = parser.ManifestParserFactory.new( From 6753121a6a5f62145eb977a0fbc384ff06787267 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 26 Nov 2021 14:13:06 +0200 Subject: [PATCH 21/56] Better cleanup package manifest fields --- platformio/package/manifest/parser.py | 67 +++++++++++---------------- tests/package/test_manager.py | 2 +- tests/package/test_manifest.py | 11 +++-- 3 files changed, 36 insertions(+), 44 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 4be71944..9ea5f4f3 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -334,7 +334,7 @@ class LibraryJsonManifestParser(BaseManifestParser): if "authors" in data: data["authors"] = self._parse_authors(data["authors"]) if "platforms" in data: - data["platforms"] = self._parse_platforms(data["platforms"]) or None + data["platforms"] = self._fix_platforms(data["platforms"]) or None if "export" in data: data["export"] = self._parse_export(data["export"]) if "dependencies" in data: @@ -367,16 +367,11 @@ class LibraryJsonManifestParser(BaseManifestParser): return [self.cleanup_author(author) for author in raw] @staticmethod - def _parse_platforms(raw): - assert isinstance(raw, list) - result = [] - # renamed platforms - for item in raw: - if item == "espressif": - item = "espressif8266" - if item not in result: - result.append(item) - return result + def _fix_platforms(items): + assert isinstance(items, list) + if "espressif" in items: + items[items.index("espressif")] = "espressif8266" + return items @staticmethod def _parse_export(raw): @@ -485,7 +480,7 @@ class LibraryPropertiesManifestParser(BaseManifestParser): repository=repository or None, description=self._parse_description(data), platforms=self._parse_platforms(data) or ["*"], - keywords=self._parse_keywords(data), + keywords=self._parse_keywords(data) or None, export=self._parse_export(), ) ) @@ -527,18 +522,17 @@ class LibraryPropertiesManifestParser(BaseManifestParser): lines[0] += "." return " ".join(lines) - @staticmethod - def _parse_keywords(properties): - result = [] - for item in re.split(r"[\s/]+", properties.get("category", "uncategorized")): - item = item.strip() - if not item: - continue - result.append(item.lower()) - return result + def _parse_keywords(self, properties): + return self.str_to_list( + re.split( + r"[\s/]+", + properties.get("category", ""), + ), + lowercase=True, + unique=True, + ) - @staticmethod - def _parse_platforms(properties): + def _parse_platforms(self, properties): result = [] platforms_map = { "avr": "atmelavr", @@ -557,9 +551,9 @@ class LibraryPropertiesManifestParser(BaseManifestParser): continue if arch == "*": return ["*"] - if arch in platforms_map and platforms_map[arch] not in result: + if arch in platforms_map: result.append(platforms_map[arch]) - return result + return self.str_to_list(result, lowercase=True, unique=True) def _parse_authors(self, properties): if "author" not in properties: @@ -655,19 +649,17 @@ class PlatformJsonManifestParser(BaseManifestParser): def parse(self, contents): data = json.loads(contents) if "keywords" in data: - data["keywords"] = self.str_to_list(data["keywords"], sep=",", unique=True) - if "frameworks" in data: - data["frameworks"] = self._parse_frameworks(data["frameworks"]) + data["keywords"] = self.str_to_list( + data["keywords"], sep=",", lowercase=True, unique=True + ) + if "frameworks" in data and not isinstance(data["frameworks"], dict): + data["frameworks"] = self.str_to_list( + data["frameworks"].keys(), lowercase=True, unique=True + ) if "packages" in data: data["dependencies"] = self._parse_dependencies(data["packages"]) return data - @staticmethod - def _parse_frameworks(raw): - if not isinstance(raw, dict): - return None - return [name.lower() for name in raw.keys()] - @staticmethod def _parse_dependencies(raw): result = [] @@ -694,16 +686,13 @@ class PackageJsonManifestParser(BaseManifestParser): data = self._parse_repository(data) return data - @staticmethod - def _parse_system(data): + def _parse_system(self, data): if "system" not in data: return data if data["system"] in ("*", ["*"], "all"): del data["system"] return data - if not isinstance(data["system"], list): - data["system"] = [data["system"]] - data["system"] = [s.strip().lower() for s in data["system"]] + data["system"] = self.str_to_list(data["system"], lowercase=True, unique=True) return data @staticmethod diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index a42110e9..4dae6302 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -202,7 +202,7 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory): lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) assert len(lm.get_installed()) == 3 pkg = lm.get_package("AsyncTCP-esphome") - assert pkg.metadata.spec.owner == "ottowinter" + assert pkg.metadata.spec.owner == "esphome" assert not lm.get_package("non-existing-package") # mbed library assert lm.install("wolfSSL", silent=True) diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 94dd29f1..33bbfd95 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -206,6 +206,7 @@ version=1.2.3 author=SomeAuthor , Maintainer Author (nickname) maintainer=Maintainer Author (nickname) sentence=This is Arduino library +category=Signal Input/Output customField=Custom Value depends=First Library (=2.0.0), Second Library (>=1.2.0), Third ignore_empty_field= @@ -229,7 +230,8 @@ includes=Arduino.h, Arduino Space.hpp {"name": "SomeAuthor", "email": "info@author.com"}, {"name": "Maintainer Author", "maintainer": True}, ], - "keywords": ["uncategorized"], + "category": "Signal Input/Output", + "keywords": ["signal", "input", "output"], "headers": ["Arduino.h", "Arduino Space.hpp"], "customField": "Custom Value", "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third", @@ -297,6 +299,7 @@ maintainer=Rocket Scream Electronics assert data["authors"] == [ {"name": "Rocket Scream Electronics", "maintainer": True} ] + assert "keywords" not in data def test_library_json_schema(): @@ -559,7 +562,7 @@ def test_platform_json_schema(): "name": "atmelavr", "title": "Atmel AVR", "description": "Atmel AVR 8- and 32-bit MCUs deliver a unique combination of performance, power efficiency and design flexibility. Optimized to speed time to market-and easily adapt to new ones-they are based on the industrys most code-efficient architecture for C and assembly programming.", - "keywords": "arduino, atmel, avr", + "keywords": "arduino, atmel, avr, MCU", "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", "license": "Apache-2.0", "engines": { @@ -619,7 +622,7 @@ def test_platform_json_schema(): "on the industrys most code-efficient architecture for C and " "assembly programming." ), - "keywords": ["arduino", "atmel", "avr"], + "keywords": ["arduino", "atmel", "avr", "mcu"], "homepage": "http://www.atmel.com/products/microcontrollers/avr/default.aspx", "license": "Apache-2.0", "repository": { @@ -648,7 +651,7 @@ def test_package_json_schema(): "description": "SCons software construction tool", "keywords": "SCons, build", "homepage": "http://www.scons.org", - "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l"], + "system": ["linux_armv6l", "linux_armv7l", "linux_armv8l", "LINUX_ARMV7L"], "version": "3.30101.0" } """ From ea1c9dec12470d0493d3e9ec618aa2e73c92bf95 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 26 Nov 2021 14:21:06 +0200 Subject: [PATCH 22/56] Typo fix --- platformio/package/manifest/parser.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 9ea5f4f3..2433e9cb 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -652,9 +652,13 @@ class PlatformJsonManifestParser(BaseManifestParser): data["keywords"] = self.str_to_list( data["keywords"], sep=",", lowercase=True, unique=True ) - if "frameworks" in data and not isinstance(data["frameworks"], dict): - data["frameworks"] = self.str_to_list( - data["frameworks"].keys(), lowercase=True, unique=True + if "frameworks" in data: + data["frameworks"] = ( + self.str_to_list( + list(data["frameworks"].keys()), lowercase=True, unique=True + ) + if isinstance(data["frameworks"], dict) + else None ) if "packages" in data: data["dependencies"] = self._parse_dependencies(data["packages"]) From 85c582bc93ca5c8726e089fde820549f12740bce Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 27 Nov 2021 15:00:10 +0200 Subject: [PATCH 23/56] Use "/v3//search" endpoint when searching for packages in registry --- platformio/clients/registry.py | 2 +- tests/package/test_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 4bd2aaf4..3a45a48e 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -117,7 +117,7 @@ class RegistryClient(HTTPClient): if page: params["page"] = int(page) return self.fetch_json_data( - "get", "/v3/packages", params=params, cache_valid="1h" + "get", "/v3/search", params=params, cache_valid="1h" ) def get_package(self, type_, owner, name, version=None): diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 4dae6302..a42110e9 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -202,7 +202,7 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory): lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) assert len(lm.get_installed()) == 3 pkg = lm.get_package("AsyncTCP-esphome") - assert pkg.metadata.spec.owner == "esphome" + assert pkg.metadata.spec.owner == "ottowinter" assert not lm.get_package("non-existing-package") # mbed library assert lm.install("wolfSSL", silent=True) From 3828e6d15ec5627bdc7efdc1a99bf96494b1971a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 29 Nov 2021 14:31:38 +0200 Subject: [PATCH 24/56] Lock "cryptography" to RUST-less 3.3.2 version --- platformio/package/manager/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index e1776ae3..9daac321 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -218,7 +218,7 @@ def get_contrib_pysite_deps(): result.extend( [ # pyopenssl depends on it, use RUST-less version - "cryptography >= 3.3, < 35.0.0", + "cryptography == 3.3.2", "pyopenssl >= 16.0.0", "service_identity >= 18.1.0", "idna >= 0.6, != 2.3", From 7bffe3993d446b4c8739d6110babd10bcd0702fb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 29 Nov 2021 20:01:27 +0200 Subject: [PATCH 25/56] Update deps --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2b81a839..ac8c217e 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ if not PY2: minimal_requirements.append("zeroconf==0.37.*") home_requirements = [ - "aiofiles==0.7.*", + "aiofiles==0.8.*", "ajsonrpc==1.*", "starlette==0.17.*", "uvicorn==0.15.*", From 414a194c9dd1cd19808f34ccb09f01a9b0801563 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 29 Nov 2021 20:02:53 +0200 Subject: [PATCH 26/56] Do not claim that library.properties packages is compatible with any dev-platform if "architectures" field is not defined --- platformio/package/manifest/parser.py | 3 +-- tests/package/test_manager.py | 4 ++-- tests/package/test_manifest.py | 4 +--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 2433e9cb..4028c276 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -479,14 +479,13 @@ class LibraryPropertiesManifestParser(BaseManifestParser): homepage=homepage, repository=repository or None, description=self._parse_description(data), - platforms=self._parse_platforms(data) or ["*"], + platforms=self._parse_platforms(data) or None, keywords=self._parse_keywords(data) or None, export=self._parse_export(), ) ) if "includes" in data: data["headers"] = self.str_to_list(data["includes"], sep=",", unique=True) - del data["includes"] if "author" in data: data["authors"] = self._parse_authors(data) for key in ("author", "maintainer"): diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index a42110e9..01bad3df 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -199,10 +199,10 @@ def test_install_from_registry(isolated_pio_core, tmpdir_factory): # Libraries lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) # library with dependencies - lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) + lm.install("AsyncMqttClient-esphome @ 0.8.6", silent=True) assert len(lm.get_installed()) == 3 pkg = lm.get_package("AsyncTCP-esphome") - assert pkg.metadata.spec.owner == "ottowinter" + assert pkg.metadata.spec.owner == "esphome" assert not lm.get_package("non-existing-package") # mbed library assert lm.install("wolfSSL", silent=True) diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 33bbfd95..216adf5d 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -80,7 +80,6 @@ def test_library_json_parser(): "keywords": ["sound", "audio", "music", "SD", "card", "playback"], "headers": ["include 1.h", "include Space.hpp"], "frameworks": "arduino", - "platforms": "atmelavr", "export": { "exclude": "audio_samples" }, @@ -100,7 +99,6 @@ def test_library_json_parser(): "headers": ["include 1.h", "include Space.hpp"], "frameworks": ["arduino"], "export": {"exclude": ["audio_samples"]}, - "platforms": ["atmelavr"], "dependencies": [ {"name": "deps1", "version": "1.0.0"}, { @@ -221,7 +219,6 @@ includes=Arduino.h, Arduino Space.hpp "version": "1.2.3", "description": "This is Arduino library", "sentence": "This is Arduino library", - "platforms": ["*"], "frameworks": ["arduino"], "export": { "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] @@ -233,6 +230,7 @@ includes=Arduino.h, Arduino Space.hpp "category": "Signal Input/Output", "keywords": ["signal", "input", "output"], "headers": ["Arduino.h", "Arduino Space.hpp"], + "includes": "Arduino.h, Arduino Space.hpp", "customField": "Custom Value", "depends": "First Library (=2.0.0), Second Library (>=1.2.0), Third", "dependencies": [ From e40b251c06b52adf3af0663a7eaf5c2887b501ff Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Dec 2021 13:13:07 +0200 Subject: [PATCH 27/56] Fixed a bug when the system environment variable does not override a project configuration option // Resolve #4125 --- HISTORY.rst | 1 + platformio/project/config.py | 2 +- tests/test_projectconf.py | 14 +++++++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 68e1f024..86f150a8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ PlatformIO Core 5 - Upgraded build engine to the SCons 4.3 (`release notes `__) - Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 `_) - Fixed an issue with the NetBeans project generator when the path to PlatformIO contains a space (`issue #4096 `_) +- Fixed a bug when the system environment variable does not override a project configuration option (`issue #4125 `_) 5.2.3 (2021-11-05) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/project/config.py b/platformio/project/config.py index d930dfa7..492bacf2 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -271,7 +271,7 @@ class ProjectConfigBase(object): if value == MISSING: value = "" value += ("\n" if value else "") + envvar_value - elif envvar_value and value == MISSING: + elif envvar_value: value = envvar_value if value == MISSING: diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index da9b7732..350bdeaa 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -226,7 +226,7 @@ def test_sysenv_options(config): "-DSYSENVDEPS1 -DSYSENVDEPS2", ] assert config.get("env:base", "upload_port") == "/dev/sysenv/port" - assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" + assert config.get("env:extra_2", "upload_port") == "/dev/sysenv/port" assert config.get("env:base", "build_unflags") == ["-DREMOVE_MACRO"] # env var as option @@ -244,10 +244,16 @@ def test_sysenv_options(config): "upload_port", ] - # sysenv - custom_core_dir = os.path.join(os.getcwd(), "custom") + # sysenv dirs + custom_core_dir = os.path.join(os.getcwd(), "custom-core") + custom_src_dir = os.path.join(os.getcwd(), "custom-src") + custom_build_dir = os.path.join(os.getcwd(), "custom-build") os.environ["PLATFORMIO_HOME_DIR"] = custom_core_dir + os.environ["PLATFORMIO_SRC_DIR"] = custom_src_dir + os.environ["PLATFORMIO_BUILD_DIR"] = custom_build_dir assert config.get("platformio", "core_dir") == os.path.realpath(custom_core_dir) + assert config.get("platformio", "src_dir") == os.path.realpath(custom_src_dir) + assert config.get("platformio", "build_dir") == os.path.realpath(custom_build_dir) # cleanup system environment variables del os.environ["PLATFORMIO_BUILD_FLAGS"] @@ -255,6 +261,8 @@ def test_sysenv_options(config): del os.environ["PLATFORMIO_UPLOAD_PORT"] del os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] del os.environ["PLATFORMIO_HOME_DIR"] + del os.environ["PLATFORMIO_SRC_DIR"] + del os.environ["PLATFORMIO_BUILD_DIR"] def test_getraw_value(config): From 014090c40786410eb04526a3ae8eba9670d0e2b7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Dec 2021 14:19:54 +0200 Subject: [PATCH 28/56] Fixed an issue when referencing "*_dir" option from a custom project configuration environment // Resolve #4110 --- HISTORY.rst | 3 ++- platformio/project/config.py | 5 ++--- platformio/project/options.py | 15 +++++++-------- tests/test_projectconf.py | 17 +++++++++++++++-- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 86f150a8..caa8d74b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,7 +16,8 @@ PlatformIO Core 5 - Upgraded build engine to the SCons 4.3 (`release notes `__) - Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 `_) - Fixed an issue with the NetBeans project generator when the path to PlatformIO contains a space (`issue #4096 `_) -- Fixed a bug when the system environment variable does not override a project configuration option (`issue #4125 `_) +- Fixed an issue when the system environment variable does not override a project configuration option (`issue #4125 `_) +- Fixed an issue when referencing "*_dir" option from a custom project configuration environment (`issue #4110 `_) 5.2.3 (2021-11-05) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/project/config.py b/platformio/project/config.py index 492bacf2..8e4ec096 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -279,9 +279,6 @@ class ProjectConfigBase(object): if value == MISSING: return None - if option_meta.validate: - value = option_meta.validate(value) - return self._expand_interpolations(value) def _expand_interpolations(self, value): @@ -318,6 +315,8 @@ class ProjectConfigBase(object): if not option_meta: return value + if option_meta.validate: + value = option_meta.validate(value) if option_meta.multiple: value = self.parse_multi_values(value or []) try: diff --git a/platformio/project/options.py b/platformio/project/options.py index d9103161..c12679b9 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -117,14 +117,13 @@ def validate_dir(path): return fs.normalize_path(path) -def validate_core_dir(path): - default_dir = ProjectOptions["platformio.core_dir"].default - win_core_dir = None - if IS_WINDOWS and path == default_dir: +def get_default_core_dir(): + path = os.path.join(fs.expanduser("~"), ".platformio") + if IS_WINDOWS: win_core_dir = os.path.splitdrive(path)[0] + "\\.platformio" if os.path.isdir(win_core_dir): - path = win_core_dir - return validate_dir(path) + return win_core_dir + return path ProjectOptions = OrderedDict( @@ -169,8 +168,8 @@ ProjectOptions = OrderedDict( ), oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR", - default=os.path.join(fs.expanduser("~"), ".platformio"), - validate=validate_core_dir, + default=get_default_core_dir(), + validate=validate_dir, ), ConfigPlatformioOption( group="directory", diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 350bdeaa..b3e85863 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -26,7 +26,8 @@ from platformio.project.exception import InvalidProjectConfError, UnknownEnvName BASE_CONFIG = """ [platformio] env_default = base, extra_2 -build_dir = ~/tmp/pio-$PROJECT_HASH +src_dir = ${custom.src_dir} +build_dir = ${custom.build_dir} extra_configs = extra_envs.ini extra_debug.ini @@ -53,6 +54,8 @@ extends = strict_ldf, monitor_custom build_flags = -D RELEASE [custom] +src_dir = source +build_dir = ~/tmp/pio-$PROJECT_HASH debug_flags = -D RELEASE lib_flags = -lc -lm extra_flags = ${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS} @@ -297,6 +300,7 @@ def test_getraw_value(config): config.getraw("custom", "debug_server") == f"\n{packages_dir}/tool-openocd/openocd\n--help" ) + assert config.getraw("platformio", "build_dir") == "~/tmp/pio-$PROJECT_HASH" def test_get_value(config): @@ -327,10 +331,16 @@ def test_get_value(config): os.path.join(DEFAULT_CORE_DIR, "packages/tool-openocd/openocd"), "--help", ] + # test relative dir + assert config.get("platformio", "src_dir") == os.path.abspath( + os.path.join(os.getcwd(), "source") + ) def test_items(config): assert config.items("custom") == [ + ("src_dir", "source"), + ("build_dir", "~/tmp/pio-$PROJECT_HASH"), ("debug_flags", "-D DEBUG=1"), ("lib_flags", "-lc -lm"), ("extra_flags", ""), @@ -473,7 +483,8 @@ def test_dump(tmpdir_factory): ( "platformio", [ - ("build_dir", "~/tmp/pio-$PROJECT_HASH"), + ("src_dir", "${custom.src_dir}"), + ("build_dir", "${custom.build_dir}"), ("extra_configs", ["extra_envs.ini", "extra_debug.ini"]), ("default_envs", ["base", "extra_2"]), ], @@ -497,6 +508,8 @@ def test_dump(tmpdir_factory): ( "custom", [ + ("src_dir", "source"), + ("build_dir", "~/tmp/pio-$PROJECT_HASH"), ("debug_flags", "-D RELEASE"), ("lib_flags", "-lc -lm"), ("extra_flags", "${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS}"), From 6e8f60a27a5a1dc05162ad33fe3490677469b956 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Dec 2021 14:20:46 +0200 Subject: [PATCH 29/56] Bump version to 5.2.4b1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index c915aafe..705e2d3a 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "4a4") +VERSION = (5, 2, "4b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 90972e9ce07f6cd3fdee409d4c85c972e173e756 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Dec 2021 14:55:48 +0200 Subject: [PATCH 30/56] Sync docs --- HISTORY.rst | 2 +- docs | 2 +- examples | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index caa8d74b..fa9aa35d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,7 +17,7 @@ PlatformIO Core 5 - Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 `_) - Fixed an issue with the NetBeans project generator when the path to PlatformIO contains a space (`issue #4096 `_) - Fixed an issue when the system environment variable does not override a project configuration option (`issue #4125 `_) -- Fixed an issue when referencing "*_dir" option from a custom project configuration environment (`issue #4110 `_) +- Fixed an issue when referencing ``*_dir`` option from a custom project configuration environment (`issue #4110 `_) 5.2.3 (2021-11-05) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 61b67f11..ad3f9b62 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 61b67f11788e257ac105cbb210aff22525c1d20b +Subproject commit ad3f9b62cf80b679b6d11162a725f2bc3462297d diff --git a/examples b/examples index b4be3d3f..d722c23d 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit b4be3d3fa4e7789549765f405614ecf24ec53a24 +Subproject commit d722c23da47639dcc92e3c9b997df61b9ee1bfe3 From 04e9f38e0e495b8449f321f73d8b207789ffe394 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Dec 2021 15:06:58 +0200 Subject: [PATCH 31/56] Check for default core dir in run-time (solves issue with tests) --- platformio/project/config.py | 2 ++ platformio/project/options.py | 4 ++-- tests/test_projectconf.py | 12 +++++++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 8e4ec096..7f2efb03 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -276,6 +276,8 @@ class ProjectConfigBase(object): if value == MISSING: value = default if default != MISSING else option_meta.default + if callable(value): + value = value() if value == MISSING: return None diff --git a/platformio/project/options.py b/platformio/project/options.py index c12679b9..ee5fd603 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -60,7 +60,7 @@ class ConfigOption(object): # pylint: disable=too-many-instance-attributes type="string", multiple=self.multiple, sysenvvar=self.sysenvvar, - default=self.default, + default=self.default() if callable(self.default) else self.default, ) if isinstance(self.type, click.ParamType): result["type"] = self.type.name @@ -168,7 +168,7 @@ ProjectOptions = OrderedDict( ), oldnames=["home_dir"], sysenvvar="PLATFORMIO_CORE_DIR", - default=get_default_core_dir(), + default=get_default_core_dir, validate=validate_dir, ), ConfigPlatformioOption( diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index b3e85863..0fada8c3 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -254,9 +254,15 @@ def test_sysenv_options(config): os.environ["PLATFORMIO_HOME_DIR"] = custom_core_dir os.environ["PLATFORMIO_SRC_DIR"] = custom_src_dir os.environ["PLATFORMIO_BUILD_DIR"] = custom_build_dir - assert config.get("platformio", "core_dir") == os.path.realpath(custom_core_dir) - assert config.get("platformio", "src_dir") == os.path.realpath(custom_src_dir) - assert config.get("platformio", "build_dir") == os.path.realpath(custom_build_dir) + assert os.path.realpath(config.get("platformio", "core_dir")) == os.path.realpath( + custom_core_dir + ) + assert os.path.realpath(config.get("platformio", "src_dir")) == os.path.realpath( + custom_src_dir + ) + assert os.path.realpath(config.get("platformio", "build_dir")) == os.path.realpath( + custom_build_dir + ) # cleanup system environment variables del os.environ["PLATFORMIO_BUILD_FLAGS"] From 55f8471aff8aa1adc44f36f3fccba571164f6594 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Dec 2021 16:34:05 +0200 Subject: [PATCH 32/56] Improved tab completion support for Bash, ZSH, and Fish shells // Resolve #4114 --- HISTORY.rst | 1 + docs | 2 +- platformio/__main__.py | 7 --- platformio/commands/system/command.py | 39 ++++------------ platformio/commands/system/completion.py | 58 +++++++++++++++--------- setup.py | 2 +- 6 files changed, 47 insertions(+), 62 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fa9aa35d..4a4bd78f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ PlatformIO Core 5 ~~~~~~~~~~~~~~~~~~ - Added support for a new ``headers`` field in `library.json `__ (declare a list of header files that can be included in a project source files using ``#include <...>`` directive) +- Improved tab completion support for Bash, ZSH, and Fish shells (`issue #4114 `_) - Improved support for projects located on a network share (`issue #3417 `_, `issue #3926 `_, `issue #4099 `_) - Upgraded build engine to the SCons 4.3 (`release notes `__) - Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 `_) diff --git a/docs b/docs index ad3f9b62..b6322c06 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ad3f9b62cf80b679b6d11162a725f2bc3462297d +Subproject commit b6322c062fd6ea58d8e6bca737a51833635d2173 diff --git a/platformio/__main__.py b/platformio/__main__.py index f7bdd61b..d3b765fb 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -24,13 +24,6 @@ from platformio import __version__, exception from platformio.commands import PlatformioCLI from platformio.compat import IS_CYGWIN, ensure_python3 -try: - import click_completion # pylint: disable=import-error - - click_completion.init() -except: # pylint: disable=bare-except - pass - @click.command( cls=PlatformioCLI, context_settings=dict(help_option_names=["-h", "--help"]) diff --git a/platformio/commands/system/command.py b/platformio/commands/system/command.py index c207d0ae..a75007d6 100644 --- a/platformio/commands/system/command.py +++ b/platformio/commands/system/command.py @@ -14,7 +14,6 @@ import json import platform -import subprocess import sys import click @@ -22,6 +21,7 @@ from tabulate import tabulate from platformio import __version__, compat, fs, proc, util from platformio.commands.system.completion import ( + ShellType, get_completion_install_path, install_completion_code, uninstall_completion_code, @@ -150,23 +150,11 @@ def system_prune(force, dry_run, cache, core_packages, platform_packages): @cli.group("completion", short_help="Shell completion support") def completion(): - # pylint: disable=import-error,import-outside-toplevel - try: - import click_completion # pylint: disable=unused-import,unused-variable - except ImportError: - click.echo("Installing dependent packages...") - subprocess.check_call( - [proc.get_pythonexe_path(), "-m", "pip", "install", "click-completion"], - ) + pass @completion.command("install", short_help="Install shell completion files/code") -@click.option( - "--shell", - default=None, - type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]), - help="The shell type, default=auto", -) +@click.argument("shell", type=click.Choice([t.value for t in ShellType])) @click.option( "--path", type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True), @@ -174,26 +162,18 @@ def completion(): "The standard installation path is used by default.", ) def completion_install(shell, path): - - import click_completion # pylint: disable=import-outside-toplevel,import-error - - shell = shell or click_completion.get_auto_shell() + shell = ShellType(shell) path = path or get_completion_install_path(shell) install_completion_code(shell, path) click.echo( "PlatformIO CLI completion has been installed for %s shell to %s \n" "Please restart a current shell session." - % (click.style(shell, fg="cyan"), click.style(path, fg="blue")) + % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue")) ) @completion.command("uninstall", short_help="Uninstall shell completion files/code") -@click.option( - "--shell", - default=None, - type=click.Choice(["fish", "bash", "zsh", "powershell", "auto"]), - help="The shell type, default=auto", -) +@click.argument("shell", type=click.Choice([t.value for t in ShellType])) @click.option( "--path", type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True), @@ -201,14 +181,11 @@ def completion_install(shell, path): "The standard installation path is used by default.", ) def completion_uninstall(shell, path): - - import click_completion # pylint: disable=import-outside-toplevel,import-error - - shell = shell or click_completion.get_auto_shell() + shell = ShellType(shell) path = path or get_completion_install_path(shell) uninstall_completion_code(shell, path) click.echo( "PlatformIO CLI completion has been uninstalled for %s shell from %s \n" "Please restart a current shell session." - % (click.style(shell, fg="cyan"), click.style(path, fg="blue")) + % (click.style(shell.name, fg="cyan"), click.style(path, fg="blue")) ) diff --git a/platformio/commands/system/completion.py b/platformio/commands/system/completion.py index 012df5fc..3ea80b99 100644 --- a/platformio/commands/system/completion.py +++ b/platformio/commands/system/completion.py @@ -13,61 +13,75 @@ # limitations under the License. import os -import subprocess +from enum import Enum import click +from platformio.compat import IS_MACOS + + +class ShellType(Enum): + FISH = "fish" + ZSH = "zsh" + BASH = "bash" + def get_completion_install_path(shell): home_dir = os.path.expanduser("~") prog_name = click.get_current_context().find_root().info_name - if shell == "fish": + if shell == ShellType.FISH: return os.path.join( home_dir, ".config", "fish", "completions", "%s.fish" % prog_name ) - if shell == "bash": - return os.path.join(home_dir, ".bash_completion") - if shell == "zsh": + if shell == ShellType.ZSH: return os.path.join(home_dir, ".zshrc") - if shell == "powershell": - return subprocess.check_output( - ["powershell", "-NoProfile", "echo $profile"] - ).strip() + if shell == ShellType.BASH: + return os.path.join(home_dir, ".bash_completion") + raise click.ClickException("%s is not supported." % shell) + + +def get_completion_code(shell): + if shell == ShellType.FISH: + return "eval (env _PIO_COMPLETE=fish_source pio)" + if shell == ShellType.ZSH: + code = "autoload -Uz compinit\ncompinit\n" if IS_MACOS else "" + return code + 'eval "$(_PIO_COMPLETE=zsh_source pio)"' + if shell == ShellType.BASH: + return 'eval "$(_PIO_COMPLETE=bash_source pio)"' raise click.ClickException("%s is not supported." % shell) def is_completion_code_installed(shell, path): - if shell == "fish" or not os.path.exists(path): + if shell == ShellType.FISH or not os.path.exists(path): return False - - import click_completion # pylint: disable=import-error,import-outside-toplevel - with open(path, encoding="utf8") as fp: - return click_completion.get_code(shell=shell) in fp.read() + return get_completion_code(shell) in fp.read() def install_completion_code(shell, path): - import click_completion # pylint: disable=import-error,import-outside-toplevel - if is_completion_code_installed(shell, path): return None - - return click_completion.install(shell=shell, path=path, append=shell != "fish") + append = shell != ShellType.FISH + with open(path, mode="a" if append else "w", encoding="utf8") as fp: + if append: + fp.write("\n\n# Begin: PlatformIO Core completion support\n") + fp.write(get_completion_code(shell)) + if append: + fp.write("\n# End: PlatformIO Core completion support\n\n") + return True def uninstall_completion_code(shell, path): if not os.path.exists(path): return True - if shell == "fish": + if shell == ShellType.FISH: os.remove(path) return True - import click_completion # pylint: disable=import-error,import-outside-toplevel - with open(path, "r+", encoding="utf8") as fp: contents = fp.read() fp.seek(0) fp.truncate() - fp.write(contents.replace(click_completion.get_code(shell=shell), "")) + fp.write(contents.replace(get_completion_code(shell), "")) return True diff --git a/setup.py b/setup.py index ac8c217e..7f0f92d4 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ from platformio.compat import PY2 minimal_requirements = [ "bottle==0.12.*", - "click>=7.1.2,<9,!=8.0.2", + "click>=8,<9,!=8.0.2", "colorama", "marshmallow%s" % (">=2,<3" if PY2 else ">=2,<4"), "pyelftools>=0.27,<1", From a1630483960e6c17a862536a820069ecc8fba538 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Dec 2021 16:34:35 +0200 Subject: [PATCH 33/56] Bump version to 5.2.4b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 705e2d3a..86d1bf41 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "4b1") +VERSION = (5, 2, "4b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 24e63e7a02cca66a798b9a1bbc41133d0197b7a4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Dec 2021 19:30:19 +0200 Subject: [PATCH 34/56] Revert "Lock "cryptography" to RUST-less 3.3.2 version" This reverts commit 3828e6d15ec5627bdc7efdc1a99bf96494b1971a. --- platformio/package/manager/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 9daac321..e1776ae3 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -218,7 +218,7 @@ def get_contrib_pysite_deps(): result.extend( [ # pyopenssl depends on it, use RUST-less version - "cryptography == 3.3.2", + "cryptography >= 3.3, < 35.0.0", "pyopenssl >= 16.0.0", "service_identity >= 18.1.0", "idna >= 0.6, != 2.3", From f13734dda4f15f74732418822bf986219317cfb6 Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 2 Dec 2021 20:05:35 +0200 Subject: [PATCH 35/56] Convert Home Directory path into a cmake-style path on Windows Resolve #4071 --- HISTORY.rst | 1 + .../project/tpls/clion/CMakeListsPrivate.txt.tpl | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4a4bd78f..a96d6374 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ PlatformIO Core 5 - Fixed an issue with the NetBeans project generator when the path to PlatformIO contains a space (`issue #4096 `_) - Fixed an issue when the system environment variable does not override a project configuration option (`issue #4125 `_) - Fixed an issue when referencing ``*_dir`` option from a custom project configuration environment (`issue #4110 `_) +- Fixed an issue with the CLion template that generated a broken CMake file if user's home directory contained an unescaped backslash (`issue #4071 `_) 5.2.3 (2021-11-05) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl index a7a04ec2..82a4b3a6 100644 --- a/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl @@ -8,15 +8,14 @@ % import os % import re % -% from platformio.compat import WINDOWS -% from platformio.project.helpers import (load_project_ide_data) +% from platformio.project.helpers import load_project_ide_data % % def _normalize_path(path): % if project_dir in path: % path = path.replace(project_dir, "${CMAKE_CURRENT_LIST_DIR}") % elif user_home_dir in path: % if "windows" in systype: -% path = path.replace(user_home_dir, "$ENV{HOMEDRIVE}$ENV{HOMEPATH}") +% path = path.replace(user_home_dir, "${ENV_HOME_PATH}") % else: % path = path.replace(user_home_dir, "$ENV{HOME}") % end @@ -63,6 +62,11 @@ SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}") SET(CMAKE_CXX_FLAGS "{{ _normalize_path(to_unix_path(cxx_flags)) }}") SET(CMAKE_C_FLAGS "{{ _normalize_path(to_unix_path(cc_flags)) }}") +# Convert "Home Directory" that may contain unescaped backslashes on Windows +% if "windows" in systype: +file(TO_CMAKE_PATH $ENV{HOMEDRIVE}$ENV{HOMEPATH} ENV_HOME_PATH) +% end + % STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") % cc_stds = STD_RE.findall(cc_flags) % cxx_stds = STD_RE.findall(cxx_flags) From c3ad3ebb578f3cd34ca5c947c230696b169571aa Mon Sep 17 00:00:00 2001 From: valeros Date: Thu, 2 Dec 2021 20:56:18 +0200 Subject: [PATCH 36/56] Properly replace Home Directory in CLion template on Windows Issue #4071 --- .../project/tpls/clion/CMakeListsPrivate.txt.tpl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl index 82a4b3a6..b8695d0e 100644 --- a/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/project/tpls/clion/CMakeListsPrivate.txt.tpl @@ -53,6 +53,11 @@ set(CMAKE_CONFIGURATION_TYPES "{{ ";".join(envs) }};" CACHE STRING "Build Types set(CMAKE_CONFIGURATION_TYPES "{{ env_name }}" CACHE STRING "Build Types reflect PlatformIO Environments" FORCE) % end +# Convert "Home Directory" that may contain unescaped backslashes on Windows +% if "windows" in systype: +file(TO_CMAKE_PATH $ENV{HOMEDRIVE}$ENV{HOMEPATH} ENV_HOME_PATH) +% end + % if svd_path: set(CLION_SVD_FILE_PATH "{{ _normalize_path(svd_path) }}" CACHE FILEPATH "Peripheral Registers Definitions File" FORCE) % end @@ -62,11 +67,6 @@ SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}") SET(CMAKE_CXX_FLAGS "{{ _normalize_path(to_unix_path(cxx_flags)) }}") SET(CMAKE_C_FLAGS "{{ _normalize_path(to_unix_path(cc_flags)) }}") -# Convert "Home Directory" that may contain unescaped backslashes on Windows -% if "windows" in systype: -file(TO_CMAKE_PATH $ENV{HOMEDRIVE}$ENV{HOMEPATH} ENV_HOME_PATH) -% end - % STD_RE = re.compile(r"\-std=[a-z\+]+(\w+)") % cc_stds = STD_RE.findall(cc_flags) % cxx_stds = STD_RE.findall(cxx_flags) From 4f4c88aca93699bad0cebe4a13e4f7aa1b6d5fcd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Dec 2021 22:16:37 +0200 Subject: [PATCH 37/56] Use SCons vars for deprecated variables --- platformio/builder/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index ce79301b..44ab9276 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -115,10 +115,10 @@ env.Replace( PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"), PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"), PROJECT_SRC_DIR=config.get("platformio", "src_dir"), - PROJECTSRC_DIR=config.get("platformio", "src_dir"), # legacy for dev/platform + PROJECTSRC_DIR="PROJECT_SRC_DIR", # legacy for dev/platform PROJECT_TEST_DIR=config.get("platformio", "test_dir"), PROJECT_DATA_DIR=config.get("platformio", "data_dir"), - PROJECTDATA_DIR=config.get("platformio", "data_dir"), # legacy for dev/platform + PROJECTDATA_DIR="$PROJECT_DATA_DIR", # legacy for dev/platform PROJECT_BUILD_DIR=config.get("platformio", "build_dir"), BUILD_CACHE_DIR=config.get("platformio", "build_cache_dir"), LIBSOURCE_DIRS=[ From b3eb81c3b484418da626e626eb09797b9a4dadf2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Dec 2021 17:01:42 +0200 Subject: [PATCH 38/56] Typo fix --- platformio/builder/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 44ab9276..6384b08d 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -115,7 +115,7 @@ env.Replace( PROJECT_LIBDEPS_DIR=config.get("platformio", "libdeps_dir"), PROJECT_INCLUDE_DIR=config.get("platformio", "include_dir"), PROJECT_SRC_DIR=config.get("platformio", "src_dir"), - PROJECTSRC_DIR="PROJECT_SRC_DIR", # legacy for dev/platform + PROJECTSRC_DIR="$PROJECT_SRC_DIR", # legacy for dev/platform PROJECT_TEST_DIR=config.get("platformio", "test_dir"), PROJECT_DATA_DIR=config.get("platformio", "data_dir"), PROJECTDATA_DIR="$PROJECT_DATA_DIR", # legacy for dev/platform From f5a23c38170862ddebbac946f323a637eab53952 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Dec 2021 17:02:05 +0200 Subject: [PATCH 39/56] Bump version to 5.2.4b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 86d1bf41..25bdc89e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "4b2") +VERSION = (5, 2, "4b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From aab42c3cffeabc542d6821bdb55c7a4743a803f7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Dec 2021 20:05:37 +0200 Subject: [PATCH 40/56] Skip library.properties "paragraph" if total len >= 1000 --- platformio/package/manifest/parser.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 4028c276..54417c0c 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -517,8 +517,11 @@ class LibraryPropertiesManifestParser(BaseManifestParser): for k in ("sentence", "paragraph"): if k in properties and properties[k] not in lines: lines.append(properties[k]) - if len(lines) == 2 and not lines[0].endswith("."): - lines[0] += "." + if len(lines) == 2: + if not lines[0].endswith("."): + lines[0] += "." + if len(lines[0]) + len(lines[1]) >= 1000: + del lines[1] return " ".join(lines) def _parse_keywords(self, properties): From 39494d18bf66cf24ae9e7b6db6c2b638985c6837 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Dec 2021 20:59:31 +0200 Subject: [PATCH 41/56] Revert "Revert "Lock "cryptography" to RUST-less 3.3.2 version"" This reverts commit 24e63e7a02cca66a798b9a1bbc41133d0197b7a4. --- platformio/package/manager/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index e1776ae3..9daac321 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -218,7 +218,7 @@ def get_contrib_pysite_deps(): result.extend( [ # pyopenssl depends on it, use RUST-less version - "cryptography >= 3.3, < 35.0.0", + "cryptography == 3.3.2", "pyopenssl >= 16.0.0", "service_identity >= 18.1.0", "idna >= 0.6, != 2.3", From 8555e83cb17b4a00aa520a754fe6656912e95e78 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 Dec 2021 15:05:25 +0200 Subject: [PATCH 42/56] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b6322c06..201759d3 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b6322c062fd6ea58d8e6bca737a51833635d2173 +Subproject commit 201759d361a59fe412d26cf8dcae5c809cac1195 From 7338a02b48aca8f8336f83869e4c23e9447d2962 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 Dec 2021 15:05:42 +0200 Subject: [PATCH 43/56] Do not pack Python bytecode by default --- platformio/package/pack.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 8688d171..2d1eea46 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -42,6 +42,7 @@ class PackagePacker(object): ".vscode", ".cache", "**/.cache", + "**/__pycache__", # VCS ".git/", ".hg/", From 5b23c9a2941e1286ef27961c484b34ddf84020dd Mon Sep 17 00:00:00 2001 From: valeros Date: Wed, 8 Dec 2021 13:48:35 +0200 Subject: [PATCH 44/56] Add basic test for CLion integration --- tests/commands/test_init.py | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 49db3560..286197fa 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -16,6 +16,10 @@ import json from os import getcwd, makedirs from os.path import getsize, isdir, isfile, join +import pytest + +from platformio import proc +from platformio.commands import platform as cli_platform from platformio.commands.boards import cli as cmd_boards from platformio.commands.project import project_init as cmd_init from platformio.project.config import ProjectConfig @@ -177,3 +181,83 @@ def test_init_incorrect_board(clirunner): assert result.exit_code == 2 assert "Error: Invalid value for" in result.output assert isinstance(result.exception, SystemExit) + + +@pytest.mark.skipif(not proc.is_ci(), reason="runs on CI") +def test_init_ide_clion(clirunner, isolated_pio_core, validate_cliresult, tmpdir): + result = clirunner.invoke( + cli_platform.platform_install, + [ + "ststm32", + "--skip-default-package", + "--with-package", + "tool-cmake", + "--with-package", + "tool-ninja", + ], + ) + + # Add extra libraries to cover cases with possible unwanted backslashes + lib_extra_dirs = isolated_pio_core.join("extra_libs").mkdir() + extra_lib = lib_extra_dirs.join("extra_lib").mkdir() + extra_lib.join("extra_lib.h").write(" ") + extra_lib.join("extra_lib.cpp").write(" ") + + with tmpdir.as_cwd(): + result = clirunner.invoke( + cmd_init, + [ + "-b", + "nucleo_f401re", + "--ide", + "clion", + "--project-option", + "framework=arduino", + "--project-option", + "lib_extra_dirs=%s" % str(lib_extra_dirs), + ], + ) + + validate_cliresult(result) + assert all(isfile(f) for f in ("CMakeLists.txt", "CMakeListsPrivate.txt")) + + tmpdir.join("src").join("main.cpp").write( + """#include +#include "extra_lib.h" +void setup(){} +void loop(){} +""" + ) + cmake_path = str( + isolated_pio_core.join("packages") + .join("tool-cmake") + .join("bin") + .join("cmake") + ) + tmpdir.join("build_dir").mkdir() + result = proc.exec_command( + [ + cmake_path, + "-DCMAKE_BUILD_TYPE=nucleo_f401re", + "-DCMAKE_MAKE_PROGRAM=%s" + % str( + isolated_pio_core.join("packages").join("tool-ninja").join("ninja") + ), + "-G", + "Ninja", + "-S", + str(tmpdir), + "-B", + "build_dir", + ] + ) + + # Check if CMake was able to generate a native project for Ninja + assert result["returncode"] == 0, result["out"] + + result = proc.exec_command( + [cmake_path, "--build", "build_dir", "--target", "Debug"] + ) + + assert result["returncode"] == 0 + assert "[SUCCESS]" in str(result["out"]) From 24fc2f7e143748c9c2e7eadb5aacbad20b58cdba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Dec 2021 18:38:16 +0200 Subject: [PATCH 45/56] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 201759d3..60dd6f2e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 201759d361a59fe412d26cf8dcae5c809cac1195 +Subproject commit 60dd6f2e85a8c38fd737c81532644b039e15339b From 5534394b06f62468c696c7383b87d7bdc06c609d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Dec 2021 18:40:07 +0200 Subject: [PATCH 46/56] Fixed an issue with wrong detecting Windows architecture when Python 32bit is used // Resolve #4134 --- HISTORY.rst | 1 + platformio/util.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a96d6374..30c9ef9f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,7 @@ PlatformIO Core 5 - Fixed an issue when the system environment variable does not override a project configuration option (`issue #4125 `_) - Fixed an issue when referencing ``*_dir`` option from a custom project configuration environment (`issue #4110 `_) - Fixed an issue with the CLion template that generated a broken CMake file if user's home directory contained an unescaped backslash (`issue #4071 `_) +- Fixed an issue with wrong detecting Windows architecture when Python 32bit is used (`issue #4134 `_) 5.2.3 (2021-11-05) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/util.py b/platformio/util.py index 8ab144a4..90bce628 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -92,8 +92,8 @@ def singleton(cls): def get_systype(): type_ = platform.system().lower() arch = platform.machine().lower() - if type_ == "windows": - arch = "amd64" if platform.architecture()[0] == "64bit" else "x86" + if type_ == "windows" and "x86" in arch: + arch = "amd64" if "64" in arch else "x86" return "%s_%s" % (type_, arch) if arch else type_ From fe69f3de04c1ff486b61cf4628cc5052113f6a68 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Dec 2021 18:40:37 +0200 Subject: [PATCH 47/56] Bump version to 5.2.4b4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 25bdc89e..cafb9706 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "4b3") +VERSION = (5, 2, "4b4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From d5db2f0eb7928d2b7413c83ebefe76afbc0f7e39 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Dec 2021 18:45:29 +0200 Subject: [PATCH 48/56] Apply formatting --- platformio/__main__.py | 1 - platformio/package/manifest/schema.py | 1 - 2 files changed, 2 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index d3b765fb..38c2ce00 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -67,7 +67,6 @@ try: def process_result(ctx, result, *_, **__): _process_result(ctx, result) - except (AttributeError, TypeError): # legacy support for CLick > 8.0.1 @cli.resultcallback() diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index cedb84ad..60d0d60c 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -33,7 +33,6 @@ if MARSHMALLOW_2: class CompatSchema(Schema): pass - else: class CompatSchema(Schema): From 8f28d1ad4353228671051dcebfd1ca846881a64b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 Dec 2021 18:45:43 +0200 Subject: [PATCH 49/56] Update uvicorn to 0.16 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f0f92d4..766a1b87 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ home_requirements = [ "aiofiles==0.8.*", "ajsonrpc==1.*", "starlette==0.17.*", - "uvicorn==0.15.*", + "uvicorn==0.16.*", "wsproto==1.0.*", ] From b7f10982c3ad215a5eec0b4cbdea5e0362a10291 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 14 Dec 2021 21:14:11 +0200 Subject: [PATCH 50/56] Update PIO Remote deps // Issue #3865 --- platformio/package/manager/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 9daac321..e9d42011 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -209,7 +209,7 @@ def get_contrib_pysite_deps(): sys_type = util.get_systype() py_version = "%d%d" % (sys.version_info.major, sys.version_info.minor) - twisted_version = "20.3.0" + twisted_version = "21.7.0" result = [ "twisted == %s" % twisted_version, ] @@ -218,7 +218,7 @@ def get_contrib_pysite_deps(): result.extend( [ # pyopenssl depends on it, use RUST-less version - "cryptography == 3.3.2", + "cryptography >= 3.3, < 35.0.0", "pyopenssl >= 16.0.0", "service_identity >= 18.1.0", "idna >= 0.6, != 2.3", From 24ea7aaede52015e7c9685f6bec783e9b516ba4e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 14 Dec 2021 22:37:42 +0200 Subject: [PATCH 51/56] Update title to PlatformIO Core --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c137fe43..ae40bcdf 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -PlatformIO -========== +PlatformIO Core +=============== .. image:: https://github.com/platformio/platformio-core/workflows/Core/badge.svg :target: https://docs.platformio.org/page/core/index.html From a4692d5457a3ff5c146bdabba89f0dde9265ab8d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 14 Dec 2021 22:38:31 +0200 Subject: [PATCH 52/56] Improved PIO Remote setup on credit-card sized computers (Raspberry Pi, BeagleBon, etc) // Resolve #3865 --- HISTORY.rst | 1 + platformio/package/manager/core.py | 26 +++++++++----------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 30c9ef9f..7d4178e4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,7 @@ PlatformIO Core 5 - Added support for a new ``headers`` field in `library.json `__ (declare a list of header files that can be included in a project source files using ``#include <...>`` directive) - Improved tab completion support for Bash, ZSH, and Fish shells (`issue #4114 `_) - Improved support for projects located on a network share (`issue #3417 `_, `issue #3926 `_, `issue #4099 `_) +- Improved PIO Remote setup on credit-card sized computers (Raspberry Pi, BeagleBon, etc) (`issue #3865 `_) - Upgraded build engine to the SCons 4.3 (`release notes `__) - Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 `_) - Fixed an issue with the NetBeans project generator when the path to PlatformIO contains a space (`issue #4096 `_) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index e9d42011..4c1751a3 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -152,8 +152,7 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): ] if "linux" in systype: args.extend(["--no-binary", ":all:"]) - for dep in get_contrib_pysite_deps(): - subprocess.check_call(args + [dep]) + subprocess.check_call(args + get_contrib_pysite_deps()) # build manifests with open( @@ -206,25 +205,18 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): def get_contrib_pysite_deps(): - sys_type = util.get_systype() - py_version = "%d%d" % (sys.version_info.major, sys.version_info.minor) - - twisted_version = "21.7.0" + twisted_version = "20.3.0" result = [ + # twisted[tls], see setup.py for %twisted_version% "twisted == %s" % twisted_version, + # pyopenssl depends on it, use RUST-less version + "cryptography >= 3.3, < 35.0.0", + "pyopenssl >= 16.0.0, <= 21.0.0", + "service_identity >= 18.1.0, <= 21.1.0 ", ] - # twisted[tls], see setup.py for %twisted_version% - result.extend( - [ - # pyopenssl depends on it, use RUST-less version - "cryptography >= 3.3, < 35.0.0", - "pyopenssl >= 16.0.0", - "service_identity >= 18.1.0", - "idna >= 0.6, != 2.3", - ] - ) - + sys_type = util.get_systype() + py_version = "%d%d" % (sys.version_info.major, sys.version_info.minor) if "windows" in sys_type: result.append("pypiwin32 == 223") # workaround for twisted wheels From d32fd72d136851eed36e1c287df4e69d94a0b4c8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 14 Dec 2021 22:38:57 +0200 Subject: [PATCH 53/56] Bump version to 5.2.4rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index cafb9706..df4c9466 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "4b4") +VERSION = (5, 2, "4rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 5098f5f42023f99bb1184ca7651007c58ebb5413 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 14 Dec 2021 22:55:48 +0200 Subject: [PATCH 54/56] Minor improvements // Issue #3865 --- platformio/package/manager/core.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 4c1751a3..4979901f 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -20,6 +20,7 @@ import sys from datetime import date from platformio import __core_packages__, exception, fs, util +from platformio.exception import UserSideException from platformio.package.exception import UnknownPackageError from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageItem, PackageSpec @@ -152,7 +153,15 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): ] if "linux" in systype: args.extend(["--no-binary", ":all:"]) - subprocess.check_call(args + get_contrib_pysite_deps()) + try: + subprocess.run(args + get_contrib_pysite_deps(), check=True) + except subprocess.CalledProcessError as exc: + if "linux" in systype: + raise UserSideException( + "\n\nPlease ensure that the next packages are installed:\n\n" + "sudo apt install python3-dev libffi-dev libssl-dev\n" + ) + raise exc # build manifests with open( @@ -212,7 +221,8 @@ def get_contrib_pysite_deps(): # pyopenssl depends on it, use RUST-less version "cryptography >= 3.3, < 35.0.0", "pyopenssl >= 16.0.0, <= 21.0.0", - "service_identity >= 18.1.0, <= 21.1.0 ", + "service_identity >= 18.1.0, <= 21.1.0", + "asdasdasdasdasdas", ] sys_type = util.get_systype() From dec38273b6ee58188e43799b437d5521cf8da04e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Dec 2021 11:59:19 +0200 Subject: [PATCH 55/56] Cleanup code --- platformio/commands/remote/command.py | 2 +- platformio/package/manager/core.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/platformio/commands/remote/command.py b/platformio/commands/remote/command.py index 03da8389..248d66d2 100644 --- a/platformio/commands/remote/command.py +++ b/platformio/commands/remote/command.py @@ -37,7 +37,7 @@ from platformio.project.exception import NotPlatformIOProjectError @click.pass_context def cli(ctx, agent): ctx.obj = agent - inject_contrib_pysite(verify_openssl=True) + inject_contrib_pysite() @cli.group("agent", short_help="Start a new agent or list active") diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 4979901f..fe479c74 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -102,7 +102,7 @@ def remove_unnecessary_core_packages(dry_run=False): return candidates -def inject_contrib_pysite(verify_openssl=False): +def inject_contrib_pysite(): # pylint: disable=import-outside-toplevel from site import addsitedir @@ -120,12 +120,10 @@ def inject_contrib_pysite(verify_openssl=False): addsitedir(contrib_pysite_dir) sys.path.insert(0, contrib_pysite_dir) - if not verify_openssl: - return True - try: # pylint: disable=import-error,unused-import,unused-variable from OpenSSL import SSL + except: # pylint: disable=bare-except build_contrib_pysite_package(contrib_pysite_dir) @@ -222,7 +220,6 @@ def get_contrib_pysite_deps(): "cryptography >= 3.3, < 35.0.0", "pyopenssl >= 16.0.0, <= 21.0.0", "service_identity >= 18.1.0, <= 21.1.0", - "asdasdasdasdasdas", ] sys_type = util.get_systype() From 9dbdf7fc8d1f1c882ca844b9ef0e320d759b353d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 15 Dec 2021 12:19:51 +0200 Subject: [PATCH 56/56] Bump version to 5.2.4 --- HISTORY.rst | 2 +- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7d4178e4..9274b86b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,7 +8,7 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** -5.2.4 (2021-??-??) +5.2.4 (2021-12-15) ~~~~~~~~~~~~~~~~~~ - Added support for a new ``headers`` field in `library.json `__ (declare a list of header files that can be included in a project source files using ``#include <...>`` directive) diff --git a/docs b/docs index 60dd6f2e..ba3fca21 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 60dd6f2e85a8c38fd737c81532644b039e15339b +Subproject commit ba3fca21eab09226c194f0a23463b3c253d2e069 diff --git a/platformio/__init__.py b/platformio/__init__.py index df4c9466..075823ca 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, "4rc1") +VERSION = (5, 2, 4) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"