From e2906e3be5700c920797553dfd911ffc36a74376 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 16:10:13 +0200 Subject: [PATCH 1/3] Refactored a workaround for a maximum command line character limitation // Resolve #3792 --- HISTORY.rst | 1 + platformio/builder/main.py | 17 +++++-- platformio/builder/tools/piomaxlen.py | 66 +++++++++++---------------- 3 files changed, 40 insertions(+), 44 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9b5218b9..f2a5bc61 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,7 @@ PlatformIO Core 5 * **Build System** - Upgraded build engine to the SCons 4.1 (`release notes `_) + - Refactored a workaround for a maximum command line character limitation (`issue #3792 `_) - Fixed an issue with Python 3.8+ on Windows when a network drive is used (`issue #3417 `_) * **Package Management System** diff --git a/platformio/builder/main.py b/platformio/builder/main.py index b1e8ffbd..6a060dd1 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -81,12 +81,19 @@ DEFAULT_ENV_OPTIONS = dict( IDE_EXTRA_DATA={}, ) +# Declare command verbose messages +command_strings = dict( + ARCOM="Archiving", + LINKCOM="Linking", + RANLIBCOM="Indexing", + ASCOM="Compiling", + ASPPCOM="Compiling", + CCCOM="Compiling", + CXXCOM="Compiling", +) if not int(ARGUMENTS.get("PIOVERBOSE", 0)): - DEFAULT_ENV_OPTIONS["ARCOMSTR"] = "Archiving $TARGET" - DEFAULT_ENV_OPTIONS["LINKCOMSTR"] = "Linking $TARGET" - DEFAULT_ENV_OPTIONS["RANLIBCOMSTR"] = "Indexing $TARGET" - for k in ("ASCOMSTR", "ASPPCOMSTR", "CCCOMSTR", "CXXCOMSTR"): - DEFAULT_ENV_OPTIONS[k] = "Compiling $TARGET" + for name, value in command_strings.items(): + DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value) env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index 04f1304f..4077273c 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -14,15 +14,18 @@ from __future__ import absolute_import -from hashlib import md5 -from os import makedirs -from os.path import isdir, isfile, join +import hashlib +import os + +from SCons.Platform import TempFileMunge # pylint: disable=import-error from platformio.compat import WINDOWS, hashlib_encode_data -# Windows CLI has limit with command length to 8192 -# Leave 2000 chars for flags and other options -MAX_LINE_LENGTH = 6000 if WINDOWS else 128072 +# There are the next limits depending on a platform: +# - Windows = 8192 +# - Unix = 131072 +# We need ~256 characters for a temporary file path +MAX_LINE_LENGTH = (8192 if WINDOWS else 131072) - 256 def long_sources_hook(env, sources): @@ -41,30 +44,14 @@ def long_sources_hook(env, sources): return '@"%s"' % _file_long_data(env, " ".join(data)) -def long_incflags_hook(env, incflags): - _incflags = env.subst(incflags).replace("\\", "/") - if len(_incflags) < MAX_LINE_LENGTH: - return incflags - - # fix space in paths - data = [] - for line in _incflags.split(" -I"): - line = line.strip() - if not line.startswith("-I"): - line = "-I" + line - data.append('-I"%s"' % line[2:]) - - return '@"%s"' % _file_long_data(env, " ".join(data)) - - def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") - if not isdir(build_dir): - makedirs(build_dir) - tmp_file = join( - build_dir, "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest() + if not os.path.isdir(build_dir): + os.makedirs(build_dir) + tmp_file = os.path.join( + build_dir, "longcmd-%s" % hashlib.md5(hashlib_encode_data(data)).hexdigest() ) - if isfile(tmp_file): + if os.path.isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: fp.write(data) @@ -76,17 +63,18 @@ def exists(_): def generate(env): - env.Replace(_long_sources_hook=long_sources_hook) - env.Replace(_long_incflags_hook=long_incflags_hook) - coms = {} - for key in ("ARCOM", "LINKCOM"): - coms[key] = env.get(key, "").replace( - "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}" - ) - for key in ("_CCCOMCOM", "ASPPCOM"): - coms[key] = env.get(key, "").replace( - "$_CPPINCFLAGS", "${_long_incflags_hook(__env__, _CPPINCFLAGS)}" - ) - env.Replace(**coms) + kwargs = dict( + _long_sources_hook=long_sources_hook, + TEMPFILE=TempFileMunge, + MAXLINELENGTH=MAX_LINE_LENGTH, + ) + + for name in ("LINKCOM", "ASCOM", "ASPPCOM", "CCCOM", "CXXCOM"): + kwargs[name] = "${TEMPFILE('%s','$%sSTR')}" % (env.get(name), name) + + kwargs["ARCOM"] = env.get("ARCOM", "").replace( + "$SOURCES", "${_long_sources_hook(__env__, SOURCES)}" + ) + env.Replace(**kwargs) return env From 7810946484330cb62dee02bcde86af6672cdfd09 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 18:47:54 +0200 Subject: [PATCH 2/3] Use project build folder for tempfile workaround with command maxlen --- platformio/__init__.py | 2 +- platformio/builder/tools/piomaxlen.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index cebcd1ed..b3ff0e42 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -50,7 +50,7 @@ __core_packages__ = { "contrib-piohome": "~3.3.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.0", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.1", "tool-cppcheck": "~1.230.0", "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.11.0", diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index 4077273c..c05ae821 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -67,6 +67,8 @@ def generate(env): _long_sources_hook=long_sources_hook, TEMPFILE=TempFileMunge, MAXLINELENGTH=MAX_LINE_LENGTH, + TEMPFILESUFFIX=".tmp", + TEMPFILEDIR="$BUILD_DIR", ) for name in ("LINKCOM", "ASCOM", "ASPPCOM", "CCCOM", "CXXCOM"): From d77dbb2cca7d245eaeb64c8b3c83db1ccfcf9450 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 27 Jan 2021 20:30:28 +0200 Subject: [PATCH 3/3] Use "TEMPFILEARGESCFUNC" for GCC workaround on Windows --- platformio/__init__.py | 2 +- platformio/builder/tools/piomaxlen.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b3ff0e42..fa2c7125 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -50,7 +50,7 @@ __core_packages__ = { "contrib-piohome": "~3.3.3", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", - "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.1", + "tool-scons": "~2.20501.7" if sys.version_info.major == 2 else "~4.40100.2", "tool-cppcheck": "~1.230.0", "tool-clangtidy": "~1.100000.0", "tool-pvs-studio": "~7.11.0", diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index c05ae821..d386bd14 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -16,8 +16,10 @@ from __future__ import absolute_import import hashlib import os +import re from SCons.Platform import TempFileMunge # pylint: disable=import-error +from SCons.Subst import quote_spaces # pylint: disable=import-error from platformio.compat import WINDOWS, hashlib_encode_data @@ -27,6 +29,16 @@ from platformio.compat import WINDOWS, hashlib_encode_data # We need ~256 characters for a temporary file path MAX_LINE_LENGTH = (8192 if WINDOWS else 131072) - 256 +WINPATHSEP_RE = re.compile(r"\\([^\"'\\]|$)") + + +def tempfile_arg_esc_func(arg): + arg = quote_spaces(arg) + if not WINDOWS: + return arg + # GCC requires double Windows slashes, let's use UNIX separator + return WINPATHSEP_RE.sub(r"/\1", arg) + def long_sources_hook(env, sources): _sources = str(sources).replace("\\", "/") @@ -67,6 +79,7 @@ def generate(env): _long_sources_hook=long_sources_hook, TEMPFILE=TempFileMunge, MAXLINELENGTH=MAX_LINE_LENGTH, + TEMPFILEARGESCFUNC=tempfile_arg_esc_func, TEMPFILESUFFIX=".tmp", TEMPFILEDIR="$BUILD_DIR", )