From a60c57ac58b4a6902f971bc0f145c89baa153c1c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Dec 2018 20:54:29 +0200 Subject: [PATCH 001/333] Initial support for Python 3.5+ // Resolve #895 Resolve #1365 --- .appveyor.yml | 2 + .pylintrc | 2 +- .travis.yml | 14 ++-- HISTORY.rst | 11 +++- platformio/__init__.py | 11 +--- platformio/__main__.py | 2 +- platformio/app.py | 44 +++++++------ platformio/builder/main.py | 10 ++- platformio/builder/tools/piolib.py | 33 ++++++---- platformio/builder/tools/piomisc.py | 4 +- platformio/builder/tools/pioplatform.py | 13 ++-- platformio/builder/tools/piowinhooks.py | 11 +++- platformio/builder/tools/platformio.py | 4 +- platformio/commands/device.py | 2 +- platformio/commands/lib.py | 12 ++-- platformio/commands/platform.py | 25 ++++---- platformio/commands/run.py | 9 +-- platformio/downloader.py | 8 ++- platformio/exception.py | 4 +- platformio/ide/projectgenerator.py | 3 +- platformio/maintenance.py | 2 +- platformio/managers/core.py | 4 +- platformio/managers/lib.py | 6 +- platformio/managers/package.py | 16 +++-- platformio/managers/platform.py | 36 +++++++---- platformio/telemetry.py | 16 +++-- platformio/unpacker.py | 4 +- platformio/util.py | 85 +++++++++++++++---------- platformio/vcsclient.py | 6 +- scripts/install_devplatforms.py | 2 +- setup.py | 4 +- tests/test_examples.py | 2 +- tests/test_maintenance.py | 2 +- tests/test_pkgmanifest.py | 2 +- tox.ini | 52 ++++++--------- 35 files changed, 265 insertions(+), 198 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index a4cc2f40..72fc32ac 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,6 +7,8 @@ platform: environment: matrix: - TOXENV: "py27" + - TOXENV: "py35" + - TOXENV: "py36" install: - cmd: git submodule update --init --recursive diff --git a/.pylintrc b/.pylintrc index aad06b28..09569a4f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -20,4 +20,4 @@ confidence= # --disable=W" # disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating -disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens +disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens,useless-object-inheritance,useless-import-alias diff --git a/.travis.yml b/.travis.yml index 589289f8..fe90e097 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,14 +6,18 @@ matrix: sudo: false python: 2.7 env: TOX_ENV=docs - - os: linux - sudo: false - python: 2.7 - env: TOX_ENV=lint - os: linux sudo: required python: 2.7 env: TOX_ENV=py27 + - os: linux + sudo: required + python: 3.5 + env: TOX_ENV=py35 + - os: linux + sudo: required + python: 3.6 + env: TOX_ENV=py36 - os: osx language: generic env: TOX_ENV=skipexamples @@ -24,7 +28,7 @@ install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox==3.0.0"; else pip install -U tox; fi # ChipKIT issue: install 32-bit support for GCC PIC32 - - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi script: - tox -e $TOX_ENV diff --git a/HISTORY.rst b/HISTORY.rst index 651451d4..80144ad7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,10 +1,19 @@ Release Notes ============= +PlatformIO 4.0 +-------------- + +4.0.0 (2019-??-??) +~~~~~~~~~~~~~~~~~~ + +* Added Python 3.5+ support + (`issue #895 `_) + PlatformIO 3.0 -------------- -3.6.4 (2018-??-??) +3.6.4 (2019-??-??) ~~~~~~~~~~~~~~~~~~ * Improved Project Generator for IDEs: diff --git a/platformio/__init__.py b/platformio/__init__.py index 07fa8a5d..8a5c9124 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -VERSION = (3, 6, "4b1") +VERSION = (4, 0, "0a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -33,10 +31,3 @@ __license__ = "Apache Software License" __copyright__ = "Copyright 2014-present PlatformIO" __apiurl__ = "https://api.platformio.org" - -if sys.version_info < (2, 7, 0) or sys.version_info >= (3, 0, 0): - msg = ("PlatformIO Core v%s does not run under Python version %s.\n" - "Minimum supported version is 2.7, please upgrade Python.\n" - "Python 3 is not yet supported.\n") - sys.stderr.write(msg % (__version__, sys.version)) - sys.exit(1) diff --git a/platformio/__main__.py b/platformio/__main__.py index 161acac3..61db78de 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -53,7 +53,7 @@ class PlatformioCLI(click.MultiCommand): # pylint: disable=R0904 if name == "platforms": from platformio.commands import platform return platform.cli - elif name == "serialports": + if name == "serialports": from platformio.commands import device return device.cli raise AttributeError() diff --git a/platformio/app.py b/platformio/app.py index 14c4de22..c7dfdc43 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -169,8 +169,11 @@ class ContentCache(object): @staticmethod def key_from_args(*args): h = hashlib.md5() - for data in args: - h.update(str(data)) + for arg in args: + if not arg: + continue + arg = str(arg) + h.update(arg if util.PY2 else arg.encode()) return h.hexdigest() def get(self, key): @@ -191,7 +194,7 @@ class ContentCache(object): if not isdir(self.cache_dir): os.makedirs(self.cache_dir) tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} - assert valid.endswith(tuple(tdmap.keys())) + assert valid.endswith(tuple(tdmap)) expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) if not self._lock_dbindex(): @@ -339,21 +342,22 @@ def is_disabled_progressbar(): def get_cid(): cid = get_state_item("cid") - if not cid: - _uid = None - if getenv("C9_UID"): - _uid = getenv("C9_UID") - elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): - try: - _uid = requests.get("{api}/user?token={token}".format( - api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), - token=getenv("USER_TOKEN"))).json().get("id") - except: # pylint: disable=bare-except - pass - cid = str( - uuid.UUID( - bytes=hashlib.md5(str(_uid if _uid else uuid.getnode())). - digest())) - if "windows" in util.get_systype() or os.getuid() > 0: - set_state_item("cid", cid) + if cid: + return cid + uid = None + if getenv("C9_UID"): + uid = getenv("C9_UID") + elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): + try: + uid = requests.get("{api}/user?token={token}".format( + api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), + token=getenv("USER_TOKEN"))).json().get("id") + except: # pylint: disable=bare-except + pass + if not uid: + uid = uuid.getnode() + cid = uuid.UUID(bytes=hashlib.md5(str(uid).encode()).digest()) + cid = str(cid) + if "windows" in util.get_systype() or os.getuid() > 0: + set_state_item("cid", cid) return cid diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 5f2856ac..2afb2289 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -92,7 +92,7 @@ DEFAULT_ENV_OPTIONS = dict( variables=commonvars, # Propagating External Environment - PIOVARIABLES=commonvars.keys(), + PIOVARIABLES=list(commonvars.keys()), ENV=environ, UNIX_TIME=int(time()), PIOHOME_DIR=util.get_home_dir(), @@ -125,9 +125,11 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)): env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) # decode common variables -for k in commonvars.keys(): +for k in list(commonvars.keys()): if k in env: env[k] = base64.b64decode(env[k]) + if isinstance(env[k], bytes): + env[k] = env[k].decode() if k in MULTILINE_VARS: env[k] = util.parse_conf_multi_values(env[k]) @@ -161,7 +163,9 @@ env['LIBSOURCE_DIRS'] = [ env.LoadPioPlatform(commonvars) env.SConscriptChdir(0) -env.SConsignFile(join("$PROJECTBUILD_DIR", ".sconsign.dblite")) +env.SConsignFile( + join("$PROJECTBUILD_DIR", + ".sconsign.dblite" if util.PY2 else ".sconsign3.dblite")) for item in env.GetExtraScripts("pre"): env.SConscript(item, exports="env") diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 17cefb0c..02fee30c 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -74,12 +74,19 @@ class LibBuilderFactory(object): if not env.IsFileWithExt( fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT): continue - with open(join(root, fname)) as f: - content = f.read() - if "Arduino.h" in content and include_re.search(content): - return ["arduino"] - elif "mbed.h" in content and include_re.search(content): - return ["mbed"] + content = "" + try: + with open(join(root, fname)) as f: + content = f.read() + except UnicodeDecodeError: + with open(join(root, fname), encoding="latin-1") as f: + content = f.read() + if not content: + continue + if "Arduino.h" in content and include_re.search(content): + return ["arduino"] + if "mbed.h" in content and include_re.search(content): + return ["mbed"] return [] @@ -183,9 +190,9 @@ class LibBuilderBase(object): @property def build_dir(self): - return join("$BUILD_DIR", - "lib%s" % hashlib.sha1(self.path).hexdigest()[:3], - basename(self.path)) + lib_hash = hashlib.sha1(self.path if util.PY2 else self.path. + encode()).hexdigest()[:3] + return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) @property def build_flags(self): @@ -227,7 +234,7 @@ class LibBuilderBase(object): @staticmethod def validate_ldf_mode(mode): - if isinstance(mode, basestring): + if isinstance(mode, util.string_types): mode = mode.strip().lower() if mode in LibBuilderBase.LDF_MODES: return mode @@ -239,7 +246,7 @@ class LibBuilderBase(object): @staticmethod def validate_compat_mode(mode): - if isinstance(mode, basestring): + if isinstance(mode, util.string_types): mode = mode.strip().lower() if mode in LibBuilderBase.COMPAT_MODES: return mode @@ -612,9 +619,9 @@ class PlatformIOLibBuilder(LibBuilderBase): def src_filter(self): if "srcFilter" in self._manifest.get("build", {}): return self._manifest.get("build").get("srcFilter") - elif self.env['SRC_FILTER']: + if self.env['SRC_FILTER']: return self.env['SRC_FILTER'] - elif self._is_arduino_manifest(): + if self._is_arduino_manifest(): return ArduinoLibBuilder.src_filter.fget(self) return LibBuilderBase.src_filter.fget(self) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index be030f39..878cb486 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -162,7 +162,7 @@ class InoToCPPConverter(object): if not prototypes: return contents - prototype_names = set([m.group(3).strip() for m in prototypes]) + prototype_names = set(m.group(3).strip() for m in prototypes) split_pos = prototypes[0].start() match_ptrs = re.search( self.PROTOPTRS_TPLRE % ("|".join(prototype_names)), @@ -212,7 +212,7 @@ def _get_compiler_type(env): output = "".join([result['out'], result['err']]).lower() if "clang" in output and "LLVM" in output: return "clang" - elif "gcc" in output: + if "gcc" in output: return "gcc" return None diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 9d37ae77..358aee7d 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -95,8 +95,10 @@ def LoadPioPlatform(env, variables): for key, value in variables.UnknownVariables().items(): if not key.startswith("BOARD_"): continue - env.Replace( - **{key.upper().replace("BUILD.", ""): base64.b64decode(value)}) + value = base64.b64decode(value) + if isinstance(value, bytes): + value = value.decode() + env.Replace(**{key.upper().replace("BUILD.", ""): value}) return # update board manifest with a custom data @@ -104,10 +106,13 @@ def LoadPioPlatform(env, variables): for key, value in variables.UnknownVariables().items(): if not key.startswith("BOARD_"): continue - board_config.update(key.lower()[6:], base64.b64decode(value)) + value = base64.b64decode(value) + if isinstance(value, bytes): + value = value.decode() + board_config.update(key.lower()[6:], value) # update default environment variables - for key in variables.keys(): + for key in list(variables.keys()): if key in env or \ not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]): continue diff --git a/platformio/builder/tools/piowinhooks.py b/platformio/builder/tools/piowinhooks.py index 7db4f943..c893ffd5 100644 --- a/platformio/builder/tools/piowinhooks.py +++ b/platformio/builder/tools/piowinhooks.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import absolute_import + from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join -from platform import system + +from platformio import util # Windows CLI has limit with command length to 8192 # Leave 2000 chars for flags and other options @@ -58,7 +61,9 @@ def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") if not isdir(build_dir): makedirs(build_dir) - tmp_file = join(build_dir, "longcmd-%s" % md5(data).hexdigest()) + tmp_file = join( + build_dir, + "longcmd-%s" % md5(data if util.PY2 else data.encode()).hexdigest()) if isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: @@ -71,7 +76,7 @@ def exists(_): def generate(env): - if system() != "Windows": + if "windows" not in util.get_systype(): return None env.Replace(_long_sources_hook=long_sources_hook) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index b93499ae..2a539e10 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -24,7 +24,7 @@ from SCons import Builder, Util from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild, DefaultEnvironment, Export, SConscript) -from platformio.util import glob_escape, pioversion_to_intstr +from platformio.util import glob_escape, pioversion_to_intstr, string_types SRC_HEADER_EXT = ["h", "hpp"] SRC_C_EXT = ["c", "cc", "cpp"] @@ -189,7 +189,7 @@ def ProcessFlags(env, flags): # pylint: disable=too-many-branches # provided with a -U option // Issue #191 undefines = [ u for u in env.get("CCFLAGS", []) - if isinstance(u, basestring) and u.startswith("-U") + if isinstance(u, string_types) and u.startswith("-U") ] if undefines: for undef in undefines: diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 450b52bd..0a8ea38f 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -44,7 +44,7 @@ def device_list( # pylint: disable=too-many-branches if mdns: data['mdns'] = util.get_mdns_services() - single_key = data.keys()[0] if len(data.keys()) == 1 else None + single_key = list(data)[0] if len(list(data)) == 1 else None if json_output: return click.echo(json.dumps(data[single_key] if single_key else data)) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index c305dad6..2f09cd9c 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -17,7 +17,6 @@ import json import time from os.path import isdir, join -from urllib import quote import click @@ -25,6 +24,11 @@ from platformio import exception, util from platformio.managers.lib import LibraryManager, get_builtin_libs from platformio.util import get_api_result +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + @click.group(short_help="Library Manager") @click.option( @@ -142,9 +146,9 @@ def lib_update(lm, libraries, only_check, json_output): manifest['versionLatest'] = latest result.append(manifest) return click.echo(json.dumps(result)) - else: - for library in libraries: - lm.update(library, only_check=only_check) + + for library in libraries: + lm.update(library, only_check=only_check) return True diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 4ccd65ff..cb69c453 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -62,7 +62,7 @@ def _original_version(version): return None if len(y) % 2 != 0: y = "0" + y - parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(len(y) / 2)] + parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(int(len(y) / 2))] return ".".join(parts) @@ -88,8 +88,8 @@ def _get_installed_platform_data(platform, docs=p.docs_url, license=p.license, forDesktop=not p.is_embedded(), - frameworks=sorted(p.frameworks.keys() if p.frameworks else []), - packages=p.packages.keys() if p.packages else []) + frameworks=sorted(list(p.frameworks) if p.frameworks else []), + packages=list(p.packages) if p.packages else []) # if dump to API # del data['version'] @@ -374,15 +374,14 @@ def platform_update(platforms, only_packages, only_check, json_output): data['versionLatest'] = latest result.append(data) return click.echo(json.dumps(result)) - else: - # cleanup cached board and platform lists - app.clean_cache() - for platform in platforms: - click.echo("Platform %s" % click.style( - pkg_dir_to_name.get(platform, platform), fg="cyan")) - click.echo("--------") - pm.update( - platform, only_packages=only_packages, only_check=only_check) - click.echo() + + # cleanup cached board and platform lists + app.clean_cache() + for platform in platforms: + click.echo("Platform %s" % click.style( + pkg_dir_to_name.get(platform, platform), fg="cyan")) + click.echo("--------") + pm.update(platform, only_packages=only_packages, only_check=only_check) + click.echo() return True diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 47f14f81..ad4e61e0 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -357,7 +357,7 @@ def _clean_build_dir(build_dir): def print_header(label, is_error=False): terminal_width, _ = click.get_terminal_size() width = len(click.unstyle(label)) - half_line = "=" * ((terminal_width - width - 2) / 2) + half_line = "=" * int((terminal_width - width - 2) / 2) click.echo("%s %s %s" % (half_line, label, half_line), err=is_error) @@ -394,7 +394,7 @@ def print_summary(results, start_time): def check_project_defopts(config): if not config.has_section("platformio"): return True - unknown = set([k for k, _ in config.items("platformio")]) - set( + unknown = set(k for k, _ in config.items("platformio")) - set( EnvironmentProcessor.KNOWN_PLATFORMIO_OPTIONS) if not unknown: return True @@ -409,7 +409,7 @@ def check_project_envs(config, environments=None): if not config.sections(): raise exception.ProjectEnvsNotAvailable() - known = set([s[4:] for s in config.sections() if s.startswith("env:")]) + known = set(s[4:] for s in config.sections() if s.startswith("env:")) unknown = set(environments or []) - known if unknown: raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) @@ -432,4 +432,5 @@ def calculate_project_hash(): # Fix issue with useless project rebuilding for case insensitive FS. # A case of disk drive can differ... chunks_to_str = chunks_to_str.lower() - return sha1(chunks_to_str).hexdigest() + return sha1(chunks_to_str if util.PY2 else chunks_to_str. + encode()).hexdigest() diff --git a/platformio/downloader.py b/platformio/downloader.py index 62c19385..d8b2fd24 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -45,14 +45,16 @@ class FileDownloader(object): if disposition and "filename=" in disposition: self._fname = disposition[disposition.index("filename=") + 9:].replace('"', "").replace("'", "") - self._fname = self._fname.encode("utf8") + if util.PY2: + self._fname = self._fname.encode("utf8") else: self._fname = [p for p in url.split("/") if p][-1] self._destination = self._fname if dest_dir: - self.set_destination( - join(dest_dir.decode(getfilesystemencoding()), self._fname)) + if util.PY2: + dest_dir = dest_dir.decode(getfilesystemencoding()) + self.set_destination(join(dest_dir, self._fname)) def set_destination(self, destination): self._destination = destination diff --git a/platformio/exception.py b/platformio/exception.py index 48f831c9..7829b648 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=not-an-iterable + class PlatformioException(Exception): @@ -20,7 +22,7 @@ class PlatformioException(Exception): def __str__(self): # pragma: no cover if self.MESSAGE: return self.MESSAGE.format(*self.args) - return Exception.__str__(self) + return super(PlatformioException, self).__str__() class ReturnErrorCode(PlatformioException): diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 8ec43aed..e5c392fa 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -116,9 +116,10 @@ class ProjectGenerator(object): os.makedirs(dst_dir) file_name = basename(tpl_path)[:-4] + contents = self._render_tpl(tpl_path) self._merge_contents( join(dst_dir, file_name), - self._render_tpl(tpl_path).encode("utf8")) + contents.encode("utf8") if util.PY2 else contents) def _render_tpl(self, tpl_path): content = "" diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 7894dbda..47105a5b 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -127,7 +127,7 @@ class Upgrader(object): if not item.endswith(".json"): continue data = util.load_json(join(boards_dir, item)) - if set(["name", "url", "vendor"]) <= set(data.keys()): + if set(["name", "url", "vendor"]) <= set(data): continue os.remove(join(boards_dir, item)) for key, value in data.items(): diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 2a516f04..8962cbb7 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -26,9 +26,9 @@ from platformio.managers.package import PackageManager CORE_PACKAGES = { "contrib-piohome": "^2.0.0", "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.0.0", + "tool-pioplus": "^2.0.2", "tool-unity": "~1.20403.0", - "tool-scons": "~2.20501.7" + "tool-scons": "~2.20501.7" if util.PY2 else "~3.30100.0" } PIOPLUS_AUTO_UPDATES_MAX = 100 diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 9042e624..eaea4412 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -120,7 +120,7 @@ class LibraryManager(BasePkgManager): # convert listed items via comma to array for key in ("keywords", "frameworks", "platforms"): if key not in manifest or \ - not isinstance(manifest[key], basestring): + not isinstance(manifest[key], util.string_types): continue manifest[key] = [ i.strip() for i in manifest[key].split(",") if i.strip() @@ -147,7 +147,7 @@ class LibraryManager(BasePkgManager): continue if item[k] == "*": del item[k] - elif isinstance(item[k], basestring): + elif isinstance(item[k], util.string_types): item[k] = [ i.strip() for i in item[k].split(",") if i.strip() ] @@ -275,7 +275,7 @@ class LibraryManager(BasePkgManager): break if not lib_info: - if filters.keys() == ["name"]: + if list(filters) == ["name"]: raise exception.LibNotFound(filters['name']) else: raise exception.LibNotFound(str(filters)) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 2f73452d..8242a649 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -71,7 +71,7 @@ class PackageRepoIterator(object): if self.package in manifest: return manifest[self.package] - return self.next() + return next(self) class PkgRepoMixin(object): @@ -544,7 +544,7 @@ class PkgInstallerMixin(object): def _install_from_tmp_dir( # pylint: disable=too-many-branches self, tmp_dir, requirements=None): tmp_manifest = self.load_manifest(tmp_dir) - assert set(["name", "version"]) <= set(tmp_manifest.keys()) + assert set(["name", "version"]) <= set(tmp_manifest) pkg_dirname = self.get_install_dirname(tmp_manifest) pkg_dir = join(self.package_dir, pkg_dirname) @@ -587,8 +587,10 @@ class PkgInstallerMixin(object): cur_manifest['version']) if "__src_url" in cur_manifest: target_dirname = "%s@src-%s" % ( - pkg_dirname, hashlib.md5( - cur_manifest['__src_url']).hexdigest()) + pkg_dirname, + hashlib.md5(cur_manifest['__src_url'] if util. + PY2 else cur_manifest['__src_url']. + encode()).hexdigest()) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: @@ -596,8 +598,10 @@ class PkgInstallerMixin(object): tmp_manifest['version']) if "__src_url" in tmp_manifest: target_dirname = "%s@src-%s" % ( - pkg_dirname, hashlib.md5( - tmp_manifest['__src_url']).hexdigest()) + pkg_dirname, + hashlib.md5(tmp_manifest['__src_url'] if util. + PY2 else tmp_manifest['__src_url']. + encode()).hexdigest()) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 5f7e0adc..76a0550c 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -18,7 +18,6 @@ import re from imp import load_source from multiprocessing import cpu_count from os.path import basename, dirname, isdir, isfile, join -from urllib import quote import click import semantic_version @@ -27,6 +26,11 @@ from platformio import __version__, app, exception, util from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + class PlatformManager(BasePkgManager): @@ -81,7 +85,7 @@ class PlatformManager(BasePkgManager): skip_default_package, silent=silent, force=force) - return self.cleanup_packages(p.packages.keys()) + return self.cleanup_packages(list(p.packages)) def uninstall(self, package, requirements=None, after_update=False): if isdir(package): @@ -101,7 +105,7 @@ class PlatformManager(BasePkgManager): if after_update: return True - return self.cleanup_packages(p.packages.keys()) + return self.cleanup_packages(list(p.packages)) def update( # pylint: disable=arguments-differ self, @@ -119,17 +123,17 @@ class PlatformManager(BasePkgManager): raise exception.UnknownPlatform(package) p = PlatformFactory.newPlatform(pkg_dir) - pkgs_before = p.get_installed_packages().keys() + pkgs_before = list(p.get_installed_packages()) missed_pkgs = set() if not only_packages: BasePkgManager.update(self, pkg_dir, requirements, only_check) p = PlatformFactory.newPlatform(pkg_dir) - missed_pkgs = set(pkgs_before) & set(p.packages.keys()) - missed_pkgs -= set(p.get_installed_packages().keys()) + missed_pkgs = set(pkgs_before) & set(p.packages) + missed_pkgs -= set(p.get_installed_packages()) p.update_packages(only_check) - self.cleanup_packages(p.packages.keys()) + self.cleanup_packages(list(p.packages)) if missed_pkgs: p.install_packages( @@ -265,7 +269,7 @@ class PlatformPackagesMixin(object): without_packages = set(self.find_pkg_names(without_packages or [])) upkgs = with_packages | without_packages - ppkgs = set(self.packages.keys()) + ppkgs = set(self.packages) if not upkgs.issubset(ppkgs): raise exception.UnknownPackage(", ".join(upkgs - ppkgs)) @@ -384,7 +388,12 @@ class PlatformRunMixin(object): # encode and append variables for key, value in variables.items(): - cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) + if util.PY2: + cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) + else: + cmd.append( + "%s=%s" % (key.upper(), base64.b64encode( + value.encode()).decode())) util.copy_pythonpath_to_osenv() result = util.exec_command( @@ -550,7 +559,7 @@ class PlatformBase( # pylint: disable=too-many-public-methods config = PlatformBoardConfig(manifest_path) if "platform" in config and config.get("platform") != self.name: return - elif "platforms" in config \ + if "platforms" in config \ and self.name not in config.get("platforms"): return config.manifest['platform'] = self.name @@ -652,7 +661,7 @@ class PlatformBoardConfig(object): self._manifest = util.load_json(manifest_path) except ValueError: raise exception.InvalidBoardManifest(manifest_path) - if not set(["name", "url", "vendor"]) <= set(self._manifest.keys()): + if not set(["name", "url", "vendor"]) <= set(self._manifest): raise exception.PlatformioException( "Please specify name, url and vendor fields for " + manifest_path) @@ -666,8 +675,7 @@ class PlatformBoardConfig(object): except KeyError: if default is not None: return default - else: - raise KeyError("Invalid board option '%s'" % path) + raise KeyError("Invalid board option '%s'" % path) def update(self, path, value): newdict = None @@ -752,7 +760,7 @@ class PlatformBoardConfig(object): return tool_name raise exception.DebugInvalidOptions( "Unknown debug tool `%s`. Please use one of `%s` or `custom`" % - (tool_name, ", ".join(sorted(debug_tools.keys())))) + (tool_name, ", ".join(sorted(list(debug_tools))))) # automatically select best tool data = {"default": [], "onboard": [], "external": []} diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 7394654c..9d379c23 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -14,7 +14,6 @@ import atexit import platform -import Queue import re import sys import threading @@ -29,6 +28,11 @@ import requests from platformio import __version__, app, exception, util +try: + import queue +except ImportError: + import Queue as queue + class TelemetryBase(object): @@ -74,12 +78,12 @@ class MeasurementProtocol(TelemetryBase): def __getitem__(self, name): if name in self.PARAMS_MAP: name = self.PARAMS_MAP[name] - return TelemetryBase.__getitem__(self, name) + return super(MeasurementProtocol, self).__getitem__(name) def __setitem__(self, name, value): if name in self.PARAMS_MAP: name = self.PARAMS_MAP[name] - TelemetryBase.__setitem__(self, name, value) + super(MeasurementProtocol, self).__setitem__(name, value) def _prefill_appinfo(self): self['av'] = __version__ @@ -178,7 +182,7 @@ class MPDataPusher(object): MAX_WORKERS = 5 def __init__(self): - self._queue = Queue.LifoQueue() + self._queue = queue.LifoQueue() self._failedque = deque() self._http_session = requests.Session() self._http_offline = False @@ -203,7 +207,7 @@ class MPDataPusher(object): try: while True: items.append(self._queue.get_nowait()) - except Queue.Empty: + except queue.Empty: pass return items @@ -384,7 +388,7 @@ def backup_reports(items): for params in items: # skip static options - for key in params.keys(): + for key in params: if key in ("v", "tid", "cid", "cd1", "cd2", "sr", "an"): del params[key] diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 8f55b42d..be0fa552 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -64,7 +64,7 @@ class ZIPArchive(ArchiveBase): @staticmethod def preserve_permissions(item, dest_dir): - attrs = item.external_attr >> 16L + attrs = item.external_attr >> 16 if attrs: chmod(join(dest_dir, item.filename), attrs) @@ -72,7 +72,7 @@ class ZIPArchive(ArchiveBase): def preserve_mtime(item, dest_dir): util.change_filemtime( join(dest_dir, item.filename), - mktime(list(item.date_time) + [0] * 3)) + mktime(tuple(item.date_time) + tuple([0, 0, 0]))) def get_items(self): return self._afo.infolist() diff --git a/platformio/util.py b/platformio/util.py index c5433ee3..a8415ab0 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -34,12 +34,15 @@ import requests from platformio import __apiurl__, __version__, exception -# pylint: disable=wrong-import-order, too-many-ancestors +# pylint: disable=too-many-ancestors -try: - import configparser as ConfigParser -except ImportError: +PY2 = sys.version_info[0] == 2 +if PY2: import ConfigParser as ConfigParser + string_types = basestring # pylint: disable=undefined-variable +else: + import configparser as ConfigParser + string_types = str class ProjectConfig(ConfigParser.ConfigParser): @@ -52,7 +55,8 @@ class ProjectConfig(ConfigParser.ConfigParser): items.append((option, self.get(section, option))) return items - def get(self, section, option, **kwargs): + def get( # pylint: disable=arguments-differ + self, section, option, **kwargs): try: value = ConfigParser.ConfigParser.get(self, section, option, **kwargs) @@ -177,6 +181,8 @@ def singleton(cls): def path_to_unicode(path): + if not PY2: + return path return path.decode(sys.getfilesystemencoding()).encode("utf-8") @@ -315,8 +321,11 @@ def get_projectbuild_dir(force=False): path = get_project_optional_dir("build_dir", join(get_project_dir(), ".pioenvs")) if "$PROJECT_HASH" in path: - path = path.replace("$PROJECT_HASH", - sha1(get_project_dir()).hexdigest()[:10]) + project_dir = get_project_dir() + path = path.replace( + "$PROJECT_HASH", + sha1(project_dir if PY2 else project_dir.encode()).hexdigest() + [:10]) try: if not isdir(path): os.makedirs(path) @@ -414,8 +423,10 @@ def exec_command(*args, **kwargs): result[s[3:]] = "\n".join(kwargs[s].get_buffer()) for k, v in result.items(): - if v and isinstance(v, basestring): - result[k].strip() + if not PY2 and isinstance(result[k], bytes): + result[k] = result[k].decode() + if v and isinstance(v, string_types): + result[k] = result[k].strip() return result @@ -445,10 +456,13 @@ def get_serial_ports(filter_hwid=False): if not p: continue if "windows" in get_systype(): - try: - d = unicode(d, errors="ignore") - except TypeError: - pass + if PY2: + try: + d = unicode( # pylint: disable=undefined-variable + d, + errors="ignore") + except TypeError: + pass if not filter_hwid or "VID:PID" in h: result.append({"port": p, "description": d, "hwid": h}) @@ -490,17 +504,17 @@ def get_logical_devices(): for device in re.findall(r"[A-Z]:\\", result): items.append({"path": device, "name": None}) return items - else: - result = exec_command(["df"]).get("out") - devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) - for line in result.split("\n"): - match = devicenamere.match(line.strip()) - if not match: - continue - items.append({ - "path": match.group(1), - "name": basename(match.group(1)) - }) + + result = exec_command(["df"]).get("out") + devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I) + for line in result.split("\n"): + match = devicenamere.match(line.strip()) + if not match: + continue + items.append({ + "path": match.group(1), + "name": basename(match.group(1)) + }) return items @@ -557,12 +571,16 @@ def get_mdns_services(): time.sleep(3) for service in mdns.get_services(): properties = None - try: - if service.properties: - json.dumps(service.properties) - properties = service.properties - except UnicodeDecodeError: - pass + if service.properties: + try: + properties = { + k.decode("utf8"): + v.decode("utf8") if isinstance(v, bytes) else v + for k, v in service.properties.items() + } + json.dumps(properties) + except UnicodeDecodeError: + properties = None items.append({ "type": @@ -570,7 +588,10 @@ def get_mdns_services(): "name": service.name, "ip": - ".".join([str(ord(c)) for c in service.address]), + ".".join([ + str(c if isinstance(c, int) else ord(c)) + for c in service.address + ]), "port": service.port, "properties": @@ -735,7 +756,7 @@ def where_is_program(program, envpath=None): for bin_dir in env.get("PATH", "").split(os.pathsep): if isfile(join(bin_dir, program)): return join(bin_dir, program) - elif isfile(join(bin_dir, "%s.exe" % program)): + if isfile(join(bin_dir, "%s.exe" % program)): return join(bin_dir, "%s.exe" % program) return program diff --git a/platformio/vcsclient.py b/platformio/vcsclient.py index 6a924370..67ef29d2 100644 --- a/platformio/vcsclient.py +++ b/platformio/vcsclient.py @@ -16,11 +16,15 @@ import re from os.path import join from subprocess import CalledProcessError, check_call from sys import modules -from urlparse import urlparse from platformio import util from platformio.exception import PlatformioException, UserSideException +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + class VCSClientFactory(object): diff --git a/scripts/install_devplatforms.py b/scripts/install_devplatforms.py index 434ab7cd..4d8daaa6 100644 --- a/scripts/install_devplatforms.py +++ b/scripts/install_devplatforms.py @@ -21,7 +21,7 @@ from platformio import util def main(): platforms = json.loads( subprocess.check_output( - ["platformio", "platform", "search", "--json-output"])) + ["platformio", "platform", "search", "--json-output"]).decode()) for platform in platforms: if platform['forDesktop']: continue diff --git a/setup.py b/setup.py index dc9b7bf4..747d6cc2 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,8 @@ setup( author_email=__email__, url=__url__, license=__license__, - python_requires='>=2.7, <3', + python_requires=", ".join([ + ">=2.7", "!=3.0.*", "!=3.1.*", "!=3.2.*", "!=3.3.*", "!=3.4.*"]), install_requires=install_requires, packages=find_packages() + ["scripts"], package_data={ @@ -65,6 +66,7 @@ setup( "Operating System :: OS Independent", "Programming Language :: C", "Programming Language :: Python", + "Programming Language :: Python :: 3", "Topic :: Software Development", "Topic :: Software Development :: Build Tools", "Topic :: Software Development :: Compilers" diff --git a/tests/test_examples.py b/tests/test_examples.py index d9588956..d1ab17d8 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -74,7 +74,7 @@ def test_run(pioproject_dir): ["platformio", "run", "-e", random.choice(env_names)]) if result['returncode'] != 0: - pytest.fail(result) + pytest.fail(str(result)) assert isdir(build_dir) diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index ec9d749a..4a69227e 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -44,7 +44,7 @@ def test_after_upgrade_2_to_3(clirunner, validate_cliresult, result = clirunner.invoke(cli_pio, ["settings", "get"]) validate_cliresult(result) - assert "upgraded to 3" in result.output + assert "upgraded to " in result.output # check PlatformIO 3.0 boards assert board_ids == set([p.basename[:-5] for p in boards.listdir()]) diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index 92da706f..e0875a16 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -28,7 +28,7 @@ def test_packages(): "https://dl.bintray.com/platformio/dl-packages/manifest.json").json() assert isinstance(pkgs_manifest, dict) items = [] - for _, variants in pkgs_manifest.iteritems(): + for _, variants in pkgs_manifest.items(): for item in variants: items.append(item) diff --git a/tox.ini b/tox.ini index 26327fd7..597c7b8f 100644 --- a/tox.ini +++ b/tox.ini @@ -13,10 +13,10 @@ # limitations under the License. [tox] -envlist = py27, docs, lint +envlist = py27, py35, py36, py37, docs -[testenv:develop] -basepython = python2.7 +[testenv] +passenv = * usedevelop = True deps = isort @@ -24,10 +24,15 @@ deps = pylint pytest pytest-xdist -commands = python --version +commands = + {envpython} --version + pylint --rcfile=./.pylintrc ./platformio + {envpython} -c "print('travis_fold:start:install_devplatforms')" + {envpython} scripts/install_devplatforms.py + {envpython} -c "print('travis_fold:end:install_devplatforms')" + py.test -v --basetemp="{envtmpdir}" tests [testenv:docs] -basepython = python2.7 deps = sphinx sphinx_rtd_theme @@ -37,44 +42,23 @@ commands = sphinx-build -W -b latex -d {envtmpdir}/doctrees docs docs/_build/latex [testenv:docslinkcheck] -basepython = python2.7 deps = sphinx sphinx_rtd_theme commands = sphinx-build -W -b linkcheck docs docs/_build/html -[testenv:lint] -basepython = python2.7 -deps = - pylint -commands = - pylint --rcfile=./.pylintrc ./platformio - -[testenv] -basepython = python2.7 -passenv = * -deps = - pytest -commands = - {envpython} --version - {envpython} -c "print 'travis_fold:start:install_devplatforms'" - {envpython} scripts/install_devplatforms.py - {envpython} -c "print 'travis_fold:end:install_devplatforms'" - py.test -v --basetemp="{envtmpdir}" tests - [testenv:skipexamples] -basepython = python2.7 deps = pytest commands = py.test -v --basetemp="{envtmpdir}" tests --ignore tests/test_examples.py -[testenv:coverage] -basepython = python2.7 -passenv = * -deps = - pytest - pytest-cov -commands = - py.test --cov=platformio --cov-report term --cov-report xml --ignore=tests/test_examples.py --ignore=tests/test_pkgmanifest.py -v tests +; [testenv:coverage] +; basepython = python2 +; passenv = * +; deps = +; pytest +; pytest-cov +; commands = +; py.test --cov=platformio --cov-report term --cov-report xml --ignore=tests/test_examples.py --ignore=tests/test_pkgmanifest.py -v tests From 74218f4f9350ea4e4e38b4d0e2bf3752ea740616 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Dec 2018 22:33:21 +0200 Subject: [PATCH 002/333] Fix PyLint warning for Windows --- platformio/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/app.py b/platformio/app.py index c7dfdc43..a83d0eec 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -358,6 +358,7 @@ def get_cid(): uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(str(uid).encode()).digest()) cid = str(cid) - if "windows" in util.get_systype() or os.getuid() > 0: + if ("windows" in util.get_systype() + or os.getuid() > 0): # yapf: disable pylint: disable=no-member set_state_item("cid", cid) return cid From aa2bc4a63b1513f82a120d10ea22cc868bbafdfc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Dec 2018 14:48:22 +0200 Subject: [PATCH 003/333] Implement "get_file_contents" helper --- platformio/builder/tools/piolib.py | 12 ++++-------- platformio/builder/tools/piomisc.py | 3 +-- platformio/commands/remote.py | 2 +- platformio/ide/projectgenerator.py | 6 ++---- platformio/util.py | 9 +++++++++ 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 02fee30c..edf343f3 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -17,6 +17,7 @@ from __future__ import absolute_import +import codecs import hashlib import os import re @@ -74,13 +75,7 @@ class LibBuilderFactory(object): if not env.IsFileWithExt( fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT): continue - content = "" - try: - with open(join(root, fname)) as f: - content = f.read() - except UnicodeDecodeError: - with open(join(root, fname), encoding="latin-1") as f: - content = f.read() + content = util.get_file_contents(join(root, fname)) if not content: continue if "Arduino.h" in content and include_re.search(content): @@ -481,7 +476,8 @@ class ArduinoLibBuilder(LibBuilderBase): manifest = {} if not isfile(join(self.path, "library.properties")): return manifest - with open(join(self.path, "library.properties")) as fp: + manifest_path = join(self.path, "library.properties") + with codecs.open(manifest_path, encoding="utf-8") as fp: for line in fp.readlines(): if "=" not in line: continue diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 878cb486..484f6228 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -76,8 +76,7 @@ class InoToCPPConverter(object): def process(self, contents): out_file = self._main_ino + ".cpp" assert self._gcc_preprocess(contents, out_file) - with open(out_file) as fp: - contents = fp.read() + contents = util.get_file_contents(out_file) contents = self._join_multiline_strings(contents) with open(out_file, "w") as fp: fp.write(self.append_prototypes(contents)) diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index 9b7ad5fb..d506d5a5 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -202,7 +202,7 @@ def device_monitor(ctx, **kwargs): sleep(0.1) if not t.is_alive(): return - kwargs['port'] = open(sock_file).read() + kwargs['port'] = util.get_file_contents(sock_file) ctx.invoke(cmd_device_monitor, **kwargs) t.join(2) finally: diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index e5c392fa..cd48ca61 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -122,10 +122,8 @@ class ProjectGenerator(object): contents.encode("utf8") if util.PY2 else contents) def _render_tpl(self, tpl_path): - content = "" - with open(tpl_path) as f: - content = f.read() - return bottle.template(content, **self._tplvars) + return bottle.template( + util.get_file_contents(tpl_path), **self._tplvars) @staticmethod def _merge_contents(dst_path, contents): diff --git a/platformio/util.py b/platformio/util.py index a8415ab0..3b5c16e9 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -852,6 +852,15 @@ def ensure_udev_rules(): return True +def get_file_contents(path): + try: + with open(path) as f: + return f.read() + except UnicodeDecodeError: + with open(path, encoding="latin-1") as f: + return f.read() + + def rmtree_(path): def _onerror(_, name, __): From 2ae41c8434f8ea9458f6ced3f7254187c6ddf507 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 9 Jan 2019 16:34:39 +0200 Subject: [PATCH 004/333] Fix "Runtime Error: Dictionary size changed during iteration" // Resolve #2003 --- HISTORY.rst | 2 ++ platformio/telemetry.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 80144ad7..2e10451e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -23,6 +23,8 @@ PlatformIO 3.0 - CLion: Improved project portability using "${CMAKE_CURRENT_LIST_DIR}" instead of full path * Fixed an issue with incorrect detecting of compatibility (LDF) between generic library and Arduino or ARM mbed frameworks +* Fixed "Runtime Error: Dictionary size changed during iteration" + (`issue #2003 `_) 3.6.3 (2018-12-12) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 9d379c23..1ced1f7d 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -388,7 +388,7 @@ def backup_reports(items): for params in items: # skip static options - for key in params: + for key in params.keys(): if key in ("v", "tid", "cid", "cd1", "cd2", "sr", "an"): del params[key] From e02d7528ade04f1731cb17222996b52b81924859 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 10 Jan 2019 19:14:03 +0200 Subject: [PATCH 005/333] Fix an error "Could not extract item..." when extracting TAR archive with symbolic items on Windows platform // Resolve #2015 --- HISTORY.rst | 2 ++ platformio/unpacker.py | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2e10451e..9ba9e6f6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -25,6 +25,8 @@ PlatformIO 3.0 * Fixed an issue with incorrect detecting of compatibility (LDF) between generic library and Arduino or ARM mbed frameworks * Fixed "Runtime Error: Dictionary size changed during iteration" (`issue #2003 `_) +* Fixed an error "Could not extract item..." when extracting TAR archive with symbolic items on Windows platform + (`issue #2015 `_) 3.6.3 (2018-12-12) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/unpacker.py b/platformio/unpacker.py index be0fa552..58132716 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -13,7 +13,7 @@ # limitations under the License. from os import chmod -from os.path import exists, islink, join +from os.path import exists, join from tarfile import open as tarfile_open from time import mktime from zipfile import ZipFile @@ -56,6 +56,10 @@ class TARArchive(ArchiveBase): def get_item_filename(self, item): return item.name + @staticmethod + def islink(item): + return item.islnk() or item.issym() + class ZIPArchive(ArchiveBase): @@ -80,6 +84,9 @@ class ZIPArchive(ArchiveBase): def get_item_filename(self, item): return item.filename + def islink(self, item): + raise NotImplementedError() + def after_extract(self, item, dest_dir): self.preserve_permissions(item, dest_dir) self.preserve_mtime(item, dest_dir) @@ -120,7 +127,9 @@ class FileUnpacker(object): for item in self._unpacker.get_items(): filename = self._unpacker.get_item_filename(item) item_path = join(dest_dir, filename) - if not islink(item_path) and not exists(item_path): - raise exception.ExtractArchiveItemError(filename, dest_dir) - + try: + if not self._unpacker.islink(item) and not exists(item_path): + raise exception.ExtractArchiveItemError(filename, dest_dir) + except NotImplementedError: + pass return True From 02937216b0483e8d4dbe8cba570e53c32407231d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 10 Jan 2019 19:33:15 +0200 Subject: [PATCH 006/333] Fix "TypeError : startswith first arg" when checking udev rules with PY3 // Resolve #2000 --- platformio/util.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index 3b5c16e9..c8637e29 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -818,14 +818,10 @@ def merge_dicts(d1, d2, path=None): def ensure_udev_rules(): def _rules_to_set(rules_path): - result = set([]) - with open(rules_path, "rb") as fp: - for line in fp.readlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - result.add(line) - return result + return set([ + l.strip() for l in get_file_contents(rules_path).split("\n") + if l.strip() and not l.startswith("#") + ]) if "linux" not in get_systype(): return None From a092f87c5070e78dc9e7319c3e6c32c8e2dc3bfc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 10 Jan 2019 21:57:39 +0200 Subject: [PATCH 007/333] Eclipse: Provide language standard to a project C/C++ indexer // Resolve #1010 --- HISTORY.rst | 2 ++ .../eclipse/.settings/language.settings.xml.tpl | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9ba9e6f6..d42f4c92 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -21,6 +21,8 @@ PlatformIO 3.0 - Use full path to PlatformIO CLI when generating a project (`issue #1674 `_) - CLion: Improved project portability using "${CMAKE_CURRENT_LIST_DIR}" instead of full path + - Eclipse: Provide language standard to a project C/C++ indexer + (`issue #1010 `_) * Fixed an issue with incorrect detecting of compatibility (LDF) between generic library and Arduino or ARM mbed frameworks * Fixed "Runtime Error: Dictionary size changed during iteration" diff --git a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl index 19740737..72895fb2 100644 --- a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl +++ b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl @@ -1,3 +1,8 @@ +% import re +% STD_RE = re.compile(r"(\-std=[a-z\+]+\d+)") +% cxx_stds = STD_RE.findall(cxx_flags) +% cxx_std = cxx_stds[-1] if cxx_stds else "" +% @@ -6,9 +11,9 @@ % if "windows" in systype: - + % else: - + % end @@ -21,9 +26,9 @@ % if "windows" in systype: - + % else: - + % end From b92a8467c9cfb32ef14bc2c746b0b1091d003db1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 10 Jan 2019 23:47:43 +0200 Subject: [PATCH 008/333] Update SCons to 3.0.3 --- platformio/managers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 8962cbb7..06183901 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -28,7 +28,7 @@ CORE_PACKAGES = { "contrib-pysite": "~2.%d%d.0" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.0.2", "tool-unity": "~1.20403.0", - "tool-scons": "~2.20501.7" if util.PY2 else "~3.30100.0" + "tool-scons": "~2.20501.7" if util.PY2 else "~3.30003.0" } PIOPLUS_AUTO_UPDATES_MAX = 100 From 286a53991c834120814e8f100e29fab7500bd8d5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 11 Jan 2019 12:52:00 +0200 Subject: [PATCH 009/333] Use GCC C++ compiler for Eclipse project indexer // Issue #1010 --- .../ide/tpls/eclipse/.settings/language.settings.xml.tpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl index 72895fb2..a119c8b2 100644 --- a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl +++ b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl @@ -11,9 +11,9 @@ % if "windows" in systype: - + % else: - + % end @@ -26,9 +26,9 @@ % if "windows" in systype: - + % else: - + % end From a750b06fc87b63cf05b0bbdb50c2f9a20de2daba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 11 Jan 2019 13:01:53 +0200 Subject: [PATCH 010/333] Fix PY3 Lint "consider-using-set-comprehension" --- platformio/util.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index c8637e29..aef0e68c 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -818,10 +818,8 @@ def merge_dicts(d1, d2, path=None): def ensure_udev_rules(): def _rules_to_set(rules_path): - return set([ - l.strip() for l in get_file_contents(rules_path).split("\n") - if l.strip() and not l.startswith("#") - ]) + return set(l.strip() for l in get_file_contents(rules_path).split("\n") + if l.strip() and not l.startswith("#")) if "linux" not in get_systype(): return None From 2b467f3fee68175ccdf0c1e78f3ca8672bfe0a84 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 11 Jan 2019 14:07:35 +0200 Subject: [PATCH 011/333] Fix PyLint warning --- platformio/util.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index aef0e68c..4ccd1916 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -815,6 +815,15 @@ def merge_dicts(d1, d2, path=None): return d1 +def get_file_contents(path): + try: + with open(path) as f: + return f.read() + except UnicodeDecodeError: + with open(path, encoding="latin-1") as f: + return f.read() + + def ensure_udev_rules(): def _rules_to_set(rules_path): @@ -846,15 +855,6 @@ def ensure_udev_rules(): return True -def get_file_contents(path): - try: - with open(path) as f: - return f.read() - except UnicodeDecodeError: - with open(path, encoding="latin-1") as f: - return f.read() - - def rmtree_(path): def _onerror(_, name, __): From c7949ecd077e6382fcaf19793c0f9f4b89936507 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 11 Jan 2019 14:11:54 +0200 Subject: [PATCH 012/333] Temporary disable checking for PlatformIO Core engine (allow PIO Core 3 dev/platforms for PIO Core 4) --- platformio/managers/package.py | 8 ++++---- platformio/managers/platform.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 8242a649..4026d3f4 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -99,10 +99,10 @@ class PkgRepoMixin(object): for v in versions: if not self.is_system_compatible(v.get("system")): continue - if "platformio" in v.get("engines", {}): - if PkgRepoMixin.PIO_VERSION not in self.parse_semver_spec( - v['engines']['platformio'], raise_exception=True): - continue + # if "platformio" in v.get("engines", {}): + # if PkgRepoMixin.PIO_VERSION not in self.parse_semver_spec( + # v['engines']['platformio'], raise_exception=True): + # continue specver = semantic_version.Version(v['version']) if reqspec and specver not in reqspec: continue diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 76a0550c..b2210427 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -474,11 +474,11 @@ class PlatformBase( # pylint: disable=too-many-public-methods self.silent = False self.verbose = False - if self.engines and "platformio" in self.engines: - if self.PIO_VERSION not in semantic_version.Spec( - self.engines['platformio']): - raise exception.IncompatiblePlatform(self.name, - str(self.PIO_VERSION)) + # if self.engines and "platformio" in self.engines: + # if self.PIO_VERSION not in semantic_version.Spec( + # self.engines['platformio']): + # raise exception.IncompatiblePlatform(self.name, + # str(self.PIO_VERSION)) @property def name(self): From 38bb2c61c1d0e4063dde7184a0cd43e798577234 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 11 Jan 2019 14:35:54 +0200 Subject: [PATCH 013/333] Docs: Sort upload protocols for board --- docs | 2 +- scripts/docspregen.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index a48c700b..e40ed515 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a48c700ba11faa1da891ddeab613192c768252a9 +Subproject commit e40ed515552cc3f78a73cfada06042bdb6210fa3 diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 721d39da..b4e5a67f 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -690,7 +690,7 @@ Uploading --------- %s supports the next uploading protocols: """ % board['name']) - for protocol in upload_protocols: + for protocol in sorted(upload_protocols): lines.append("* ``%s``" % protocol) lines.append(""" Default protocol is ``%s``""" % variables['upload_protocol']) From 3019e357245650f0333f374a2f87dc805979943b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 11 Jan 2019 16:52:01 +0200 Subject: [PATCH 014/333] Initial support for STM8 --- README.rst | 1 + docs | 2 +- examples | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fa295e95..76e1665c 100644 --- a/README.rst +++ b/README.rst @@ -96,6 +96,7 @@ Development Platforms * `Samsung ARTIK `_ * `Silicon Labs EFM32 `_ * `ST STM32 `_ +* `ST STM8 `_ * `Teensy `_ * `TI MSP430 `_ * `TI Tiva `_ diff --git a/docs b/docs index e40ed515..92337c5e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e40ed515552cc3f78a73cfada06042bdb6210fa3 +Subproject commit 92337c5e67ca1b46a1cba05b05827c0e1e579d44 diff --git a/examples b/examples index 99f177e5..02874d2b 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 99f177e5be7c595ac99167fd7b302ea117d64b42 +Subproject commit 02874d2b1ac84465db623dbf3b0d6e111258b3e7 From 88d06b4437b5f70914f26aef08e5caf0c5b8144e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 12 Jan 2019 01:08:13 +0200 Subject: [PATCH 015/333] Ignore examples for ststm8 on Linux --- tests/test_examples.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index d1ab17d8..e2404d07 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -37,7 +37,8 @@ def pytest_generate_tests(metafunc): if not p.is_embedded(): continue # issue with "version `CXXABI_1.3.9' not found (required by sdcc)" - if "linux" in util.get_systype() and p.name == "intel_mcs51": + if "linux" in util.get_systype() and p.name in ("intel_mcs51", + "ststm8"): continue examples_dir = join(p.get_dir(), "examples") assert isdir(examples_dir) From 7e8349d45e28997bdc9ea3c6aea8b72330fa6dab Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 16 Jan 2019 15:34:31 +0200 Subject: [PATCH 016/333] Docs: Sync Atmel AVR boards --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 92337c5e..ae5928df 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 92337c5e67ca1b46a1cba05b05827c0e1e579d44 +Subproject commit ae5928dfb0ee79f62b01377ba416a99aaee1c261 From 35f96a534aee4ade4f5327945a6544e2f20d9391 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Jan 2019 02:07:46 +0200 Subject: [PATCH 017/333] Drop "do-not-modify-files-here.url" from build_dir --- platformio/util.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index 4ccd1916..0898abc9 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -329,13 +329,6 @@ def get_projectbuild_dir(force=False): try: if not isdir(path): os.makedirs(path) - dontmod_path = join(path, "do-not-modify-files-here.url") - if not isfile(dontmod_path): - with open(dontmod_path, "w") as fp: - fp.write(""" -[InternetShortcut] -URL=https://docs.platformio.org/page/projectconf/section_platformio.html#build-dir -""") except Exception as e: # pylint: disable=broad-except if not force: raise Exception(e) From b6ccda356857a702db24223e9dbde32d03ee7b14 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Jan 2019 17:56:21 +0200 Subject: [PATCH 018/333] Fix cmd_lib test --- tests/commands/test_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 7fc6ca90..b67cec39 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -170,7 +170,7 @@ def test_global_lib_list(clirunner, validate_cliresult): ] versions2 = [ 'ArduinoJson@5.8.2', 'ArduinoJson@5.10.1', 'AsyncMqttClient@0.8.2', - 'AsyncTCP@1.0.1', 'NeoPixelBus@2.2.4', 'PJON@07fe9aa', 'PJON@1fb26fd', + 'NeoPixelBus@2.2.4', 'PJON@07fe9aa', 'PJON@1fb26fd', 'PubSubClient@bef5814', 'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0' ] assert set(versions1) >= set(versions2) From 63075c960796876c657b74e56a160303ea1223ef Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Jan 2019 18:04:52 +0200 Subject: [PATCH 019/333] CLion: Improve project portability using "${CMAKE_CURRENT_LIST_DIR}" instead of USER_HOME --- platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index 65423e22..a1c84d68 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -1,12 +1,12 @@ % def _normalize_path(path): -% if user_home_dir in path: +% if project_dir in path: +% path = path.replace(project_dir, "${CMAKE_CURRENT_LIST_DIR}") +% elif user_home_dir in path: % if "windows" in systype: % path = path.replace(user_home_dir, "$ENV{HOMEDRIVE}$ENV{HOMEPATH}") % else: % path = path.replace(user_home_dir, "$ENV{HOME}") % end -% elif project_dir in path: -% path = path.replace(project_dir, "${CMAKE_CURRENT_LIST_DIR}") % end % return path.replace("\\", "/") % end From eedc1c3ccde46c4719ea1d02485b9b61bfa784dd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 17 Jan 2019 18:18:28 +0200 Subject: [PATCH 020/333] Bump version to 4.0.0a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 8a5c9124..dacfa306 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a1") +VERSION = (4, 0, "0a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From d7d981f5224c1659ee987d7f7142ba38db38a616 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 22 Jan 2019 21:05:38 +0200 Subject: [PATCH 021/333] Add "Variable Format" section to VSCode --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index ae5928df..c1865c17 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ae5928dfb0ee79f62b01377ba416a99aaee1c261 +Subproject commit c1865c1795351d6854b9262d3391f2cc20935ee0 From 3cc996d89f58e4f94ea3144823f84b5e84292a06 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 22 Jan 2019 21:59:26 +0200 Subject: [PATCH 022/333] Fix "ValueError: invalid literal for int() with base 10" // Resolve #2058 --- platformio/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/app.py b/platformio/app.py index a83d0eec..75420311 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -229,9 +229,9 @@ class ContentCache(object): newlines = [] with open(self._db_path) as fp: for line in fp.readlines(): + line = line.strip() if "=" not in line: continue - line = line.strip() expire, path = line.split("=") if time() < int(expire) and isfile(path) and \ path not in paths_for_delete: From eb57e14ac1adba8626fca2d7526d6a437b8c1a45 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 23 Jan 2019 17:54:45 +0200 Subject: [PATCH 023/333] Fix "ValueError: invalid literal for int() with base 10" // Resolve #2061 --- platformio/app.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 75420311..247fb65c 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -233,10 +233,13 @@ class ContentCache(object): if "=" not in line: continue expire, path = line.split("=") - if time() < int(expire) and isfile(path) and \ - path not in paths_for_delete: - newlines.append(line) - continue + try: + if time() < int(expire) and isfile(path) and \ + path not in paths_for_delete: + newlines.append(line) + continue + except ValueError: + pass found = True if isfile(path): try: From d2c545eb275da18e33f5a755cc54fceff8450545 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 23 Jan 2019 20:55:18 +0200 Subject: [PATCH 024/333] Bump version to 4.0.0a3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index dacfa306..7880c2f6 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a2") +VERSION = (4, 0, "0a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 27ac17e8c3f657930193e78980fa821ca524e548 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 23 Jan 2019 21:24:29 +0200 Subject: [PATCH 025/333] Release docs to 3.6.4 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index c1865c17..12cf9691 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c1865c1795351d6854b9262d3391f2cc20935ee0 +Subproject commit 12cf9691dcecdc1518e0b6ad280d9e8c7bbbc882 From 5aed0efd61492342229e2569b7c7feff66c46a65 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 29 Jan 2019 22:20:41 +0200 Subject: [PATCH 026/333] Docs: Add Alorium Hinj board --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 12cf9691..50b4d450 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 12cf9691dcecdc1518e0b6ad280d9e8c7bbbc882 +Subproject commit 50b4d450eda383f445a0689d237ff18d1cb5c952 From 527efe335928472baa61fbd9aeb63c907eebf5c1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 30 Jan 2019 18:00:12 +0200 Subject: [PATCH 027/333] RISV-GAP, document AutoTiler --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 50b4d450..53f5a048 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 50b4d450eda383f445a0689d237ff18d1cb5c952 +Subproject commit 53f5a048071f7de171c18c9f5b5a7429fa8a402d From 17efa8904759f2f4e83dce2a4bea12e0d5c84a5c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 30 Jan 2019 18:00:40 +0200 Subject: [PATCH 028/333] Disable STM8 from CI tests --- tests/test_examples.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index e2404d07..3f34433b 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -34,11 +34,13 @@ def pytest_generate_tests(metafunc): # dev/platforms for manifest in PlatformManager().get_installed(): p = PlatformFactory.newPlatform(manifest['__pkg_dir']) - if not p.is_embedded(): - continue - # issue with "version `CXXABI_1.3.9' not found (required by sdcc)" - if "linux" in util.get_systype() and p.name in ("intel_mcs51", - "ststm8"): + ignore_conds = [ + not p.is_embedded(), + p.name == "ststm8", + # issue with "version `CXXABI_1.3.9' not found (required by sdcc)" + "linux" in util.get_systype() and p.name == "intel_mcs51" + ] + if any(ignore_conds): continue examples_dir = join(p.get_dir(), "examples") assert isdir(examples_dir) From e23ca2c10992f10b1906ad616248aa2ef999db23 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 31 Jan 2019 16:42:44 +0200 Subject: [PATCH 029/333] Docs: Allow to control firmware optimization for Teensy --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 53f5a048..adb6e55d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 53f5a048071f7de171c18c9f5b5a7429fa8a402d +Subproject commit adb6e55d9680f0c84415c38f629abe321fe7f6b6 From 31d4706abd66f0ec2510709c23ae97bec5460450 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Feb 2019 19:21:12 +0200 Subject: [PATCH 030/333] Debug: Update wiring scheme for Mini-Module --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index adb6e55d..4aa008f1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit adb6e55d9680f0c84415c38f629abe321fe7f6b6 +Subproject commit 4aa008f1f92a5c87267e709f8374989e5c8bb451 From 08000a6f622273a86cebc8e36fe426086a7f9f2e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Feb 2019 19:01:51 +0200 Subject: [PATCH 031/333] Sync docs & examples --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 4aa008f1..8e604864 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4aa008f1f92a5c87267e709f8374989e5c8bb451 +Subproject commit 8e60486406818bf97433471d53222ae5fa76e8eb diff --git a/examples b/examples index 02874d2b..1351ff7f 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 02874d2b1ac84465db623dbf3b0d6e111258b3e7 +Subproject commit 1351ff7fa80e2e3c90d7498014af58909cd705aa From 47aa63fc04abb109e03c7b1fe6b543baeadb349d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Feb 2019 20:36:45 +0200 Subject: [PATCH 032/333] Update docs for mbed framework --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 8e604864..b557a582 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8e60486406818bf97433471d53222ae5fa76e8eb +Subproject commit b557a58266d1ba2f93fbf2ef1b33bcd4522c064c From f1b809f8dde24061d5d29cc0ad250f0b3c710f94 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 9 Feb 2019 19:11:51 +0200 Subject: [PATCH 033/333] Improve debugging instruction for ESP32 tutorial // Resolve #1871 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b557a582..a8b2afa2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b557a58266d1ba2f93fbf2ef1b33bcd4522c064c +Subproject commit a8b2afa251a549e31c7524447850c60e7f2e2726 From 5add5f1cfb2257306823fef14b157e8815a452cd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Feb 2019 15:51:35 +0200 Subject: [PATCH 034/333] Document lwIP profiles for EPS8266 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index a8b2afa2..48775363 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a8b2afa251a549e31c7524447850c60e7f2e2726 +Subproject commit 48775363dd73e5f725f701f16998ecb08c6b207f From 38dfa40e3235808e094d3686084363028919a3c7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 18 Feb 2019 18:57:37 +0200 Subject: [PATCH 035/333] Docs: Sync Atmel SAM boards --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 48775363..ba68bc0b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 48775363dd73e5f725f701f16998ecb08c6b207f +Subproject commit ba68bc0b0d14ec079623358186ed047475f37bb3 From 234bb75b9b3a27370a92ab8be8a4aa9d4a4ad99e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 19 Feb 2019 18:11:21 +0200 Subject: [PATCH 036/333] Sync docs & examples --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index ba68bc0b..505d797d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ba68bc0b0d14ec079623358186ed047475f37bb3 +Subproject commit 505d797d847514e913dfd05d0c62eeb576177fbb diff --git a/examples b/examples index 1351ff7f..74367905 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 1351ff7fa80e2e3c90d7498014af58909cd705aa +Subproject commit 74367905af9f82213dfb0d645d56c84b6cfab455 From 6059e0dce614014cecd1015321f0588090cb4cd1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 19 Feb 2019 20:12:07 +0200 Subject: [PATCH 037/333] Fix an issue with slow updating of PlatformIO Core packages on Windows --- HISTORY.rst | 1 + platformio/managers/core.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a0e3f46f..3443639f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ PlatformIO 4.0 * Added Python 3.5+ support (`issue #895 `_) +* Fixed an issue with slow updating of PlatformIO Core packages on Windows PlatformIO 3.0 -------------- diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 06183901..3fd7f1cd 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -106,9 +106,10 @@ def update_core_packages(only_check=False, silent=False): def shutdown_piohome_servers(): port = 8010 - while port < 8100: + while port < 8050: try: - requests.get("http://127.0.0.1:%d?__shutdown__=1" % port) + requests.get( + "http://127.0.0.1:%d?__shutdown__=1" % port, timeout=0.01) except: # pylint: disable=bare-except pass port += 1 From e477d1fbcf67cfe4e52243ca527980e6810b541e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 19 Feb 2019 23:25:13 +0200 Subject: [PATCH 038/333] Docs: Move images from root of _static to _static/images --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 505d797d..43a79535 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 505d797d847514e913dfd05d0c62eeb576177fbb +Subproject commit 43a79535077b88ed9bf5be24507b699e35aa5b36 From 2e4c9411af9f1d3ace5f4d90a8d61ecadc13a181 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Feb 2019 00:00:41 +0200 Subject: [PATCH 039/333] Docs: Update VSCode Custom Task example --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 43a79535..eef5b841 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 43a79535077b88ed9bf5be24507b699e35aa5b36 +Subproject commit eef5b841a7b3c4bb1dc711be50e3f20d681d1554 From dc4a5df2afb6cbeef311d4c595b46982847689af Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Feb 2019 00:13:56 +0200 Subject: [PATCH 040/333] Drop support for RFduino --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index eef5b841..a9691e11 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit eef5b841a7b3c4bb1dc711be50e3f20d681d1554 +Subproject commit a9691e11c5b379bc0fdc2c67a57f4b236307e235 From 6575ddab2654e7f12cd02e18748afe34ca728f27 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 21 Feb 2019 00:25:14 +0200 Subject: [PATCH 041/333] Improve docs for `pio ci` command when multiple `--project-option` are used --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index a9691e11..944b8f3f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a9691e11c5b379bc0fdc2c67a57f4b236307e235 +Subproject commit 944b8f3f6dfb80e81263b340e947b9b6a42a149c From 59b27d5d0a65b11ed31d4abe77d079430f9ceccc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 21 Feb 2019 22:53:33 +0200 Subject: [PATCH 042/333] Document switching between Arduino cores --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 944b8f3f..147be01b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 944b8f3f6dfb80e81263b340e947b9b6a42a149c +Subproject commit 147be01b31428c8fa2022bbc635e0b082e4c1098 From 65f30dd7b487f3601199e709cf521b0bcad2123f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Feb 2019 01:10:41 +0200 Subject: [PATCH 043/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 147be01b..3bfbe649 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 147be01b31428c8fa2022bbc635e0b082e4c1098 +Subproject commit 3bfbe6497fc05f43de1c8ae4f706ec0578ac04d8 From 1bed2650f3c07a196954037e669ba9a5480bcd1e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Feb 2019 14:44:13 +0200 Subject: [PATCH 044/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 3bfbe649..6270f547 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3bfbe6497fc05f43de1c8ae4f706ec0578ac04d8 +Subproject commit 6270f547f8388d91f4c1e1744cb4ae2c0b395d0a From 5a707a4849b9506350ae035dd2aca8d0ae464d0b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Feb 2019 16:00:24 +0200 Subject: [PATCH 045/333] CI only for Python 2.7 & 3.6 --- .appveyor.yml | 1 - .travis.yml | 4 ---- 2 files changed, 5 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 72fc32ac..e071eaa8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,7 +7,6 @@ platform: environment: matrix: - TOXENV: "py27" - - TOXENV: "py35" - TOXENV: "py36" install: diff --git a/.travis.yml b/.travis.yml index fe90e097..07226606 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,10 +10,6 @@ matrix: sudo: required python: 2.7 env: TOX_ENV=py27 - - os: linux - sudo: required - python: 3.5 - env: TOX_ENV=py35 - os: linux sudo: required python: 3.6 From 522b42c2a926d89bd3942b47d40acded1fd45400 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Feb 2019 18:39:54 +0200 Subject: [PATCH 046/333] Fix an issue when `platformio ci` recompiles project if ``--keep-build-dir`` option is passed // Resolve #2109 --- HISTORY.rst | 2 ++ platformio/commands/ci.py | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3443639f..b5b339af 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,8 @@ PlatformIO 4.0 * Added Python 3.5+ support (`issue #895 `_) * Fixed an issue with slow updating of PlatformIO Core packages on Windows +* Fixed an issue when `platformio ci `__ recompiles project if ``--keep-build-dir`` option is passed + (`issue #2109 `_) PlatformIO 3.0 -------------- diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 0abf72d2..a5f28256 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -84,7 +84,11 @@ def cli( # pylint: disable=too-many-arguments try: app.set_session_var("force_option", True) - _clean_dir(build_dir) + + if not keep_build_dir and isdir(build_dir): + util.rmtree_(build_dir) + if not isdir(build_dir): + makedirs(build_dir) for dir_name, patterns in dict(lib=lib, src=src).items(): if not patterns: @@ -116,11 +120,6 @@ def cli( # pylint: disable=too-many-arguments util.rmtree_(build_dir) -def _clean_dir(dirpath): - util.rmtree_(dirpath) - makedirs(dirpath) - - def _copy_contents(dst_dir, contents): items = {"dirs": set(), "files": set()} From 42c22758bb05c14f30351ecf92831e7ae646215f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Feb 2019 21:04:43 +0200 Subject: [PATCH 047/333] Use NodeMCUv2 as default board for wiring example --- examples | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples b/examples index 74367905..6a0e7d47 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 74367905af9f82213dfb0d645d56c84b6cfab455 +Subproject commit 6a0e7d4794e66d1f78f8c440baae7b92ba9378ca From e0b9d080fa099c2ead66867a48d039d4ae08a148 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Feb 2019 22:34:36 +0200 Subject: [PATCH 048/333] Project Generator: add new targets for CLion IDE "BUILD_VERBOSE" and "MONITOR" (serial port monitor) // Resolve #359 --- HISTORY.rst | 2 ++ docs | 2 +- platformio/ide/tpls/clion/CMakeLists.txt.tpl | 12 ++++++++++++ platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl | 4 ++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index b5b339af..c9384b85 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ PlatformIO 4.0 * Added Python 3.5+ support (`issue #895 `_) +* Project Generator: added new targets for CLion IDE "BUILD_VERBOSE" and "MONITOR" (serial port monitor) + (`issue #359 `_) * Fixed an issue with slow updating of PlatformIO Core packages on Windows * Fixed an issue when `platformio ci `__ recompiles project if ``--keep-build-dir`` option is passed (`issue #2109 `_) diff --git a/docs b/docs index 6270f547..11ed4498 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6270f547f8388d91f4c1e1744cb4ae2c0b395d0a +Subproject commit 11ed449808bb8202d1212113afce25b4f3ebfbc5 diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 63274c43..7b17a983 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -9,6 +9,12 @@ add_custom_target( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) +add_custom_target( + PLATFORMIO_BUILD_VERBOSE ALL + COMMAND ${PLATFORMIO_CMD} -f -c clion run --verbose + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + add_custom_target( PLATFORMIO_UPLOAD ALL COMMAND ${PLATFORMIO_CMD} -f -c clion run --target upload @@ -21,6 +27,12 @@ add_custom_target( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) +add_custom_target( + PLATFORMIO_MONITOR ALL + COMMAND ${PLATFORMIO_CMD} -f -c clion device monitor + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + add_custom_target( PLATFORMIO_TEST ALL COMMAND ${PLATFORMIO_CMD} -f -c clion test diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index a1c84d68..0f7c053e 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -1,3 +1,7 @@ +# !!! WARNING !!! +# PLEASE DO NOT MODIFY THIS FILE! +# USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags + % def _normalize_path(path): % if project_dir in path: % path = path.replace(project_dir, "${CMAKE_CURRENT_LIST_DIR}") From ab914e1566888f789431f1cb4ea00fc2e6c5f7fe Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Feb 2019 11:45:14 +0200 Subject: [PATCH 049/333] PyLin fix --- platformio/commands/ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index a5f28256..0b5b3e4e 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -73,7 +73,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument @click.option("-O", "--project-option", multiple=True) @click.option("-v", "--verbose", is_flag=True) @click.pass_context -def cli( # pylint: disable=too-many-arguments +def cli( # pylint: disable=too-many-arguments, too-many-branches ctx, src, lib, exclude, board, build_dir, keep_build_dir, project_conf, project_option, verbose): From fb9fe8c77c7efa5c2c149ada3d61132f4cb4f33e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Mar 2019 13:03:22 +0200 Subject: [PATCH 050/333] Document `upload_protocol = espota` option --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 11ed4498..1516fb3e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 11ed449808bb8202d1212113afce25b4f3ebfbc5 +Subproject commit 1516fb3e5d7407889daf32beebade39d2085c02e From e39d4f5c32febcf8ea7dc8f6aa51681c316d4cdf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Mar 2019 13:12:16 +0200 Subject: [PATCH 051/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 1516fb3e..d78b6617 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1516fb3e5d7407889daf32beebade39d2085c02e +Subproject commit d78b6617f14f0e15a763aa4bd451413efeb31c9a From 4659a44bd6d0b79b16b35ec3e2cc4ad8df26656f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Mar 2019 13:50:34 +0200 Subject: [PATCH 052/333] New boards: Adafruit Bluefruit nRF52832 Feather, Feather nRF52840 Express, and Nordic nRF52840-DK (Adafruit BSP) --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index d78b6617..6a0d15dd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d78b6617f14f0e15a763aa4bd451413efeb31c9a +Subproject commit 6a0d15dd8402b52baeaf53fe1518f8b4bc2c09d6 diff --git a/examples b/examples index 6a0e7d47..d9d47747 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 6a0e7d4794e66d1f78f8c440baae7b92ba9378ca +Subproject commit d9d4774772149eb9aaebaf7daee9f2e87a93f0ed From f1549e1f0eaac4d231f00f64ff62bff89349d9f1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 5 Mar 2019 13:20:19 +0200 Subject: [PATCH 053/333] Docs: Sync new boards by STM32 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 6a0d15dd..e3bc009a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6a0d15dd8402b52baeaf53fe1518f8b4bc2c09d6 +Subproject commit e3bc009a4480997fb5a318e4efd4cca7a5a25170 From 9911a0423245af1d295a196ec8938aa1f43dc570 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Mar 2019 12:29:03 +0200 Subject: [PATCH 054/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index e3bc009a..9b0c63fc 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e3bc009a4480997fb5a318e4efd4cca7a5a25170 +Subproject commit 9b0c63fcb69014ebe07b0cd356d3216fc8f73b18 From 1ce913ea74576918a6683fb14f44a1ae6bff65bc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Mar 2019 21:04:01 +0200 Subject: [PATCH 055/333] Sync docs --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 9b0c63fc..eefd1f0f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9b0c63fcb69014ebe07b0cd356d3216fc8f73b18 +Subproject commit eefd1f0f93e5d82b09d7e51fb121b270602e4c89 diff --git a/examples b/examples index d9d47747..9c16f551 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit d9d4774772149eb9aaebaf7daee9f2e87a93f0ed +Subproject commit 9c16f551d72e37aa4c2f28555487e2fb23408b5d From 5a72e3f2a1ede9835eebf88b30ca3f3361728f49 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 6 Mar 2019 22:39:24 +0200 Subject: [PATCH 056/333] Update "dl.bintray.com" IP address --- platformio/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/util.py b/platformio/util.py index 0898abc9..57c7b16e 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -696,7 +696,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): PING_INTERNET_IPS = [ "192.30.253.113", # github.com - "159.122.18.156", # dl.bintray.com + "18.195.111.75", # dl.bintray.com "193.222.52.25" # dl.platformio.org ] From 237d55208c41b84dc2291d096c83cbdfa3e1dd91 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Mar 2019 12:35:34 +0200 Subject: [PATCH 057/333] Fix "Unnecessary "else/elif" after "raise"" by PyLint --- platformio/commands/upgrade.py | 4 +--- platformio/managers/lib.py | 3 +-- platformio/managers/package.py | 2 +- platformio/util.py | 5 ++--- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 98430483..ec4fe864 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -84,9 +84,7 @@ WARNING! Don't use `sudo` for the rest PlatformIO commands. fg="yellow", err=True) raise exception.ReturnErrorCode(1) - else: - raise exception.UpgradeError("\n".join( - [str(cmd), r['out'], r['err']])) + raise exception.UpgradeError("\n".join([str(cmd), r['out'], r['err']])) return True diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index eaea4412..3ca89d49 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -277,8 +277,7 @@ class LibraryManager(BasePkgManager): if not lib_info: if list(filters) == ["name"]: raise exception.LibNotFound(filters['name']) - else: - raise exception.LibNotFound(str(filters)) + raise exception.LibNotFound(str(filters)) if not silent: click.echo("Found: %s" % click.style( "https://platformio.org/lib/show/{id}/{name}".format( diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 4026d3f4..05f330d2 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -481,7 +481,7 @@ class PkgInstallerMixin(object): if versions is None: util.internet_on(raise_exception=True) raise exception.UnknownPackage(name) - elif not pkgdata: + if not pkgdata: raise exception.UndefinedPackageVersion(requirements or "latest", util.get_systype()) return pkg_dir diff --git a/platformio/util.py b/platformio/util.py index 57c7b16e..62882353 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -643,10 +643,9 @@ def _get_api_result( except requests.exceptions.HTTPError as e: if result and "message" in result: raise exception.APIRequestError(result['message']) - elif result and "errors" in result: + if result and "errors" in result: raise exception.APIRequestError(result['errors'][0]['title']) - else: - raise exception.APIRequestError(e) + raise exception.APIRequestError(e) except ValueError: raise exception.APIRequestError( "Invalid response: %s" % r.text.encode("utf-8")) From 3f52a6d5bac7f451d9bd978f29cf2672cd44f1a9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Mar 2019 12:36:17 +0200 Subject: [PATCH 058/333] YAPF 0.26.0 --- platformio/builder/tools/piolib.py | 4 ++-- platformio/commands/platform.py | 4 ++-- platformio/commands/run.py | 4 ++-- platformio/managers/package.py | 8 ++++---- platformio/managers/platform.py | 6 ++---- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index edf343f3..2613c868 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -185,8 +185,8 @@ class LibBuilderBase(object): @property def build_dir(self): - lib_hash = hashlib.sha1(self.path if util.PY2 else self.path. - encode()).hexdigest()[:3] + lib_hash = hashlib.sha1( + self.path if util.PY2 else self.path.encode()).hexdigest()[:3] return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) @property diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index cb69c453..2d007d4a 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -365,8 +365,8 @@ def platform_update(platforms, only_packages, only_check, json_output): if not pkg_dir: continue latest = pm.outdated(pkg_dir, requirements) - if (not latest and not PlatformFactory.newPlatform(pkg_dir). - are_outdated_packages()): + if (not latest and not PlatformFactory.newPlatform( + pkg_dir).are_outdated_packages()): continue data = _get_installed_platform_data( pkg_dir, with_boards=False, expose_packages=False) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index ad4e61e0..7cac3eca 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -432,5 +432,5 @@ def calculate_project_hash(): # Fix issue with useless project rebuilding for case insensitive FS. # A case of disk drive can differ... chunks_to_str = chunks_to_str.lower() - return sha1(chunks_to_str if util.PY2 else chunks_to_str. - encode()).hexdigest() + return sha1( + chunks_to_str if util.PY2 else chunks_to_str.encode()).hexdigest() diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 05f330d2..516a951c 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -589,8 +589,8 @@ class PkgInstallerMixin(object): target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5(cur_manifest['__src_url'] if util. - PY2 else cur_manifest['__src_url']. - encode()).hexdigest()) + PY2 else cur_manifest['__src_url'].encode( + )).hexdigest()) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: @@ -600,8 +600,8 @@ class PkgInstallerMixin(object): target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5(tmp_manifest['__src_url'] if util. - PY2 else tmp_manifest['__src_url']. - encode()).hexdigest()) + PY2 else tmp_manifest['__src_url'].encode( + )).hexdigest()) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index b2210427..851011a9 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -642,10 +642,8 @@ class PlatformBase( # pylint: disable=too-many-public-methods if not isdir(libcore_dir): continue storages.append({ - "name": - "%s-core-%s" % (opts['package'], item), - "path": - libcore_dir + "name": "%s-core-%s" % (opts['package'], item), + "path": libcore_dir }) return storages From 4c8df44a5ae415e7ef4e1841a8e8541e961e60e4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Mar 2019 12:40:55 +0200 Subject: [PATCH 059/333] Fix an issue when ``$PROJECT_HASH`` template was not expanded for the other directory ``***_dir`` options in "platformio.ini" // Resolve #2170 --- HISTORY.rst | 43 +++++++++++++++++++++++++------------------ docs | 2 +- platformio/util.py | 27 +++++++++++++++------------ 3 files changed, 41 insertions(+), 31 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c9384b85..b0e5b2bc 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,14 +9,21 @@ PlatformIO 4.0 * Added Python 3.5+ support (`issue #895 `_) + + +PlatformIO 3.0 +-------------- + +3.6.5 (2019-??-??) +~~~~~~~~~~~~~~~~~~ + * Project Generator: added new targets for CLion IDE "BUILD_VERBOSE" and "MONITOR" (serial port monitor) (`issue #359 `_) * Fixed an issue with slow updating of PlatformIO Core packages on Windows * Fixed an issue when `platformio ci `__ recompiles project if ``--keep-build-dir`` option is passed (`issue #2109 `_) - -PlatformIO 3.0 --------------- +* Fixed an issue when ``$PROJECT_HASH`` template was not expanded for the other directory ``***_dir`` options in `"platformio.ini" (Project Configuration File) `__ + (`issue #2170 `_) 3.6.4 (2019-01-23) ~~~~~~~~~~~~~~~~~~ @@ -60,8 +67,8 @@ PlatformIO 3.0 ~~~~~~~~~~~~~~~~~~ * Generate an `include `__ and `test `__ directories with a README file when initializing a new project -* Support in-line comments for multi-line value (``lib_deps``, ``build_flags``, etc) in `“platformio.ini” (Project Configuration File) `__ -* Added ``$PROJECT_HASH`` template variable for `build_dir `__. One of the use cases is setting a global storage for project artifacts using `PLATFORMIO_BUILD_DIR `__ system environment variable. For example, ``/tmp/pio-build/$PROJECT_HASH`` (Unix) or ``%TEMP%/pio-build/$PROJECT_HASH`` (Windows) +* Support in-line comments for multi-line value (``lib_deps``, ``build_flags``, etc) in `"platformio.ini" (Project Configuration File) `__ +* Added ``$PROJECT_HASH`` template variable for `build_dir `__. One of the use cases is setting a global storage for project artifacts using `PLATFORMIO_BUILD_DIR `__ system environment variable. For example, ``/tmp/pio-build/$PROJECT_HASH`` (Unix) or ``$[sysenv.TEMP}/pio-build/$PROJECT_HASH`` (Windows) * Improved a loading speed of PIO Home "Recent News" * Improved `PIO Unified Debugger `__ for "mbed" framework and fixed issue with missed local variables * Introduced `"Release" and "Debug" Build Configurations `__ @@ -145,7 +152,7 @@ PlatformIO 3.0 * Simplify configuration for `PIO Unit Testing `__: separate main program from a test build process, drop requirement for ``#ifdef UNIT_TEST`` guard -* Override any option from board manifest in `“platformio.ini” (Project Configuration File) `__ +* Override any option from board manifest in `"platformio.ini" (Project Configuration File) `__ (`issue #1612 `_) * Configure a custom path to SVD file using `debug_svd_path `__ option @@ -208,7 +215,7 @@ PlatformIO 3.0 (`issue #1274 `_) * Configure a custom firmware/program name in build directory (`example `__) * Renamed ``envs_dir`` option to ``build_dir`` - in `“platformio.ini” (Project Configuration File) `__ + in `"platformio.ini" (Project Configuration File) `__ * Refactored code without "arrow" dependency (resolve issue with "ImportError: No module named backports.functools_lru_cache") * Improved support of PIO Unified Debugger for Eclipse Oxygen @@ -243,7 +250,7 @@ PlatformIO 3.0 folder for project's header files (`issue #1107 `_) * Depend on development platform using VCS URL (Git, Mercurial and Subversion) - instead of a name in `“platformio.ini” (Project Configuration File) `__. + instead of a name in `"platformio.ini" (Project Configuration File) `__. Drop support for ``*_stage`` dev/platform names (use VCS URL instead). * Reinstall/redownload package with a new ``-f, --force`` option for `platformio lib install `__ @@ -264,7 +271,7 @@ PlatformIO 3.0 - Parse library source file in pair with a header when they have the same name (`issue #1175 `_) - Handle library dependencies defined as VCS or SemVer in - `“platformio.ini” (Project Configuration File) `__ + `"platformio.ini" (Project Configuration File) `__ (`issue #1155 `_) - Added option to configure library `Compatible Mode `__ using `library.json `__ @@ -300,7 +307,7 @@ PlatformIO 3.0 * Use a root of library when filtering source code using `library.json `__ and ``srcFilter`` field -* Added ``monitor_*`` options to white-list for `“platformio.ini” (Project Configuration File) `__ +* Added ``monitor_*`` options to white-list for `"platformio.ini" (Project Configuration File) `__ (`issue #982 `_) * Do not ask for board ID when initialize project for desktop platform * Handle broken PIO Core state and create new one @@ -321,18 +328,18 @@ PlatformIO 3.0 - Integration with `Eclipse `__ and `Sublime Text `__ * Filter `PIO Unit Testing `__ - tests using a new ``test_filter`` option in `“platformio.ini” (Project Configuration File) `__ + tests using a new ``test_filter`` option in `"platformio.ini" (Project Configuration File) `__ or `platformio test --filter `__ command (`issue #934 `_) * Custom ``test_transport`` for `PIO Unit Testing `__ Engine -* Configure Serial Port Monitor in `“platformio.ini” (Project Configuration File) `__ +* Configure Serial Port Monitor in `"platformio.ini" (Project Configuration File) `__ (`issue #787 `_) * New `monitor `__ target which allows to launch Serial Monitor automatically after successful "build" or "upload" operations (`issue #788 `_) * Project generator for `VIM `__ -* Multi-line support for the different options in `“platformio.ini” (Project Configuration File) `__, +* Multi-line support for the different options in `"platformio.ini" (Project Configuration File) `__, such as: ``build_flags``, ``build_unflags``, etc. (`issue #889 `_) * Handle dynamic ``SRC_FILTER`` environment variable from @@ -344,7 +351,7 @@ PlatformIO 3.0 that were installed from repository * Add support for ``.*cc`` extension (`issue #939 `_) -* Handle ``env_default`` in `“platformio.ini” (Project Configuration File) `__ +* Handle ``env_default`` in `"platformio.ini" (Project Configuration File) `__ when re-initializing a project (`issue #950 `_) * Use root directory for PIO Home when path contains non-ascii characters @@ -442,10 +449,10 @@ PlatformIO 3.0 (`issue #808 `_, `issue #467 `_) * Inject system environment variables to configuration settings in - `“platformio.ini” (Project Configuration File) `__ + `"platformio.ini" (Project Configuration File) `__ (`issue #792 `_) * Custom boards per project with ``boards_dir`` option in - `“platformio.ini” (Project Configuration File) `__ + `"platformio.ini" (Project Configuration File) `__ (`issue #515 `_) * Unix shell-style wildcards for `upload_port `_ (`issue #839 `_) @@ -456,7 +463,7 @@ PlatformIO 3.0 * Added new `LDF Modes `__: ``chain+`` and ``deep+`` and set ``chain+`` as default * Added global ``lib_extra_dirs`` option to ``[platformio]`` section for - `“platformio.ini” (Project Configuration File) `__ + `"platformio.ini" (Project Configuration File) `__ (`issue #842 `_) * Enabled caching by default for API requests and Library Manager (see `enable_cache `__ setting) * Native integration with VIM/Neovim using `neomake-platformio `__ plugin @@ -479,7 +486,7 @@ PlatformIO 3.0 3.1.0 (2016-09-19) ~~~~~~~~~~~~~~~~~~ -* New! Dynamic variables/templates for `“platformio.ini” (Project Configuration File) `__ +* New! Dynamic variables/templates for `"platformio.ini" (Project Configuration File) `__ (`issue #705 `_) * Summary about processed environments (`issue #777 `_) diff --git a/docs b/docs index eefd1f0f..768ccfd2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit eefd1f0f93e5d82b09d7e51fb121b270602e4c89 +Subproject commit 768ccfd2b4a3e508addb48f7f6d1fdebad5d946c diff --git a/platformio/util.py b/platformio/util.py index 62882353..d3dbb965 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -209,28 +209,37 @@ def pioversion_to_intstr(): def get_project_optional_dir(name, default=None): - data = None + paths = None var_name = "PLATFORMIO_%s" % name.upper() if var_name in os.environ: - data = os.getenv(var_name) + paths = os.getenv(var_name) else: try: config = load_project_config() if (config.has_section("platformio") and config.has_option("platformio", name)): - data = config.get("platformio", name) + paths = config.get("platformio", name) except exception.NotPlatformIOProject: pass - if not data: + if not paths: return default items = [] - for item in data.split(", "): + for item in paths.split(", "): if item.startswith("~"): item = expanduser(item) items.append(abspath(item)) - return ", ".join(items) + paths = ", ".join(items) + + while "$PROJECT_HASH" in paths: + project_dir = get_project_dir() + paths = paths.replace( + "$PROJECT_HASH", + sha1(project_dir if PY2 else project_dir.encode()).hexdigest() + [:10]) + + return paths def get_home_dir(): @@ -320,12 +329,6 @@ def get_projectboards_dir(): def get_projectbuild_dir(force=False): path = get_project_optional_dir("build_dir", join(get_project_dir(), ".pioenvs")) - if "$PROJECT_HASH" in path: - project_dir = get_project_dir() - path = path.replace( - "$PROJECT_HASH", - sha1(project_dir if PY2 else project_dir.encode()).hexdigest() - [:10]) try: if not isdir(path): os.makedirs(path) From 15a1cbf95ab6d3f51560c33d4bd52ef92b156494 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Mar 2019 14:09:50 +0200 Subject: [PATCH 060/333] Tag docs with v3.6.5 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 768ccfd2..a8b7286e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 768ccfd2b4a3e508addb48f7f6d1fdebad5d946c +Subproject commit a8b7286ea4d4526a34359c6fe876944a20dd3c88 From 8280fd557bc87357c370332d146c140ea16e1d09 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Mar 2019 14:12:45 +0200 Subject: [PATCH 061/333] Bump version to 4.0.0a4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 7880c2f6..0dcc38c1 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a3") +VERSION = (4, 0, "0a4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From a9288e5a5bbeb32c95294802e4cd4af857704f13 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 7 Mar 2019 17:26:22 +0200 Subject: [PATCH 062/333] Docs: Cosmetic changes to PIO Unified Debugging and Unit Testing --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index a8b7286e..d36d9622 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a8b7286ea4d4526a34359c6fe876944a20dd3c88 +Subproject commit d36d96227ec41b16b7d2e253aab69dc68f48bad4 From 9a6d148bdca28135bb45fe67465a510cb25d3ddd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 8 Mar 2019 18:01:35 +0200 Subject: [PATCH 063/333] Update docs for PIO Plus products --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index d36d9622..74c80fd2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit d36d96227ec41b16b7d2e253aab69dc68f48bad4 +Subproject commit 74c80fd2ff59e3ec218545e61cba64eef2abc9a8 From 03aec79cc1cd62122377941edce454f02eec23f0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 13 Mar 2019 22:22:48 +0200 Subject: [PATCH 064/333] Drop mbed OS support for Adafruit boards --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 74c80fd2..3b2fe3de 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 74c80fd2ff59e3ec218545e61cba64eef2abc9a8 +Subproject commit 3b2fe3de42cd27d6d9c62a3300c8ab772c115fd6 From b3ac567b536047683b5ae2b66b3821e93ae0be80 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 14 Mar 2019 14:46:44 +0200 Subject: [PATCH 065/333] Docs: New article "Getting started with the STM32F407VG and STM32Cube" --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 3b2fe3de..7349afe0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3b2fe3de42cd27d6d9c62a3300c8ab772c115fd6 +Subproject commit 7349afe05572deec8cdf28378d8339519af9615d From 0d904ad1cceb0844e94135904b32c74fe7355a78 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 14 Mar 2019 20:12:18 +0200 Subject: [PATCH 066/333] Fix "FileExistsError" when "platformio ci" command is used in pair with "--keep-build-dir" option --- HISTORY.rst | 5 +++++ platformio/commands/ci.py | 4 ++-- tests/commands/test_ci.py | 32 +++++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e4f7a9db..782c749f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,11 @@ PlatformIO 4.0 PlatformIO 3.0 -------------- +3.6.6 (2019-??-??) +~~~~~~~~~~~~~~~~~~ + +* Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option + 3.6.5 (2019-03-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 0b5b3e4e..13224fac 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -55,7 +55,6 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument "--build-dir", default=mkdtemp, type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, @@ -134,7 +133,8 @@ def _copy_contents(dst_dir, contents): if dst_dir_name == "src" and len(items['dirs']) == 1: copytree(list(items['dirs']).pop(), dst_dir, symlinks=True) else: - makedirs(dst_dir) + if not isdir(dst_dir): + makedirs(dst_dir) for d in items['dirs']: copytree(d, join(dst_dir, basename(d)), symlinks=True) diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index ec4077ba..67ff4420 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os.path import join +from os.path import isfile, join from platformio.commands.ci import cli as cmd_ci from platformio.commands.lib import cli as cmd_lib @@ -32,6 +32,36 @@ def test_ci_boards(clirunner, validate_cliresult): validate_cliresult(result) +def test_ci_build_dir(clirunner, tmpdir_factory, validate_cliresult): + build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) + result = clirunner.invoke(cmd_ci, [ + join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", + "--build-dir", build_dir + ]) + validate_cliresult(result) + assert not isfile(join(build_dir, "platformio.ini")) + + +def test_ci_keep_build_dir(clirunner, tmpdir_factory, validate_cliresult): + build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) + result = clirunner.invoke(cmd_ci, [ + join("examples", "wiring-blink", "src", "main.cpp"), "-b", "uno", + "--build-dir", build_dir, "--keep-build-dir" + ]) + validate_cliresult(result) + assert isfile(join(build_dir, "platformio.ini")) + + # 2nd attempt + result = clirunner.invoke(cmd_ci, [ + join("examples", "wiring-blink", "src", "main.cpp"), "-b", "metro", + "--build-dir", build_dir, "--keep-build-dir" + ]) + validate_cliresult(result) + + assert "board: uno" in result.output + assert "board: metro" in result.output + + def test_ci_project_conf(clirunner, validate_cliresult): project_dir = join("examples", "wiring-blink") result = clirunner.invoke(cmd_ci, [ From d4af985eb8786f6531e319e837fcc8d7b3e33ece Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 14 Mar 2019 20:12:52 +0200 Subject: [PATCH 067/333] Bump version to 4.0.0a5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 0dcc38c1..c2868e58 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a4") +VERSION = (4, 0, "0a5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From d9e6111ac3b19721a3c0ee9bb8cc3d5a6add4c44 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Mar 2019 18:49:20 +0200 Subject: [PATCH 068/333] Fix error with conflicting declaration of a prototype (Arduino sketch preprocessor) --- HISTORY.rst | 1 + platformio/builder/tools/piomisc.py | 11 +++++++++-- tests/ino2cpp/basic/basic.ino | 8 ++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 782c749f..44b32ed0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ PlatformIO 3.0 3.6.6 (2019-??-??) ~~~~~~~~~~~~~~~~~~ +* Fixed error with conflicting declaration of a prototype (Arduino sketch preprocessor) * Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option 3.6.5 (2019-03-07) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 484f6228..e6f846e0 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -33,10 +33,10 @@ class InoToCPPConverter(object): PROTOTYPE_RE = re.compile( r"""^( (?:template\<.*\>\s*)? # template - ([a-z_\d\&]+\*?\s+){1,2} # return type + ([a-z_\d\&]+\*?\s+){1,2} # return type ([a-z_\d]+\s*) # name of prototype \([a-z_,\.\*\&\[\]\s\d]*\) # arguments - )\s*\{ # must end with { + )\s*(\{|;) # must end with `{` or `;` """, re.X | re.M | re.I) DETECTMAIN_RE = re.compile(r"void\s+(setup|loop)\s*\(", re.M | re.I) PROTOPTRS_TPLRE = r"\([^&\(]*&(%s)[^\)]*\)" @@ -161,6 +161,13 @@ class InoToCPPConverter(object): if not prototypes: return contents + # skip already declared prototypes + declared = set( + m.group(1).strip() for m in prototypes if m.group(4) == ";") + prototypes = [ + m for m in prototypes if m.group(1).strip() not in declared + ] + prototype_names = set(m.group(3).strip() for m in prototypes) split_pos = prototypes[0].start() match_ptrs = re.search( diff --git a/tests/ino2cpp/basic/basic.ino b/tests/ino2cpp/basic/basic.ino index 0c5d7511..fcec8edd 100644 --- a/tests/ino2cpp/basic/basic.ino +++ b/tests/ino2cpp/basic/basic.ino @@ -49,4 +49,12 @@ void fooCallback(){ } +extern "C" { +void some_extern(const char *fmt, ...); +}; + +void some_extern(const char *fmt, ...) { + +} + // юнікод From 4ca71e7df1c28a687893bcf3aa5dfb8609598ce6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Mar 2019 21:16:02 +0200 Subject: [PATCH 069/333] Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode // Resolve #2243 --- HISTORY.rst | 4 +++- platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 44b32ed0..b7629c20 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,7 +16,9 @@ PlatformIO 3.0 3.6.6 (2019-??-??) ~~~~~~~~~~~~~~~~~~ -* Fixed error with conflicting declaration of a prototype (Arduino sketch preprocessor) +* Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode + (`issue #2243 `_) +* Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor) * Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option 3.6.5 (2019-03-07) diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index 5f0343de..75d77ffd 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -1,7 +1,9 @@ { - "!!! WARNING !!!": "PLEASE DO NOT MODIFY THIS FILE! USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags", "configurations": [ { + "name": "!!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags" + }, + { % import platform % from os.path import commonprefix, dirname % From dea655184153c4cb2bfdfa2854ac9c4e0136d613 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Mar 2019 21:16:33 +0200 Subject: [PATCH 070/333] Bump version to 4.0.0a6 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index c2868e58..a2ced22a 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a5") +VERSION = (4, 0, "0a6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 10a7367b33f15f365a58de9698fc17c36cbb9f58 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 22 Mar 2019 21:24:43 +0200 Subject: [PATCH 071/333] Fix "SameFileError" when CI is used in pair with --keep-build-dir // Resolve #2227 --- platformio/commands/ci.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 13224fac..8fb1bf89 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -145,7 +145,10 @@ def _copy_contents(dst_dir, contents): dst_dir = join(dst_dir, mkdtemp(dir=dst_dir)) for f in items['files']: - copyfile(f, join(dst_dir, basename(f))) + dst_file = join(dst_dir, basename(f)) + if f == dst_file: + continue + copyfile(f, dst_file) def _exclude_contents(dst_dir, patterns): From 3a77bed0d4e69e22b753a8bc91bcedf048eaa075 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Mar 2019 14:24:43 +0200 Subject: [PATCH 072/333] Fix an issue when PlatformIO Build System does not pick up "mbed_lib.json" files from libraries // Resolve #2164 --- HISTORY.rst | 2 ++ platformio/builder/tools/piolib.py | 57 +++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index b7629c20..7bd5d9de 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,8 @@ PlatformIO 3.0 * Project Generator: fixed a warning "Property !!! WARNING !!! is not allowed" for VSCode (`issue #2243 `_) +* Fixed an issue when PlatformIO Build System does not pick up "mbed_lib.json" files from libraries + (`issue #2164 `_) * Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor) * Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 2613c868..2c4b3054 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -71,6 +71,8 @@ class LibBuilderFactory(object): # check source files for root, _, files in os.walk(path, followlinks=True): + if "mbed_lib.json" in files: + return ["mbed"] for fname in files: if not env.IsFileWithExt( fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT): @@ -262,7 +264,6 @@ class LibBuilderBase(object): def process_extra_options(self): with util.cd(self.path): - self.env.ProcessUnFlags(self.build_unflags) self.env.ProcessFlags(self.build_flags) if self.extra_script: self.env.SConscriptChdir(1) @@ -272,6 +273,7 @@ class LibBuilderBase(object): "env": self.env, "pio_lib_builder": self }) + self.env.ProcessUnFlags(self.build_unflags) def process_dependencies(self): if not self.dependencies: @@ -591,6 +593,59 @@ class MbedLibBuilder(LibBuilderBase): def is_frameworks_compatible(self, frameworks): return util.items_in_list(frameworks, ["mbed"]) + @property + def build_flags(self): + return self._mbed_lib_json_to_build_flags() + + def _mbed_lib_json_to_build_flags(self): # pylint: disable=too-many-locals + json_files = [ + join(root, "mbed_lib.json") + for root, _, files in os.walk(self.path) + if "mbed_lib.json" in files + ] + if not json_files: + return None + + build_flags = [] + cppdefines = str(self.env.Flatten(self.env.subst("$CPPDEFINES"))) + for p in json_files: + manifest = util.load_json(p) + + # default macros + build_flags.extend(["-D" + m for m in manifest.get("macros", [])]) + + macros = {} + # configuration items + for key, options in manifest.get("config", {}).items(): + if "value" not in options: + continue + macros[key] = dict( + name=options.get("macro_name"), value=options.get("value")) + # overrode items per target + for target, options in manifest.get("target_overrides", + {}).items(): + if target != "*" and "TARGET_" + target not in cppdefines: + continue + build_flags.extend( + ["-D" + m for m in options.get("target.macros_add", [])]) + for key, value in options.items(): + if not key.startswith("target.") and key in macros: + macros[key]['value'] = value + for key, macro in macros.items(): + name = macro['name'] + value = macro['value'] + if not name: + name = key + if "." not in name: + name = manifest.get("name") + "." + name + name = re.sub(r"[^a-z\d]+", "_", name, flags=re.I).upper() + name = "MBED_CONF_" + name + if isinstance(value, bool): + value = 1 if value else 0 + build_flags.append("-D%s=%s" % (name, value)) + + return build_flags + class PlatformIOLibBuilder(LibBuilderBase): From cae708f2d7037b52196fd1024077e221cd8a4cfc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Mar 2019 17:44:01 +0200 Subject: [PATCH 073/333] Fix an issue with incorrect order of project "include" and "src" paths in ``CPPPATH`` // Resolve #1914 --- HISTORY.rst | 2 ++ platformio/builder/tools/piolib.py | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7bd5d9de..935f1322 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,8 @@ PlatformIO 3.0 (`issue #2164 `_) * Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor) * Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option +* Fixed an issue with incorrect order of project "include" and "src" paths in ``CPPPATH`` + (`issue #1914 `_) 3.6.5 (2019-03-07) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 2c4b3054..f64251ea 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -179,10 +179,11 @@ class LibBuilderBase(object): if isdir(join(self.path, "src")) else self.path) def get_include_dirs(self): - items = [self.src_dir] + items = [] include_dir = self.include_dir if include_dir and include_dir not in items: items.append(include_dir) + items.append(self.src_dir) return items @property @@ -759,10 +760,11 @@ class ProjectAsLibBuilder(LibBuilderBase): return self.env.subst("$PROJECTSRC_DIR") def get_include_dirs(self): - include_dirs = LibBuilderBase.get_include_dirs(self) + include_dirs = [] project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR") if isdir(project_include_dir): include_dirs.append(project_include_dir) + include_dirs.extend(LibBuilderBase.get_include_dirs(self)) return include_dirs def get_search_files(self): @@ -830,8 +832,9 @@ class ProjectAsLibBuilder(LibBuilderBase): def build(self): self._is_built = True # do not build Project now + result = LibBuilderBase.build(self) self.env.PrependUnique(CPPPATH=self.get_include_dirs()) - return LibBuilderBase.build(self) + return result def GetLibBuilders(env): # pylint: disable=too-many-branches From 51cb87bc6f2a854063b3fa6b89913712800fe28c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Mar 2019 17:44:30 +0200 Subject: [PATCH 074/333] Bump version to 4.0.0a7 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index a2ced22a..584acb8b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a6") +VERSION = (4, 0, "0a7") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From c141189883f16b1479de8bab7f8c35392fc5a3c0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 24 Mar 2019 23:54:53 +0200 Subject: [PATCH 075/333] Append configuration settings from "mbed_lib.json" to "mbed_config.h" // Resolve #2164 --- platformio/builder/tools/piolib.py | 136 ++++++++++++++++++++--------- 1 file changed, 94 insertions(+), 42 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index f64251ea..221df7e8 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=no-member, no-self-use, unused-argument +# pylint: disable=no-member, no-self-use, unused-argument, too-many-lines # pylint: disable=too-many-instance-attributes, too-many-public-methods from __future__ import absolute_import @@ -594,58 +594,110 @@ class MbedLibBuilder(LibBuilderBase): def is_frameworks_compatible(self, frameworks): return util.items_in_list(frameworks, ["mbed"]) - @property - def build_flags(self): - return self._mbed_lib_json_to_build_flags() + def process_extra_options(self): + self._process_mbed_lib_confs() + return super(MbedLibBuilder, self).process_extra_options() - def _mbed_lib_json_to_build_flags(self): # pylint: disable=too-many-locals - json_files = [ + def _process_mbed_lib_confs(self): + mbed_lib_paths = [ join(root, "mbed_lib.json") for root, _, files in os.walk(self.path) if "mbed_lib.json" in files ] - if not json_files: + if not mbed_lib_paths: return None - build_flags = [] + mbed_config_path = None + for p in self.env.get("CPPPATH"): + mbed_config_path = join(self.env.subst(p), "mbed_config.h") + if isfile(mbed_config_path): + break + else: + mbed_config_path = None + if not mbed_config_path: + return None + + macros = {} + for mbed_lib_path in mbed_lib_paths: + macros.update(self._mbed_lib_conf_parse_macros(mbed_lib_path)) + + self._mbed_conf_append_macros(mbed_config_path, macros) + return True + + @staticmethod + def _mbed_normalize_macro(macro): + name = macro + value = None + if "=" in macro: + name, value = macro.split("=", 1) + return dict(name=name, value=value) + + def _mbed_lib_conf_parse_macros(self, mbed_lib_path): + macros = {} cppdefines = str(self.env.Flatten(self.env.subst("$CPPDEFINES"))) - for p in json_files: - manifest = util.load_json(p) + manifest = util.load_json(mbed_lib_path) - # default macros - build_flags.extend(["-D" + m for m in manifest.get("macros", [])]) + # default macros + for macro in manifest.get("macros", []): + macro = self._mbed_normalize_macro(macro) + macros[macro['name']] = macro - macros = {} - # configuration items - for key, options in manifest.get("config", {}).items(): - if "value" not in options: + # configuration items + for key, options in manifest.get("config", {}).items(): + if "value" not in options: + continue + macros[key] = dict( + name=options.get("macro_name"), value=options.get("value")) + + # overrode items per target + for target, options in manifest.get("target_overrides", {}).items(): + if target != "*" and "TARGET_" + target not in cppdefines: + continue + for macro in options.get("target.macros_add", []): + macro = self._mbed_normalize_macro(macro) + macros[macro['name']] = macro + for key, value in options.items(): + if not key.startswith("target.") and key in macros: + macros[key]['value'] = value + + # normalize macro names + for key, macro in macros.items(): + if not macro['name']: + macro['name'] = key + if "." not in macro['name']: + macro['name'] = "%s.%s" % (manifest.get("name"), + macro['name']) + macro['name'] = re.sub( + r"[^a-z\d]+", "_", macro['name'], flags=re.I).upper() + macro['name'] = "MBED_CONF_" + macro['name'] + if isinstance(macro['value'], bool): + macro['value'] = 1 if macro['value'] else 0 + + return {macro["name"]: macro["value"] for macro in macros.values()} + + def _mbed_conf_append_macros(self, mbed_config_path, macros): + lines = [] + with open(mbed_config_path) as fp: + for line in fp.readlines(): + line = line.strip() + if line == "#endif": + lines.append( + "// PlatformIO Library Dependency Finder (LDF)") + lines.extend([ + "#define %s %s" % (name, + value if value is not None else "") + for name, value in macros.items() + ]) + lines.append("") + if not line.startswith("#define"): + lines.append(line) continue - macros[key] = dict( - name=options.get("macro_name"), value=options.get("value")) - # overrode items per target - for target, options in manifest.get("target_overrides", - {}).items(): - if target != "*" and "TARGET_" + target not in cppdefines: - continue - build_flags.extend( - ["-D" + m for m in options.get("target.macros_add", [])]) - for key, value in options.items(): - if not key.startswith("target.") and key in macros: - macros[key]['value'] = value - for key, macro in macros.items(): - name = macro['name'] - value = macro['value'] - if not name: - name = key - if "." not in name: - name = manifest.get("name") + "." + name - name = re.sub(r"[^a-z\d]+", "_", name, flags=re.I).upper() - name = "MBED_CONF_" + name - if isinstance(value, bool): - value = 1 if value else 0 - build_flags.append("-D%s=%s" % (name, value)) - - return build_flags + tokens = line.split() + if len(tokens) < 2 or tokens[1] not in macros: + lines.append(line) + lines.append("") + with open(mbed_config_path, "w") as fp: + fp.write("\n".join(lines)) class PlatformIOLibBuilder(LibBuilderBase): From 2d4421e8e580fbe2bab0e2d6d89fb449a75c4c74 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Mar 2019 16:04:05 +0200 Subject: [PATCH 076/333] New article "Automated unit testing in the metal" --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7349afe0..1ed66f3c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7349afe05572deec8cdf28378d8339519af9615d +Subproject commit 1ed66f3cc6cc3a31f6eaf77e1db9a0ab55062ded From 8f29d951cb982bcb9d369229f2aa24c5336a59e8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 29 Mar 2019 12:55:05 +0200 Subject: [PATCH 077/333] YAPF 0.26.0 --- platformio/commands/ci.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 8fb1bf89..e1b60087 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -55,10 +55,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument "--build-dir", default=mkdtemp, type=click.Path( - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) + file_okay=False, dir_okay=True, writable=True, resolve_path=True)) @click.option("--keep-build-dir", is_flag=True) @click.option( "-C", From e089c4a546da3b1bd0a56c5a44c00a96f5fdad63 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 29 Mar 2019 21:26:45 +0200 Subject: [PATCH 078/333] Project Generator: fixed a VSCode C/C++'s "Cannot find" warning when CPPPATH folder does not exist --- HISTORY.rst | 5 +++++ .../ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fb17416e..17b53191 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,11 @@ PlatformIO 4.0 PlatformIO 3.0 -------------- +3.6.7 (2019-??-??) +~~~~~~~~~~~~~~~~~~ + +* Project Generator: fixed a VSCode C/C++'s "Cannot find" warning when CPPPATH folder does not exist + 3.6.6 (2019-03-29) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index 75d77ffd..c91ab548 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -5,7 +5,7 @@ }, { % import platform -% from os.path import commonprefix, dirname +% from os.path import commonprefix, dirname, isdir % % systype = platform.system().lower() % @@ -15,7 +15,7 @@ % % cleaned_includes = [] % for include in includes: -% if "toolchain-" not in dirname(commonprefix([include, cc_path])): +% if "toolchain-" not in dirname(commonprefix([include, cc_path])) and isdir(include): % cleaned_includes.append(include) % end % end @@ -65,7 +65,7 @@ % if cxx_stds: "cppStandard": "c++{{ cxx_stds[-1] }}", % end - "compilerPath": "{{! _escape(cc_path) }} {{! _escape(cc_m_flags) }}" + "compilerPath": "\"{{! _escape(cc_path) }}\" {{! _escape(cc_m_flags) }}" } ] } \ No newline at end of file From bb17630571be2c5e4e23246f3143913e4356ed98 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 30 Mar 2019 13:00:10 +0200 Subject: [PATCH 079/333] Set manifest version of VSCode C/C++ configuration file --- platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index c91ab548..1658bcb4 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -67,5 +67,6 @@ % end "compilerPath": "\"{{! _escape(cc_path) }}\" {{! _escape(cc_m_flags) }}" } - ] + ], + "version": 4 } \ No newline at end of file From ff5da3c3ccb378772e073a1020d3a7fcee72d7e4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 30 Mar 2019 13:54:29 +0200 Subject: [PATCH 080/333] Use stable dev/platforms for CI --- scripts/install_devplatforms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_devplatforms.py b/scripts/install_devplatforms.py index 4d8daaa6..a7a2668e 100644 --- a/scripts/install_devplatforms.py +++ b/scripts/install_devplatforms.py @@ -34,7 +34,7 @@ def main(): and platform['name'] == "aceinna_imu"): continue subprocess.check_call( - ["platformio", "platform", "install", platform['repository']]) + ["platformio", "platform", "install", platform['name']]) if __name__ == "__main__": From c1d01dbe349da5a86446f65c871581050db2e4b2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Apr 2019 18:35:00 +0300 Subject: [PATCH 081/333] Update info about PIO IDE for VSCode --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 3a6c69a1..aa145a1c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3a6c69a1ae227479321de75184aa8678104fc37f +Subproject commit aa145a1c59557cb88f6fa4dede1fdac6e15987b1 From cf13ec4035a561715fb3cd8cb6446eb796a1753d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Apr 2019 18:50:40 +0300 Subject: [PATCH 082/333] Fix an "IndexError: list index out of range" for Arduino sketch preprocessor // Resolve #2268 --- HISTORY.rst | 2 ++ platformio/builder/tools/piomisc.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 17b53191..f165c8d6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,8 @@ PlatformIO 3.0 ~~~~~~~~~~~~~~~~~~ * Project Generator: fixed a VSCode C/C++'s "Cannot find" warning when CPPPATH folder does not exist +* Fixed an "IndexError: list index out of range" for Arduino sketch preprocessor + (`issue #2268 `_) 3.6.6 (2019-03-29) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index e6f846e0..1118ac35 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -157,9 +157,7 @@ class InoToCPPConverter(object): return total def append_prototypes(self, contents): - prototypes = self._parse_prototypes(contents) - if not prototypes: - return contents + prototypes = self._parse_prototypes(contents) or [] # skip already declared prototypes declared = set( @@ -168,6 +166,9 @@ class InoToCPPConverter(object): m for m in prototypes if m.group(1).strip() not in declared ] + if not prototypes: + return contents + prototype_names = set(m.group(3).strip() for m in prototypes) split_pos = prototypes[0].start() match_ptrs = re.search( From e992e156bf61581ecf331fdca8ed22324049d48a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 5 Apr 2019 19:48:04 +0300 Subject: [PATCH 083/333] Fix "ValueError: invalid literal for int() with base 10" for click.get_terminal_size --- platformio/telemetry.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 1ced1f7d..3f9937cf 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -70,7 +70,11 @@ class MeasurementProtocol(TelemetryBase): self['tid'] = self.TID self['cid'] = app.get_cid() - self['sr'] = "%dx%d" % click.get_terminal_size() + try: + self['sr'] = "%dx%d" % click.get_terminal_size() + except ValueError: + pass + self._prefill_screen_name() self._prefill_appinfo() self._prefill_custom_data() From e9b847894268c204965f558e643565f77d70ac14 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 7 Apr 2019 00:52:20 +0300 Subject: [PATCH 084/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index aa145a1c..8fc59b09 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit aa145a1c59557cb88f6fa4dede1fdac6e15987b1 +Subproject commit 8fc59b09a2de0034bc93422775aab9f61a23da7c From a51a03843d9f025e78a4d3c18cde4940055d37ec Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 16 Apr 2019 00:33:21 +0300 Subject: [PATCH 085/333] Docs: Use jQuery from Sphinx theme (fix search) --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 8fc59b09..6b7cee0a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8fc59b09a2de0034bc93422775aab9f61a23da7c +Subproject commit 6b7cee0a32b6d529d8d0208dcd241111b014c151 From 41ab97203a46271246deacfbad36fb627d0cdeb2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 16 Apr 2019 13:55:47 +0300 Subject: [PATCH 086/333] Docs: Add community video tutorials --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 6b7cee0a..706ae16d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6b7cee0a32b6d529d8d0208dcd241111b014c151 +Subproject commit 706ae16d481e26c0ad91682ba5fdb9dbfb76aaac From 6501c1f171ecc93f92ae577fdb23c93e4915c7e9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 17 Apr 2019 20:05:51 +0300 Subject: [PATCH 087/333] Cleanup PING_INTERNET_IPS --- platformio/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/util.py b/platformio/util.py index d3dbb965..cdba2501 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -698,7 +698,6 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): PING_INTERNET_IPS = [ "192.30.253.113", # github.com - "18.195.111.75", # dl.bintray.com "193.222.52.25" # dl.platformio.org ] From c084db16194d916ffc05cfe5f450420eaf8e3a7a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 17 Apr 2019 20:09:05 +0300 Subject: [PATCH 088/333] Enable AppVeyor CI only for Windows x64 --- .appveyor.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index e071eaa8..ee0986dc 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,7 +1,6 @@ build: off platform: - - x86 - x64 environment: From f63041a40263959ea7333f7340943c3fac80f436 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 17 Apr 2019 23:20:45 +0300 Subject: [PATCH 089/333] Add `get_original_version` to public utils API --- platformio/commands/platform.py | 14 +------------- platformio/util.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 2d007d4a..bf7ecf75 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -54,18 +54,6 @@ def _get_registry_platforms(): return platforms -def _original_version(version): - if version.count(".") != 2: - return None - _, y = version.split(".")[:2] - if int(y) < 100: - return None - if len(y) % 2 != 0: - y = "0" + y - parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(int(len(y) / 2))] - return ".".join(parts) - - def _get_platform_data(*args, **kwargs): try: return _get_installed_platform_data(*args, **kwargs) @@ -122,7 +110,7 @@ def _get_installed_platform_data(platform, continue item[key] = value if key == "version": - item["originalVersion"] = _original_version(value) + item["originalVersion"] = util.get_original_version(value) data['packages'].append(item) return data diff --git a/platformio/util.py b/platformio/util.py index cdba2501..dd280ba2 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -849,6 +849,18 @@ def ensure_udev_rules(): return True +def get_original_version(version): + if version.count(".") != 2: + return None + _, y = version.split(".")[:2] + if int(y) < 100: + return None + if len(y) % 2 != 0: + y = "0" + y + parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(int(len(y) / 2))] + return ".".join(parts) + + def rmtree_(path): def _onerror(_, name, __): From 963eabc3f582507a8bde065f2644616895ce6be9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 18 Apr 2019 01:01:26 +0300 Subject: [PATCH 090/333] Docs: Rename "Download" button to INSTALL --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 706ae16d..1532572c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 706ae16d481e26c0ad91682ba5fdb9dbfb76aaac +Subproject commit 1532572c6259b98b687ca61abe80b35f8bc00093 From 3d96e584fb92bad96448d66c382afc47164e4928 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 18 Apr 2019 14:17:22 +0300 Subject: [PATCH 091/333] Fix an issue when invalid "env_default" results into unhandled errors // Resolve #2265 --- HISTORY.rst | 2 ++ platformio/commands/init.py | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f165c8d6..99c41cb5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,8 @@ PlatformIO 3.0 * Project Generator: fixed a VSCode C/C++'s "Cannot find" warning when CPPPATH folder does not exist * Fixed an "IndexError: list index out of range" for Arduino sketch preprocessor (`issue #2268 `_) +* Fixed an issue when invalid "env_default" in `"platformio.ini" (Project Configuration File) `__ results into unhandled errors + (`issue #2265 `_) 3.6.6 (2019-03-29) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/commands/init.py b/platformio/commands/init.py index e90bc2fd..e7127313 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -23,6 +23,7 @@ import click from platformio import exception, util from platformio.commands.platform import \ platform_install as cli_platform_install +from platformio.commands.run import check_project_envs from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager @@ -130,10 +131,11 @@ def get_best_envname(project_dir, boards=None): config = util.load_project_config(project_dir) env_default = None if config.has_option("platformio", "env_default"): - env_default = config.get("platformio", - "env_default").split(", ")[0].strip() + env_default = util.parse_conf_multi_values( + config.get("platformio", "env_default")) + check_project_envs(config, env_default) if env_default: - return env_default + return env_default[0] section = None for section in config.sections(): if not section.startswith("env:"): From 21a36f8ee946bb6c2490ea05eb489e9f8d988667 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 18 Apr 2019 16:20:01 +0300 Subject: [PATCH 092/333] Fix UnicodeEncodeError when converting INO to CPP --- platformio/builder/tools/piomisc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 1118ac35..45212415 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -58,7 +58,7 @@ class InoToCPPConverter(object): assert nodes lines = [] for node in nodes: - contents = node.get_text_contents() + contents = util.get_file_contents(node.get_path()) _lines = [ '# 1 "%s"' % node.get_path().replace("\\", "/"), contents ] From 40d1bb204c2b2200bc136bcbd915afee20699f2a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Apr 2019 13:26:25 +0300 Subject: [PATCH 093/333] Docs: New boards by ESP32 dev/platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 1532572c..26068f27 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1532572c6259b98b687ca61abe80b35f8bc00093 +Subproject commit 26068f27c518b7f4b79660a9d4cad6e3db34f60c From f1da5442792903af6bdd6d51b7c98b3aaeeefc6c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Apr 2019 19:56:16 +0300 Subject: [PATCH 094/333] Initial commit of PIO Unified Debugger --- .isort.cfg | 2 +- platformio/__main__.py | 16 +- platformio/commands/debug.py | 42 ---- platformio/commands/debug/__init__.py | 15 ++ platformio/commands/debug/client.py | 271 ++++++++++++++++++++++ platformio/commands/debug/command.py | 135 +++++++++++ platformio/commands/debug/helpers.py | 317 ++++++++++++++++++++++++++ platformio/commands/debug/initcfgs.py | 109 +++++++++ platformio/commands/debug/process.py | 72 ++++++ platformio/commands/debug/server.py | 110 +++++++++ platformio/managers/core.py | 9 + platformio/util.py | 7 +- 12 files changed, 1051 insertions(+), 54 deletions(-) delete mode 100644 platformio/commands/debug.py create mode 100644 platformio/commands/debug/__init__.py create mode 100644 platformio/commands/debug/client.py create mode 100644 platformio/commands/debug/command.py create mode 100644 platformio/commands/debug/helpers.py create mode 100644 platformio/commands/debug/initcfgs.py create mode 100644 platformio/commands/debug/process.py create mode 100644 platformio/commands/debug/server.py diff --git a/.isort.cfg b/.isort.cfg index 9b58f629..1ee0d412 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=79 -known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial +known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial, twisted diff --git a/platformio/__main__.py b/platformio/__main__.py index 61db78de..0252aba6 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -14,7 +14,7 @@ import os import sys -from os.path import join +from os.path import isdir, isfile, join from platform import system from traceback import format_exc @@ -24,15 +24,19 @@ from platformio import __version__, exception, maintenance from platformio.util import get_source_dir -class PlatformioCLI(click.MultiCommand): # pylint: disable=R0904 +class PlatformioCLI(click.MultiCommand): def list_commands(self, ctx): cmds = [] - for filename in os.listdir(join(get_source_dir(), "commands")): - if filename.startswith("__init__"): + commands_dir = join(get_source_dir(), "commands") + for name in os.listdir(commands_dir): + if name.startswith("__init__"): continue - if filename.endswith(".py"): - cmds.append(filename[:-3]) + if (isdir(join(commands_dir, name)) + and isfile(join(commands_dir, name, "command.py"))): + cmds.append(name) + elif name.endswith(".py"): + cmds.append(name[:-3]) cmds.sort() return cmds diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py deleted file mode 100644 index e43aeed1..00000000 --- a/platformio/commands/debug.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -from os import getcwd - -import click - -from platformio.managers.core import pioplus_call - - -@click.command( - "debug", - context_settings=dict(ignore_unknown_options=True), - short_help="PIO Unified Debugger") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("--environment", "-e", metavar="") -@click.option("--verbose", "-v", is_flag=True) -@click.option("--interface", type=click.Choice(["gdb"])) -@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) -def cli(*args, **kwargs): # pylint: disable=unused-argument - pioplus_call(sys.argv[1:]) diff --git a/platformio/commands/debug/__init__.py b/platformio/commands/debug/__init__.py new file mode 100644 index 00000000..7fba44c2 --- /dev/null +++ b/platformio/commands/debug/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.debug.command import cli diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py new file mode 100644 index 00000000..a0bcc61b --- /dev/null +++ b/platformio/commands/debug/client.py @@ -0,0 +1,271 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import re +import time +from hashlib import sha1 +from os.path import abspath, basename, dirname, isdir, join, splitext +from tempfile import mkdtemp + +from twisted.internet import protocol, reactor, stdio, task + +from platformio import app, exception, util +from platformio.commands.debug import helpers, initcfgs +from platformio.commands.debug.process import BaseProcess +from platformio.commands.debug.server import DebugServer +from platformio.telemetry import MeasurementProtocol + +LOG_FILE = None + + +class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes + + PIO_SRC_NAME = ".pioinit" + INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" + + def __init__(self, project_dir, args, debug_options, env_options): + self.project_dir = project_dir + self.args = list(args) + self.debug_options = debug_options + self.env_options = env_options + + self._debug_server = DebugServer(debug_options, env_options) + self._session_id = None + + if not isdir(util.get_cache_dir()): + os.makedirs(util.get_cache_dir()) + self._gdbsrc_dir = mkdtemp( + dir=util.get_cache_dir(), prefix=".piodebug-") + + self._target_is_run = False + self._last_server_activity = 0 + self._auto_continue_timer = None + + def spawn(self, gdb_path, prog_path): + session_hash = gdb_path + prog_path + self._session_id = sha1( + session_hash if util.PY2 else session_hash.encode()).hexdigest() + self._kill_previous_session() + + patterns = { + "PROJECT_DIR": helpers.escape_path(self.project_dir), + "PROG_PATH": helpers.escape_path(prog_path), + "PROG_DIR": helpers.escape_path(dirname(prog_path)), + "PROG_NAME": basename(splitext(prog_path)[0]), + "DEBUG_PORT": self.debug_options['port'], + "UPLOAD_PROTOCOL": self.debug_options['upload_protocol'], + "INIT_BREAK": self.debug_options['init_break'] or "", + "LOAD_CMD": self.debug_options['load_cmd'] or "", + } + + self._debug_server.spawn(patterns) + + if not patterns['DEBUG_PORT']: + patterns['DEBUG_PORT'] = self._debug_server.get_debug_port() + self.generate_pioinit(self._gdbsrc_dir, patterns) + + # start GDB client + args = [ + "piogdb", + "-q", + "--directory", self._gdbsrc_dir, + "--directory", self.project_dir, + "-l", "10" + ] # yapf: disable + args.extend(self.args) + if not gdb_path: + raise exception.DebugInvalidOptions("GDB client is not configured") + gdb_data_dir = self._get_data_dir(gdb_path) + if gdb_data_dir: + args.extend(["--data-directory", gdb_data_dir]) + args.append(patterns['PROG_PATH']) + + return reactor.spawnProcess( + self, gdb_path, args, path=self.project_dir, env=os.environ) + + @staticmethod + def _get_data_dir(gdb_path): + if "msp430" in gdb_path: + return None + gdb_data_dir = abspath(join(dirname(gdb_path), "..", "share", "gdb")) + return gdb_data_dir if isdir(gdb_data_dir) else None + + def generate_pioinit(self, dst_dir, patterns): + server_exe = (self.debug_options.get("server") or {}).get( + "executable", "").lower() + if "jlink" in server_exe: + cfg = initcfgs.GDB_JLINK_INIT_CONFIG + elif "st-util" in server_exe: + cfg = initcfgs.GDB_STUTIL_INIT_CONFIG + elif "mspdebug" in server_exe: + cfg = initcfgs.MSPDEBUG_INIT_CONFIG + elif self.debug_options['require_debug_port']: + cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG + else: + cfg = initcfgs.GDB_DEFAULT_INIT_CONFIG + commands = cfg.split("\n") + + if self.debug_options['init_cmds']: + commands = self.debug_options['init_cmds'] + commands.extend(self.debug_options['extra_cmds']) + + if not any("define pio_reset_target" in cmd for cmd in commands): + commands = [ + "define pio_reset_target", + " echo Warning! Undefined pio_reset_target command\\n", + " mon reset", + "end" + ] + commands # yapf: disable + if not any("define pio_reset_halt_target" in cmd for cmd in commands): + commands = [ + "define pio_reset_halt_target", + " echo Warning! Undefined pio_reset_halt_target command\\n", + " mon reset halt", + "end" + ] + commands # yapf: disable + if not any("define pio_restart_target" in cmd for cmd in commands): + commands += [ + "define pio_restart_target", + " pio_reset_halt_target", + " $INIT_BREAK", + " %s" % ("continue" if patterns['INIT_BREAK'] else "next"), + "end" + ] # yapf: disable + + banner = [ + "echo PlatformIO Unified Debugger > http://bit.ly/pio-debug\\n", + "echo PlatformIO: Initializing remote target...\\n" + ] + footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] + commands = banner + commands + footer + + with open(join(dst_dir, self.PIO_SRC_NAME), "w") as fp: + fp.write("\n".join(self.apply_patterns(commands, patterns))) + + def connectionMade(self): + self._lock_session(self.transport.pid) + + p = protocol.Protocol() + p.dataReceived = self.onStdInData + stdio.StandardIO(p) + + def onStdInData(self, data): + self._last_server_activity = time.time() + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + + if b"-exec-run" in data: + if self._target_is_run: + token, _ = data.split(b"-", 1) + self.outReceived(token + b"^running\n") + return + data = data.replace(b"-exec-run", b"-exec-continue") + + if b"-exec-continue" in data: + self._target_is_run = True + if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"): + self.transport.write(b"pio_reset_target\n") + self.transport.write(data) + + def processEnded(self, reason): # pylint: disable=unused-argument + self._unlock_session() + if self._gdbsrc_dir and isdir(self._gdbsrc_dir): + util.rmtree_(self._gdbsrc_dir) + if self._debug_server: + self._debug_server.terminate() + + reactor.stop() + + def outReceived(self, data): + self._last_server_activity = time.time() + super(GDBClient, self).outReceived(data) + self._handle_error(data) + # go to init break automatically + if self.INIT_COMPLETED_BANNER.encode() in data: + self._auto_continue_timer = task.LoopingCall( + self._auto_exec_continue) + self._auto_continue_timer.start(0.1) + + def errReceived(self, data): + super(GDBClient, self).errReceived(data) + self._handle_error(data) + + def console_log(self, msg): + if helpers.is_mi_mode(self.args): + self.outReceived(('~"%s\\n"\n' % msg).encode()) + else: + self.outReceived(("%s\n" % msg).encode()) + + def _auto_exec_continue(self): + auto_exec_delay = 0.5 # in seconds + if self._last_server_activity > (time.time() - auto_exec_delay): + return + if self._auto_continue_timer: + self._auto_continue_timer.stop() + self._auto_continue_timer = None + + if not self.debug_options['init_break'] or self._target_is_run: + return + self.console_log( + "PlatformIO: Resume the execution to `debug_init_break = %s`" % + self.debug_options['init_break']) + self.transport.write(b"0-exec-continue\n" if helpers. + is_mi_mode(self.args) else b"continue\n") + self._target_is_run = True + + def _handle_error(self, data): + if (self.PIO_SRC_NAME.encode() not in data + or b"Error in sourced" not in data): + return + configuration = {"debug": self.debug_options, "env": self.env_options} + exd = re.sub(r'\\(?!")', "/", json.dumps(configuration)) + exd = re.sub(r'"(?:[a-z]\:)?((/[^"/]+)+)"', lambda m: '"%s"' % join( + *m.group(1).split("/")[-2:]), exd, re.I | re.M) + mp = MeasurementProtocol() + mp['exd'] = "DebugGDBPioInitError: %s" % exd + mp['exf'] = 1 + mp.send("exception") + self.transport.loseConnection() + + def _kill_previous_session(self): + assert self._session_id + pid = None + with app.ContentCache() as cc: + pid = cc.get(self._session_id) + cc.delete(self._session_id) + if not pid: + return + if "windows" in util.get_systype(): + kill = ["Taskkill", "/PID", pid, "/F"] + else: + kill = ["kill", pid] + try: + util.exec_command(kill) + except: # pylint: disable=bare-except + pass + + def _lock_session(self, pid): + if not self._session_id: + return + with app.ContentCache() as cc: + cc.set(self._session_id, str(pid), "1h") + + def _unlock_session(self): + if not self._session_id: + return + with app.ContentCache() as cc: + cc.delete(self._session_id) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py new file mode 100644 index 00000000..918aeb60 --- /dev/null +++ b/platformio/commands/debug/command.py @@ -0,0 +1,135 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-arguments, too-many-statements +# pylint: disable=too-many-locals, too-many-branches + +import os +from os.path import isfile + +import click + +from platformio import exception, util +from platformio.commands.debug import helpers +from platformio.managers.core import inject_contrib_pysite + + +@click.command( + "debug", + context_settings=dict(ignore_unknown_options=True), + short_help="PIO Unified Debugger") +@click.option( + "-d", + "--project-dir", + default=os.getcwd, + type=click.Path( + exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("--environment", "-e", metavar="") +@click.option("--verbose", "-v", is_flag=True) +@click.option("--interface", type=click.Choice(["gdb"])) +@click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) +@click.pass_context +def cli(ctx, project_dir, environment, verbose, interface, __unprocessed): + try: + util.ensure_udev_rules() + except NameError: + pass + except exception.InvalidUdevRules as e: + for line in str(e).split("\n") + [""]: + click.echo( + ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") % + line) + + if not util.is_platformio_project(project_dir) and os.getenv("CWD"): + project_dir = os.getenv("CWD") + + with util.cd(project_dir): + env_name = helpers.check_env_name(project_dir, environment) + env_options = helpers.get_env_options(project_dir, env_name) + if not set(env_options.keys()) >= set(["platform", "board"]): + raise exception.ProjectEnvsNotAvailable() + debug_options = helpers.validate_debug_options(ctx, env_options) + assert debug_options + + if not interface: + return helpers.predebug_project(ctx, project_dir, env_name, False, + verbose) + + configuration = helpers.load_configuration(ctx, project_dir, env_name) + if not configuration: + raise exception.DebugInvalidOptions( + "Could not load debug configuration") + + if "--version" in __unprocessed: + result = util.exec_command([configuration['gdb_path'], "--version"]) + if result['returncode'] == 0: + return click.echo(result['out']) + raise exception.PlatformioException("\n".join( + [result['out'], result['err']])) + + debug_options['load_cmd'] = helpers.configure_esp32_load_cmd( + debug_options, configuration) + + rebuild_prog = False + preload = debug_options['load_cmd'] == "preload" + load_mode = debug_options['load_mode'] + if load_mode == "always": + rebuild_prog = ( + preload + or not helpers.has_debug_symbols(configuration['prog_path'])) + elif load_mode == "modified": + rebuild_prog = ( + helpers.is_prog_obsolete(configuration['prog_path']) + or not helpers.has_debug_symbols(configuration['prog_path'])) + else: + rebuild_prog = not isfile(configuration['prog_path']) + + if preload or (not rebuild_prog and load_mode != "always"): + # don't load firmware through debug server + debug_options['load_cmd'] = None + + if rebuild_prog: + if helpers.is_mi_mode(__unprocessed): + output = helpers.GDBBytesIO() + click.echo('~"Preparing firmware for debugging...\\n"') + with helpers.capture_std_streams(output): + helpers.predebug_project(ctx, project_dir, env_name, preload, + verbose) + output.close() + else: + click.echo("Preparing firmware for debugging...") + helpers.predebug_project(ctx, project_dir, env_name, preload, + verbose) + + # save SHA sum of newly created prog + if load_mode == "modified": + helpers.is_prog_obsolete(configuration['prog_path']) + + if not isfile(configuration['prog_path']): + raise exception.DebugInvalidOptions("Program/firmware is missed") + + # run debugging client + inject_contrib_pysite() + from platformio.commands.debug.client import GDBClient, reactor + + client = GDBClient(project_dir, __unprocessed, debug_options, env_options) + client.spawn(configuration['gdb_path'], configuration['prog_path']) + + reactor.run() + + return True diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py new file mode 100644 index 00000000..42a4ef40 --- /dev/null +++ b/platformio/commands/debug/helpers.py @@ -0,0 +1,317 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import sys +import time +from contextlib import contextmanager +from fnmatch import fnmatch +from hashlib import sha1 +from io import BytesIO +from os.path import isfile + +from platformio import VERSION, exception, util +from platformio.commands.platform import \ + platform_install as cmd_platform_install +from platformio.commands.run import cli as cmd_run +from platformio.managers.platform import PlatformFactory + + +class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods + + STDOUT = sys.stdout + + def write(self, text): + for line in text.strip().split("\n"): + self.STDOUT.write('~"%s\\n"\n' % line) + self.STDOUT.flush() + + +def is_mi_mode(args): + return "--interpreter" in " ".join(args) + + +def escape_path(path): + return path.replace("\\", "/") + + +def check_env_name(project_dir, environment): + config = util.load_project_config(project_dir) + envs = [] + for section in config.sections(): + if section.startswith("env:"): + envs.append(section[4:]) + if not envs: + raise exception.ProjectEnvsNotAvailable() + if not environment and config.has_option("platformio", "env_default"): + environment = config.get("platformio", "env_default").split(", ")[0] + if environment: + if environment in envs: + return environment + raise exception.UnknownEnvNames(environment, envs) + return envs[0] + + +def get_env_options(project_dir, environment): + config = util.load_project_config(project_dir) + options = {} + for k, v in config.items("env:%s" % environment): + options[k] = v + return options + + +def validate_debug_options(cmd_ctx, env_options): + + def _cleanup_cmds(cmds): + if not cmds: + return [] + if not isinstance(cmds, list): + cmds = cmds.split("\n") + return [c.strip() for c in cmds if c.strip()] + + try: + platform = PlatformFactory.newPlatform(env_options['platform']) + except exception.UnknownPlatform: + cmd_ctx.invoke( + cmd_platform_install, + platforms=[env_options['platform']], + skip_default_package=True) + platform = PlatformFactory.newPlatform(env_options['platform']) + + board_config = platform.board_config(env_options['board']) + tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) + tool_settings = board_config.get("debug", {}).get("tools", {}).get( + tool_name, {}) + server_options = None + + # specific server per a system + if isinstance(tool_settings.get("server", {}), list): + for item in tool_settings['server'][:]: + tool_settings['server'] = item + if util.get_systype() in item.get("system", []): + break + + # user overwrites debug server + if env_options.get("debug_server"): + server_options = { + "cwd": None, + "executable": None, + "arguments": env_options.get("debug_server") + } + if not isinstance(server_options['arguments'], list): + server_options['arguments'] = server_options['arguments'].split( + "\n") + server_options['arguments'] = [ + arg.strip() for arg in server_options['arguments'] if arg.strip() + ] + server_options['executable'] = server_options['arguments'][0] + server_options['arguments'] = server_options['arguments'][1:] + elif "server" in tool_settings: + server_package = tool_settings['server'].get("package") + server_package_dir = platform.get_package_dir( + server_package) if server_package else None + if server_package and not server_package_dir: + platform.install_packages( + with_packages=[server_package], + skip_default_package=True, + silent=True) + server_package_dir = platform.get_package_dir(server_package) + server_options = dict( + cwd=server_package_dir if server_package else None, + executable=tool_settings['server'].get("executable"), + arguments=[ + a.replace("$PACKAGE_DIR", escape_path(server_package_dir)) + if server_package_dir else a + for a in tool_settings['server'].get("arguments", []) + ]) + + extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds")) + extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds"))) + result = dict( + tool=tool_name, + upload_protocol=env_options.get( + "upload_protocol", + board_config.get("upload", {}).get("protocol")), + load_cmd=env_options.get("debug_load_cmd", + tool_settings.get("load_cmd", "load")), + load_mode=env_options.get("debug_load_mode", + tool_settings.get("load_mode", "always")), + init_break=env_options.get( + "debug_init_break", tool_settings.get("init_break", + "tbreak main")), + init_cmds=_cleanup_cmds( + env_options.get("debug_init_cmds", + tool_settings.get("init_cmds"))), + extra_cmds=extra_cmds, + require_debug_port=tool_settings.get("require_debug_port", False), + port=reveal_debug_port( + env_options.get("debug_port", tool_settings.get("port")), + tool_name, tool_settings), + server=server_options) + return result + + +def predebug_project(ctx, project_dir, env_name, preload, verbose): + ctx.invoke( + cmd_run, + project_dir=project_dir, + environment=[env_name], + target=["__debug"] + (["upload"] if preload else []), + verbose=verbose) + if preload: + time.sleep(5) + + +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr + + +def load_configuration(ctx, project_dir, env_name): + output = BytesIO() + with capture_std_streams(output): + ctx.invoke( + cmd_run, + project_dir=project_dir, + environment=[env_name], + target=["idedata"]) + result = output.getvalue().decode() + output.close() + if '"includes":' not in result: + return None + for line in result.split("\n"): + line = line.strip() + if line.startswith('{"') and "cxx_path" in line: + return json.loads(line[:line.rindex("}") + 1]) + return None + + +def configure_esp32_load_cmd(debug_options, configuration): + ignore_conds = [ + debug_options['load_cmd'] != "load", + "xtensa-esp32" not in configuration.get("cc_path", ""), + not configuration.get("flash_extra_images"), not all([ + isfile(item['path']) + for item in configuration.get("flash_extra_images") + ]) + ] + if any(ignore_conds): + return debug_options['load_cmd'] + + mon_cmds = [ + 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( + path=escape_path(item['path']), offset=item['offset']) + for item in configuration.get("flash_extra_images") + ] + mon_cmds.append('monitor program_esp32 "{%s.bin}" 0x10000 verify' % + escape_path(configuration['prog_path'][:-4])) + return "\n".join(mon_cmds) + + +def has_debug_symbols(prog_path): + if not isfile(prog_path): + return False + matched = { + b".debug_info": False, + b".debug_abbrev": False, + b" -Og": False, + b" -g": False, + b"__PLATFORMIO_DEBUG__": (3, 6) > VERSION[:2] + } + with open(prog_path, "rb") as fp: + last_data = b"" + while True: + data = fp.read(1024) + if not data: + break + for pattern, found in matched.items(): + if found: + continue + if pattern in last_data + data: + matched[pattern] = True + last_data = data + return all(matched.values()) + + +def is_prog_obsolete(prog_path): + prog_hash_path = prog_path + ".sha1" + if not isfile(prog_path): + return True + shasum = sha1() + with open(prog_path, "rb") as fp: + while True: + data = fp.read(1024) + if not data: + break + shasum.update(data) + new_digest = shasum.hexdigest() + old_digest = None + if isfile(prog_hash_path): + with open(prog_hash_path, "r") as fp: + old_digest = fp.read() + if new_digest == old_digest: + return False + with open(prog_hash_path, "w") as fp: + fp.write(new_digest) + return True + + +def reveal_debug_port(env_debug_port, tool_name, tool_settings): + + def _get_pattern(): + if not env_debug_port: + return None + if set(["*", "?", "[", "]"]) & set(env_debug_port): + return env_debug_port + return None + + def _is_match_pattern(port): + pattern = _get_pattern() + if not pattern: + return True + return fnmatch(port, pattern) + + def _look_for_serial_port(hwids): + for item in util.get_serialports(filter_hwid=True): + if not _is_match_pattern(item['port']): + continue + port = item['port'] + if tool_name.startswith("blackmagic"): + if "windows" in util.get_systype() and \ + port.startswith("COM") and len(port) > 4: + port = "\\\\.\\%s" % port + if "GDB" in item['description']: + return port + for hwid in hwids: + hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") + if hwid_str in item['hwid']: + return port + return None + + if env_debug_port and not _get_pattern(): + return env_debug_port + if not tool_settings.get("require_debug_port"): + return None + + debug_port = _look_for_serial_port(tool_settings.get("hwids", [])) + if not debug_port: + raise exception.DebugInvalidOptions( + "Please specify `debug_port` for environment") + return debug_port diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py new file mode 100644 index 00000000..81af4a21 --- /dev/null +++ b/platformio/commands/debug/initcfgs.py @@ -0,0 +1,109 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GDB_DEFAULT_INIT_CONFIG = """ +define pio_reset_halt_target + monitor reset halt +end + +define pio_reset_target + monitor reset +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +pio_reset_halt_target +$LOAD_CMD +monitor init +pio_reset_halt_target +""" + +GDB_STUTIL_INIT_CONFIG = """ +define pio_reset_halt_target + monitor halt + monitor reset +end + +define pio_reset_target + monitor reset +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +pio_reset_halt_target +$LOAD_CMD +pio_reset_halt_target +""" + +GDB_JLINK_INIT_CONFIG = """ +define pio_reset_halt_target + monitor halt + monitor reset +end + +define pio_reset_target + monitor reset +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +pio_reset_halt_target +$LOAD_CMD +pio_reset_halt_target +""" + +GDB_BLACKMAGIC_INIT_CONFIG = """ +define pio_reset_halt_target + set language c + set *0xE000ED0C = 0x05FA0004 + set $busy = (*0xE000ED0C & 0x4) + while ($busy) + set $busy = (*0xE000ED0C & 0x4) + end + set language auto +end + +define pio_reset_target + pio_reset_halt_target +end + +target extended-remote $DEBUG_PORT +monitor swdp_scan +attach 1 +set mem inaccessible-by-default off +$INIT_BREAK +$LOAD_CMD + +set language c +set *0xE000ED0C = 0x05FA0004 +set $busy = (*0xE000ED0C & 0x4) +while ($busy) + set $busy = (*0xE000ED0C & 0x4) +end +set language auto +""" + +MSPDEBUG_INIT_CONFIG = """ +define pio_reset_halt_target +end + +define pio_reset_target +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +monitor erase +$LOAD_CMD +pio_reset_halt_target +""" diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py new file mode 100644 index 00000000..3f1fbd22 --- /dev/null +++ b/platformio/commands/debug/process.py @@ -0,0 +1,72 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import click +from twisted.internet import protocol + +from platformio import util +from platformio.commands.debug import helpers + +LOG_FILE = None + + +class BaseProcess(protocol.ProcessProtocol, object): + + STDOUT_CHUNK_SIZE = 2048 + + COMMON_PATTERNS = { + "PLATFORMIO_HOME_DIR": helpers.escape_path(util.get_home_dir()), + "PYTHONEXE": os.getenv("PYTHONEXEPATH", "") + } + + def apply_patterns(self, source, patterns=None): + _patterns = self.COMMON_PATTERNS.copy() + _patterns.update(patterns or {}) + + def _replace(text): + for key, value in _patterns.items(): + pattern = "$%s" % key + text = text.replace(pattern, value or "") + return text + + if isinstance(source, util.string_types): + source = _replace(source) + elif isinstance(source, (list, dict)): + items = enumerate(source) if isinstance(source, + list) else source.items() + for key, value in items: + if isinstance(value, util.string_types): + source[key] = _replace(value) + elif isinstance(value, (list, dict)): + source[key] = self.apply_patterns(value, patterns) + + return source + + def outReceived(self, data): + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + while data: + chunk = data[:self.STDOUT_CHUNK_SIZE] + click.echo(chunk, nl=False) + data = data[self.STDOUT_CHUNK_SIZE:] + + @staticmethod + def errReceived(data): + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + click.echo(data, nl=False, err=True) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py new file mode 100644 index 00000000..977fe812 --- /dev/null +++ b/platformio/commands/debug/server.py @@ -0,0 +1,110 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from os.path import isdir, isfile, join + +from twisted.internet import reactor + +from platformio import exception, util +from platformio.commands.debug import helpers +from platformio.commands.debug.process import BaseProcess + + +class DebugServer(BaseProcess): + + def __init__(self, debug_options, env_options): + self.debug_options = debug_options + self.env_options = env_options + + self._debug_port = None + self._transport = None + + def spawn(self, patterns): # pylint: disable=too-many-branches + systype = util.get_systype() + server = self.debug_options.get("server") + if not server: + return None + server = self.apply_patterns(server, patterns) + server_executable = server['executable'] + if not server_executable: + return None + if server['cwd']: + server_executable = join(server['cwd'], server_executable) + if ("windows" in systype and not server_executable.endswith(".exe") + and isfile(server_executable + ".exe")): + server_executable = server_executable + ".exe" + + if not isfile(server_executable): + server_executable = util.where_is_program(server_executable) + if not isfile(server_executable): + raise exception.DebugInvalidOptions( + "\nCould not launch Debug Server '%s'. Please check that it " + "is installed and is included in a system PATH\n\n" + "See documentation or contact contact@platformio.org:\n" + "http://docs.platformio.org/page/plus/debugging.html\n" % + server_executable) + + self._debug_port = ":3333" + openocd_pipe_allowed = all([ + not self.debug_options['port'], "openocd" in server_executable, + self.env_options['platform'] != "riscv" + ]) + if openocd_pipe_allowed: + args = [] + if server['cwd']: + args.extend(["-s", helpers.escape_path(server['cwd'])]) + args.extend([ + "-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled" + ]) + args.extend(server['arguments']) + str_args = " ".join( + [arg if arg.startswith("-") else '"%s"' % arg for arg in args]) + self._debug_port = '| "%s" %s' % ( + helpers.escape_path(server_executable), str_args) + else: + env = os.environ.copy() + # prepend server "lib" folder to LD path + if ("windows" not in systype and server['cwd'] + and isdir(join(server['cwd'], "lib"))): + ld_key = ("DYLD_LIBRARY_PATH" + if "darwin" in systype else "LD_LIBRARY_PATH") + env[ld_key] = join(server['cwd'], "lib") + if os.environ.get(ld_key): + env[ld_key] = "%s:%s" % (env[ld_key], + os.environ.get(ld_key)) + # prepend BIN to PATH + if server['cwd'] and isdir(join(server['cwd'], "bin")): + env['PATH'] = "%s%s%s" % ( + join(server['cwd'], "bin"), os.pathsep, + os.environ.get("PATH", os.environ.get("Path", ""))) + + self._transport = reactor.spawnProcess( + self, + server_executable, [server_executable] + server['arguments'], + path=server['cwd'], + env=env) + if "mspdebug" in server_executable.lower(): + self._debug_port = ":2000" + elif "jlink" in server_executable.lower(): + self._debug_port = ":2331" + + return self._transport + + def get_debug_port(self): + return self._debug_port + + def terminate(self): + if self._transport: + self._transport.signalProcess("KILL") diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 3fd7f1cd..a6598361 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -115,6 +115,15 @@ def shutdown_piohome_servers(): port += 1 +def inject_contrib_pysite(): + from site import addsitedir + contrib_pysite_dir = get_core_package_dir("contrib-pysite") + if contrib_pysite_dir in sys.path: + return + addsitedir(contrib_pysite_dir) + sys.path.insert(0, contrib_pysite_dir) + + def pioplus_call(args, **kwargs): if "windows" in util.get_systype() and sys.version_info < (2, 7, 6): raise exception.PlatformioException( diff --git a/platformio/util.py b/platformio/util.py index dd280ba2..08592355 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -518,11 +518,8 @@ def get_mdns_services(): try: import zeroconf except ImportError: - from site import addsitedir - from platformio.managers.core import get_core_package_dir - contrib_pysite_dir = get_core_package_dir("contrib-pysite") - addsitedir(contrib_pysite_dir) - sys.path.insert(0, contrib_pysite_dir) + from platformio.managers.core import inject_contrib_pysite + inject_contrib_pysite() import zeroconf class mDNSListener(object): From c7d8b50474e2c6e4847739401fa50fc704bd4d7e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Apr 2019 19:58:34 +0300 Subject: [PATCH 095/333] Use generic GDB_MSPDEBUG_INIT_CONFIG --- platformio/commands/debug/client.py | 2 +- platformio/commands/debug/initcfgs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index a0bcc61b..7e9aebb9 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -111,7 +111,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes elif "st-util" in server_exe: cfg = initcfgs.GDB_STUTIL_INIT_CONFIG elif "mspdebug" in server_exe: - cfg = initcfgs.MSPDEBUG_INIT_CONFIG + cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG elif self.debug_options['require_debug_port']: cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG else: diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index 81af4a21..62f36c0d 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -94,7 +94,7 @@ end set language auto """ -MSPDEBUG_INIT_CONFIG = """ +GDB_MSPDEBUG_INIT_CONFIG = """ define pio_reset_halt_target end From 948a977fa543a7f70eebaabaa5f836323f0906cf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Apr 2019 20:33:31 +0300 Subject: [PATCH 096/333] Initial commit of PIO Unit Testing --- platformio/commands/test.py | 67 --------- platformio/commands/test/__init__.py | 15 ++ platformio/commands/test/command.py | 181 +++++++++++++++++++++++ platformio/commands/test/embedded.py | 133 +++++++++++++++++ platformio/commands/test/native.py | 39 +++++ platformio/commands/test/processor.py | 200 ++++++++++++++++++++++++++ platformio/exception.py | 9 ++ 7 files changed, 577 insertions(+), 67 deletions(-) delete mode 100644 platformio/commands/test.py create mode 100644 platformio/commands/test/__init__.py create mode 100644 platformio/commands/test/command.py create mode 100644 platformio/commands/test/embedded.py create mode 100644 platformio/commands/test/native.py create mode 100644 platformio/commands/test/processor.py diff --git a/platformio/commands/test.py b/platformio/commands/test.py deleted file mode 100644 index 4c6414c9..00000000 --- a/platformio/commands/test.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -from os import getcwd - -import click - -from platformio.managers.core import pioplus_call - - -@click.command("test", short_help="Local Unit Testing") -@click.option("--environment", "-e", multiple=True, metavar="") -@click.option( - "--filter", - "-f", - multiple=True, - metavar="", - help="Filter tests by a pattern") -@click.option( - "--ignore", - "-i", - multiple=True, - metavar="", - help="Ignore tests by a pattern") -@click.option("--upload-port") -@click.option("--test-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option("--without-building", is_flag=True) -@click.option("--without-uploading", is_flag=True) -@click.option( - "--no-reset", - is_flag=True, - help="Disable software reset via Serial.DTR/RST") -@click.option( - "--monitor-rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state for Serial Monitor") -@click.option( - "--monitor-dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state for Serial Monitor") -@click.option("--verbose", "-v", is_flag=True) -def cli(*args, **kwargs): # pylint: disable=unused-argument - pioplus_call(sys.argv[1:]) diff --git a/platformio/commands/test/__init__.py b/platformio/commands/test/__init__.py new file mode 100644 index 00000000..6d4c3e2b --- /dev/null +++ b/platformio/commands/test/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.test.command import cli diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py new file mode 100644 index 00000000..49da7bba --- /dev/null +++ b/platformio/commands/test/command.py @@ -0,0 +1,181 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fnmatch import fnmatch +from os import getcwd, listdir +from os.path import isdir, join +from time import time + +import click + +from platformio import exception, util +from platformio.commands.run import check_project_envs, print_header +from platformio.commands.test.embedded import EmbeddedTestProcessor +from platformio.commands.test.native import NativeTestProcessor + + +@click.command("test", short_help="Unit Testing") +@click.option("--environment", "-e", multiple=True, metavar="") +@click.option( + "--filter", + "-f", + multiple=True, + metavar="", + help="Filter tests by a pattern") +@click.option( + "--ignore", + "-i", + multiple=True, + metavar="", + help="Ignore tests by a pattern") +@click.option("--upload-port") +@click.option("--test-port") +@click.option( + "-d", + "--project-dir", + default=getcwd, + type=click.Path( + exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("--without-building", is_flag=True) +@click.option("--without-uploading", is_flag=True) +@click.option("--without-testing", is_flag=True) +@click.option("--no-reset", is_flag=True) +@click.option( + "--monitor-rts", + default=None, + type=click.IntRange(0, 1), + help="Set initial RTS line state for Serial Monitor") +@click.option( + "--monitor-dtr", + default=None, + type=click.IntRange(0, 1), + help="Set initial DTR line state for Serial Monitor") +@click.option("--verbose", "-v", is_flag=True) +@click.pass_context +def cli( # pylint: disable=redefined-builtin + ctx, environment, ignore, filter, upload_port, test_port, project_dir, + without_building, without_uploading, without_testing, no_reset, + monitor_rts, monitor_dtr, verbose): + with util.cd(project_dir): + test_dir = util.get_projecttest_dir() + if not isdir(test_dir): + raise exception.TestDirNotExists(test_dir) + test_names = get_test_names(test_dir) + projectconf = util.load_project_config() + env_default = None + if projectconf.has_option("platformio", "env_default"): + env_default = util.parse_conf_multi_values( + projectconf.get("platformio", "env_default")) + assert check_project_envs(projectconf, environment or env_default) + + click.echo("Verbose mode can be enabled via `-v, --verbose` option") + click.echo("Collected %d items" % len(test_names)) + + start_time = time() + results = [] + for testname in test_names: + for section in projectconf.sections(): + if not section.startswith("env:"): + continue + + # filter and ignore patterns + patterns = dict(filter=list(filter), ignore=list(ignore)) + for key in patterns: + if projectconf.has_option(section, "test_%s" % key): + patterns[key].extend([ + p.strip() + for p in projectconf.get(section, "test_%s" % + key).split(", ") + if p.strip() + ]) + + envname = section[4:] + skip_conditions = [ + environment and envname not in environment, + not environment and env_default + and envname not in env_default, + testname != "*" and patterns['filter'] and + not any([fnmatch(testname, p) + for p in patterns['filter']]), + testname != "*" + and any([fnmatch(testname, p) + for p in patterns['ignore']]), + ] + if any(skip_conditions): + results.append((None, testname, envname)) + continue + + cls = (NativeTestProcessor if projectconf.get( + section, "platform") == "native" else + EmbeddedTestProcessor) + tp = cls( + ctx, testname, envname, + dict( + project_config=projectconf, + project_dir=project_dir, + upload_port=upload_port, + test_port=test_port, + without_building=without_building, + without_uploading=without_uploading, + without_testing=without_testing, + no_reset=no_reset, + monitor_rts=monitor_rts, + monitor_dtr=monitor_dtr, + verbose=verbose)) + results.append((tp.process(), testname, envname)) + + if without_testing: + return + + click.echo() + print_header("[%s]" % click.style("TEST SUMMARY")) + + passed = True + for result in results: + status, testname, envname = result + status_str = click.style("PASSED", fg="green") + if status is False: + passed = False + status_str = click.style("FAILED", fg="red") + elif status is None: + status_str = click.style("IGNORED", fg="yellow") + + click.echo( + "test/%s/env:%s\t[%s]" % (click.style(testname, fg="yellow"), + click.style(envname, fg="cyan"), + status_str), + err=status is False) + + print_header( + "[%s] Took %.2f seconds" % ( + (click.style("PASSED", fg="green", bold=True) if passed else + click.style("FAILED", fg="red", bold=True)), time() - start_time), + is_error=not passed) + + if not passed: + raise exception.ReturnErrorCode(1) + + +def get_test_names(test_dir): + names = [] + for item in sorted(listdir(test_dir)): + if isdir(join(test_dir, item)): + names.append(item) + if not names: + names = ["*"] + return names diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py new file mode 100644 index 00000000..39e8555d --- /dev/null +++ b/platformio/commands/test/embedded.py @@ -0,0 +1,133 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import sleep + +import click +import serial + +from platformio import exception, util +from platformio.commands.test.processor import TestProcessorBase +from platformio.managers.platform import PlatformFactory + + +class EmbeddedTestProcessor(TestProcessorBase): + + SERIAL_TIMEOUT = 600 + + def process(self): + if not self.options['without_building']: + self.print_progress("Building... (1/3)") + target = ["__test"] + if self.options['without_uploading']: + target.append("checkprogsize") + self.build_or_upload(target) + + if not self.options['without_uploading']: + self.print_progress("Uploading... (2/3)") + target = ["upload"] + if self.options['without_building']: + target.append("nobuild") + else: + target.append("__test") + self.build_or_upload(target) + + if self.options['without_testing']: + return None + + self.print_progress("Testing... (3/3)") + return self.run() + + def run(self): + click.echo("If you don't see any output for the first 10 secs, " + "please reset board (press reset button)") + click.echo() + + try: + ser = serial.Serial( + baudrate=self.get_baudrate(), timeout=self.SERIAL_TIMEOUT) + ser.port = self.get_test_port() + ser.rts = self.options['monitor_rts'] + ser.dtr = self.options['monitor_dtr'] + ser.open() + except serial.SerialException as e: + click.secho(str(e), fg="red", err=True) + return False + + if not self.options['no_reset']: + ser.flushInput() + ser.setDTR(False) + ser.setRTS(False) + sleep(0.1) + ser.setDTR(True) + ser.setRTS(True) + sleep(0.1) + + while True: + line = ser.readline().strip() + + # fix non-ascii output from device + for i, c in enumerate(line[::-1]): + if not isinstance(c, int): + c = ord(c) + if c > 127: + line = line[-i:] + break + + if not line: + continue + if isinstance(line, bytes): + line = line.decode("utf8") + self.on_run_out(line) + if all([l in line for l in ("Tests", "Failures", "Ignored")]): + break + ser.close() + return not self._run_failed + + def get_test_port(self): + # if test port is specified manually or in config + if self.options.get("test_port"): + return self.options.get("test_port") + if self.env_options.get("test_port"): + return self.env_options.get("test_port") + + assert set(["platform", "board"]) & set(self.env_options.keys()) + p = PlatformFactory.newPlatform(self.env_options['platform']) + board_hwids = p.board_config(self.env_options['board']).get( + "build.hwids", []) + port = None + elapsed = 0 + while elapsed < 5 and not port: + for item in util.get_serialports(): + port = item['port'] + for hwid in board_hwids: + hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") + if hwid_str in item['hwid']: + return port + + # check if port is already configured + try: + serial.Serial(port, timeout=self.SERIAL_TIMEOUT).close() + except serial.SerialException: + port = None + + if not port: + sleep(0.25) + elapsed += 0.25 + + if not port: + raise exception.PlatformioException( + "Please specify `test_port` for environment or use " + "global `--test-port` option.") + return port diff --git a/platformio/commands/test/native.py b/platformio/commands/test/native.py new file mode 100644 index 00000000..0945b10f --- /dev/null +++ b/platformio/commands/test/native.py @@ -0,0 +1,39 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os.path import join + +from platformio import util +from platformio.commands.test.processor import TestProcessorBase + + +class NativeTestProcessor(TestProcessorBase): + + def process(self): + if not self.options['without_building']: + self.print_progress("Building... (1/2)") + self.build_or_upload(["__test"]) + if self.options['without_testing']: + return None + self.print_progress("Testing... (2/2)") + return self.run() + + def run(self): + with util.cd(self.options['project_dir']): + build_dir = util.get_projectbuild_dir() + result = util.exec_command([join(build_dir, self.env_name, "program")], + stdout=util.AsyncPipe(self.on_run_out), + stderr=util.AsyncPipe(self.on_run_out)) + assert "returncode" in result + return result['returncode'] == 0 and not self._run_failed diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py new file mode 100644 index 00000000..7c3b84eb --- /dev/null +++ b/platformio/commands/test/processor.py @@ -0,0 +1,200 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +from os import remove +from os.path import isdir, isfile, join +from string import Template + +import click + +from platformio import exception, util +from platformio.commands.run import cli as cmd_run +from platformio.commands.run import print_header + +TRANSPORT_OPTIONS = { + "arduino": { + "include": "#include ", + "object": "", + "putchar": "Serial.write(c)", + "flush": "Serial.flush()", + "begin": "Serial.begin($baudrate)", + "end": "Serial.end()" + }, + "mbed": { + "include": "#include ", + "object": "Serial pc(USBTX, USBRX);", + "putchar": "pc.putc(c)", + "flush": "", + "begin": "pc.baud($baudrate)", + "end": "" + }, + "energia": { + "include": "#include ", + "object": "", + "putchar": "Serial.write(c)", + "flush": "Serial.flush()", + "begin": "Serial.begin($baudrate)", + "end": "Serial.end()" + }, + "espidf": { + "include": "#include ", + "object": "", + "putchar": "putchar(c)", + "flush": "fflush(stdout)", + "begin": "", + "end": "" + }, + "native": { + "include": "#include ", + "object": "", + "putchar": "putchar(c)", + "flush": "fflush(stdout)", + "begin": "", + "end": "" + }, + "custom": { + "include": '#include "unittest_transport.h"', + "object": "", + "putchar": "unittest_uart_putchar(c)", + "flush": "unittest_uart_flush()", + "begin": "unittest_uart_begin()", + "end": "unittest_uart_end()" + } +} + + +class TestProcessorBase(object): + + DEFAULT_BAUDRATE = 115200 + + def __init__(self, cmd_ctx, testname, envname, options): + self.cmd_ctx = cmd_ctx + self.cmd_ctx.meta['piotest_processor'] = True + self.test_name = testname + self.options = options + self.env_name = envname + self.env_options = { + k: v + for k, v in options['project_config'].items("env:" + envname) + } + self._run_failed = False + self._outputcpp_generated = False + + def get_transport(self): + transport = self.env_options.get("framework") + if self.env_options.get("platform") == "native": + transport = "native" + if "test_transport" in self.env_options: + transport = self.env_options['test_transport'] + if transport not in TRANSPORT_OPTIONS: + raise exception.PlatformioException( + "Unknown Unit Test transport `%s`" % transport) + return transport.lower() + + def get_baudrate(self): + return int(self.env_options.get("test_speed", self.DEFAULT_BAUDRATE)) + + def print_progress(self, text, is_error=False): + click.echo() + print_header( + "[test/%s] %s" % (click.style( + self.test_name, fg="yellow", bold=True), text), + is_error=is_error) + + def build_or_upload(self, target): + if not self._outputcpp_generated: + self.generate_outputcpp(util.get_projecttest_dir()) + self._outputcpp_generated = True + + if self.test_name != "*": + self.cmd_ctx.meta['piotest'] = self.test_name + + if not self.options['verbose']: + click.echo("Please wait...") + + return self.cmd_ctx.invoke( + cmd_run, + project_dir=self.options['project_dir'], + upload_port=self.options['upload_port'], + silent=not self.options['verbose'], + environment=[self.env_name], + disable_auto_clean="nobuild" in target, + target=target) + + def process(self): + raise NotImplementedError + + def run(self): + raise NotImplementedError + + def on_run_out(self, line): + if line.endswith(":PASS"): + click.echo( + "%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green"))) + elif ":FAIL" in line: + self._run_failed = True + click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red"))) + else: + click.echo(line) + + def generate_outputcpp(self, test_dir): + assert isdir(test_dir) + + cpp_tpl = "\n".join([ + "$include", + "#include ", + "", + "$object", + "", + "void output_start(unsigned int baudrate)", + "{", + " $begin;", + "}", + "", + "void output_char(int c)", + "{", + " $putchar;", + "}", + "", + "void output_flush(void)", + "{", + " $flush;", + "}", + "", + "void output_complete(void)", + "{", + " $end;", + "}" + ]) # yapf: disable + + def delete_tmptest_file(file_): + try: + remove(file_) + except: # pylint: disable=bare-except + if isfile(file_): + click.secho( + "Warning: Could not remove temporary file '%s'. " + "Please remove it manually." % file_, + fg="yellow") + + tpl = Template(cpp_tpl).substitute( + TRANSPORT_OPTIONS[self.get_transport()]) + data = Template(tpl).substitute(baudrate=self.get_baudrate()) + + tmp_file = join(test_dir, "output_export.cpp") + with open(tmp_file, "w") as f: + f.write(data) + + atexit.register(delete_tmptest_file, tmp_file) diff --git a/platformio/exception.py b/platformio/exception.py index 7829b648..77837d7a 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -292,3 +292,12 @@ class DebugSupportError(PlatformioException): class DebugInvalidOptions(PlatformioException): pass + + +class TestDirNotExists(PlatformioException): + + MESSAGE = "A test folder '{0}' does not exist.\nPlease create 'test' "\ + "directory in project's root and put a test set.\n"\ + "More details about Unit "\ + "Testing: http://docs.platformio.org/page/plus/"\ + "unit-testing.html" From 3032cade17909bf0b5f08d1c182a2692e66e3416 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 19 Apr 2019 20:46:28 +0300 Subject: [PATCH 097/333] PyLint fixes --- platformio/commands/test/command.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index 49da7bba..a4a3d627 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-arguments, too-many-locals, too-many-branches + from fnmatch import fnmatch from os import getcwd, listdir from os.path import isdir, join From 65354e995d8511caa36b24a8ff337090dbbd5394 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 22 Apr 2019 21:07:28 +0300 Subject: [PATCH 098/333] Initial commit of PIO Home --- .isort.cfg | 2 +- platformio/commands/home/__init__.py | 15 + platformio/commands/home/command.py | 100 +++++++ platformio/commands/home/helpers.py | 69 +++++ platformio/commands/home/rpc/__init__.py | 13 + .../commands/home/rpc/handlers/__init__.py | 13 + platformio/commands/home/rpc/handlers/app.py | 83 +++++ platformio/commands/home/rpc/handlers/ide.py | 42 +++ platformio/commands/home/rpc/handlers/misc.py | 194 ++++++++++++ platformio/commands/home/rpc/handlers/os.py | 153 ++++++++++ .../commands/home/rpc/handlers/piocore.py | 83 +++++ .../commands/home/rpc/handlers/project.py | 283 ++++++++++++++++++ platformio/commands/home/rpc/server.py | 70 +++++ platformio/commands/{home.py => home/web.py} | 29 +- 14 files changed, 1133 insertions(+), 16 deletions(-) create mode 100644 platformio/commands/home/__init__.py create mode 100644 platformio/commands/home/command.py create mode 100644 platformio/commands/home/helpers.py create mode 100644 platformio/commands/home/rpc/__init__.py create mode 100644 platformio/commands/home/rpc/handlers/__init__.py create mode 100644 platformio/commands/home/rpc/handlers/app.py create mode 100644 platformio/commands/home/rpc/handlers/ide.py create mode 100644 platformio/commands/home/rpc/handlers/misc.py create mode 100644 platformio/commands/home/rpc/handlers/os.py create mode 100644 platformio/commands/home/rpc/handlers/piocore.py create mode 100644 platformio/commands/home/rpc/handlers/project.py create mode 100644 platformio/commands/home/rpc/server.py rename platformio/commands/{home.py => home/web.py} (54%) diff --git a/.isort.cfg b/.isort.cfg index 1ee0d412..5f6d6207 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=79 -known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial, twisted +known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,bs4,jsonrpc diff --git a/platformio/commands/home/__init__.py b/platformio/commands/home/__init__.py new file mode 100644 index 00000000..a889291e --- /dev/null +++ b/platformio/commands/home/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.home.command import cli diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py new file mode 100644 index 00000000..0685670a --- /dev/null +++ b/platformio/commands/home/command.py @@ -0,0 +1,100 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +from os.path import isdir + +import click + +from platformio import exception +from platformio.managers.core import (get_core_package_dir, + inject_contrib_pysite) + + +@click.command("home", short_help="PIO Home") +@click.option("--port", type=int, default=8008, help="HTTP port, default=8008") +@click.option( + "--host", + default="127.0.0.1", + help="HTTP host, default=127.0.0.1. " + "You can open PIO Home for inbound connections with --host=0.0.0.0") +@click.option("--no-open", is_flag=True) # pylint: disable=too-many-locals +def cli(port, host, no_open): + # import contrib modules + inject_contrib_pysite() + from autobahn.twisted.resource import WebSocketResource + from twisted.internet import reactor + from twisted.web import server + from platformio.commands.home.rpc.handlers.app import AppRPC + from platformio.commands.home.rpc.handlers.ide import IDERPC + from platformio.commands.home.rpc.handlers.misc import MiscRPC + from platformio.commands.home.rpc.handlers.os import OSRPC + from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC + from platformio.commands.home.rpc.handlers.project import ProjectRPC + from platformio.commands.home.rpc.server import JSONRPCServerFactory + from platformio.commands.home.web import WebRoot + + factory = JSONRPCServerFactory() + factory.addHandler(AppRPC(), namespace="app") + factory.addHandler(IDERPC(), namespace="ide") + factory.addHandler(MiscRPC(), namespace="misc") + factory.addHandler(OSRPC(), namespace="os") + factory.addHandler(PIOCoreRPC(), namespace="core") + factory.addHandler(ProjectRPC(), namespace="project") + + contrib_dir = get_core_package_dir("contrib-piohome") + if not isdir(contrib_dir): + raise exception.PlatformioException("Invalid path to PIO Home Contrib") + root = WebRoot(contrib_dir) + root.putChild(b"wsrpc", WebSocketResource(factory)) + site = server.Site(root) + + # hook for `platformio-node-helpers` + if host == "__do_not_start__": + return + + # if already started + already_started = False + socket.setdefaulttimeout(1) + try: + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) + already_started = True + except: # pylint: disable=bare-except + pass + + home_url = "http://%s:%d" % (host, port) + if not no_open: + if already_started: + click.launch(home_url) + else: + reactor.callLater(1, lambda: click.launch(home_url)) + + click.echo("\n".join([ + "", + " ___I_", + " /\\-_--\\ PlatformIO Home", + "/ \\_-__\\", + "|[]| [] | %s" % home_url, + "|__|____|______________%s" % ("_" * len(host)), + ])) + click.echo("") + click.echo("Open PIO Home in your browser by this URL => %s" % home_url) + + if already_started: + return + + click.echo("PIO Home has been started. Press Ctrl+C to shutdown.") + + reactor.listenTCP(port, site, interface=host) + reactor.run() diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py new file mode 100644 index 00000000..abac9416 --- /dev/null +++ b/platformio/commands/home/helpers.py @@ -0,0 +1,69 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=keyword-arg-before-vararg, arguments-differ + +import os +import socket + +import requests +from twisted.internet import threads +from twisted.internet.defer import ensureDeferred + +from platformio import util + + +class AsyncSession(requests.Session): + + def __init__(self, n=None, *args, **kwargs): + if n: + from twisted.internet import reactor + pool = reactor.getThreadPool() + pool.adjustPoolsize(0, n) + + super(AsyncSession, self).__init__(*args, **kwargs) + + def request(self, *args, **kwargs): + func = super(AsyncSession, self).request + return threads.deferToThread(func, *args, **kwargs) + + def wrap(self, *args, **kwargs): # pylint: disable=no-self-use + return ensureDeferred(*args, **kwargs) + + +@util.memoized(expire=5000) +def requests_session(): + return AsyncSession(n=5) + + +@util.memoized() +def get_core_fullpath(): + return util.where_is_program( + "platformio" + (".exe" if "windows" in util.get_systype() else "")) + + +@util.memoized(expire=10000) +def is_twitter_blocked(): + ip = "104.244.42.1" + timeout = 2 + try: + if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): + requests.get( + "http://%s" % ip, allow_redirects=False, timeout=timeout) + else: + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) + return False + except: # pylint: disable=bare-except + pass + return True diff --git a/platformio/commands/home/rpc/__init__.py b/platformio/commands/home/rpc/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/home/rpc/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/home/rpc/handlers/__init__.py b/platformio/commands/home/rpc/handlers/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py new file mode 100644 index 00000000..ce95a218 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/app.py @@ -0,0 +1,83 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import json +from os.path import expanduser, isfile, join + +from platformio import __version__, app, exception, util + + +class AppRPC(object): + + APPSTATE_PATH = join(util.get_home_dir(), "homestate.json") + + @staticmethod + def load_state(): + state = None + try: + if isfile(AppRPC.APPSTATE_PATH): + state = util.load_json(AppRPC.APPSTATE_PATH) + except exception.PlatformioException: + pass + if not isinstance(state, dict): + state = {} + storage = state.get("storage", {}) + + # base data + caller_id = app.get_session_var("caller_id") + storage['cid'] = app.get_cid() + storage['coreVersion'] = __version__ + storage['coreSystype'] = util.get_systype() + storage['coreCaller'] = (str(caller_id).lower() if caller_id else None) + storage['coreSettings'] = { + name: { + "description": data['description'], + "default_value": data['value'], + "value": app.get_setting(name) + } + for name, data in app.DEFAULT_SETTINGS.items() + } + + # encode to UTF-8 + for key in storage['coreSettings']: + if not key.endswith("dir"): + continue + storage['coreSettings'][key][ + 'default_value'] = util.path_to_unicode( + storage['coreSettings'][key]['default_value']) + storage['coreSettings'][key]['value'] = util.path_to_unicode( + storage['coreSettings'][key]['value']) + storage['homeDir'] = util.path_to_unicode(expanduser("~")) + storage['projectsDir'] = storage['coreSettings']['projects_dir'][ + 'value'] + + # skip non-existing recent projects + storage['recentProjects'] = [ + p for p in storage.get("recentProjects", []) + if util.is_platformio_project(p) + ] + + state['storage'] = storage + return state + + @staticmethod + def get_state(): + return AppRPC.load_state() + + def save_state(self, state): + with open(self.APPSTATE_PATH, "w") as fp: + json.dump(state, fp) + return True diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py new file mode 100644 index 00000000..d5c2bf74 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from jsonrpc.exceptions import JSONRPCDispatchException +from twisted.internet import defer + + +class IDERPC(object): + + def __init__(self): + self._queue = [] + + def send_command(self, command, params): + if not self._queue: + raise JSONRPCDispatchException( + code=4005, message="PIO Home IDE agent is not started") + while self._queue: + self._queue.pop().callback({ + "id": time.time(), + "method": command, + "params": params + }) + + def listen_commands(self): + self._queue.append(defer.Deferred()) + return self._queue[-1] + + def open_project(self, project_dir): + return self.send_command("open_project", project_dir) diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py new file mode 100644 index 00000000..d7deff19 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -0,0 +1,194 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +import time + +from bs4 import BeautifulSoup +from twisted.internet import defer, reactor + +from platformio import app +from platformio.commands.home import helpers +from platformio.commands.home.rpc.handlers.os import OSRPC + + +class MiscRPC(object): + + def load_latest_tweets(self, username): + cache_key = "piohome_latest_tweets_%s" % username + cache_valid = "7d" + with app.ContentCache() as cc: + cache_data = cc.get(cache_key) + if cache_data: + cache_data = json.loads(cache_data) + # automatically update cache in background every 12 hours + if cache_data['time'] < (time.time() - (3600 * 12)): + reactor.callLater(5, self._preload_latest_tweets, username, + cache_key, cache_valid) + return cache_data['result'] + + result = self._preload_latest_tweets(username, cache_key, cache_valid) + return result + + @defer.inlineCallbacks + def _preload_latest_tweets(self, username, cache_key, cache_valid): + result = yield self._fetch_tweets(username) + with app.ContentCache() as cc: + cc.set(cache_key, + json.dumps({ + "time": int(time.time()), + "result": result + }), cache_valid) + defer.returnValue(result) + + @defer.inlineCallbacks + def _fetch_tweets(self, username): + api_url = ("https://twitter.com/i/profiles/show/%s/timeline/tweets?" + "include_available_features=1&include_entities=1&" + "include_new_items_bar=true") % username + if helpers.is_twitter_blocked(): + api_url = self._get_proxed_uri(api_url) + html_or_json = yield OSRPC.fetch_content( + api_url, headers=self._get_twitter_headers(username)) + # issue with PIO Core < 3.5.3 and ContentCache + if not isinstance(html_or_json, dict): + html_or_json = json.loads(html_or_json) + assert "items_html" in html_or_json + soup = BeautifulSoup(html_or_json['items_html'], "html.parser") + tweet_nodes = soup.find_all( + "div", attrs={ + "class": "tweet", + "data-tweet-id": True + }) + result = yield defer.DeferredList( + [self._parse_tweet_node(node, username) for node in tweet_nodes], + consumeErrors=True) + defer.returnValue([r[1] for r in result if r[0]]) + + @defer.inlineCallbacks + def _parse_tweet_node(self, tweet, username): + # remove non-visible items + for node in tweet.find_all(class_=["invisible", "u-hidden"]): + node.decompose() + twitter_url = "https://twitter.com" + time_node = tweet.find("span", attrs={"data-time": True}) + text_node = tweet.find(class_="tweet-text") + quote_text_node = tweet.find(class_="QuoteTweet-text") + if quote_text_node and not text_node.get_text().strip(): + text_node = quote_text_node + photos = [ + node.get("data-image-url") for node in (tweet.find_all(class_=[ + "AdaptiveMedia-photoContainer", "QuoteMedia-photoContainer" + ]) or []) + ] + urls = [ + node.get("data-expanded-url") + for node in (quote_text_node or text_node).find_all( + class_="twitter-timeline-link", + attrs={"data-expanded-url": True}) + ] + + # fetch data from iframe card + if (not photos or not urls) and tweet.get("data-card2-type"): + iframe_node = tweet.find( + "div", attrs={"data-full-card-iframe-url": True}) + if iframe_node: + iframe_card = yield self._fetch_iframe_card( + twitter_url + iframe_node.get("data-full-card-iframe-url"), + username) + if not photos and iframe_card['photo']: + photos.append(iframe_card['photo']) + if not urls and iframe_card['url']: + urls.append(iframe_card['url']) + if iframe_card['text_node']: + text_node = iframe_card['text_node'] + + if not photos: + photos.append(tweet.find("img", class_="avatar").get("src")) + + def _fetch_text(text_node): + text = text_node.decode_contents(formatter="html").strip() + text = re.sub(r'href="/', 'href="%s/' % twitter_url, text) + if "

" not in text and "", text) + return text + + defer.returnValue({ + "tweetId": + tweet.get("data-tweet-id"), + "tweetUrl": + twitter_url + tweet.get("data-permalink-path"), + "author": + tweet.get("data-name"), + "time": + int(time_node.get("data-time")), + "timeFormatted": + time_node.string, + "text": + _fetch_text(text_node), + "entries": { + "urls": + urls, + "photos": [ + self._get_proxed_uri(uri) + if helpers.is_twitter_blocked() else uri for uri in photos + ] + }, + "isPinned": + "user-pinned" in tweet.get("class") + }) + + @defer.inlineCallbacks + def _fetch_iframe_card(self, url, username): + if helpers.is_twitter_blocked(): + url = self._get_proxed_uri(url) + html = yield OSRPC.fetch_content( + url, headers=self._get_twitter_headers(username), cache_valid="7d") + soup = BeautifulSoup(html, "html.parser") + photo_node = soup.find("img", attrs={"data-src": True}) + url_node = soup.find("a", class_="TwitterCard-container") + text_node = soup.find("div", class_="SummaryCard-content") + if text_node: + text_node.find( + "span", class_="SummaryCard-destination").decompose() + defer.returnValue({ + "photo": + photo_node.get("data-src") if photo_node else None, + "text_node": + text_node, + "url": + url_node.get("href") if url_node else None + }) + + @staticmethod + def _get_proxed_uri(uri): + index = uri.index("://") + return "https://dl.platformio.org/__prx__/" + uri[index + 3:] + + @staticmethod + def _get_twitter_headers(username): + return { + "Accept": + "application/json, text/javascript, */*; q=0.01", + "Referer": + "https://twitter.com/%s" % username, + "User-Agent": + ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit" + "/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8"), + "X-Twitter-Active-User": + "yes", + "X-Requested-With": + "XMLHttpRequest" + } diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py new file mode 100644 index 00000000..f581c469 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/os.py @@ -0,0 +1,153 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import glob +import os +import shutil +import sys +from functools import cmp_to_key +from os.path import expanduser, isdir, isfile, join + +import click +from twisted.internet import defer + +from platformio import app, util +from platformio.commands.home import helpers + + +class OSRPC(object): + + @staticmethod + @defer.inlineCallbacks + def fetch_content(uri, data=None, headers=None, cache_valid=None): + timeout = 2 + if not headers: + headers = { + "User-Agent": + ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) " + "AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 " + "Safari/603.3.8") + } + cache_key = (app.ContentCache.key_from_args(uri, data) + if cache_valid else None) + with app.ContentCache() as cc: + if cache_key: + result = cc.get(cache_key) + if result is not None: + defer.returnValue(result) + + # check internet before and resolve issue with 60 seconds timeout + util.internet_on(raise_exception=True) + + session = helpers.requests_session() + if data: + r = yield session.post( + uri, data=data, headers=headers, timeout=timeout) + else: + r = yield session.get(uri, headers=headers, timeout=timeout) + + r.raise_for_status() + result = r.text + if cache_valid: + with app.ContentCache() as cc: + cc.set(cache_key, result, cache_valid) + defer.returnValue(result) + + def request_content(self, uri, data=None, headers=None, cache_valid=None): + if uri.startswith('http'): + return self.fetch_content(uri, data, headers, cache_valid) + if isfile(uri): + with open(uri) as fp: + return fp.read() + return None + + @staticmethod + def open_url(url): + return click.launch(url) + + @staticmethod + def reveal_file(path): + return click.launch( + path.encode(sys.getfilesystemencoding()) if util.PY2 else path, + locate=True) + + @staticmethod + def is_file(path): + return isfile(path) + + @staticmethod + def is_dir(path): + return isdir(path) + + @staticmethod + def make_dirs(path): + return os.makedirs(path) + + @staticmethod + def rename(src, dst): + return os.rename(src, dst) + + @staticmethod + def copy(src, dst): + return shutil.copytree(src, dst) + + @staticmethod + def glob(pathnames, root=None): + if not isinstance(pathnames, list): + pathnames = [pathnames] + result = set() + for pathname in pathnames: + result |= set( + glob.glob(join(root, pathname) if root else pathname)) + return list(result) + + @staticmethod + def list_dir(path): + + def _cmp(x, y): + if x[1] and not y[1]: + return -1 + if not x[1] and y[1]: + return 1 + if x[0].lower() > y[0].lower(): + return 1 + if x[0].lower() < y[0].lower(): + return -1 + return 0 + + items = [] + if path.startswith("~"): + path = expanduser(path) + if not isdir(path): + return items + for item in os.listdir(path): + try: + item_is_dir = isdir(join(path, item)) + if item_is_dir: + os.listdir(join(path, item)) + items.append((item, item_is_dir)) + except OSError: + pass + return sorted(items, key=cmp_to_key(_cmp)) + + @staticmethod + def get_logical_devices(): + items = [] + for item in util.get_logical_devices(): + if item['name']: + item['name'] = util.path_to_unicode(item['name']) + items.append(item) + return items diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py new file mode 100644 index 00000000..85255a9c --- /dev/null +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -0,0 +1,83 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import json +import os +import re +import sys + +from jsonrpc.exceptions import JSONRPCDispatchException +from twisted.internet.utils import getProcessOutputAndValue + +from platformio import __version__, util +from platformio.commands.home import helpers + + +class PIOCoreRPC(object): + + @staticmethod + def call(args, options=None): + json_output = "--json-output" in args + try: + args = [ + arg.encode(sys.getfilesystemencoding()) if isinstance( + arg, util.string_types) else str(arg) for arg in args + ] + except UnicodeError: + raise JSONRPCDispatchException( + code=4002, message="PIO Core: non-ASCII chars in arguments") + d = getProcessOutputAndValue( + helpers.get_core_fullpath(), + args, + path=(options or {}).get("cwd"), + env={k: v + for k, v in os.environ.items() if "%" not in k}) + d.addCallback(PIOCoreRPC._call_callback, json_output) + d.addErrback(PIOCoreRPC._call_errback) + return d + + @staticmethod + def _call_callback(result, json_output=False): + result = list(result) + assert len(result) == 3 + for i in (0, 1): + result[i] = result[i].decode(sys.getfilesystemencoding()).strip() + out, err, code = result + text = ("%s\n\n%s" % (out, err)).strip() + if code != 0: + raise Exception(text) + + if not json_output: + return text + + try: + return json.loads(out) + except ValueError as e: + if "sh: " in out: + return json.loads( + re.sub(r"^sh: [^\n]+$", "", out, flags=re.M).strip()) + raise e + + @staticmethod + def _call_errback(failure): + raise JSONRPCDispatchException( + code=4003, + message="PIO Core Call Error", + data=failure.getErrorMessage()) + + @staticmethod + def version(): + return __version__ diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py new file mode 100644 index 00000000..02961ed8 --- /dev/null +++ b/platformio/commands/home/rpc/handlers/project.py @@ -0,0 +1,283 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os +import shutil +import sys +import time +from os.path import (basename, expanduser, getmtime, isdir, isfile, join, + realpath, sep) + +from jsonrpc.exceptions import JSONRPCDispatchException + +from platformio import exception, util +from platformio.commands.home.rpc.handlers.app import AppRPC +from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.ide.projectgenerator import ProjectGenerator +from platformio.managers.platform import PlatformManager + +try: + from configparser import Error as ConfigParserError +except ImportError: + from ConfigParser import Error as ConfigParserError + + +class ProjectRPC(object): + + @staticmethod + def _get_projects(project_dirs=None): + + def _get_project_data(project_dir): + data = {"boards": [], "libExtraDirs": []} + config = util.load_project_config(project_dir) + + if config.has_section("platformio") and \ + config.has_option("platformio", "lib_extra_dirs"): + data['libExtraDirs'].extend( + util.parse_conf_multi_values( + config.get("platformio", "lib_extra_dirs"))) + + for section in config.sections(): + if not section.startswith("env:"): + continue + if config.has_option(section, "board"): + data['boards'].append(config.get(section, "board")) + if config.has_option(section, "lib_extra_dirs"): + data['libExtraDirs'].extend( + util.parse_conf_multi_values( + config.get(section, "lib_extra_dirs"))) + + # resolve libExtraDirs paths + with util.cd(project_dir): + data['libExtraDirs'] = [ + expanduser(d) if d.startswith("~") else realpath(d) + for d in data['libExtraDirs'] + ] + + # skip non existing folders + data['libExtraDirs'] = [ + d for d in data['libExtraDirs'] if isdir(d) + ] + + return data + + def _path_to_name(path): + return (sep).join(path.split(sep)[-2:]) + + if not project_dirs: + project_dirs = AppRPC.load_state()['storage']['recentProjects'] + + result = [] + pm = PlatformManager() + for project_dir in project_dirs: + data = {} + boards = [] + try: + data = _get_project_data(project_dir) + except exception.NotPlatformIOProject: + continue + except ConfigParserError: + pass + + for board_id in data.get("boards", []): + name = board_id + try: + name = pm.board_config(board_id)['name'] + except (exception.UnknownBoard, exception.UnknownPlatform): + pass + boards.append({"id": board_id, "name": name}) + + result.append({ + "path": + project_dir, + "name": + _path_to_name(project_dir), + "modified": + int(getmtime(project_dir)), + "boards": + boards, + "extraLibStorages": [{ + "name": _path_to_name(d), + "path": d + } for d in data.get("libExtraDirs", [])] + }) + return result + + def get_projects(self, project_dirs=None): + return self._get_projects(project_dirs) + + def init(self, board, framework, project_dir): + assert project_dir + state = AppRPC.load_state() + if not isdir(project_dir): + os.makedirs(project_dir) + args = ["init", "--project-dir", project_dir, "--board", board] + if framework: + args.extend(["--project-option", "framework = %s" % framework]) + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args) + d.addCallback(self._generate_project_main, project_dir, framework) + return d + + @staticmethod + def _generate_project_main(_, project_dir, framework): + main_content = None + if framework == "arduino": + main_content = "\n".join([ + "#include ", + "", + "void setup() {", + " // put your setup code here, to run once:", + "}", + "", + "void loop() {", + " // put your main code here, to run repeatedly:", + "}" + "" + ]) # yapf: disable + elif framework == "mbed": + main_content = "\n".join([ + "#include ", + "", + "int main() {", + "", + " // put your setup code here, to run once:", + "", + " while(1) {", + " // put your main code here, to run repeatedly:", + " }", + "}", + "" + ]) # yapf: disable + if not main_content: + return project_dir + with util.cd(project_dir): + src_dir = util.get_projectsrc_dir() + main_path = join(src_dir, "main.cpp") + if isfile(main_path): + return project_dir + if not isdir(src_dir): + os.makedirs(src_dir) + with open(main_path, "w") as f: + f.write(main_content.strip()) + return project_dir + + def import_arduino(self, board, use_arduino_libs, arduino_project_dir): + # don't import PIO Project + if util.is_platformio_project(arduino_project_dir): + return arduino_project_dir + + is_arduino_project = any([ + isfile( + join(arduino_project_dir, + "%s.%s" % (basename(arduino_project_dir), ext))) + for ext in ("ino", "pde") + ]) + if not is_arduino_project: + raise JSONRPCDispatchException( + code=4000, + message="Not an Arduino project: %s" % arduino_project_dir) + + state = AppRPC.load_state() + project_dir = join(state['storage']['projectsDir'].decode("utf-8"), + time.strftime("%y%m%d-%H%M%S-") + board) + if not isdir(project_dir): + os.makedirs(project_dir) + args = ["init", "--project-dir", project_dir, "--board", board] + args.extend(["--project-option", "framework = arduino"]) + if use_arduino_libs: + args.extend([ + "--project-option", + "lib_extra_dirs = ~/Documents/Arduino/libraries" + ]) + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args) + d.addCallback(self._finalize_arduino_import, project_dir, + arduino_project_dir) + return d + + @staticmethod + def _finalize_arduino_import(_, project_dir, arduino_project_dir): + with util.cd(project_dir): + src_dir = util.get_projectsrc_dir() + if isdir(src_dir): + util.rmtree_(src_dir) + shutil.copytree( + arduino_project_dir.encode(sys.getfilesystemencoding()), + src_dir) + return project_dir + + @staticmethod + def get_project_examples(): + result = [] + for manifest in PlatformManager().get_installed(): + examples_dir = join(manifest['__pkg_dir'], "examples") + if not isdir(examples_dir): + continue + items = [] + for project_dir, _, __ in os.walk(examples_dir): + project_description = None + try: + config = util.load_project_config(project_dir) + if config.has_section("platformio") and \ + config.has_option("platformio", "description"): + project_description = config.get( + "platformio", "description") + except (exception.NotPlatformIOProject, + exception.InvalidProjectConf): + continue + + path_tokens = project_dir.split(sep) + items.append({ + "name": + "/".join(path_tokens[path_tokens.index("examples") + 1:]), + "path": + project_dir, + "description": + project_description + }) + result.append({ + "platform": { + "title": manifest['title'], + "version": manifest['version'] + }, + "items": sorted(items, key=lambda item: item['name']) + }) + return sorted(result, key=lambda data: data['platform']['title']) + + @staticmethod + def import_pio(project_dir): + if not project_dir or not util.is_platformio_project(project_dir): + raise JSONRPCDispatchException( + code=4001, + message="Not an PlatformIO project: %s" % project_dir) + new_project_dir = join( + AppRPC.load_state()['storage']['projectsDir'].decode("utf-8"), + time.strftime("%y%m%d-%H%M%S-") + basename(project_dir)) + shutil.copytree(project_dir, new_project_dir) + + state = AppRPC.load_state() + args = ["init", "--project-dir", new_project_dir] + if (state['storage']['coreCaller'] and state['storage']['coreCaller'] + in ProjectGenerator.get_supported_ides()): + args.extend(["--ide", state['storage']['coreCaller']]) + d = PIOCoreRPC.call(args) + d.addCallback(lambda _: new_project_dir) + return d diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py new file mode 100644 index 00000000..cd4f05b9 --- /dev/null +++ b/platformio/commands/home/rpc/server.py @@ -0,0 +1,70 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import jsonrpc +from autobahn.twisted.websocket import (WebSocketServerFactory, + WebSocketServerProtocol) +from jsonrpc.exceptions import JSONRPCDispatchException +from twisted.internet import defer + + +class JSONRPCServerProtocol(WebSocketServerProtocol): + + def onMessage(self, payload, isBinary): # pylint: disable=unused-argument + # print("> %s" % payload) + response = jsonrpc.JSONRPCResponseManager.handle( + payload, self.factory.dispatcher).data + # if error + if "result" not in response: + self.sendJSONResponse(response) + return None + + d = defer.maybeDeferred(lambda: response['result']) + d.addCallback(self._callback, response) + d.addErrback(self._errback, response) + + return None + + def _callback(self, result, response): + response['result'] = result + self.sendJSONResponse(response) + + def _errback(self, failure, response): + if isinstance(failure.value, JSONRPCDispatchException): + e = failure.value + else: + e = JSONRPCDispatchException( + code=4999, message=failure.getErrorMessage()) + del response["result"] + response['error'] = e.error._data # pylint: disable=protected-access + print(response['error']) + self.sendJSONResponse(response) + + def sendJSONResponse(self, response): + # print("< %s" % response) + self.sendMessage(json.dumps(response).encode("utf8")) + + +class JSONRPCServerFactory(WebSocketServerFactory): + + protocol = JSONRPCServerProtocol + + def __init__(self): + super(JSONRPCServerFactory, self).__init__() + self.dispatcher = jsonrpc.Dispatcher() + + def addHandler(self, handler, namespace): + self.dispatcher.build_method_map(handler, prefix="%s." % namespace) diff --git a/platformio/commands/home.py b/platformio/commands/home/web.py similarity index 54% rename from platformio/commands/home.py rename to platformio/commands/home/web.py index cd6b86f6..313ce084 100644 --- a/platformio/commands/home.py +++ b/platformio/commands/home/web.py @@ -12,20 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -import click - -from platformio.managers.core import pioplus_call +from twisted.internet import reactor +from twisted.web import static -@click.command("home", short_help="PIO Home") -@click.option("--port", type=int, default=8008, help="HTTP port, default=8008") -@click.option( - "--host", - default="127.0.0.1", - help="HTTP host, default=127.0.0.1. " - "You can open PIO Home for inbound connections with --host=0.0.0.0") -@click.option("--no-open", is_flag=True) -def cli(*args, **kwargs): # pylint: disable=unused-argument - pioplus_call(sys.argv[1:]) +class WebRoot(static.File): + + def render_GET(self, request): + if request.args.get("__shutdown__", False): + reactor.stop() + return "Server has been stopped" + + request.setHeader("cache-control", + "no-cache, no-store, must-revalidate") + request.setHeader("pragma", "no-cache") + request.setHeader("expires", "0") + return static.File.render_GET(self, request) From 137a5d1c42bc4531ef5456413584588ab9b38141 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Apr 2019 00:49:53 +0300 Subject: [PATCH 099/333] Improve debugging in `debug_load_mode = modified` and fix an issue with useless project rebuilding --- HISTORY.rst | 1 + platformio/builder/tools/piomisc.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 99c41cb5..75810755 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ PlatformIO 3.0 3.6.7 (2019-??-??) ~~~~~~~~~~~~~~~~~~ +* `PIO Unified Debugger `__: improved debugging in ``debug_load_mode = modified`` and fixed an issue with useless project rebuilding * Project Generator: fixed a VSCode C/C++'s "Cannot find" warning when CPPPATH folder does not exist * Fixed an "IndexError: list index out of range" for Arduino sketch preprocessor (`issue #2268 `_) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 45212415..19a6a2ef 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -296,8 +296,7 @@ def ProcessDebug(env): if not env.subst("$PIODEBUGFLAGS"): env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"]) env.Append( - PIODEBUGFLAGS=["-D__PLATFORMIO_DEBUG__"], - BUILD_FLAGS=env.get("PIODEBUGFLAGS", [])) + BUILD_FLAGS=list(env['PIODEBUGFLAGS']) + ["-D__PLATFORMIO_DEBUG__"]) unflags = ["-Os"] for level in [0, 1, 2]: for flag in ("O", "g", "ggdb"): From d3c3491a91baa37f93d8981ff8ffaac23e23fa64 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Apr 2019 01:07:21 +0300 Subject: [PATCH 100/333] Fix links in changelog --- HISTORY.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 75810755..fa11ace5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,7 +16,7 @@ PlatformIO 3.0 3.6.7 (2019-??-??) ~~~~~~~~~~~~~~~~~~ -* `PIO Unified Debugger `__: improved debugging in ``debug_load_mode = modified`` and fixed an issue with useless project rebuilding +* `PIO Unified Debugger `__: improved debugging in ``debug_load_mode = modified`` and fixed an issue with useless project rebuilding * Project Generator: fixed a VSCode C/C++'s "Cannot find" warning when CPPPATH folder does not exist * Fixed an "IndexError: list index out of range" for Arduino sketch preprocessor (`issue #2268 `_) @@ -31,7 +31,7 @@ PlatformIO 3.0 * Fixed an issue when PlatformIO Build System does not pick up "mbed_lib.json" files from libraries (`issue #2164 `_) * Fixed an error with conflicting declaration of a prototype (Arduino sketch preprocessor) -* Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option +* Fixed "FileExistsError" when `platformio ci `__ command is used in pair with ``--keep-build-dir`` option * Fixed an issue with incorrect order of project "include" and "src" paths in ``CPPPATH`` (`issue #1914 `_) @@ -41,7 +41,7 @@ PlatformIO 3.0 * Project Generator: added new targets for CLion IDE "BUILD_VERBOSE" and "MONITOR" (serial port monitor) (`issue #359 `_) * Fixed an issue with slow updating of PlatformIO Core packages on Windows -* Fixed an issue when `platformio ci `__ recompiles project if ``--keep-build-dir`` option is passed +* Fixed an issue when `platformio ci `__ recompiles project if ``--keep-build-dir`` option is passed (`issue #2109 `_) * Fixed an issue when ``$PROJECT_HASH`` template was not expanded for the other directory ``***_dir`` options in `"platformio.ini" (Project Configuration File) `__ (`issue #2170 `_) @@ -89,9 +89,9 @@ PlatformIO 3.0 * Generate an `include `__ and `test `__ directories with a README file when initializing a new project * Support in-line comments for multi-line value (``lib_deps``, ``build_flags``, etc) in `"platformio.ini" (Project Configuration File) `__ -* Added ``$PROJECT_HASH`` template variable for `build_dir `__. One of the use cases is setting a global storage for project artifacts using `PLATFORMIO_BUILD_DIR `__ system environment variable. For example, ``/tmp/pio-build/$PROJECT_HASH`` (Unix) or ``$[sysenv.TEMP}/pio-build/$PROJECT_HASH`` (Windows) +* Added ``$PROJECT_HASH`` template variable for `build_dir `__. One of the use cases is setting a global storage for project artifacts using `PLATFORMIO_BUILD_DIR `__ system environment variable. For example, ``/tmp/pio-build/$PROJECT_HASH`` (Unix) or ``$[sysenv.TEMP}/pio-build/$PROJECT_HASH`` (Windows) * Improved a loading speed of PIO Home "Recent News" -* Improved `PIO Unified Debugger `__ for "mbed" framework and fixed issue with missed local variables +* Improved `PIO Unified Debugger `__ for "mbed" framework and fixed issue with missed local variables * Introduced `"Release" and "Debug" Build Configurations `__ * Build project in "Debug Mode" including debugging information with a new ``debug`` target using `platformio run `__ command or `targets `__ option in ``platformio.ini``. The last option allows avoiding project rebuilding between "Run/Debug" modes. (`issue #1833 `_) @@ -149,7 +149,7 @@ PlatformIO 3.0 build environment (`issue #1665 `_) * Handle "architectures" data from "library.properties" manifest in - `lib_compat_mode = strict `__ + `lib_compat_mode = strict `__ * Added workaround for Python SemVer package's `issue #61 `_ with caret range and pre-releases * Replaced conflicted "env" pattern by "sysenv" for `"platformio.ini" Dynamic Variables" `__ (`issue #1705 `_) @@ -177,7 +177,7 @@ PlatformIO 3.0 (`issue #1612 `_) * Configure a custom path to SVD file using `debug_svd_path `__ option -* Custom project `description `_ +* Custom project `description `_ which will be used by `PlatformIO Home `_ * Updated Unity tool to 2.4.3 * Improved support for Black Magic Probe in "uploader" mode @@ -203,9 +203,9 @@ PlatformIO 3.0 - Multiple themes (Dark & Light) - Ability to specify a name for new project -* Control `PIO Unified Debugger `__ +* Control `PIO Unified Debugger `__ and its firmware loading mode using - `debug_load_mode `__ option + `debug_load_mode `__ option * Added aliases (off, light, strict) for `LDF Compatibility Mode `__ * Search for a library using PIO Library Registry ID ``id:X`` (e.g. ``pio lib search id:13``) From a59efc2fc03af8d67588c7273e83a6be0a2e73a7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Apr 2019 12:29:16 +0300 Subject: [PATCH 101/333] Update history with upcoming 3.6.7 release --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index fa11ace5..48ef369c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,7 +13,7 @@ PlatformIO 4.0 PlatformIO 3.0 -------------- -3.6.7 (2019-??-??) +3.6.7 (2019-04-23) ~~~~~~~~~~~~~~~~~~ * `PIO Unified Debugger `__: improved debugging in ``debug_load_mode = modified`` and fixed an issue with useless project rebuilding From 9bdc85fd52ec1c45a88f25fb5d42cbf6ee1aa5dd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 23 Apr 2019 12:49:36 +0300 Subject: [PATCH 102/333] Bump version to 4.0.0a8 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 584acb8b..c1f0846d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a7") +VERSION = (4, 0, "0a8") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From e9df6166eeaf22544cd886409a1f8c9f73900492 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 24 Apr 2019 00:07:27 +0300 Subject: [PATCH 103/333] Update SCons to 3.0.5 --- platformio/managers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 2f55aec4..05822f7a 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -29,7 +29,7 @@ CORE_PACKAGES = { "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.1.4", "tool-unity": "~1.20403.0", - "tool-scons": "~2.20501.7" if util.PY2 else "~3.30003.0" + "tool-scons": "~2.20501.7" if util.PY2 else "~3.30005.0" } PIOPLUS_AUTO_UPDATES_MAX = 100 From ba6d120cf4966eddc40da68feb69369d05818e79 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 24 Apr 2019 00:14:53 +0300 Subject: [PATCH 104/333] Docs: Sync Atmel AVR dev/platform --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 0c29f967..30a7a537 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0c29f9671f44b5221e7cac15e93ebb40f9db88c1 +Subproject commit 30a7a53749867bea57fe4978fa27ac9e69e9d60a From eab2fd91fd4ae3bca108bc90025187802c4d87b9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 25 Apr 2019 12:49:22 +0300 Subject: [PATCH 105/333] Lowercase SHA sum for package manager --- platformio/downloader.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/platformio/downloader.py b/platformio/downloader.py index d8b2fd24..0b0ef5df 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -100,7 +100,7 @@ class FileDownloader(object): raise FDSizeMismatch(_dlsize, self._fname, self.get_size()) if not sha1: - return + return None dlsha1 = None try: @@ -113,11 +113,12 @@ class FileDownloader(object): dlsha1 = result['out'] except (OSError, ValueError): pass - - if dlsha1: - dlsha1 = dlsha1[1:41] if dlsha1.startswith("\\") else dlsha1[:40] - if sha1 != dlsha1: - raise FDSHASumMismatch(dlsha1, self._fname, sha1) + if not dlsha1: + return None + dlsha1 = dlsha1[1:41] if dlsha1.startswith("\\") else dlsha1[:40] + if sha1.lower() != dlsha1.lower(): + raise FDSHASumMismatch(dlsha1, self._fname, sha1) + return True def _preserve_filemtime(self, lmdate): timedata = parsedate_tz(lmdate) From 48c1aeae035ab5afad9811ec621018b4059a8991 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 May 2019 13:11:03 +0300 Subject: [PATCH 106/333] Fix "systemd-udevd" warnings in 99-platformio-udev.rules // Resolve #2442 --- HISTORY.rst | 6 + scripts/99-platformio-udev.rules | 202 +++++++++++++++---------------- 2 files changed, 107 insertions(+), 101 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 48ef369c..96025861 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,12 @@ PlatformIO 4.0 PlatformIO 3.0 -------------- +3.6.8 (2019-??-??) +~~~~~~~~~~~~~~~~~~ + +* Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ + (`issue #2442 `_) + 3.6.7 (2019-04-23) ~~~~~~~~~~~~~~~~~~ diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 99755db1..f3249682 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -84,176 +84,176 @@ SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic GDB Server" SUBSYSTEM=="tty", ATTRS{interface}=="Black Magic UART Port" # opendous and estick -ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="204f", MODE="0666" # Original FT232/FT245 VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6001", MODE="0666" # Original FT2232 VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6010", MODE="0666" # Original FT4232 VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE="0666" # Original FT232H VID:PID -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6014", MODE="0666" # DISTORTEC JTAG-lock-pick Tiny 2 -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8220", MODE="0666" # TUMPA, TUMPA Lite -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a98", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="8a99", MODE="0666" # XDS100v2 -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="a6d0", MODE="0666" # Xverve Signalyzer Tool (DT-USB-ST), Signalyzer LITE (DT-USB-SLITE) -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca0", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bca1", MODE="0666" # TI/Luminary Stellaris Evaluation Board FTDI (several) -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcd9", MODE="0666" # TI/Luminary Stellaris In-Circuit Debug Interface FTDI (ICDI) Board -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bcda", MODE="0666" # egnite Turtelizer 2 -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="bdc8", MODE="0666" # Section5 ICEbear -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c140", MODE="0666" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="c141", MODE="0666" # Amontec JTAGkey and JTAGkey-tiny -ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0403", ATTRS{idProduct}=="cff8", MODE="0666" # TI ICDI -ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0451", ATTRS{idProduct}=="c32a", MODE="0666" # STLink v1 -ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3744", MODE="0666" # STLink v2 -ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0483", ATTRS{idProduct}=="3748", MODE="0666" # STLink v2-1 -ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374b", MODE="0666" # Hilscher NXHX Boards -ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0640", ATTRS{idProduct}=="0028", MODE="0666" # Hitex STR9-comStick -ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002c", MODE="0666" # Hitex STM32-PerformanceStick -ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0640", ATTRS{idProduct}=="002d", MODE="0666" # Altera USB Blaster -ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="09fb", ATTRS{idProduct}=="6001", MODE="0666" # Amontec JTAGkey-HiSpeed -ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="0fbb", ATTRS{idProduct}=="1000", MODE="0666" # SEGGER J-Link -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="660", GROUP="plugdev", TAG+="uaccess" -ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0101", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0102", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0103", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0104", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0105", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0107", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="0108", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1010", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1011", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1012", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1013", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1014", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1015", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1016", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1017", MODE="0666" +ATTRS{idVendor}=="1366", ATTRS{idProduct}=="1018", MODE="0666" # Raisonance RLink -ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="138e", ATTRS{idProduct}=="9000", MODE="0666" # Debug Board for Neo1973 -ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="1457", ATTRS{idProduct}=="5118", MODE="0666" # Olimex ARM-USB-OCD -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0003", MODE="0666" # Olimex ARM-USB-OCD-TINY -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="0004", MODE="0666" # Olimex ARM-JTAG-EW -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="001e", MODE="0666" # Olimex ARM-USB-OCD-TINY-H -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002a", MODE="0666" # Olimex ARM-USB-OCD-H -ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="15ba", ATTRS{idProduct}=="002b", MODE="0666" # USBprog with OpenOCD firmware -ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="1781", ATTRS{idProduct}=="0c63", MODE="0666" # TI/Luminary Stellaris In-Circuit Debug Interface (ICDI) Board -ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666" # Marvell Sheevaplug -ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="9e88", ATTRS{idProduct}=="9e8f", MODE="0666" # Keil Software, Inc. ULink -ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="0666" # CMSIS-DAP compatible adapters -ATTRS{product}=="*CMSIS-DAP*", MODE="660", GROUP="plugdev", TAG+="uaccess" +ATTRS{product}=="*CMSIS-DAP*", MODE="0666" #SEGGER J-LIK -ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="666" -ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="0666" +ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="0666" From 41ff1b0188d19c7faa116265b7cb1b31521e1536 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 May 2019 16:56:44 +0300 Subject: [PATCH 107/333] Support for Kendryte K210 // Resolve #2233 --- README.rst | 6 ++++++ docs | 2 +- examples | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 76e1665c..ee1f6896 100644 --- a/README.rst +++ b/README.rst @@ -78,6 +78,7 @@ Registry Development Platforms --------------------- +* `Aceinna IMU `_ * `Atmel AVR `_ * `Atmel SAM `_ * `Espressif 32 `_ @@ -86,6 +87,7 @@ Development Platforms * `Infineon XMC `_ * `Intel ARC32 `_ * `Intel MCS-51 (8051) `_ +* `Kendryte K210 `_ * `Lattice iCE40 `_ * `Maxim 32 `_ * `Microchip PIC32 `_ @@ -93,6 +95,7 @@ Development Platforms * `Nordic nRF52 `_ * `NXP LPC `_ * `RISC-V `_ +* `RISC-V GAP `_ * `Samsung ARTIK `_ * `Silicon Labs EFM32 `_ * `ST STM32 `_ @@ -112,8 +115,11 @@ Frameworks * `ESP-IDF `_ * `ESP8266 Non-OS SDK `_ * `ESP8266 RTOS SDK `_ +* `Freedom E SDK `_ +* `Kendryte Standalone SDK `_ * `libOpenCM3 `_ * `mbed `_ +* `PULP OS `_ * `Pumbaa `_ * `Simba `_ * `SPL `_ diff --git a/docs b/docs index 30a7a537..4350844c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 30a7a53749867bea57fe4978fa27ac9e69e9d60a +Subproject commit 4350844cafefed1787db7d360710ada8d892df31 diff --git a/examples b/examples index 9c16f551..46bac491 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 9c16f551d72e37aa4c2f28555487e2fb23408b5d +Subproject commit 46bac491dc59bfdc3baf8edafe091cd8b30b0d10 From 8e55c9e4d07a2b9a86bf3a5d9d3015b6d19514fe Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 May 2019 21:03:36 +0300 Subject: [PATCH 108/333] Include external configuration files with "extra_configs" option // Resolve #1590 --- .pylintrc | 2 +- HISTORY.rst | 4 +- docs | 2 +- platformio/commands/run.py | 2 +- platformio/exception.py | 2 +- platformio/project/__init__.py | 13 ++++ platformio/project/config.py | 106 +++++++++++++++++++++++++++++++++ platformio/util.py | 62 ++----------------- tests/test_projectconf.py | 74 +++++++++++++++++++++++ 9 files changed, 205 insertions(+), 62 deletions(-) create mode 100644 platformio/project/__init__.py create mode 100644 platformio/project/config.py create mode 100644 tests/test_projectconf.py diff --git a/.pylintrc b/.pylintrc index 09569a4f..bbaef24a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -20,4 +20,4 @@ confidence= # --disable=W" # disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating -disable=locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens,useless-object-inheritance,useless-import-alias +disable=fixme,locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens,useless-object-inheritance,useless-import-alias diff --git a/HISTORY.rst b/HISTORY.rst index 96025861..bd899a08 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,8 +7,10 @@ PlatformIO 4.0 4.0.0 (2019-??-??) ~~~~~~~~~~~~~~~~~~ -* Added Python 3.5+ support +* Python 3 support (`issue #895 `_) +* Include external configuration files with "extra_configs" option + (`issue #1590 `_) PlatformIO 3.0 -------------- diff --git a/docs b/docs index 4350844c..9a51b847 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4350844cafefed1787db7d360710ada8d892df31 +Subproject commit 9a51b847424c85444f4b9c38efad9d83f72353be diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 7cac3eca..b480ae2a 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -130,7 +130,7 @@ class EnvironmentProcessor(object): KNOWN_PLATFORMIO_OPTIONS = [ "description", "env_default", "home_dir", "lib_dir", "libdeps_dir", "include_dir", "src_dir", "build_dir", "data_dir", "test_dir", - "boards_dir", "lib_extra_dirs" + "boards_dir", "lib_extra_dirs", "extra_configs" ] KNOWN_ENV_OPTIONS = [ diff --git a/platformio/exception.py b/platformio/exception.py index 7829b648..92ffd6c5 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -207,7 +207,7 @@ class InvalidLibConfURL(PlatformioException): class InvalidProjectConf(PlatformioException): - MESSAGE = "Invalid `platformio.ini`, project configuration file: '{0}'" + MESSAGE = ("Invalid '{0}' (project configuration file): '{1}'") class BuildScriptNotFound(PlatformioException): diff --git a/platformio/project/__init__.py b/platformio/project/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/project/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/project/config.py b/platformio/project/config.py new file mode 100644 index 00000000..9883ffb7 --- /dev/null +++ b/platformio/project/config.py @@ -0,0 +1,106 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os +import re + +import click + +from platformio import exception + +try: + import ConfigParser as ConfigParser +except ImportError: + import configparser as ConfigParser + + +class ProjectConfig(object): + + VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") + + _parser = None + _parsed = [] + + @staticmethod + def parse_multi_values(items): + result = [] + if not items: + return result + inline_comment_re = re.compile(r"\s+;.*$") + for item in items.split("\n" if "\n" in items else ", "): + item = item.strip() + # comment + if not item or item.startswith((";", "#")): + continue + if ";" in item: + item = inline_comment_re.sub("", item).strip() + result.append(item) + return result + + def __init__(self, path): + self.path = path + self._parsed = [] + self._parser = ConfigParser.ConfigParser() + self.read(path) + + def read(self, path): + if path in self._parsed: + return + self._parsed.append(path) + try: + self._parser.read(path) + except ConfigParser.Error as e: + raise exception.InvalidProjectConf(path, str(e)) + + # load extra configs + if (not self._parser.has_section("platformio") + or not self._parser.has_option("platformio", "extra_configs")): + return + extra_configs = self.parse_multi_values( + self.get("platformio", "extra_configs")) + for pattern in extra_configs: + for item in glob.glob(pattern): + self.read(item) + + def __getattr__(self, name): + return getattr(self._parser, name) + + def items(self, section): + items = [] + for option in self._parser.options(section): + items.append((option, self._parser.get(section, option))) + return items + + def get(self, section, option): + try: + value = self._parser.get(section, option) + except ConfigParser.Error as e: + raise exception.InvalidProjectConf(self.path, str(e)) + if "${" not in value or "}" not in value: + return value + return self.VARTPL_RE.sub(self._re_sub_handler, value) + + def _re_sub_handler(self, match): + section, option = match.group(1), match.group(2) + if section in ("env", + "sysenv") and not self._parser.has_section(section): + if section == "env": + click.secho( + "Warning! Access to system environment variable via " + "`${{env.{0}}}` is deprecated. Please use " + "`${{sysenv.{0}}}` instead".format(option), + fg="yellow") + return os.getenv(option) + return self._parser.get(section, option) diff --git a/platformio/util.py b/platformio/util.py index dd280ba2..1adcd936 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -33,52 +33,17 @@ import click import requests from platformio import __apiurl__, __version__, exception +from platformio.project.config import ProjectConfig # pylint: disable=too-many-ancestors PY2 = sys.version_info[0] == 2 if PY2: - import ConfigParser as ConfigParser string_types = basestring # pylint: disable=undefined-variable else: - import configparser as ConfigParser string_types = str -class ProjectConfig(ConfigParser.ConfigParser): - - VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") - - def items(self, section, **_): # pylint: disable=arguments-differ - items = [] - for option in ConfigParser.ConfigParser.options(self, section): - items.append((option, self.get(section, option))) - return items - - def get( # pylint: disable=arguments-differ - self, section, option, **kwargs): - try: - value = ConfigParser.ConfigParser.get(self, section, option, - **kwargs) - except ConfigParser.Error as e: - raise exception.InvalidProjectConf(str(e)) - if "${" not in value or "}" not in value: - return value - return self.VARTPL_RE.sub(self._re_sub_handler, value) - - def _re_sub_handler(self, match): - section, option = match.group(1), match.group(2) - if section in ("env", "sysenv") and not self.has_section(section): - if section == "env": - click.secho( - "Warning! Access to system environment variable via " - "`${{env.{0}}}` is deprecated. Please use " - "`${{sysenv.{0}}}` instead".format(option), - fg="yellow") - return os.getenv(option) - return self.get(section, option) - - class AsyncPipe(Thread): def __init__(self, outcallback=None): @@ -347,34 +312,17 @@ def get_projectdata_dir(): "data")) -def load_project_config(path=None): +def load_project_config(path=None): # FIXME: if not path or isdir(path): path = join(path or get_project_dir(), "platformio.ini") if not isfile(path): raise exception.NotPlatformIOProject( dirname(path) if path.endswith("platformio.ini") else path) - cp = ProjectConfig() - try: - cp.read(path) - except ConfigParser.Error as e: - raise exception.InvalidProjectConf(str(e)) - return cp + return ProjectConfig(path) -def parse_conf_multi_values(items): - result = [] - if not items: - return result - inline_comment_re = re.compile(r"\s+;.*$") - for item in items.split("\n" if "\n" in items else ", "): - item = item.strip() - # comment - if not item or item.startswith((";", "#")): - continue - if ";" in item: - item = inline_comment_re.sub("", item).strip() - result.append(item) - return result +def parse_conf_multi_values(items): # FIXME: + return ProjectConfig.parse_multi_values(items) def change_filemtime(path, mtime): diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py new file mode 100644 index 00000000..ba10d1c4 --- /dev/null +++ b/tests/test_projectconf.py @@ -0,0 +1,74 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.project.config import ProjectConfig + +BASE_CONFIG = """ +[platformio] +extra_configs = + extra_envs.ini + extra_debug.ini + +[common] +debug_flags = -D RELEASE +lib_flags = -lc -lm + +[env:esp-wrover-kit] +platform = espressif32 +framework = espidf +board = esp-wrover-kit +build_flags = ${common.debug_flags} +""" + +EXTRA_ENVS_CONFIG = """ +[env:esp32dev] +platform = espressif32 +framework = espidf +board = esp32dev +build_flags = ${common.lib_flags} ${common.debug_flags} + +[env:lolin32] +platform = espressif32 +framework = espidf +board = lolin32 +build_flags = ${common.debug_flags} +""" + +EXTRA_DEBUG_CONFIG = """ +# Override base "common.debug_flags" +[common] +debug_flags = -D DEBUG=1 + +[env:lolin32] +build_flags = -Og +""" + + +def test_parser(tmpdir): + tmpdir.join("platformio.ini").write(BASE_CONFIG) + tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG) + tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG) + + config = None + with tmpdir.as_cwd(): + config = ProjectConfig(tmpdir.join("platformio.ini").strpath) + assert config + + assert config.sections() == [ + "platformio", "common", "env:esp-wrover-kit", "env:esp32dev", + "env:lolin32" + ] + assert config.get("common", "debug_flags") == "-D DEBUG=1" + assert config.get("env:esp32dev", "build_flags") == "-lc -lm -D DEBUG=1" + assert config.get("env:lolin32", "build_flags") == "-Og" From 2c0e0b26194ff60f24772d399102811fb9cabef6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 May 2019 21:14:08 +0300 Subject: [PATCH 109/333] Bump version to 4.0.0a9 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index c1f0846d..dbc2bf6e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a8") +VERSION = (4, 0, "0a9") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 7f607b742fd98ba591fb384f2d1471fb0da13f89 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 4 May 2019 12:23:23 +0300 Subject: [PATCH 110/333] Fix issue when handling dynamic variables --- platformio/project/config.py | 6 +++--- tests/test_projectconf.py | 23 +++++++++++++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 9883ffb7..61e22473 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -80,7 +80,7 @@ class ProjectConfig(object): def items(self, section): items = [] for option in self._parser.options(section): - items.append((option, self._parser.get(section, option))) + items.append((option, self.get(section, option))) return items def get(self, section, option): @@ -95,7 +95,7 @@ class ProjectConfig(object): def _re_sub_handler(self, match): section, option = match.group(1), match.group(2) if section in ("env", - "sysenv") and not self._parser.has_section(section): + "sysenv") and not self.has_section(section): if section == "env": click.secho( "Warning! Access to system environment variable via " @@ -103,4 +103,4 @@ class ProjectConfig(object): "`${{sysenv.{0}}}` instead".format(option), fg="yellow") return os.getenv(option) - return self._parser.get(section, option) + return self.get(section, option) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index ba10d1c4..f47ae9af 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os from platformio.project.config import ProjectConfig BASE_CONFIG = """ @@ -23,12 +24,13 @@ extra_configs = [common] debug_flags = -D RELEASE lib_flags = -lc -lm +extra_flags = ${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS} [env:esp-wrover-kit] platform = espressif32 framework = espidf board = esp-wrover-kit -build_flags = ${common.debug_flags} +build_flags = ${common.debug_flags} ${common.extra_flags} """ EXTRA_ENVS_CONFIG = """ @@ -42,7 +44,7 @@ build_flags = ${common.lib_flags} ${common.debug_flags} platform = espressif32 framework = espidf board = lolin32 -build_flags = ${common.debug_flags} +build_flags = ${common.debug_flags} ${common.extra_flags} """ EXTRA_DEBUG_CONFIG = """ @@ -65,10 +67,27 @@ def test_parser(tmpdir): config = ProjectConfig(tmpdir.join("platformio.ini").strpath) assert config + # sections assert config.sections() == [ "platformio", "common", "env:esp-wrover-kit", "env:esp32dev", "env:lolin32" ] + + # sysenv + assert config.get("common", "extra_flags") == "" + os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] = "-L /usr/local/lib" + assert config.get("common", "extra_flags") == "-L /usr/local/lib" + + # get assert config.get("common", "debug_flags") == "-D DEBUG=1" assert config.get("env:esp32dev", "build_flags") == "-lc -lm -D DEBUG=1" assert config.get("env:lolin32", "build_flags") == "-Og" + assert config.get("env:esp-wrover-kit", + "build_flags") == ("-D DEBUG=1 -L /usr/local/lib") + + # items + assert config.items("env:esp32dev") == [("platform", "espressif32"), + ("framework", "espidf"), + ("board", "esp32dev"), + ("build_flags", + "-lc -lm -D DEBUG=1")] From e4dca378743563de22a654c828468a058963de4a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 4 May 2019 12:23:51 +0300 Subject: [PATCH 111/333] Bump version to 4.0.0a10 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index dbc2bf6e..6d00c736 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a9") +VERSION = (4, 0, "0a10") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 94e580bf4ecd8b3019aa86987526e893d5b82c2f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 4 May 2019 13:36:27 +0300 Subject: [PATCH 112/333] Override default source and include directories for a library via "library.json" manifest using "includeDir" and "srcDir" fields --- HISTORY.rst | 5 +++-- docs | 2 +- platformio/builder/tools/piolib.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bd899a08..0ce2bb74 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,8 +9,9 @@ PlatformIO 4.0 * Python 3 support (`issue #895 `_) -* Include external configuration files with "extra_configs" option +* Include external configuration files with `extra_configs `__ option (`issue #1590 `_) +* Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields PlatformIO 3.0 -------------- @@ -18,7 +19,7 @@ PlatformIO 3.0 3.6.8 (2019-??-??) ~~~~~~~~~~~~~~~~~~ -* Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ +* Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) 3.6.7 (2019-04-23) diff --git a/docs b/docs index 9a51b847..570187b4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9a51b847424c85444f4b9c38efad9d83f72353be +Subproject commit 570187b4afdf8295880cd37177596e023df4d117 diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 221df7e8..54961eb0 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -719,6 +719,18 @@ class PlatformIOLibBuilder(LibBuilderBase): def _is_arduino_manifest(self): return isfile(join(self.path, "library.properties")) + @property + def include_dir(self): + if "includeDir" in self._manifest.get("build", {}): + return self._manifest.get("build").get("includeDir") + return LibBuilderBase.include_dir.fget(self) + + @property + def src_dir(self): + if "srcDir" in self._manifest.get("build", {}): + return self._manifest.get("build").get("srcDir") + return LibBuilderBase.src_dir.fget(self) + @property def src_filter(self): if "srcFilter" in self._manifest.get("build", {}): From 95b9ae9f2400a8b411bccea1cec960a259d29af0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 May 2019 13:38:48 +0300 Subject: [PATCH 113/333] Docs: Sync Atmel SAM boards --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 570187b4..e7d578dc 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 570187b4afdf8295880cd37177596e023df4d117 +Subproject commit e7d578dc24082763478179ccec3b9a4220dd783b From 4fdd51e19094583758e9e1963ef97fd59c0d4dd1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 May 2019 13:26:10 +0300 Subject: [PATCH 114/333] Docs: Sync MCS51 boards --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index e7d578dc..5df8f4b0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e7d578dc24082763478179ccec3b9a4220dd783b +Subproject commit 5df8f4b066cebfa41e46db058936b2aa8298a70b From 1b4f94590708b6269f4ba7f9bd205dafde2eb4bc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 May 2019 15:59:09 +0300 Subject: [PATCH 115/333] Remove support for renamed dev/platforms --- platformio/commands/run.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index b480ae2a..817df353 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -168,8 +168,6 @@ class EnvironmentProcessor(object): "board_flash_mode": "board_build.flash_mode" } - RENAMED_PLATFORMS = {"espressif": "espressif8266"} - def __init__( self, # pylint: disable=R0913 cmd_ctx, @@ -231,14 +229,6 @@ class EnvironmentProcessor(object): "`%s` instead." % (k, self.RENAMED_OPTIONS[k]), fg="yellow") k = self.RENAMED_OPTIONS[k] - # process renamed platforms - if k == "platform" and v in self.RENAMED_PLATFORMS: - click.secho( - "Warning! Platform `%s` is deprecated and will be " - "removed in the next release! Please use " - "`%s` instead." % (v, self.RENAMED_PLATFORMS[v]), - fg="yellow") - v = self.RENAMED_PLATFORMS[v] # warn about unknown options unknown_conditions = [ From c235974eb63253f2a501fa4289bc2ac25bbd8f96 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 May 2019 17:51:50 +0300 Subject: [PATCH 116/333] Switch to the new ProjectConfig API --- platformio/commands/ci.py | 5 +- platformio/commands/device.py | 38 ++---- platformio/commands/init.py | 22 ++-- platformio/commands/run.py | 99 ++------------ platformio/exception.py | 167 ++++++++++++++---------- platformio/ide/projectgenerator.py | 14 +- platformio/project/config.py | 199 +++++++++++++++++++++++++++-- platformio/telemetry.py | 9 +- platformio/util.py | 7 +- tests/conftest.py | 6 +- tests/test_projectconf.py | 11 +- 11 files changed, 354 insertions(+), 223 deletions(-) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index e1b60087..18845592 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -25,6 +25,7 @@ from platformio.commands.init import cli as cmd_init from platformio.commands.init import validate_boards from platformio.commands.run import cli as cmd_run from platformio.exception import CIBuildEnvsEmpty +from platformio.project.config import ProjectConfig def validate_path(ctx, param, value): # pylint: disable=unused-argument @@ -161,8 +162,8 @@ def _exclude_contents(dst_dir, patterns): def _copy_project_conf(build_dir, project_conf): - config = util.load_project_config(project_conf) + config = ProjectConfig(project_conf, parse_extra=False) if config.has_section("platformio"): config.remove_section("platformio") - with open(join(build_dir, "platformio.ini"), "w") as fp: + with open(join(build_dir, "platformio.ini"), "wb") as fp: config.write(fp) diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 0a8ea38f..42bd4f6c 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -15,11 +15,13 @@ import json import sys from os import getcwd +from os.path import join import click from serial.tools import miniterm from platformio import exception, util +from platformio.project.config import ProjectConfig @click.group(short_help="Monitor device or list existing") @@ -161,11 +163,10 @@ def device_list( # pylint: disable=too-many-branches help="Load configuration from `platformio.ini` and specified environment") def device_monitor(**kwargs): # pylint: disable=too-many-branches try: - project_options = get_project_options(kwargs['project_dir'], + monitor_options = get_project_options(kwargs['project_dir'], kwargs['environment']) - monitor_options = {k: v for k, v in project_options or []} if monitor_options: - for k in ("port", "baud", "speed", "rts", "dtr"): + for k in ("port", "speed", "rts", "dtr"): k2 = "monitor_%s" % k if k == "speed": k = "baud" @@ -205,24 +206,13 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches raise exception.MinitermException(e) -def get_project_options(project_dir, environment): - config = util.load_project_config(project_dir) - if not config.sections(): - return None - - known_envs = [s[4:] for s in config.sections() if s.startswith("env:")] - if environment: - if environment in known_envs: - return config.items("env:%s" % environment) - raise exception.UnknownEnvNames(environment, ", ".join(known_envs)) - - if not known_envs: - return None - - if config.has_option("platformio", "env_default"): - env_default = config.get("platformio", - "env_default").split(", ")[0].strip() - if env_default and env_default in known_envs: - return config.items("env:%s" % env_default) - - return config.items("env:%s" % known_envs[0]) +def get_project_options(project_dir, environment=None): + config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) + config.validate(envs=[environment] if environment else None) + if not environment: + default_envs = config.default_envs() + if default_envs: + environment = default_envs[0] + else: + environment = config.envs()[0] + return {k: v for k, v in config.items(env=environment)} diff --git a/platformio/commands/init.py b/platformio/commands/init.py index e7127313..db45ddfc 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -23,9 +23,9 @@ import click from platformio import exception, util from platformio.commands.platform import \ platform_install as cli_platform_install -from platformio.commands.run import check_project_envs from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager +from platformio.project.config import ProjectConfig def validate_boards(ctx, param, value): # pylint: disable=W0613 @@ -128,14 +128,11 @@ def cli( def get_best_envname(project_dir, boards=None): - config = util.load_project_config(project_dir) - env_default = None - if config.has_option("platformio", "env_default"): - env_default = util.parse_conf_multi_values( - config.get("platformio", "env_default")) - check_project_envs(config, env_default) - if env_default: - return env_default[0] + config = ProjectConfig(join(project_dir, "platformio.ini")) + config.validate() + default_envs = config.default_envs() + if default_envs: + return default_envs[0] section = None for section in config.sections(): if not section.startswith("env:"): @@ -369,7 +366,8 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, used_boards = [] used_platforms = [] - config = util.load_project_config(project_dir) + config_path = join(project_dir, "platformio.ini") + config = util.load_project_config(config_path) for section in config.sections(): cond = [ section.startswith("env:"), @@ -409,10 +407,12 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, if not content: return - with open(join(project_dir, "platformio.ini"), "a") as f: + with open(config_path, "a") as f: content.append("") f.write("\n".join(content)) + ProjectConfig.reset_instances() + def _install_dependent_platforms(ctx, platforms): installed_platforms = [ diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 817df353..baff6a54 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -26,6 +26,7 @@ from platformio.commands.platform import \ platform_install as cmd_platform_install from platformio.managers.lib import LibraryManager, is_builtin_lib from platformio.managers.platform import PlatformFactory +from platformio.project.config import ProjectConfig # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @@ -54,9 +55,6 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose, if isfile(project_dir): project_dir = util.find_project_dir_above(project_dir) - if not util.is_platformio_project(project_dir): - raise exception.NotPlatformIOProject(project_dir) - with util.cd(project_dir): # clean obsolete build dir if not disable_auto_clean: @@ -69,25 +67,17 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose, util.get_projectbuild_dir(force=True), fg="yellow") - config = util.load_project_config() - env_default = None - if config.has_option("platformio", "env_default"): - env_default = util.parse_conf_multi_values( - config.get("platformio", "env_default")) - - check_project_defopts(config) - check_project_envs(config, environment or env_default) + config = ProjectConfig.get_instance( + join(project_dir, "platformio.ini")) + config.validate() results = [] start_time = time() - for section in config.sections(): - if not section.startswith("env:"): - continue - - envname = section[4:] + default_envs = config.default_envs() + for envname in config.envs(): skipenv = any([ environment and envname not in environment, not environment - and env_default and envname not in env_default + and default_envs and envname not in default_envs ]) if skipenv: results.append((envname, None)) @@ -97,7 +87,7 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose, click.echo() options = {} - for k, v in config.items(section): + for k, v in config.items(env=envname): options[k] = v if "piotest" not in options and "piotest" in ctx.meta: options['piotest'] = ctx.meta['piotest'] @@ -127,26 +117,6 @@ class EnvironmentProcessor(object): DEFAULT_DUMP_OPTIONS = ("platform", "framework", "board") - KNOWN_PLATFORMIO_OPTIONS = [ - "description", "env_default", "home_dir", "lib_dir", "libdeps_dir", - "include_dir", "src_dir", "build_dir", "data_dir", "test_dir", - "boards_dir", "lib_extra_dirs", "extra_configs" - ] - - KNOWN_ENV_OPTIONS = [ - "platform", "framework", "board", "build_flags", "src_build_flags", - "build_unflags", "src_filter", "extra_scripts", "targets", - "upload_port", "upload_protocol", "upload_speed", "upload_flags", - "upload_resetmethod", "lib_deps", "lib_ignore", "lib_extra_dirs", - "lib_ldf_mode", "lib_compat_mode", "lib_archive", "piotest", - "test_transport", "test_filter", "test_ignore", "test_port", - "test_speed", "test_build_project_src", "debug_tool", "debug_port", - "debug_init_cmds", "debug_extra_cmds", "debug_server", - "debug_init_break", "debug_load_cmd", "debug_load_mode", - "debug_svd_path", "monitor_port", "monitor_speed", "monitor_rts", - "monitor_dtr" - ] - IGNORE_BUILD_OPTIONS = [ "test_transport", "test_filter", "test_ignore", "test_port", "test_speed", "debug_port", "debug_init_cmds", "debug_extra_cmds", @@ -157,17 +127,6 @@ class EnvironmentProcessor(object): REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"} - RENAMED_OPTIONS = { - "lib_use": "lib_deps", - "lib_force": "lib_deps", - "extra_script": "extra_scripts", - "monitor_baud": "monitor_speed", - "board_mcu": "board_build.mcu", - "board_f_cpu": "board_build.f_cpu", - "board_f_flash": "board_build.f_flash", - "board_flash_mode": "board_build.flash_mode" - } - def __init__( self, # pylint: disable=R0913 cmd_ctx, @@ -201,7 +160,6 @@ class EnvironmentProcessor(object): self.name, fg="cyan", bold=True), "; ".join(env_dump))) click.secho("-" * terminal_width, bold=True) - self.options = self._validate_options(self.options) result = self._run() is_error = result['returncode'] != 0 @@ -218,31 +176,6 @@ class EnvironmentProcessor(object): return not is_error - def _validate_options(self, options): - result = {} - for k, v in options.items(): - # process obsolete options - if k in self.RENAMED_OPTIONS: - click.secho( - "Warning! `%s` option is deprecated and will be " - "removed in the next release! Please use " - "`%s` instead." % (k, self.RENAMED_OPTIONS[k]), - fg="yellow") - k = self.RENAMED_OPTIONS[k] - - # warn about unknown options - unknown_conditions = [ - k not in self.KNOWN_ENV_OPTIONS, not k.startswith("custom_"), - not k.startswith("board_") - ] - if all(unknown_conditions): - click.secho( - "Detected non-PlatformIO `%s` option in `[env:%s]` section" - % (k, self.name), - fg="yellow") - result[k] = v - return result - def get_build_variables(self): variables = {"pioenv": self.name} if self.upload_port: @@ -381,21 +314,7 @@ def print_summary(results, start_time): is_error=not successed) -def check_project_defopts(config): - if not config.has_section("platformio"): - return True - unknown = set(k for k, _ in config.items("platformio")) - set( - EnvironmentProcessor.KNOWN_PLATFORMIO_OPTIONS) - if not unknown: - return True - click.secho( - "Warning! Ignore unknown `%s` option in `[platformio]` section" % - ", ".join(unknown), - fg="yellow") - return False - - -def check_project_envs(config, environments=None): +def check_project_envs(config, environments=None): # FIXME: Remove if not config.sections(): raise exception.ProjectEnvsNotAvailable() diff --git a/platformio/exception.py b/platformio/exception.py index 92ffd6c5..26131a5d 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -42,11 +42,16 @@ class UserSideException(PlatformioException): pass -class AbortedByUser(PlatformioException): +class AbortedByUser(UserSideException): MESSAGE = "Aborted by user" +# +# Development Platform +# + + class UnknownPlatform(PlatformioException): MESSAGE = "Unknown development platform '{0}'" @@ -85,54 +90,75 @@ class UnknownFramework(PlatformioException): MESSAGE = "Unknown framework '{0}'" -class UnknownPackage(PlatformioException): +# Package Manager + + +class PlatformIOPackageException(PlatformioException): + pass + + +class UnknownPackage(PlatformIOPackageException): MESSAGE = "Detected unknown package '{0}'" -class MissingPackageManifest(PlatformioException): +class MissingPackageManifest(PlatformIOPackageException): MESSAGE = "Could not find one of '{0}' manifest files in the package" -class UndefinedPackageVersion(PlatformioException): +class UndefinedPackageVersion(PlatformIOPackageException): MESSAGE = ("Could not find a version that satisfies the requirement '{0}'" " for your system '{1}'") -class PackageInstallError(PlatformioException): +class PackageInstallError(PlatformIOPackageException): MESSAGE = ("Could not install '{0}' with version requirements '{1}' " "for your system '{2}'.\n\n" "Please try this solution -> http://bit.ly/faq-package-manager") -class ExtractArchiveItemError(PlatformioException): +class ExtractArchiveItemError(PlatformIOPackageException): MESSAGE = ( "Could not extract `{0}` to `{1}`. Try to disable antivirus " "tool or check this solution -> http://bit.ly/faq-package-manager") -class FDUnrecognizedStatusCode(PlatformioException): +class UnsupportedArchiveType(PlatformIOPackageException): + + MESSAGE = "Can not unpack file '{0}'" + + +class FDUnrecognizedStatusCode(PlatformIOPackageException): MESSAGE = "Got an unrecognized status code '{0}' when downloaded {1}" -class FDSizeMismatch(PlatformioException): +class FDSizeMismatch(PlatformIOPackageException): MESSAGE = ("The size ({0:d} bytes) of downloaded file '{1}' " "is not equal to remote size ({2:d} bytes)") -class FDSHASumMismatch(PlatformioException): +class FDSHASumMismatch(PlatformIOPackageException): MESSAGE = ("The 'sha1' sum '{0}' of downloaded file '{1}' " "is not equal to remote '{2}'") -class NotPlatformIOProject(PlatformioException): +# +# Project +# + + +class PlatformIOProjectException(PlatformioException): + pass + + +class NotPlatformIOProject(PlatformIOProjectException): MESSAGE = ( "Not a PlatformIO project. `platformio.ini` file has not been " @@ -140,26 +166,82 @@ class NotPlatformIOProject(PlatformioException): "please use `platformio init` command") -class UndefinedEnvPlatform(PlatformioException): +class InvalidProjectConf(PlatformIOProjectException): + + MESSAGE = ("Invalid '{0}' (project configuration file): '{1}'") + + +class UndefinedEnvPlatform(PlatformIOProjectException): MESSAGE = "Please specify platform for '{0}' environment" -class UnsupportedArchiveType(PlatformioException): - - MESSAGE = "Can not unpack file '{0}'" - - -class ProjectEnvsNotAvailable(PlatformioException): +class ProjectEnvsNotAvailable(PlatformIOProjectException): MESSAGE = "Please setup environments in `platformio.ini` file" -class UnknownEnvNames(PlatformioException): +class UnknownEnvNames(PlatformIOProjectException): # FIXME: UnknownProjectEnvs MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" +# +# Library +# + + +class LibNotFound(PlatformioException): + + MESSAGE = ("Library `{0}` has not been found in PlatformIO Registry.\n" + "You can ignore this message, if `{0}` is a built-in library " + "(included in framework, SDK). E.g., SPI, Wire, etc.") + + +class NotGlobalLibDir(UserSideException): + + MESSAGE = ( + "The `{0}` is not a PlatformIO project.\n\n" + "To manage libraries in global storage `{1}`,\n" + "please use `platformio lib --global {2}` or specify custom storage " + "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" + "Check `platformio lib --help` for details.") + + +class InvalidLibConfURL(PlatformioException): + + MESSAGE = "Invalid library config URL '{0}'" + + +# +# UDEV Rules +# + + +class InvalidUdevRules(PlatformioException): + pass + + +class MissedUdevRules(InvalidUdevRules): + + MESSAGE = ( + "Warning! Please install `99-platformio-udev.rules`. \nMode details: " + "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules") + + +class OutdatedUdevRules(InvalidUdevRules): + + MESSAGE = ( + "Warning! Your `{0}` are outdated. Please update or reinstall them." + "\n Mode details: https://docs.platformio.org" + "/en/latest/faq.html#platformio-udev-rules") + + +# +# Misc +# + + class GetSerialPortsError(PlatformioException): MESSAGE = "No implementation for your platform ('{0}') available" @@ -175,7 +257,7 @@ class APIRequestError(PlatformioException): MESSAGE = "[API] {0}" -class InternetIsOffline(PlatformioException): +class InternetIsOffline(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" @@ -183,33 +265,6 @@ class InternetIsOffline(PlatformioException): "to install all dependencies and toolchains.") -class LibNotFound(PlatformioException): - - MESSAGE = ("Library `{0}` has not been found in PlatformIO Registry.\n" - "You can ignore this message, if `{0}` is a built-in library " - "(included in framework, SDK). E.g., SPI, Wire, etc.") - - -class NotGlobalLibDir(PlatformioException): - - MESSAGE = ( - "The `{0}` is not a PlatformIO project.\n\n" - "To manage libraries in global storage `{1}`,\n" - "please use `platformio lib --global {2}` or specify custom storage " - "`platformio lib --storage-dir /path/to/storage/ {2}`.\n" - "Check `platformio lib --help` for details.") - - -class InvalidLibConfURL(PlatformioException): - - MESSAGE = "Invalid library config URL '{0}'" - - -class InvalidProjectConf(PlatformioException): - - MESSAGE = ("Invalid '{0}' (project configuration file): '{1}'") - - class BuildScriptNotFound(PlatformioException): MESSAGE = "Invalid path '{0}' to build script" @@ -237,25 +292,6 @@ class CIBuildEnvsEmpty(PlatformioException): "predefined environments using `--project-conf` option") -class InvalidUdevRules(PlatformioException): - pass - - -class MissedUdevRules(InvalidUdevRules): - - MESSAGE = ( - "Warning! Please install `99-platformio-udev.rules`. \nMode details: " - "https://docs.platformio.org/en/latest/faq.html#platformio-udev-rules") - - -class OutdatedUdevRules(InvalidUdevRules): - - MESSAGE = ( - "Warning! Your `{0}` are outdated. Please update or reinstall them." - "\n Mode details: https://docs.platformio.org" - "/en/latest/faq.html#platformio-udev-rules") - - class UpgradeError(PlatformioException): MESSAGE = """{0} @@ -290,5 +326,4 @@ class DebugSupportError(PlatformioException): class DebugInvalidOptions(PlatformioException): - pass diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index cd48ca61..26774a70 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -23,6 +23,7 @@ from click.testing import CliRunner from platformio import exception, util from platformio.commands.run import cli as cmd_run +from platformio.project.config import ProjectConfig class ProjectGenerator(object): @@ -44,14 +45,13 @@ class ProjectGenerator(object): @util.memoized() def get_project_env(self): data = {} - config = util.load_project_config(self.project_dir) - for section in config.sections(): - if not section.startswith("env:"): + config = ProjectConfig.get_instance( + join(self.project_dir, "platformio.ini")) + for env in config.envs(): + if self.env_name != env: continue - if self.env_name != section[4:]: - continue - data = {"env_name": section[4:]} - for k, v in config.items(section): + data = {"env_name": self.env_name} + for k, v in config.items(env=env): data[k] = v return data diff --git a/platformio/project/config.py b/platformio/project/config.py index 61e22473..f827020c 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -15,6 +15,7 @@ import glob import os import re +from os.path import isfile import click @@ -25,11 +26,99 @@ try: except ImportError: import configparser as ConfigParser +KNOWN_PLATFORMIO_OPTIONS = [ + "description", + "env_default", + "extra_configs", + + # Dirs + "home_dir", + "lib_dir", + "libdeps_dir", + "include_dir", + "src_dir", + "build_dir", + "data_dir", + "test_dir", + "boards_dir", + "lib_extra_dirs" +] + +KNOWN_ENV_OPTIONS = [ + # Generic + "platform", + "framework", + "board", + "targets", + + # Build + "build_flags", + "src_build_flags", + "build_unflags", + "src_filter", + + # Upload + "upload_port", + "upload_protocol", + "upload_speed", + "upload_flags", + "upload_resetmethod", + + # Monitor + "monitor_port", + "monitor_speed", + "monitor_rts", + "monitor_dtr", + + # Library + "lib_deps", + "lib_ignore", + "lib_extra_dirs", + "lib_ldf_mode", + "lib_compat_mode", + "lib_archive", + + # Test + "piotest", + "test_filter", + "test_ignore", + "test_port", + "test_speed", + "test_transport", + "test_build_project_src", + + # Debug + "debug_tool", + "debug_init_break", + "debug_init_cmds", + "debug_extra_cmds", + "debug_load_cmd", + "debug_load_mode", + "debug_server", + "debug_port", + "debug_svd_path", + + # Other + "extra_scripts" +] + +RENAMED_OPTIONS = { + "lib_use": "lib_deps", + "lib_force": "lib_deps", + "extra_script": "extra_scripts", + "monitor_baud": "monitor_speed", + "board_mcu": "board_build.mcu", + "board_f_cpu": "board_build.f_cpu", + "board_f_flash": "board_build.f_flash", + "board_flash_mode": "board_build.flash_mode" +} + class ProjectConfig(object): VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") + _instances = {} _parser = None _parsed = [] @@ -49,13 +138,30 @@ class ProjectConfig(object): result.append(item) return result - def __init__(self, path): + @staticmethod + def get_instance(path): + if not isfile(path): + raise exception.NotPlatformIOProject(path) + if path not in ProjectConfig._instances: + ProjectConfig._instances[path] = ProjectConfig(path) + return ProjectConfig._instances[path] + + @staticmethod + def reset_instances(): + ProjectConfig._instances = {} + + def __init__(self, path, parse_extra=True): + if not isfile(path): + raise exception.NotPlatformIOProject(path) self.path = path self._parsed = [] self._parser = ConfigParser.ConfigParser() - self.read(path) + self.read(path, parse_extra) - def read(self, path): + def __getattr__(self, name): + return getattr(self._parser, name) + + def read(self, path, parse_extra=True): if path in self._parsed: return self._parsed.append(path) @@ -64,6 +170,9 @@ class ProjectConfig(object): except ConfigParser.Error as e: raise exception.InvalidProjectConf(path, str(e)) + if not parse_extra: + return + # load extra configs if (not self._parser.has_section("platformio") or not self._parser.has_option("platformio", "extra_configs")): @@ -74,14 +183,18 @@ class ProjectConfig(object): for item in glob.glob(pattern): self.read(item) - def __getattr__(self, name): - return getattr(self._parser, name) + def options(self, section=None, env=None): + assert section or env + if not section: + section = "env:" + env + return self._parser.options(section) - def items(self, section): - items = [] - for option in self._parser.options(section): - items.append((option, self.get(section, option))) - return items + def items(self, section=None, env=None): + assert section or env + if not section: + section = "env:" + env + return [(option, self.get(section, option)) + for option in self.options(section)] def get(self, section, option): try: @@ -95,7 +208,7 @@ class ProjectConfig(object): def _re_sub_handler(self, match): section, option = match.group(1), match.group(2) if section in ("env", - "sysenv") and not self.has_section(section): + "sysenv") and not self._parser.has_section(section): if section == "env": click.secho( "Warning! Access to system environment variable via " @@ -104,3 +217,67 @@ class ProjectConfig(object): fg="yellow") return os.getenv(option) return self.get(section, option) + + def envs(self): + return [s[4:] for s in self._parser.sections() if s.startswith("env:")] + + def default_envs(self): + if not self._parser.has_option("platformio", "env_default"): + return [] + return self.parse_multi_values(self.get("platformio", "env_default")) + + def validate(self, envs=None): + # check envs + known = set(self.envs()) + if not known: + raise exception.ProjectEnvsNotAvailable() + + unknown = set((envs or []) + self.default_envs()) - known + if unknown: + raise exception.UnknownEnvNames(", ".join(unknown), + ", ".join(known)) + return self.validate_options() + + def validate_options(self): + warnings = set() + # check [platformio] section + if self._parser.has_section("platformio"): + unknown = set(k for k, _ in self.items("platformio")) - set( + KNOWN_PLATFORMIO_OPTIONS) + if unknown: + warnings.add( + "Ignore unknown `%s` options in `[platformio]` section" % + ", ".join(unknown)) + + # check [env:*] sections + for section in self._parser.sections(): + if not section.startswith("env:"): + continue + for option in self._parser.options(section): + # obsolete + if option in RENAMED_OPTIONS: + warnings.add( + "`%s` option in `[%s]` section is deprecated and will " + "be removed in the next release! Please use `%s` " + "instead" % (option, section, RENAMED_OPTIONS[option])) + # rename on-the-fly + self._parser.set(section, RENAMED_OPTIONS[option], + self._parser.get(section, option)) + self._parser.remove_option(section, option) + continue + + # unknown + unknown_conditions = [ + option not in KNOWN_ENV_OPTIONS, + not option.startswith("custom_"), + not option.startswith("board_") + ] # yapf: disable + if all(unknown_conditions): + warnings.add( + "Detected non-PlatformIO `%s` option in `[%s]` section" + % (option, section)) + + for warning in warnings: + click.secho("Warning! %s" % warning, fg="yellow") + + return True diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 3f9937cf..12e4a034 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -342,12 +342,9 @@ def on_exception(e): return text.strip() skip_conditions = [ - isinstance(e, cls) - for cls in (IOError, exception.ReturnErrorCode, - exception.AbortedByUser, exception.NotGlobalLibDir, - exception.InternetIsOffline, - exception.NotPlatformIOProject, - exception.UserSideException) + isinstance(e, cls) for cls in (IOError, exception.ReturnErrorCode, + exception.UserSideException, + exception.PlatformIOProjectException) ] try: skip_conditions.append("[API] Account: " in str(e)) diff --git a/platformio/util.py b/platformio/util.py index 1adcd936..85a55dc6 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -180,7 +180,8 @@ def get_project_optional_dir(name, default=None): paths = os.getenv(var_name) else: try: - config = load_project_config() + config = ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini")) if (config.has_section("platformio") and config.has_option("platformio", name)): paths = config.get("platformio", name) @@ -312,7 +313,7 @@ def get_projectdata_dir(): "data")) -def load_project_config(path=None): # FIXME: +def load_project_config(path=None): # FIXME: Remove if not path or isdir(path): path = join(path or get_project_dir(), "platformio.ini") if not isfile(path): @@ -321,7 +322,7 @@ def load_project_config(path=None): # FIXME: return ProjectConfig(path) -def parse_conf_multi_values(items): # FIXME: +def parse_conf_multi_values(items): # FIXME: Remove return ProjectConfig.parse_multi_values(items) diff --git a/tests/conftest.py b/tests/conftest.py index 02d16069..d53d92e8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,8 +24,10 @@ from platformio import util def validate_cliresult(): def decorator(result): - assert result.exit_code == 0, result.output - assert not result.exception, result.output + assert result.exit_code == 0, "{} => {}".format( + result.exception, result.output) + assert not result.exception, "{} => {}".format(result.exception, + result.output) return decorator diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index f47ae9af..f6228b88 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -13,10 +13,12 @@ # limitations under the License. import os + from platformio.project.config import ProjectConfig BASE_CONFIG = """ [platformio] +env_default = esp32dev, lolin32 extra_configs = extra_envs.ini extra_debug.ini @@ -86,8 +88,15 @@ def test_parser(tmpdir): "build_flags") == ("-D DEBUG=1 -L /usr/local/lib") # items - assert config.items("env:esp32dev") == [("platform", "espressif32"), + assert config.items("common") == [("debug_flags", "-D DEBUG=1"), + ("lib_flags", "-lc -lm"), + ("extra_flags", "-L /usr/local/lib")] + assert config.items(env="esp32dev") == [("platform", "espressif32"), ("framework", "espidf"), ("board", "esp32dev"), ("build_flags", "-lc -lm -D DEBUG=1")] + + # envs + assert config.envs() == ["esp-wrover-kit", "esp32dev", "lolin32"] + assert config.default_envs() == ["esp32dev", "lolin32"] From 6cd4484be919dfee2333e1c96b35dee39f60ffa3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 May 2019 19:57:24 +0300 Subject: [PATCH 117/333] Init new project using new ProjectConfig API --- platformio/commands/ci.py | 3 +-- platformio/commands/init.py | 42 +++++++++++++---------------------- platformio/project/config.py | 37 +++++++++++++++++++++++------- platformio/projectconftpl.ini | 9 -------- platformio/util.py | 14 +++++------- setup.py | 1 - 6 files changed, 51 insertions(+), 55 deletions(-) delete mode 100644 platformio/projectconftpl.ini diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 18845592..f9e93ace 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -165,5 +165,4 @@ def _copy_project_conf(build_dir, project_conf): config = ProjectConfig(project_conf, parse_extra=False) if config.has_section("platformio"): config.remove_section("platformio") - with open(join(build_dir, "platformio.ini"), "wb") as fp: - config.write(fp) + config.save(join(build_dir, "platformio.ini")) diff --git a/platformio/commands/init.py b/platformio/commands/init.py index db45ddfc..f08a52c2 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -16,7 +16,6 @@ from os import getcwd, makedirs from os.path import isdir, isfile, join -from shutil import copyfile import click @@ -89,7 +88,8 @@ def cli( "platformio.ini", fg="cyan")) is_new_project = not util.is_platformio_project(project_dir) - init_base_project(project_dir) + if is_new_project: + init_base_project(project_dir) if board: fill_project_envs(ctx, project_dir, board, project_option, env_prefix, @@ -144,13 +144,7 @@ def get_best_envname(project_dir, boards=None): def init_base_project(project_dir): - if util.is_platformio_project(project_dir): - return - - copyfile( - join(util.get_source_dir(), "projectconftpl.ini"), - join(project_dir, "platformio.ini")) - + ProjectConfig(join(project_dir, "platformio.ini")).save() with util.cd(project_dir): dir_to_readme = [ (util.get_projectsrc_dir(), None), @@ -362,12 +356,9 @@ def init_cvs_ignore(project_dir): def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, force_download): - content = [] + config = ProjectConfig( + join(project_dir, "platformio.ini"), parse_extra=False) used_boards = [] - used_platforms = [] - - config_path = join(project_dir, "platformio.ini") - config = util.load_project_config(config_path) for section in config.sections(): cond = [ section.startswith("env:"), @@ -377,12 +368,15 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, used_boards.append(config.get(section, "board")) pm = PlatformManager() + used_platforms = [] + modified = False for id_ in board_ids: board_config = pm.board_config(id_) used_platforms.append(board_config['platform']) if id_ in used_boards: continue used_boards.append(id_) + modified = True envopts = {"platform": board_config['platform'], "board": id_} # find default framework for board @@ -396,22 +390,18 @@ def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, _name, _value = item.split("=", 1) envopts[_name.strip()] = _value.strip() - content.append("") - content.append("[env:%s%s]" % (env_prefix, id_)) - for name, value in envopts.items(): - content.append("%s = %s" % (name, value)) + section = "env:%s%s" % (env_prefix, id_) + config.add_section(section) + + for option, value in envopts.items(): + config.set(section, option, value) if force_download and used_platforms: _install_dependent_platforms(ctx, used_platforms) - if not content: - return - - with open(config_path, "a") as f: - content.append("") - f.write("\n".join(content)) - - ProjectConfig.reset_instances() + if modified: + config.save() + config.reset_instances() def _install_dependent_platforms(ctx, platforms): diff --git a/platformio/project/config.py b/platformio/project/config.py index f827020c..46a99425 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -26,6 +26,18 @@ try: except ImportError: import configparser as ConfigParser +CONFIG_HEADER = """ +; PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; https://docs.platformio.org/page/projectconf.html +""" + KNOWN_PLATFORMIO_OPTIONS = [ "description", "env_default", @@ -118,6 +130,7 @@ class ProjectConfig(object): VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") + expand_interpolations = True _instances = {} _parser = None _parsed = [] @@ -140,8 +153,6 @@ class ProjectConfig(object): @staticmethod def get_instance(path): - if not isfile(path): - raise exception.NotPlatformIOProject(path) if path not in ProjectConfig._instances: ProjectConfig._instances[path] = ProjectConfig(path) return ProjectConfig._instances[path] @@ -150,13 +161,13 @@ class ProjectConfig(object): def reset_instances(): ProjectConfig._instances = {} - def __init__(self, path, parse_extra=True): - if not isfile(path): - raise exception.NotPlatformIOProject(path) + def __init__(self, path, parse_extra=True, expand_interpolations=True): self.path = path + self.expand_interpolations = expand_interpolations self._parsed = [] self._parser = ConfigParser.ConfigParser() - self.read(path, parse_extra) + if isfile(path): + self.read(path, parse_extra) def __getattr__(self, name): return getattr(self._parser, name) @@ -197,6 +208,8 @@ class ProjectConfig(object): for option in self.options(section)] def get(self, section, option): + if not self.expand_interpolations: + return self._parser.get(section, option) try: value = self._parser.get(section, option) except ConfigParser.Error as e: @@ -227,6 +240,8 @@ class ProjectConfig(object): return self.parse_multi_values(self.get("platformio", "env_default")) def validate(self, envs=None): + if not isfile(self.path): + raise exception.NotPlatformIOProject(self.path) # check envs known = set(self.envs()) if not known: @@ -246,7 +261,7 @@ class ProjectConfig(object): KNOWN_PLATFORMIO_OPTIONS) if unknown: warnings.add( - "Ignore unknown `%s` options in `[platformio]` section" % + "Ignore unknown `%s` options in section `[platformio]`" % ", ".join(unknown)) # check [env:*] sections @@ -257,7 +272,7 @@ class ProjectConfig(object): # obsolete if option in RENAMED_OPTIONS: warnings.add( - "`%s` option in `[%s]` section is deprecated and will " + "`%s` option in section `[%s]` is deprecated and will " "be removed in the next release! Please use `%s` " "instead" % (option, section, RENAMED_OPTIONS[option])) # rename on-the-fly @@ -281,3 +296,9 @@ class ProjectConfig(object): click.secho("Warning! %s" % warning, fg="yellow") return True + + def save(self, path=None): + with open(path or self.path, "wb") as fp: + fp.write(CONFIG_HEADER.strip()) + fp.write("\n\n") + self._parser.write(fp) diff --git a/platformio/projectconftpl.ini b/platformio/projectconftpl.ini deleted file mode 100644 index ebc510b2..00000000 --- a/platformio/projectconftpl.ini +++ /dev/null @@ -1,9 +0,0 @@ -; PlatformIO Project Configuration File -; -; Build options: build flags, source filter -; Upload options: custom upload port, speed and extra flags -; Library options: dependencies, extra library storages -; Advanced options: extra scripting -; -; Please visit documentation for the other options and examples -; https://docs.platformio.org/page/projectconf.html diff --git a/platformio/util.py b/platformio/util.py index 85a55dc6..8df17ec9 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -179,15 +179,11 @@ def get_project_optional_dir(name, default=None): if var_name in os.environ: paths = os.getenv(var_name) else: - try: - config = ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini")) - if (config.has_section("platformio") - and config.has_option("platformio", name)): - paths = config.get("platformio", name) - except exception.NotPlatformIOProject: - pass - + config = ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini")) + if (config.has_section("platformio") + and config.has_option("platformio", name)): + paths = config.get("platformio", name) if not paths: return default diff --git a/setup.py b/setup.py index 747d6cc2..667a2842 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,6 @@ setup( packages=find_packages() + ["scripts"], package_data={ "platformio": [ - "projectconftpl.ini", "ide/tpls/*/.*.tpl", "ide/tpls/*/*.tpl", "ide/tpls/*/*/*.tpl", From 7ddd22209f99ff2e8429c966596910d0a43ebc3a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 May 2019 21:16:42 +0300 Subject: [PATCH 118/333] Enable PyLint "import-error" --- .pylintrc | 2 +- platformio/builder/main.py | 11 ++++++++--- platformio/builder/tools/pioide.py | 2 +- platformio/builder/tools/piolib.py | 6 ++++-- platformio/builder/tools/piomisc.py | 4 ++-- platformio/builder/tools/pioplatform.py | 2 +- platformio/builder/tools/pioupload.py | 2 +- platformio/builder/tools/platformio.py | 9 ++++++--- 8 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.pylintrc b/.pylintrc index bbaef24a..82c3fb7a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -20,4 +20,4 @@ confidence= # --disable=W" # disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating -disable=fixme,locally-disabled,missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,import-error,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens,useless-object-inheritance,useless-import-alias +disable=missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens,useless-object-inheritance,useless-import-alias,fixme diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 2afb2289..2d070104 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -19,9 +19,14 @@ from os import environ from os.path import expanduser, join from time import time -from SCons.Script import (ARGUMENTS, COMMAND_LINE_TARGETS, DEFAULT_TARGETS, - AllowSubstExceptions, AlwaysBuild, Default, - DefaultEnvironment, Variables) +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error +from SCons.Script import DEFAULT_TARGETS # pylint: disable=import-error +from SCons.Script import AllowSubstExceptions # pylint: disable=import-error +from SCons.Script import AlwaysBuild # pylint: disable=import-error +from SCons.Script import Default # pylint: disable=import-error +from SCons.Script import DefaultEnvironment # pylint: disable=import-error +from SCons.Script import Variables # pylint: disable=import-error from platformio import util diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 0e6bf31a..db496741 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -18,7 +18,7 @@ from glob import glob from os import environ from os.path import abspath, isfile, join -from SCons.Defaults import processDefines +from SCons.Defaults import processDefines # pylint: disable=import-error from platformio import util from platformio.managers.core import get_core_package_dir diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 54961eb0..0651eddd 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -26,8 +26,10 @@ from glob import glob from os.path import (basename, commonprefix, dirname, isdir, isfile, join, realpath, sep) -import SCons.Scanner -from SCons.Script import ARGUMENTS, COMMAND_LINE_TARGETS, DefaultEnvironment +import SCons.Scanner # pylint: disable=import-error +from SCons.Script import ARGUMENTS # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error +from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, util from platformio.builder.tools import platformio as piotool diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 19a6a2ef..01a797ea 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -21,8 +21,8 @@ from os import environ, remove, walk from os.path import basename, isdir, isfile, join, realpath, relpath, sep from tempfile import mkstemp -from SCons.Action import Action -from SCons.Script import ARGUMENTS +from SCons.Action import Action # pylint: disable=import-error +from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import util from platformio.managers.core import get_core_package_dir diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 358aee7d..6427b33c 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -18,7 +18,7 @@ import base64 import sys from os.path import isdir, isfile, join -from SCons.Script import COMMAND_LINE_TARGETS +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import exception, util from platformio.managers.platform import PlatformFactory diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index f9628085..0d5a55ea 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -22,7 +22,7 @@ from os.path import isfile, join from shutil import copyfile from time import sleep -from SCons.Script import ARGUMENTS +from SCons.Script import ARGUMENTS # pylint: disable=import-error from serial import Serial, SerialException from platformio import exception, util diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 2a539e10..41f32700 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -20,9 +20,12 @@ from glob import glob from os import sep, walk from os.path import basename, dirname, isdir, join, realpath -from SCons import Builder, Util -from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild, - DefaultEnvironment, Export, SConscript) +from SCons import Builder, Util # pylint: disable=import-error +from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error +from SCons.Script import AlwaysBuild # pylint: disable=import-error +from SCons.Script import DefaultEnvironment # pylint: disable=import-error +from SCons.Script import Export # pylint: disable=import-error +from SCons.Script import SConscript # pylint: disable=import-error from platformio.util import glob_escape, pioversion_to_intstr, string_types From 07a2a49d931105809db4297e97ad421dd586c082 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 May 2019 22:13:21 +0300 Subject: [PATCH 119/333] Refactor project helpers --- platformio/builder/main.py | 24 ++--- platformio/builder/tools/piolib.py | 4 +- platformio/commands/device.py | 2 +- platformio/commands/init.py | 44 +++++---- platformio/commands/lib.py | 26 +++--- platformio/commands/run.py | 45 +++------- platformio/exception.py | 7 -- platformio/ide/projectgenerator.py | 15 ++-- platformio/managers/platform.py | 3 +- platformio/project/config.py | 16 ++-- platformio/project/helpers.py | 139 +++++++++++++++++++++++++++++ platformio/util.py | 109 ++-------------------- tests/commands/test_init.py | 2 +- 13 files changed, 236 insertions(+), 200 deletions(-) create mode 100644 platformio/project/helpers.py diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 2d070104..c656da0b 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -29,6 +29,10 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error from platformio import util +from platformio.project.helpers import ( + get_project_dir, get_project_optional_dir, get_projectbuild_dir, + get_projectdata_dir, get_projectinclude_dir, get_projectlib_dir, + get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir) AllowSubstExceptions(NameError) @@ -101,19 +105,19 @@ DEFAULT_ENV_OPTIONS = dict( ENV=environ, UNIX_TIME=int(time()), PIOHOME_DIR=util.get_home_dir(), - PROJECT_DIR=util.get_project_dir(), - PROJECTINCLUDE_DIR=util.get_projectinclude_dir(), - PROJECTSRC_DIR=util.get_projectsrc_dir(), - PROJECTTEST_DIR=util.get_projecttest_dir(), - PROJECTDATA_DIR=util.get_projectdata_dir(), - PROJECTBUILD_DIR=util.get_projectbuild_dir(), + PROJECT_DIR=get_project_dir(), + PROJECTINCLUDE_DIR=get_projectinclude_dir(), + PROJECTSRC_DIR=get_projectsrc_dir(), + PROJECTTEST_DIR=get_projecttest_dir(), + PROJECTDATA_DIR=get_projectdata_dir(), + PROJECTBUILD_DIR=get_projectbuild_dir(), BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"), BUILDSRC_DIR=join("$BUILD_DIR", "src"), BUILDTEST_DIR=join("$BUILD_DIR", "test"), LIBPATH=["$BUILD_DIR"], LIBSOURCE_DIRS=[ - util.get_projectlib_dir(), - util.get_projectlibdeps_dir(), + get_projectlib_dir(), + get_projectlibdeps_dir(), join("$PIOHOME_DIR", "lib") ], PROGNAME="program", @@ -156,10 +160,10 @@ for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPTS", env.Append(**{var: util.parse_conf_multi_values(environ.get(k))}) # Configure extra library source directories for LDF -if util.get_project_optional_dir("lib_extra_dirs"): +if get_project_optional_dir("lib_extra_dirs"): env.Prepend( LIBSOURCE_DIRS=util.parse_conf_multi_values( - util.get_project_optional_dir("lib_extra_dirs"))) + get_project_optional_dir("lib_extra_dirs"))) env.Prepend(LIBSOURCE_DIRS=env.get("LIB_EXTRA_DIRS", [])) env['LIBSOURCE_DIRS'] = [ expanduser(d) if d.startswith("~") else d for d in env['LIBSOURCE_DIRS'] diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 0651eddd..e1a0b50b 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -46,8 +46,8 @@ class LibBuilderFactory(object): clsname = "PlatformIOLibBuilder" else: used_frameworks = LibBuilderFactory.get_used_frameworks(env, path) - common_frameworks = ( - set(env.get("PIOFRAMEWORK", [])) & set(used_frameworks)) + common_frameworks = (set(env.get("PIOFRAMEWORK", [])) + & set(used_frameworks)) if common_frameworks: clsname = "%sLibBuilder" % list(common_frameworks)[0].title() elif used_frameworks: diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 42bd4f6c..e75660ea 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -215,4 +215,4 @@ def get_project_options(project_dir, environment=None): environment = default_envs[0] else: environment = config.envs()[0] - return {k: v for k, v in config.items(env=environment)} + return config.items(env=environment, as_dict=True) diff --git a/platformio/commands/init.py b/platformio/commands/init.py index f08a52c2..70f8a8e4 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -25,6 +25,9 @@ from platformio.commands.platform import \ from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig +from platformio.project.helpers import ( + get_projectinclude_dir, get_projectlib_dir, get_projectsrc_dir, + get_projecttest_dir, is_platformio_project) def validate_boards(ctx, param, value): # pylint: disable=W0613 @@ -87,7 +90,7 @@ def cli( click.echo("%s - Project Configuration File" % click.style( "platformio.ini", fg="cyan")) - is_new_project = not util.is_platformio_project(project_dir) + is_new_project = not is_platformio_project(project_dir) if is_new_project: init_base_project(project_dir) @@ -96,10 +99,8 @@ def cli( ide is not None) if ide: - env_name = get_best_envname(project_dir, board) - if not env_name: - raise exception.BoardNotDefined() - pg = ProjectGenerator(project_dir, ide, env_name) + pg = ProjectGenerator(project_dir, ide, + get_best_envname(project_dir, board)) pg.generate() if is_new_project: @@ -130,27 +131,34 @@ def cli( def get_best_envname(project_dir, boards=None): config = ProjectConfig(join(project_dir, "platformio.ini")) config.validate() + + envname = None default_envs = config.default_envs() if default_envs: - return default_envs[0] - section = None - for section in config.sections(): - if not section.startswith("env:"): - continue - elif config.has_option(section, "board") and (not boards or config.get( - section, "board") in boards): - break - return section[4:] if section else None + envname = default_envs[0] + if not boards: + return envname + + for env in config.envs(): + if not boards: + return env + if not envname: + envname = env + items = config.items(env=env, as_dict=True) + if "board" in items and items.get("board") in boards: + return env + + return envname def init_base_project(project_dir): ProjectConfig(join(project_dir, "platformio.ini")).save() with util.cd(project_dir): dir_to_readme = [ - (util.get_projectsrc_dir(), None), - (util.get_projectinclude_dir(), init_include_readme), - (util.get_projectlib_dir(), init_lib_readme), - (util.get_projecttest_dir(), init_test_readme), + (get_projectsrc_dir(), None), + (get_projectinclude_dir(), init_include_readme), + (get_projectlib_dir(), init_lib_readme), + (get_projecttest_dir(), init_test_readme), ] for (path, cb) in dir_to_readme: if isdir(path): diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 2f09cd9c..7caf1271 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -22,7 +22,8 @@ import click from platformio import exception, util from platformio.managers.lib import LibraryManager, get_builtin_libs -from platformio.util import get_api_result +from platformio.project.helpers import ( + get_project_dir, get_projectlibdeps_dir, is_platformio_project) try: from urllib.parse import quote @@ -58,8 +59,8 @@ def cli(ctx, **options): if not storage_dir: if options['global']: storage_dir = join(util.get_home_dir(), "lib") - elif util.is_platformio_project(): - storage_dir = util.get_projectlibdeps_dir() + elif is_platformio_project(): + storage_dir = get_projectlibdeps_dir() elif util.is_ci(): storage_dir = join(util.get_home_dir(), "lib") click.secho( @@ -67,12 +68,12 @@ def cli(ctx, **options): "Please use `platformio lib --global %s` command to remove " "this warning." % ctx.invoked_subcommand, fg="yellow") - elif util.is_platformio_project(storage_dir): + elif is_platformio_project(storage_dir): with util.cd(storage_dir): - storage_dir = util.get_projectlibdeps_dir() + storage_dir = get_projectlibdeps_dir() - if not storage_dir and not util.is_platformio_project(): - raise exception.NotGlobalLibDir(util.get_project_dir(), + if not storage_dir and not is_platformio_project(): + raise exception.NotGlobalLibDir(get_project_dir(), join(util.get_home_dir(), "lib"), ctx.invoked_subcommand) @@ -211,7 +212,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): for value in values: query.append('%s:"%s"' % (key, value)) - result = get_api_result( + result = util.get_api_result( "/v2/lib/search", dict(query=" ".join(query), page=page), cache_valid="1d") @@ -258,7 +259,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): time.sleep(5) elif not click.confirm("Show next libraries?"): break - result = get_api_result( + result = util.get_api_result( "/v2/lib/search", { "query": " ".join(query), "page": int(result['page']) + 1 @@ -317,7 +318,7 @@ def lib_show(library, json_output): }, silent=json_output, interactive=not json_output) - lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") + lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") if json_output: return click.echo(json.dumps(lib)) @@ -393,7 +394,8 @@ def lib_register(config_url): and not config_url.startswith("https://")): raise exception.InvalidLibConfURL(config_url) - result = get_api_result("/lib/register", data=dict(config_url=config_url)) + result = util.get_api_result( + "/lib/register", data=dict(config_url=config_url)) if "message" in result and result['message']: click.secho( result['message'], @@ -404,7 +406,7 @@ def lib_register(config_url): @cli.command("stats", short_help="Library Registry Statistics") @click.option("--json-output", is_flag=True) def lib_stats(json_output): - result = get_api_result("/lib/stats", cache_valid="1h") + result = util.get_api_result("/lib/stats", cache_valid="1h") if json_output: return click.echo(json.dumps(result)) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index baff6a54..17f088da 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -12,14 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from hashlib import sha1 -from os import getcwd, makedirs, walk +from os import getcwd, makedirs from os.path import getmtime, isdir, isfile, join from time import time import click -from platformio import __version__, exception, telemetry, util +from platformio import exception, telemetry, util from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.commands.lib import lib_install as cmd_lib_install from platformio.commands.platform import \ @@ -27,6 +26,9 @@ from platformio.commands.platform import \ from platformio.managers.lib import LibraryManager, is_builtin_lib from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig +from platformio.project.helpers import ( + calculate_project_hash, find_project_dir_above, get_project_dir, + get_projectbuild_dir, get_projectlibdeps_dir) # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @@ -53,18 +55,18 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose, disable_auto_clean): # find project directory on upper level if isfile(project_dir): - project_dir = util.find_project_dir_above(project_dir) + project_dir = find_project_dir_above(project_dir) with util.cd(project_dir): # clean obsolete build dir if not disable_auto_clean: try: - _clean_build_dir(util.get_projectbuild_dir()) + _clean_build_dir(get_projectbuild_dir()) except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " "it manually to avoid build issues" % - util.get_projectbuild_dir(force=True), + get_projectbuild_dir(force=True), fg="yellow") config = ProjectConfig.get_instance( @@ -86,9 +88,7 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose, if not silent and results: click.echo() - options = {} - for k, v in config.items(env=envname): - options[k] = v + options = config.items(env=envname, as_dict=True) if "piotest" not in options and "piotest" in ctx.meta: options['piotest'] = ctx.meta['piotest'] @@ -239,7 +239,7 @@ class EnvironmentProcessor(object): def _autoinstall_libdeps(ctx, libraries, verbose=False): if not libraries: return - storage_dir = util.get_projectlibdeps_dir() + storage_dir = get_projectlibdeps_dir() ctx.obj = LibraryManager(storage_dir) if verbose: click.echo("Library Storage: " + storage_dir) @@ -258,9 +258,8 @@ def _clean_build_dir(build_dir): proj_hash = calculate_project_hash() # if project's config is modified - if (isdir(build_dir) - and getmtime(join(util.get_project_dir(), - "platformio.ini")) > getmtime(build_dir)): + if (isdir(build_dir) and getmtime( + join(get_project_dir(), "platformio.ini")) > getmtime(build_dir)): util.rmtree_(build_dir) # check project structure @@ -323,23 +322,3 @@ def check_project_envs(config, environments=None): # FIXME: Remove if unknown: raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) return True - - -def calculate_project_hash(): - check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") - chunks = [__version__] - for d in (util.get_projectsrc_dir(), util.get_projectlib_dir()): - if not isdir(d): - continue - for root, _, files in walk(d): - for f in files: - path = join(root, f) - if path.endswith(check_suffixes): - chunks.append(path) - chunks_to_str = ",".join(sorted(chunks)) - if "windows" in util.get_systype(): - # Fix issue with useless project rebuilding for case insensitive FS. - # A case of disk drive can differ... - chunks_to_str = chunks_to_str.lower() - return sha1( - chunks_to_str if util.PY2 else chunks_to_str.encode()).hexdigest() diff --git a/platformio/exception.py b/platformio/exception.py index 26131a5d..4c4af06f 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -68,13 +68,6 @@ class PlatformNotInstalledYet(PlatformioException): "Use `platformio platform install {0}` command") -class BoardNotDefined(PlatformioException): - - MESSAGE = ( - "You need to specify board ID using `-b` or `--board` option. " - "Supported boards list is available via `platformio boards` command") - - class UnknownBoard(PlatformioException): MESSAGE = "Unknown board ID '{0}'" diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 26774a70..9cbf3028 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -24,6 +24,8 @@ from click.testing import CliRunner from platformio import exception, util from platformio.commands.run import cli as cmd_run from platformio.project.config import ProjectConfig +from platformio.project.helpers import ( + get_projectlib_dir, get_projectlibdeps_dir, get_projectsrc_dir) class ProjectGenerator(object): @@ -50,9 +52,8 @@ class ProjectGenerator(object): for env in config.envs(): if self.env_name != env: continue - data = {"env_name": self.env_name} - for k, v in config.items(env=env): - data[k] = v + data = config.items(env=env, as_dict=True) + data['env_name'] = self.env_name return data def get_project_build_data(self): @@ -89,7 +90,7 @@ class ProjectGenerator(object): def get_src_files(self): result = [] with util.cd(self.project_dir): - for root, _, files in os.walk(util.get_projectsrc_dir()): + for root, _, files in os.walk(get_projectsrc_dir()): for f in files: result.append(relpath(join(root, f))) return result @@ -141,9 +142,9 @@ class ProjectGenerator(object): "src_files": self.get_src_files(), "user_home_dir": abspath(expanduser("~")), "project_dir": self.project_dir, - "project_src_dir": util.get_projectsrc_dir(), - "project_lib_dir": util.get_projectlib_dir(), - "project_libdeps_dir": util.get_projectlibdeps_dir(), + "project_src_dir": get_projectsrc_dir(), + "project_lib_dir": get_projectlib_dir(), + "project_libdeps_dir": get_projectlibdeps_dir(), "systype": util.get_systype(), "platformio_path": self._fix_os_path( sys.argv[0] if isfile(sys.argv[0]) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 851011a9..bfcda3f2 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -25,6 +25,7 @@ import semantic_version from platformio import __version__, app, exception, util from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager +from platformio.project.helpers import get_projectboards_dir try: from urllib.parse import quote @@ -566,7 +567,7 @@ class PlatformBase( # pylint: disable=too-many-public-methods self._BOARDS_CACHE[board_id] = config bdirs = [ - util.get_projectboards_dir(), + get_projectboards_dir(), join(util.get_home_dir(), "boards"), join(self.get_dir(), "boards"), ] diff --git a/platformio/project/config.py b/platformio/project/config.py index 46a99425..737ea89a 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -26,8 +26,7 @@ try: except ImportError: import configparser as ConfigParser -CONFIG_HEADER = """ -; PlatformIO Project Configuration File +CONFIG_HEADER = """;PlatformIO Project Configuration File ; ; Build options: build flags, source filter ; Upload options: custom upload port, speed and extra flags @@ -36,6 +35,7 @@ CONFIG_HEADER = """ ; ; Please visit documentation for the other options and examples ; https://docs.platformio.org/page/projectconf.html + """ KNOWN_PLATFORMIO_OPTIONS = [ @@ -200,10 +200,15 @@ class ProjectConfig(object): section = "env:" + env return self._parser.options(section) - def items(self, section=None, env=None): + def items(self, section=None, env=None, as_dict=False): assert section or env if not section: section = "env:" + env + if as_dict: + return { + option: self.get(section, option) + for option in self.options(section) + } return [(option, self.get(section, option)) for option in self.options(section)] @@ -298,7 +303,6 @@ class ProjectConfig(object): return True def save(self, path=None): - with open(path or self.path, "wb") as fp: - fp.write(CONFIG_HEADER.strip()) - fp.write("\n\n") + with open(path or self.path, "w") as fp: + fp.write(CONFIG_HEADER) self._parser.write(fp) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py new file mode 100644 index 00000000..d7c19595 --- /dev/null +++ b/platformio/project/helpers.py @@ -0,0 +1,139 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +from hashlib import sha1 +from os import walk +from os.path import abspath, dirname, expanduser, isdir, isfile, join + +from platformio import __version__ +from platformio.project.config import ProjectConfig + +PY2 = sys.version_info[0] == 2 + + +def get_project_dir(): + return os.getcwd() + + +def is_platformio_project(project_dir=None): + if not project_dir: + project_dir = get_project_dir() + return isfile(join(project_dir, "platformio.ini")) + + +def find_project_dir_above(path): + if isfile(path): + path = dirname(path) + if is_platformio_project(path): + return path + if isdir(dirname(path)): + return find_project_dir_above(dirname(path)) + return None + + +def get_project_optional_dir(name, default=None): + paths = None + var_name = "PLATFORMIO_%s" % name.upper() + if var_name in os.environ: + paths = os.getenv(var_name) + else: + config = ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini")) + if (config.has_section("platformio") + and config.has_option("platformio", name)): + paths = config.get("platformio", name) + if not paths: + return default + + items = [] + for item in paths.split(", "): + if item.startswith("~"): + item = expanduser(item) + items.append(abspath(item)) + paths = ", ".join(items) + + while "$PROJECT_HASH" in paths: + project_dir = get_project_dir() + paths = paths.replace( + "$PROJECT_HASH", + sha1(project_dir if PY2 else project_dir.encode()).hexdigest() + [:10]) + + return paths + + +def get_projectlib_dir(): + return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) + + +def get_projectlibdeps_dir(): + return get_project_optional_dir("libdeps_dir", + join(get_project_dir(), ".piolibdeps")) + + +def get_projectsrc_dir(): + return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) + + +def get_projectinclude_dir(): + return get_project_optional_dir("include_dir", + join(get_project_dir(), "include")) + + +def get_projecttest_dir(): + return get_project_optional_dir("test_dir", join(get_project_dir(), + "test")) + + +def get_projectboards_dir(): + return get_project_optional_dir("boards_dir", + join(get_project_dir(), "boards")) + + +def get_projectbuild_dir(force=False): + path = get_project_optional_dir("build_dir", + join(get_project_dir(), ".pioenvs")) + try: + if not isdir(path): + os.makedirs(path) + except Exception as e: # pylint: disable=broad-except + if not force: + raise Exception(e) + return path + + +def get_projectdata_dir(): + return get_project_optional_dir("data_dir", join(get_project_dir(), + "data")) + + +def calculate_project_hash(): + check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") + chunks = [__version__] + for d in (get_projectsrc_dir(), get_projectlib_dir()): + if not isdir(d): + continue + for root, _, files in walk(d): + for f in files: + path = join(root, f) + if path.endswith(check_suffixes): + chunks.append(path) + chunks_to_str = ",".join(sorted(chunks)) + if sys.platform == "win32": + # Fix issue with useless project rebuilding for case insensitive FS. + # A case of disk drive can differ... + chunks_to_str = chunks_to_str.lower() + return sha1(chunks_to_str if PY2 else chunks_to_str.encode()).hexdigest() diff --git a/platformio/util.py b/platformio/util.py index 8df17ec9..a68d7607 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-ancestors + import json import os import platform @@ -23,7 +25,6 @@ import sys import time from functools import wraps from glob import glob -from hashlib import sha1 from os.path import (abspath, basename, dirname, expanduser, isdir, isfile, join, normpath, splitdrive) from shutil import rmtree @@ -34,8 +35,12 @@ import requests from platformio import __apiurl__, __version__, exception from platformio.project.config import ProjectConfig +from platformio.project.helpers import ( # pylint: disable=unused-import + get_project_dir, get_project_optional_dir, get_projectboards_dir, + get_projectbuild_dir, get_projectlib_dir, get_projectsrc_dir, + get_projecttest_dir, is_platformio_project) -# pylint: disable=too-many-ancestors +# FIXME: check platformio.project.helpers imports PY2 = sys.version_info[0] == 2 if PY2: @@ -173,37 +178,6 @@ def pioversion_to_intstr(): return [int(i) for i in vermatch.group(1).split(".")[:3]] -def get_project_optional_dir(name, default=None): - paths = None - var_name = "PLATFORMIO_%s" % name.upper() - if var_name in os.environ: - paths = os.getenv(var_name) - else: - config = ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini")) - if (config.has_section("platformio") - and config.has_option("platformio", name)): - paths = config.get("platformio", name) - if not paths: - return default - - items = [] - for item in paths.split(", "): - if item.startswith("~"): - item = expanduser(item) - items.append(abspath(item)) - paths = ", ".join(items) - - while "$PROJECT_HASH" in paths: - project_dir = get_project_dir() - paths = paths.replace( - "$PROJECT_HASH", - sha1(project_dir if PY2 else project_dir.encode()).hexdigest() - [:10]) - - return paths - - def get_home_dir(): home_dir = get_project_optional_dir("home_dir", join(expanduser("~"), ".platformio")) @@ -240,75 +214,6 @@ def get_source_dir(): return dirname(curpath) -def get_project_dir(): - return os.getcwd() - - -def find_project_dir_above(path): - if isfile(path): - path = dirname(path) - if is_platformio_project(path): - return path - if isdir(dirname(path)): - return find_project_dir_above(dirname(path)) - return None - - -def is_platformio_project(project_dir=None): - if not project_dir: - project_dir = get_project_dir() - return isfile(join(project_dir, "platformio.ini")) - - -def get_projectlib_dir(): - return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) - - -def get_projectlibdeps_dir(): - return get_project_optional_dir("libdeps_dir", - join(get_project_dir(), ".piolibdeps")) - - -def get_projectsrc_dir(): - return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) - - -def get_projectinclude_dir(): - return get_project_optional_dir("include_dir", - join(get_project_dir(), "include")) - - -def get_projecttest_dir(): - return get_project_optional_dir("test_dir", join(get_project_dir(), - "test")) - - -def get_projectboards_dir(): - return get_project_optional_dir("boards_dir", - join(get_project_dir(), "boards")) - - -def get_projectbuild_dir(force=False): - path = get_project_optional_dir("build_dir", - join(get_project_dir(), ".pioenvs")) - try: - if not isdir(path): - os.makedirs(path) - except Exception as e: # pylint: disable=broad-except - if not force: - raise Exception(e) - return path - - -# compatibility with PIO Core+ -get_projectpioenvs_dir = get_projectbuild_dir - - -def get_projectdata_dir(): - return get_project_optional_dir("data_dir", join(get_project_dir(), - "data")) - - def load_project_config(path=None): # FIXME: Remove if not path or isdir(path): path = join(path or get_project_dir(), "platformio.ini") diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index eecb25f3..1d7d699e 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -58,7 +58,7 @@ def test_init_ide_without_board(clirunner, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke(cmd_init, ["--ide", "atom"]) assert result.exit_code == -1 - assert isinstance(result.exception, exception.BoardNotDefined) + assert isinstance(result.exception, exception.ProjectEnvsNotAvailable) def test_init_ide_atom(clirunner, validate_cliresult, tmpdir): From 45d4b926787be6c894d7666df8533ff1cec2f535 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 May 2019 23:00:01 +0300 Subject: [PATCH 120/333] Export get_projectdata_dir for util, fix riscv_gap dev/platform --- platformio/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index a68d7607..49739c14 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -37,8 +37,8 @@ from platformio import __apiurl__, __version__, exception from platformio.project.config import ProjectConfig from platformio.project.helpers import ( # pylint: disable=unused-import get_project_dir, get_project_optional_dir, get_projectboards_dir, - get_projectbuild_dir, get_projectlib_dir, get_projectsrc_dir, - get_projecttest_dir, is_platformio_project) + get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir, + get_projectsrc_dir, get_projecttest_dir, is_platformio_project) # FIXME: check platformio.project.helpers imports From 947e31ca8d41bcbe7b8b0fd112b2fedc127d3b33 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 7 May 2019 23:51:46 +0300 Subject: [PATCH 121/333] Fix some PyLint errors --- .pylintrc | 33 +++++++++++---------------------- platformio/commands/upgrade.py | 3 +-- platformio/exception.py | 2 -- platformio/managers/lib.py | 2 +- platformio/util.py | 4 +--- 5 files changed, 14 insertions(+), 30 deletions(-) diff --git a/.pylintrc b/.pylintrc index 82c3fb7a..180a05b8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,23 +1,12 @@ [MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -# disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating - -disable=missing-docstring,invalid-name,too-few-public-methods,redefined-variable-type,similarities,unsupported-membership-test,unsubscriptable-object,ungrouped-imports,cyclic-import,superfluous-parens,useless-object-inheritance,useless-import-alias,fixme +disable= + missing-docstring, + ungrouped-imports, + invalid-name, + cyclic-import, + duplicate-code, + superfluous-parens, + too-few-public-methods, + useless-object-inheritance, + useless-import-alias, + fixme diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index ec4fe864..c47ff8dc 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -43,11 +43,10 @@ def cli(dev): get_pip_package(to_develop)], ["platformio", "--version"]) cmd = None - r = None + r = {} try: for cmd in cmds: cmd = [util.get_pythonexe_path(), "-m"] + cmd - r = None r = util.exec_command(cmd) # try pip with disabled cache diff --git a/platformio/exception.py b/platformio/exception.py index 4c4af06f..bba8955a 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=not-an-iterable - class PlatformioException(Exception): diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 3ca89d49..4cbb34a7 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -164,7 +164,7 @@ class LibraryManager(BasePkgManager): semver_spec = self.parse_semver_spec( requirements) if requirements else None - item = None + item = {} for v in versions: semver_new = self.parse_semver_version(v['name']) diff --git a/platformio/util.py b/platformio/util.py index 49739c14..8055cc4d 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-ancestors - import json import os import platform @@ -464,7 +462,7 @@ def _get_api_result( auth=None): from platformio.app import get_setting - result = None + result = {} r = None verify_ssl = sys.version_info >= (2, 7, 9) From 693304590c918a66e2c3facf49b3e02eb61e5b3d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 May 2019 12:41:11 +0300 Subject: [PATCH 122/333] Fix PyLint "not-an-iterable" error --- platformio/exception.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/exception.py b/platformio/exception.py index bba8955a..6f1dd4cf 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -19,7 +19,9 @@ class PlatformioException(Exception): def __str__(self): # pragma: no cover if self.MESSAGE: - return self.MESSAGE.format(*self.args) + return self.MESSAGE.format( + *self.args # pylint: disable=not-an-iterable + ) return super(PlatformioException, self).__str__() From 4f98a3fd42e4f0e5ec9dcc9c9ce3f6ae1c2b247f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 May 2019 20:19:39 +0300 Subject: [PATCH 123/333] Share common (global) options between declared build environments using "[env]" section // Resolve #1643 Resolve #790 --- HISTORY.rst | 2 + docs | 2 +- platformio/exception.py | 5 +- platformio/project/config.py | 39 +++++++++++---- tests/test_projectconf.py | 92 +++++++++++++++++++++--------------- 5 files changed, 87 insertions(+), 53 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0ce2bb74..6991bb7d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ PlatformIO 4.0 * Python 3 support (`issue #895 `_) +* Share common (global) options between declared build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) `__ + (`issue #1643 `_) * Include external configuration files with `extra_configs `__ option (`issue #1590 `_) * Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields diff --git a/docs b/docs index 5df8f4b0..45a3b5a3 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5df8f4b066cebfa41e46db058936b2aa8298a70b +Subproject commit 45a3b5a365b9ee651087192f0a42dc876b3afeb8 diff --git a/platformio/exception.py b/platformio/exception.py index 6f1dd4cf..a7713b99 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -19,9 +19,8 @@ class PlatformioException(Exception): def __str__(self): # pragma: no cover if self.MESSAGE: - return self.MESSAGE.format( - *self.args # pylint: disable=not-an-iterable - ) + return self.MESSAGE.format(*self.args # pylint: disable=not-an-iterable + ) return super(PlatformioException, self).__str__() diff --git a/platformio/project/config.py b/platformio/project/config.py index 737ea89a..757ecd0a 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -13,6 +13,7 @@ # limitations under the License. import glob +import json import os import re from os.path import isfile @@ -198,7 +199,22 @@ class ProjectConfig(object): assert section or env if not section: section = "env:" + env - return self._parser.options(section) + options = self._parser.options(section) + + # handle global options from [env] + if ((env or section.startswith("env:")) + and self._parser.has_section("env")): + for option in self._parser.options("env"): + if option not in options: + options.append(option) + + return options + + def has_option(self, section, option): + if self._parser.has_option(section, option): + return True + return (section.startswith("env:") and self._parser.has_section("env") + and self._parser.has_option("env", option)) def items(self, section=None, env=None, as_dict=False): assert section or env @@ -215,24 +231,21 @@ class ProjectConfig(object): def get(self, section, option): if not self.expand_interpolations: return self._parser.get(section, option) + try: value = self._parser.get(section, option) + except ConfigParser.NoOptionError: + value = self._parser.get("env", option) except ConfigParser.Error as e: raise exception.InvalidProjectConf(self.path, str(e)) + if "${" not in value or "}" not in value: return value return self.VARTPL_RE.sub(self._re_sub_handler, value) def _re_sub_handler(self, match): section, option = match.group(1), match.group(2) - if section in ("env", - "sysenv") and not self._parser.has_section(section): - if section == "env": - click.secho( - "Warning! Access to system environment variable via " - "`${{env.{0}}}` is deprecated. Please use " - "`${{sysenv.{0}}}` instead".format(option), - fg="yellow") + if section == "sysenv": return os.getenv(option) return self.get(section, option) @@ -271,7 +284,7 @@ class ProjectConfig(object): # check [env:*] sections for section in self._parser.sections(): - if not section.startswith("env:"): + if section != "env" and not section.startswith("env:"): continue for option in self._parser.options(section): # obsolete @@ -302,6 +315,12 @@ class ProjectConfig(object): return True + def to_json(self): + result = {} + for section in self.sections(): + result[section] = self.items(section, as_dict=True) + return json.dumps(result) + def save(self, path=None): with open(path or self.path, "w") as fp: fp.write(CONFIG_HEADER) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index f6228b88..d58ee0ce 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -18,43 +18,42 @@ from platformio.project.config import ProjectConfig BASE_CONFIG = """ [platformio] -env_default = esp32dev, lolin32 +env_default = base, extra_2 extra_configs = extra_envs.ini extra_debug.ini -[common] +# global options per [env:*] +[env] +monitor_speed = 115200 +lib_deps = Lib1, Lib2 +lib_ignore = ${custom.lib_ignore} + +[custom] debug_flags = -D RELEASE lib_flags = -lc -lm extra_flags = ${sysenv.__PIO_TEST_CNF_EXTRA_FLAGS} +lib_ignore = LibIgnoreCustom -[env:esp-wrover-kit] -platform = espressif32 -framework = espidf -board = esp-wrover-kit -build_flags = ${common.debug_flags} ${common.extra_flags} +[env:base] +build_flags = ${custom.debug_flags} ${custom.extra_flags} """ EXTRA_ENVS_CONFIG = """ -[env:esp32dev] -platform = espressif32 -framework = espidf -board = esp32dev -build_flags = ${common.lib_flags} ${common.debug_flags} +[env:extra_1] +build_flags = ${custom.lib_flags} ${custom.debug_flags} -[env:lolin32] -platform = espressif32 -framework = espidf -board = lolin32 -build_flags = ${common.debug_flags} ${common.extra_flags} +[env:extra_2] +build_flags = ${custom.debug_flags} ${custom.extra_flags} +lib_ignore = ${env.lib_ignore}, Lib3 """ EXTRA_DEBUG_CONFIG = """ -# Override base "common.debug_flags" -[common] +# Override original "custom.debug_flags" +[custom] debug_flags = -D DEBUG=1 -[env:lolin32] +[env:extra_2] build_flags = -Og """ @@ -71,32 +70,47 @@ def test_parser(tmpdir): # sections assert config.sections() == [ - "platformio", "common", "env:esp-wrover-kit", "env:esp32dev", - "env:lolin32" + "platformio", "env", "custom", "env:base", "env:extra_1", "env:extra_2" ] + # envs + assert config.envs() == ["base", "extra_1", "extra_2"] + assert config.default_envs() == ["base", "extra_2"] + + # options + assert config.options(env="base") == [ + "build_flags", "monitor_speed", "lib_deps", "lib_ignore" + ] + + # has_option + assert config.has_option("env:base", "monitor_speed") + assert not config.has_option("custom", "monitor_speed") + # sysenv - assert config.get("common", "extra_flags") == "" + assert config.get("custom", "extra_flags") == "" os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] = "-L /usr/local/lib" - assert config.get("common", "extra_flags") == "-L /usr/local/lib" + assert config.get("custom", "extra_flags") == "-L /usr/local/lib" # get - assert config.get("common", "debug_flags") == "-D DEBUG=1" - assert config.get("env:esp32dev", "build_flags") == "-lc -lm -D DEBUG=1" - assert config.get("env:lolin32", "build_flags") == "-Og" - assert config.get("env:esp-wrover-kit", + assert config.get("custom", "debug_flags") == "-D DEBUG=1" + assert config.get("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1" + assert config.get("env:extra_2", "build_flags") == "-Og" + assert config.get("env:extra_2", "monitor_speed") == "115200" + assert config.get("env:base", "build_flags") == ("-D DEBUG=1 -L /usr/local/lib") # items - assert config.items("common") == [("debug_flags", "-D DEBUG=1"), + assert config.items("custom") == [("debug_flags", "-D DEBUG=1"), ("lib_flags", "-lc -lm"), - ("extra_flags", "-L /usr/local/lib")] - assert config.items(env="esp32dev") == [("platform", "espressif32"), - ("framework", "espidf"), - ("board", "esp32dev"), - ("build_flags", - "-lc -lm -D DEBUG=1")] - - # envs - assert config.envs() == ["esp-wrover-kit", "esp32dev", "lolin32"] - assert config.default_envs() == ["esp32dev", "lolin32"] + ("extra_flags", "-L /usr/local/lib"), + ("lib_ignore", "LibIgnoreCustom")] + assert config.items(env="extra_1") == [("build_flags", + "-lc -lm -D DEBUG=1"), + ("monitor_speed", "115200"), + ("lib_deps", "Lib1, Lib2"), + ("lib_ignore", "LibIgnoreCustom")] + assert config.items(env="extra_2") == [("build_flags", "-Og"), + ("lib_ignore", + "LibIgnoreCustom, Lib3"), + ("monitor_speed", "115200"), + ("lib_deps", "Lib1, Lib2")] From f63fe1699b9dae403242bf6799fa1324fffcc202 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 May 2019 20:20:56 +0300 Subject: [PATCH 124/333] Bump version to 4.0.0a11 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6d00c736..2ce1d9e4 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a10") +VERSION = (4, 0, "0a11") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 7687a0a929d8a60fa30f43d9e7bf0d53b4ff4aa6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 8 May 2019 21:02:23 +0300 Subject: [PATCH 125/333] Fix PyLint "not-an-iterable" error --- platformio/exception.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/exception.py b/platformio/exception.py index a7713b99..ac7288cf 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -19,8 +19,9 @@ class PlatformioException(Exception): def __str__(self): # pragma: no cover if self.MESSAGE: - return self.MESSAGE.format(*self.args # pylint: disable=not-an-iterable - ) + # pylint: disable=not-an-iterable + return self.MESSAGE.format(*self.args) + return super(PlatformioException, self).__str__() From 62b80c396bc2eb669430acb41b55f9fecb8a4283 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 May 2019 00:51:28 +0300 Subject: [PATCH 126/333] Added support for the latest Python "Click" package (CLI Builder) // Resolve #349 --- HISTORY.rst | 2 ++ platformio/__main__.py | 38 +------------------------- platformio/commands/__init__.py | 48 +++++++++++++++++++++++++++++++++ platformio/commands/lib.py | 3 ++- platformio/maintenance.py | 20 +++++++------- platformio/managers/core.py | 2 +- platformio/telemetry.py | 9 ++++--- setup.py | 2 +- tests/commands/test_init.py | 2 +- tests/commands/test_lib.py | 3 +++ tests/commands/test_platform.py | 4 +-- 11 files changed, 75 insertions(+), 58 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6991bb7d..77c93c52 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,8 @@ PlatformIO 4.0 * Include external configuration files with `extra_configs `__ option (`issue #1590 `_) * Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields +* Added support for the latest Python "Click" package (CLI Builder) + (`issue #349 `_) PlatformIO 3.0 -------------- diff --git a/platformio/__main__.py b/platformio/__main__.py index 61db78de..9138fe3c 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -14,49 +14,13 @@ import os import sys -from os.path import join from platform import system from traceback import format_exc import click from platformio import __version__, exception, maintenance -from platformio.util import get_source_dir - - -class PlatformioCLI(click.MultiCommand): # pylint: disable=R0904 - - def list_commands(self, ctx): - cmds = [] - for filename in os.listdir(join(get_source_dir(), "commands")): - if filename.startswith("__init__"): - continue - if filename.endswith(".py"): - cmds.append(filename[:-3]) - cmds.sort() - return cmds - - def get_command(self, ctx, cmd_name): - mod = None - try: - mod = __import__("platformio.commands." + cmd_name, None, None, - ["cli"]) - except ImportError: - try: - return self._handle_obsolate_command(cmd_name) - except AttributeError: - raise click.UsageError('No such command "%s"' % cmd_name, ctx) - return mod.cli - - @staticmethod - def _handle_obsolate_command(name): - if name == "platforms": - from platformio.commands import platform - return platform.cli - if name == "serialports": - from platformio.commands import device - return device.cli - raise AttributeError() +from platformio.commands import PlatformioCLI @click.command( diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index b0514903..48dd394e 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -11,3 +11,51 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import os +from os.path import dirname, join + +import click + + +class PlatformioCLI(click.MultiCommand): + + leftover_args = [] + + def invoke(self, ctx): + PlatformioCLI.leftover_args = ctx.args + if hasattr(ctx, "protected_args"): + PlatformioCLI.leftover_args = ctx.protected_args + ctx.args + return super(PlatformioCLI, self).invoke(ctx) + + def list_commands(self, ctx): + cmds = [] + for filename in os.listdir(join(dirname(__file__), "commands")): + if filename.startswith("__init__"): + continue + if filename.endswith(".py"): + cmds.append(filename[:-3]) + cmds.sort() + return cmds + + def get_command(self, ctx, cmd_name): + mod = None + try: + mod = __import__("platformio.commands." + cmd_name, None, None, + ["cli"]) + except ImportError: + try: + return self._handle_obsolate_command(cmd_name) + except AttributeError: + raise click.UsageError('No such command "%s"' % cmd_name, ctx) + return mod.cli + + @staticmethod + def _handle_obsolate_command(name): + if name == "platforms": + from platformio.commands import platform + return platform.cli + if name == "serialports": + from platformio.commands import device + return device.cli + raise AttributeError() diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 7caf1271..c52350b1 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -21,6 +21,7 @@ from os.path import isdir, join import click from platformio import exception, util +from platformio.commands import PlatformioCLI from platformio.managers.lib import LibraryManager, get_builtin_libs from platformio.project.helpers import ( get_project_dir, get_projectlibdeps_dir, is_platformio_project) @@ -78,7 +79,7 @@ def cli(ctx, **options): ctx.invoked_subcommand) ctx.obj = LibraryManager(storage_dir) - if "--json-output" not in ctx.args: + if "--json-output" not in PlatformioCLI.leftover_args: click.echo("Library Storage: " + storage_dir) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 47105a5b..8841b7f5 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -22,6 +22,7 @@ import click import semantic_version from platformio import __version__, app, exception, telemetry, util +from platformio.commands import PlatformioCLI from platformio.commands.lib import lib_update as cmd_lib_update from platformio.commands.platform import \ platform_install as cmd_platform_install @@ -40,12 +41,12 @@ def on_platformio_start(ctx, force, caller): set_caller(caller) telemetry.on_command() - if not in_silence(ctx): + if not in_silence(): after_upgrade(ctx) -def on_platformio_end(ctx, result): # pylint: disable=W0613 - if in_silence(ctx): +def on_platformio_end(ctx, result): # pylint: disable=unused-argument + if in_silence(): return try: @@ -64,14 +65,11 @@ def on_platformio_exception(e): telemetry.on_exception(e) -def in_silence(ctx=None): - ctx = ctx or app.get_session_var("command_ctx") - if not ctx: - return True - return ctx.args and any([ - ctx.args[0] == "debug" and "--interpreter" in " ".join(ctx.args), - ctx.args[0] == "upgrade", "--json-output" in ctx.args, - "--version" in ctx.args +def in_silence(): + args = PlatformioCLI.leftover_args + return args and any([ + args[0] == "debug" and "--interpreter" in " ".join(args), + args[0] == "upgrade", "--json-output" in args, "--version" in args ]) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 05822f7a..be9e3a1d 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -27,7 +27,7 @@ CORE_PACKAGES = { "contrib-piohome": "^2.0.1", "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.1.4", + "tool-pioplus": "^2.1.5", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if util.PY2 else "~3.30005.0" } diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 12e4a034..987d5899 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -27,6 +27,7 @@ import click import requests from platformio import __version__, app, exception, util +from platformio.commands import PlatformioCLI try: import queue @@ -133,10 +134,10 @@ class MeasurementProtocol(TelemetryBase): return _arg return None - if not app.get_session_var("command_ctx"): - return - ctx_args = app.get_session_var("command_ctx").args - args = [str(s).lower() for s in ctx_args if not str(s).startswith("-")] + args = [ + str(arg).lower() for arg in PlatformioCLI.leftover_args + if not str(arg).startswith("-") + ] if not args: return cmd_path = args[:1] diff --git a/setup.py b/setup.py index 667a2842..eb667d72 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ from platformio import (__author__, __description__, __email__, __license__, install_requires = [ "bottle<0.13", - "click>=5,<6", + "click>=5,<8", "colorama", "pyserial>=3,<4,!=3.3", "requests>=2.4.0,<3", diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 1d7d699e..4e8eac09 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -57,7 +57,7 @@ def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir): def test_init_ide_without_board(clirunner, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke(cmd_init, ["--ide", "atom"]) - assert result.exit_code == -1 + assert result.exit_code != 0 assert isinstance(result.exception, exception.ProjectEnvsNotAvailable) diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index b67cec39..47e5a709 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -16,8 +16,11 @@ import json import re from platformio import exception +from platformio.commands import PlatformioCLI from platformio.commands.lib import cli as cmd_lib +PlatformioCLI.leftover_args = ["--json-output"] # hook for click + def test_search(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["search", "DHT22"]) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index d327ef37..72941574 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -38,14 +38,14 @@ def test_search_raw_output(clirunner, validate_cliresult): def test_install_unknown_version(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["atmelavr@99.99.99"]) - assert result.exit_code == -1 + assert result.exit_code != 0 assert isinstance(result.exception, exception.UndefinedPackageVersion) def test_install_unknown_from_registry(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["unknown-platform"]) - assert result.exit_code == -1 + assert result.exit_code != 0 assert isinstance(result.exception, exception.UnknownPackage) From 71f606912a8dc28196f567c524c09783a7fcb6c3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 May 2019 14:14:19 +0300 Subject: [PATCH 127/333] Implement ProjectConfig.getlist() --- platformio/project/config.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index 757ecd0a..d2e77bb7 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -189,8 +189,7 @@ class ProjectConfig(object): if (not self._parser.has_section("platformio") or not self._parser.has_option("platformio", "extra_configs")): return - extra_configs = self.parse_multi_values( - self.get("platformio", "extra_configs")) + extra_configs = self.getlist("platformio", "extra_configs") for pattern in extra_configs: for item in glob.glob(pattern): self.read(item) @@ -249,13 +248,16 @@ class ProjectConfig(object): return os.getenv(option) return self.get(section, option) + def getlist(self, section, option): + return self.parse_multi_values(self.get(section, option)) + def envs(self): return [s[4:] for s in self._parser.sections() if s.startswith("env:")] def default_envs(self): if not self._parser.has_option("platformio", "env_default"): return [] - return self.parse_multi_values(self.get("platformio", "env_default")) + return self.getlist("platformio", "env_default") def validate(self, envs=None): if not isfile(self.path): From 32d317d3cb9d6ab3a07f65ed4e0e6bd75f2a14f9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 May 2019 18:39:27 +0300 Subject: [PATCH 128/333] Fix PlatformIO CLI --- platformio/commands/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index 48dd394e..83c88e11 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. import os -from os.path import dirname, join +from os.path import dirname import click @@ -30,7 +30,7 @@ class PlatformioCLI(click.MultiCommand): def list_commands(self, ctx): cmds = [] - for filename in os.listdir(join(dirname(__file__), "commands")): + for filename in os.listdir(dirname(__file__)): if filename.startswith("__init__"): continue if filename.endswith(".py"): From 2b5ac57fd02e5e20f738f9060456542f69eeff95 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 9 May 2019 18:40:19 +0300 Subject: [PATCH 129/333] Bump version to 4.0.0a12 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 2ce1d9e4..232433f9 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a11") +VERSION = (4, 0, "0a12") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From c79b3ff7f19284be29977e07c1c6aade8a2ccf66 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 00:01:10 +0300 Subject: [PATCH 130/333] =?UTF-8?q?Override=20default=20=E2=80=9Cplatformi?= =?UTF-8?q?o.ini=E2=80=9D=20with=20a=20custom=20using=20"-c,=20--project-c?= =?UTF-8?q?onf"=20option=20//=20Resolve=20#1913?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 4 +++- docs | 2 +- platformio/commands/ci.py | 2 +- platformio/commands/run.py | 15 ++++++++++++--- platformio/managers/core.py | 2 +- tests/commands/test_lib.py | 31 +++++++++++++------------------ tests/test_misc.py | 6 ++++++ 7 files changed, 37 insertions(+), 25 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 77c93c52..f5a953f6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,8 +11,10 @@ PlatformIO 4.0 (`issue #895 `_) * Share common (global) options between declared build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) `__ (`issue #1643 `_) -* Include external configuration files with `extra_configs `__ option +* Include external configuration files in `"platformio.ini" (Project Configuration File) `__ with `extra_configs `__ option (`issue #1590 `_) +* Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands + (`issue #1913 `_) * Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields * Added support for the latest Python "Click" package (CLI Builder) (`issue #349 `_) diff --git a/docs b/docs index 45a3b5a3..c5a33bc0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 45a3b5a365b9ee651087192f0a42dc876b3afeb8 +Subproject commit c5a33bc006dfedfa2d1a2ab1a340615467b885fd diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index f9e93ace..9fd157e3 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -59,7 +59,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument file_okay=False, dir_okay=True, writable=True, resolve_path=True)) @click.option("--keep-build-dir", is_flag=True) @click.option( - "-C", + "-c", "--project-conf", type=click.Path( exists=True, diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 17f088da..fe497f3b 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -47,12 +47,21 @@ from platformio.project.helpers import ( dir_okay=True, writable=True, resolve_path=True)) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--disable-auto-clean", is_flag=True) @click.pass_context -def cli(ctx, environment, target, upload_port, project_dir, silent, verbose, - disable_auto_clean): +def cli(ctx, environment, target, upload_port, project_dir, project_conf, + silent, verbose, disable_auto_clean): # find project directory on upper level if isfile(project_dir): project_dir = find_project_dir_above(project_dir) @@ -70,7 +79,7 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose, fg="yellow") config = ProjectConfig.get_instance( - join(project_dir, "platformio.ini")) + project_conf or join(project_dir, "platformio.ini")) config.validate() results = [] diff --git a/platformio/managers/core.py b/platformio/managers/core.py index be9e3a1d..0dd84552 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -27,7 +27,7 @@ CORE_PACKAGES = { "contrib-piohome": "^2.0.1", "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.1.5", + "tool-pioplus": "^2.2.0", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if util.PY2 else "~3.30005.0" } diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 47e5a709..4b75d81b 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -61,7 +61,6 @@ def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_lib, [ "-g", "install", - "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip", "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", "SomeLib=http://dl.platformio.org/libraries/archives/0/9540.tar.gz", @@ -77,10 +76,7 @@ def test_global_install_archive(clirunner, validate_cliresult, assert result.exit_code != 0 items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] - items2 = [ - "RadioHead-1.62", "ArduinoJson", "SomeLib_ID54", - "OneWire_ID1", "ESP32WebServer" - ] + items2 = ["ArduinoJson", "SomeLib_ID54", "OneWire_ID1", "ESP32WebServer"] assert set(items1) >= set(items2) @@ -126,7 +122,7 @@ def test_install_duplicates(clirunner, validate_cliresult, without_internet): # archive result = clirunner.invoke(cmd_lib, [ "-g", "install", - "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip" + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip" ]) validate_cliresult(result) assert "is already installed" in result.output @@ -148,7 +144,7 @@ def test_global_lib_list(clirunner, validate_cliresult): ("Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", "Version: 5.10.1", "Source: git+https://github.com/gioblu/PJON.git#3.0", - "Version: 1fb26fd", "RadioHead-1.62") + "Version: 1fb26fd") ]) result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) @@ -161,10 +157,9 @@ def test_global_lib_list(clirunner, validate_cliresult): items1 = [i['name'] for i in json.loads(result.output)] items2 = [ "ESP32WebServer", "ArduinoJson", "ArduinoJson", "ArduinoJson", - "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "SomeLib", - "ESPAsyncTCP", "NeoPixelBus", "OneWire", "PJON", "PJON", - "PubSubClient", "RFcontrol", "RadioHead-1.62", "platformio-libmirror", - "rs485-nodeproto" + "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "SomeLib", "ESPAsyncTCP", + "NeoPixelBus", "OneWire", "PJON", "PJON", "PubSubClient", "RFcontrol", + "platformio-libmirror", "rs485-nodeproto" ] assert sorted(items1) == sorted(items2) @@ -172,9 +167,9 @@ def test_global_lib_list(clirunner, validate_cliresult): "{name}@{version}".format(**item) for item in json.loads(result.output) ] versions2 = [ - 'ArduinoJson@5.8.2', 'ArduinoJson@5.10.1', 'AsyncMqttClient@0.8.2', - 'NeoPixelBus@2.2.4', 'PJON@07fe9aa', 'PJON@1fb26fd', - 'PubSubClient@bef5814', 'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0' + "ArduinoJson@5.8.2", "ArduinoJson@5.10.1", "AsyncMqttClient@0.8.2", + "NeoPixelBus@2.2.4", "PJON@07fe9aa", "PJON@1fb26fd", + "PubSubClient@bef5814", "RFcontrol@77d4eb3f8a" ] assert set(versions1) >= set(versions2) @@ -205,7 +200,7 @@ def test_global_lib_update(clirunner, validate_cliresult): # update rest libraries result = clirunner.invoke(cmd_lib, ["-g", "update"]) validate_cliresult(result) - assert result.output.count("[Detached]") == 6 + assert result.output.count("[Detached]") == 5 assert result.output.count("[Up-to-date]") == 11 assert "Uninstalling RFcontrol @ 77d4eb3f8a" in result.output @@ -235,10 +230,10 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ - "RadioHead-1.62", "rs485-nodeproto", "platformio-libmirror", + "rs485-nodeproto", "platformio-libmirror", "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", - "ESPAsyncTCP_ID305", "SomeLib_ID54", "NeoPixelBus_ID547", - "PJON", "AsyncMqttClient_ID346", "ArduinoJson_ID64", + "ESPAsyncTCP_ID305", "SomeLib_ID54", "NeoPixelBus_ID547", "PJON", + "AsyncMqttClient_ID346", "ArduinoJson_ID64", "PJON@src-79de467ebe19de18287becff0a1fb42d", "ESP32WebServer" ] assert set(items1) == set(items2) diff --git a/tests/test_misc.py b/tests/test_misc.py index b918767b..8f6fccf2 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -18,6 +18,12 @@ import requests from platformio import exception, util +def test_platformio_cli(): + result = util.exec_command(["pio", "--help"]) + assert result['returncode'] == 0 + assert "Usage: pio [OPTIONS] COMMAND [ARGS]..." in result['out'] + + def test_ping_internet_ips(): for ip in util.PING_INTERNET_IPS: requests.get("http://%s" % ip, allow_redirects=False, timeout=2) From 131144ec347e051d6998c258fb631392855d6f8a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 13:00:53 +0300 Subject: [PATCH 131/333] Resolve PyLint "import-error" --- platformio/commands/debug/client.py | 5 ++++- platformio/commands/debug/process.py | 2 +- platformio/commands/debug/server.py | 2 +- platformio/commands/home/command.py | 2 ++ platformio/commands/home/helpers.py | 8 ++++---- platformio/commands/home/rpc/handlers/ide.py | 6 +++--- platformio/commands/home/rpc/handlers/misc.py | 4 ++-- platformio/commands/home/rpc/handlers/os.py | 2 +- platformio/commands/home/rpc/handlers/piocore.py | 10 +++++----- platformio/commands/home/rpc/handlers/project.py | 6 +++--- platformio/commands/home/rpc/server.py | 2 ++ platformio/commands/home/web.py | 4 ++-- 12 files changed, 30 insertions(+), 23 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 7e9aebb9..ac97243f 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -20,7 +20,10 @@ from hashlib import sha1 from os.path import abspath, basename, dirname, isdir, join, splitext from tempfile import mkdtemp -from twisted.internet import protocol, reactor, stdio, task +from twisted.internet import protocol # pylint: disable=import-error +from twisted.internet import reactor # pylint: disable=import-error +from twisted.internet import stdio # pylint: disable=import-error +from twisted.internet import task # pylint: disable=import-error from platformio import app, exception, util from platformio.commands.debug import helpers, initcfgs diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index 3f1fbd22..0079e18c 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -15,7 +15,7 @@ import os import click -from twisted.internet import protocol +from twisted.internet import protocol # pylint: disable=import-error from platformio import util from platformio.commands.debug import helpers diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index 977fe812..ca5c2a2e 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -15,7 +15,7 @@ import os from os.path import isdir, isfile, join -from twisted.internet import reactor +from twisted.internet import reactor # pylint: disable=import-error from platformio import exception, util from platformio.commands.debug import helpers diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 0685670a..917187fb 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -33,9 +33,11 @@ from platformio.managers.core import (get_core_package_dir, def cli(port, host, no_open): # import contrib modules inject_contrib_pysite() + # pylint: disable=import-error from autobahn.twisted.resource import WebSocketResource from twisted.internet import reactor from twisted.web import server + # pylint: enable=import-error from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.ide import IDERPC from platformio.commands.home.rpc.handlers.misc import MiscRPC diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index abac9416..20b9503c 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -18,8 +18,9 @@ import os import socket import requests -from twisted.internet import threads -from twisted.internet.defer import ensureDeferred +from twisted.internet import defer # pylint: disable=import-error +from twisted.internet import reactor # pylint: disable=import-error +from twisted.internet import threads # pylint: disable=import-error from platformio import util @@ -28,7 +29,6 @@ class AsyncSession(requests.Session): def __init__(self, n=None, *args, **kwargs): if n: - from twisted.internet import reactor pool = reactor.getThreadPool() pool.adjustPoolsize(0, n) @@ -39,7 +39,7 @@ class AsyncSession(requests.Session): return threads.deferToThread(func, *args, **kwargs) def wrap(self, *args, **kwargs): # pylint: disable=no-self-use - return ensureDeferred(*args, **kwargs) + return defer.ensureDeferred(*args, **kwargs) @util.memoized(expire=5000) diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index d5c2bf74..5f8f31a4 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -14,8 +14,8 @@ import time -from jsonrpc.exceptions import JSONRPCDispatchException -from twisted.internet import defer +import jsonrpc # pylint: disable=import-error +from twisted.internet import defer # pylint: disable=import-error class IDERPC(object): @@ -25,7 +25,7 @@ class IDERPC(object): def send_command(self, command, params): if not self._queue: - raise JSONRPCDispatchException( + raise jsonrpc.exceptions.JSONRPCDispatchException( code=4005, message="PIO Home IDE agent is not started") while self._queue: self._queue.pop().callback({ diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index d7deff19..d3b53383 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -16,8 +16,8 @@ import json import re import time -from bs4 import BeautifulSoup -from twisted.internet import defer, reactor +from bs4 import BeautifulSoup # pylint: disable=import-error +from twisted.internet import defer, reactor # pylint: disable=import-error from platformio import app from platformio.commands.home import helpers diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index f581c469..780dbec8 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -22,7 +22,7 @@ from functools import cmp_to_key from os.path import expanduser, isdir, isfile, join import click -from twisted.internet import defer +from twisted.internet import defer # pylint: disable=import-error from platformio import app, util from platformio.commands.home import helpers diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 85255a9c..80a642cb 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -19,8 +19,8 @@ import os import re import sys -from jsonrpc.exceptions import JSONRPCDispatchException -from twisted.internet.utils import getProcessOutputAndValue +import jsonrpc # pylint: disable=import-error +from twisted.internet import utils # pylint: disable=import-error from platformio import __version__, util from platformio.commands.home import helpers @@ -37,9 +37,9 @@ class PIOCoreRPC(object): arg, util.string_types) else str(arg) for arg in args ] except UnicodeError: - raise JSONRPCDispatchException( + raise jsonrpc.exceptions.JSONRPCDispatchException( code=4002, message="PIO Core: non-ASCII chars in arguments") - d = getProcessOutputAndValue( + d = utils.getProcessOutputAndValue( helpers.get_core_fullpath(), args, path=(options or {}).get("cwd"), @@ -73,7 +73,7 @@ class PIOCoreRPC(object): @staticmethod def _call_errback(failure): - raise JSONRPCDispatchException( + raise jsonrpc.exceptions.JSONRPCDispatchException( code=4003, message="PIO Core Call Error", data=failure.getErrorMessage()) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 02961ed8..2a4c4032 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -21,7 +21,7 @@ import time from os.path import (basename, expanduser, getmtime, isdir, isfile, join, realpath, sep) -from jsonrpc.exceptions import JSONRPCDispatchException +import jsonrpc # pylint: disable=import-error from platformio import exception, util from platformio.commands.home.rpc.handlers.app import AppRPC @@ -189,7 +189,7 @@ class ProjectRPC(object): for ext in ("ino", "pde") ]) if not is_arduino_project: - raise JSONRPCDispatchException( + raise jsonrpc.exceptions.JSONRPCDispatchException( code=4000, message="Not an Arduino project: %s" % arduino_project_dir) @@ -265,7 +265,7 @@ class ProjectRPC(object): @staticmethod def import_pio(project_dir): if not project_dir or not util.is_platformio_project(project_dir): - raise JSONRPCDispatchException( + raise jsonrpc.exceptions.JSONRPCDispatchException( code=4001, message="Not an PlatformIO project: %s" % project_dir) new_project_dir = join( diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index cd4f05b9..bf339994 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=import-error + import json import jsonrpc diff --git a/platformio/commands/home/web.py b/platformio/commands/home/web.py index 313ce084..df48b1fa 100644 --- a/platformio/commands/home/web.py +++ b/platformio/commands/home/web.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from twisted.internet import reactor -from twisted.web import static +from twisted.internet import reactor # pylint: disable=import-error +from twisted.web import static # pylint: disable=import-error class WebRoot(static.File): From 76818448e29c029f4524dd90bdc2d844b794326b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 13:01:52 +0300 Subject: [PATCH 132/333] Ensure PIO Home mimetypes are known --- platformio/commands/home/command.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 917187fb..d1b7d607 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import mimetypes import socket from os.path import isdir @@ -58,6 +59,12 @@ def cli(port, host, no_open): contrib_dir = get_core_package_dir("contrib-piohome") if not isdir(contrib_dir): raise exception.PlatformioException("Invalid path to PIO Home Contrib") + + # Ensure PIO Home mimetypes are known + mimetypes.add_type("text/html", ".html") + mimetypes.add_type("text/css", ".css") + mimetypes.add_type("application/javascript", ".js") + root = WebRoot(contrib_dir) root.putChild(b"wsrpc", WebSocketResource(factory)) site = server.Site(root) From d9b0364aa8e8e5c1d20c1f396a8b30ec1b3a3439 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 13:12:41 +0300 Subject: [PATCH 133/333] Allow overriding a default project "platformio.ini" configuration file --- platformio/commands/debug/command.py | 18 +++++++++++++++--- platformio/commands/debug/helpers.py | 8 ++++---- platformio/commands/test/command.py | 15 ++++++++++++--- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 918aeb60..c604d8b9 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -39,12 +39,22 @@ from platformio.managers.core import inject_contrib_pysite dir_okay=True, writable=True, resolve_path=True)) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) @click.option("--environment", "-e", metavar="") @click.option("--verbose", "-v", is_flag=True) @click.option("--interface", type=click.Choice(["gdb"])) @click.argument("__unprocessed", nargs=-1, type=click.UNPROCESSED) @click.pass_context -def cli(ctx, project_dir, environment, verbose, interface, __unprocessed): +def cli(ctx, project_dir, project_conf, environment, verbose, interface, + __unprocessed): try: util.ensure_udev_rules() except NameError: @@ -59,8 +69,10 @@ def cli(ctx, project_dir, environment, verbose, interface, __unprocessed): project_dir = os.getenv("CWD") with util.cd(project_dir): - env_name = helpers.check_env_name(project_dir, environment) - env_options = helpers.get_env_options(project_dir, env_name) + env_name = helpers.check_env_name(project_conf or project_dir, + environment) + env_options = helpers.get_env_options(project_conf or project_dir, + env_name) if not set(env_options.keys()) >= set(["platform", "board"]): raise exception.ProjectEnvsNotAvailable() debug_options = helpers.validate_debug_options(ctx, env_options) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 42a4ef40..757d3ad9 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -46,8 +46,8 @@ def escape_path(path): return path.replace("\\", "/") -def check_env_name(project_dir, environment): - config = util.load_project_config(project_dir) +def check_env_name(project_conf, environment): + config = util.load_project_config(project_conf) envs = [] for section in config.sections(): if section.startswith("env:"): @@ -63,8 +63,8 @@ def check_env_name(project_dir, environment): return envs[0] -def get_env_options(project_dir, environment): - config = util.load_project_config(project_dir) +def get_env_options(project_conf, environment): + config = util.load_project_config(project_conf) options = {} for k, v in config.items("env:%s" % environment): options[k] = v diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index a4a3d627..47f8571f 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -53,6 +53,15 @@ from platformio.commands.test.native import NativeTestProcessor dir_okay=True, writable=True, resolve_path=True)) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @click.option("--without-testing", is_flag=True) @@ -71,14 +80,14 @@ from platformio.commands.test.native import NativeTestProcessor @click.pass_context def cli( # pylint: disable=redefined-builtin ctx, environment, ignore, filter, upload_port, test_port, project_dir, - without_building, without_uploading, without_testing, no_reset, - monitor_rts, monitor_dtr, verbose): + project_conf, without_building, without_uploading, without_testing, + no_reset, monitor_rts, monitor_dtr, verbose): with util.cd(project_dir): test_dir = util.get_projecttest_dir() if not isdir(test_dir): raise exception.TestDirNotExists(test_dir) test_names = get_test_names(test_dir) - projectconf = util.load_project_config() + projectconf = util.load_project_config(project_conf) env_default = None if projectconf.has_option("platformio", "env_default"): env_default = util.parse_conf_multi_values( From 0b2d7806183c817fbb7854c2245cb7b538fe09b2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 15:45:52 +0300 Subject: [PATCH 134/333] Switch to the new ProjectConfig API --- platformio/commands/debug/command.py | 13 +++--- platformio/commands/debug/helpers.py | 26 ++---------- .../commands/home/rpc/handlers/project.py | 19 ++++----- platformio/commands/test/command.py | 42 ++++++++----------- platformio/commands/test/processor.py | 6 +-- 5 files changed, 38 insertions(+), 68 deletions(-) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index c604d8b9..7e1bf62e 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -16,13 +16,14 @@ # pylint: disable=too-many-locals, too-many-branches import os -from os.path import isfile +from os.path import isfile, join import click from platformio import exception, util from platformio.commands.debug import helpers from platformio.managers.core import inject_contrib_pysite +from platformio.project.config import ProjectConfig @click.command( @@ -69,10 +70,12 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, project_dir = os.getenv("CWD") with util.cd(project_dir): - env_name = helpers.check_env_name(project_conf or project_dir, - environment) - env_options = helpers.get_env_options(project_conf or project_dir, - env_name) + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(envs=[environment] if environment else None) + + env_name = environment or helpers.get_default_debug_env(config) + env_options = config.items(env=env_name, as_dict=True) if not set(env_options.keys()) >= set(["platform", "board"]): raise exception.ProjectEnvsNotAvailable() debug_options = helpers.validate_debug_options(ctx, env_options) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 757d3ad9..139411ea 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -46,29 +46,9 @@ def escape_path(path): return path.replace("\\", "/") -def check_env_name(project_conf, environment): - config = util.load_project_config(project_conf) - envs = [] - for section in config.sections(): - if section.startswith("env:"): - envs.append(section[4:]) - if not envs: - raise exception.ProjectEnvsNotAvailable() - if not environment and config.has_option("platformio", "env_default"): - environment = config.get("platformio", "env_default").split(", ")[0] - if environment: - if environment in envs: - return environment - raise exception.UnknownEnvNames(environment, envs) - return envs[0] - - -def get_env_options(project_conf, environment): - config = util.load_project_config(project_conf) - options = {} - for k, v in config.items("env:%s" % environment): - options[k] = v - return options +def get_default_debug_env(config): + default_envs = config.default_envs() + return default_envs[0] if default_envs else config.envs()[0] def validate_debug_options(cmd_ctx, env_options): diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 2a4c4032..c3f48fcc 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -28,11 +28,7 @@ from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager - -try: - from configparser import Error as ConfigParserError -except ImportError: - from ConfigParser import Error as ConfigParserError +from platformio.project.config import ProjectConfig class ProjectRPC(object): @@ -42,7 +38,8 @@ class ProjectRPC(object): def _get_project_data(project_dir): data = {"boards": [], "libExtraDirs": []} - config = util.load_project_config(project_dir) + config = ProjectConfig(join(project_dir, "platformio.ini")) + config.validate(validate_options=False) if config.has_section("platformio") and \ config.has_option("platformio", "lib_extra_dirs"): @@ -87,10 +84,8 @@ class ProjectRPC(object): boards = [] try: data = _get_project_data(project_dir) - except exception.NotPlatformIOProject: + except exception.PlatformIOProjectException: continue - except ConfigParserError: - pass for board_id in data.get("boards", []): name = board_id @@ -235,13 +230,13 @@ class ProjectRPC(object): for project_dir, _, __ in os.walk(examples_dir): project_description = None try: - config = util.load_project_config(project_dir) + config = ProjectConfig(join(project_dir, "platformio.ini")) + config.validate(validate_options=False) if config.has_section("platformio") and \ config.has_option("platformio", "description"): project_description = config.get( "platformio", "description") - except (exception.NotPlatformIOProject, - exception.InvalidProjectConf): + except exception.PlatformIOProjectException: continue path_tokens = project_dir.split(sep) diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index 47f8571f..2fc2eca1 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -22,9 +22,10 @@ from time import time import click from platformio import exception, util -from platformio.commands.run import check_project_envs, print_header +from platformio.commands.run import print_header from platformio.commands.test.embedded import EmbeddedTestProcessor from platformio.commands.test.native import NativeTestProcessor +from platformio.project.config import ProjectConfig @click.command("test", short_help="Unit Testing") @@ -87,39 +88,32 @@ def cli( # pylint: disable=redefined-builtin if not isdir(test_dir): raise exception.TestDirNotExists(test_dir) test_names = get_test_names(test_dir) - projectconf = util.load_project_config(project_conf) - env_default = None - if projectconf.has_option("platformio", "env_default"): - env_default = util.parse_conf_multi_values( - projectconf.get("platformio", "env_default")) - assert check_project_envs(projectconf, environment or env_default) + + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(envs=environment) click.echo("Verbose mode can be enabled via `-v, --verbose` option") click.echo("Collected %d items" % len(test_names)) - start_time = time() results = [] + start_time = time() + default_envs = config.default_envs() for testname in test_names: - for section in projectconf.sections(): - if not section.startswith("env:"): - continue + for envname in config.envs(): + section = "env:%s" % envname # filter and ignore patterns patterns = dict(filter=list(filter), ignore=list(ignore)) for key in patterns: - if projectconf.has_option(section, "test_%s" % key): - patterns[key].extend([ - p.strip() - for p in projectconf.get(section, "test_%s" % - key).split(", ") - if p.strip() - ]) + if config.has_option(section, "test_%s" % key): + patterns[key].extend( + config.getlist(section, "test_%s" % key)) - envname = section[4:] skip_conditions = [ environment and envname not in environment, - not environment and env_default - and envname not in env_default, + not environment and default_envs + and envname not in default_envs, testname != "*" and patterns['filter'] and not any([fnmatch(testname, p) for p in patterns['filter']]), @@ -131,13 +125,13 @@ def cli( # pylint: disable=redefined-builtin results.append((None, testname, envname)) continue - cls = (NativeTestProcessor if projectconf.get( - section, "platform") == "native" else + cls = (NativeTestProcessor + if config.get(section, "platform") == "native" else EmbeddedTestProcessor) tp = cls( ctx, testname, envname, dict( - project_config=projectconf, + project_config=config, project_dir=project_dir, upload_port=upload_port, test_port=test_port, diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 7c3b84eb..4a4bc9dd 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -85,10 +85,8 @@ class TestProcessorBase(object): self.test_name = testname self.options = options self.env_name = envname - self.env_options = { - k: v - for k, v in options['project_config'].items("env:" + envname) - } + self.env_options = options['project_config'].items( + env=envname, as_dict=True) self._run_failed = False self._outputcpp_generated = False From 446176bf5e57b53678b8eb271c150c7b76679d4a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 15:47:17 +0300 Subject: [PATCH 135/333] Allow to skip ProjectConfig option validation with new "validate_options" argument --- platformio/project/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index d2e77bb7..d4f82c1b 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -259,7 +259,7 @@ class ProjectConfig(object): return [] return self.getlist("platformio", "env_default") - def validate(self, envs=None): + def validate(self, envs=None, validate_options=True): if not isfile(self.path): raise exception.NotPlatformIOProject(self.path) # check envs @@ -267,11 +267,11 @@ class ProjectConfig(object): if not known: raise exception.ProjectEnvsNotAvailable() - unknown = set((envs or []) + self.default_envs()) - known + unknown = set(list(envs or []) + self.default_envs()) - known if unknown: raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) - return self.validate_options() + return self.validate_options() if validate_options else True def validate_options(self): warnings = set() From b710bbd80ed39472f8e20434c5ee20c35312c9ca Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 15:47:17 +0300 Subject: [PATCH 136/333] Allow to skip ProjectConfig option validation with new "validate_options" argument --- platformio/project/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index d2e77bb7..d4f82c1b 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -259,7 +259,7 @@ class ProjectConfig(object): return [] return self.getlist("platformio", "env_default") - def validate(self, envs=None): + def validate(self, envs=None, validate_options=True): if not isfile(self.path): raise exception.NotPlatformIOProject(self.path) # check envs @@ -267,11 +267,11 @@ class ProjectConfig(object): if not known: raise exception.ProjectEnvsNotAvailable() - unknown = set((envs or []) + self.default_envs()) - known + unknown = set(list(envs or []) + self.default_envs()) - known if unknown: raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) - return self.validate_options() + return self.validate_options() if validate_options else True def validate_options(self): warnings = set() From ce4ed18ceb194f4c086b532a31b33c4be32ecf09 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 15:49:01 +0300 Subject: [PATCH 137/333] Check unknown build environment passed by user --- platformio/commands/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index fe497f3b..a7327d34 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -80,7 +80,7 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, config = ProjectConfig.get_instance( project_conf or join(project_dir, "platformio.ini")) - config.validate() + config.validate(environment) results = [] start_time = time() From 834206ff2090feb58c7b406d6c55478ae2c9f119 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 17:26:10 +0300 Subject: [PATCH 138/333] Move Python or Platform dependent code to "compat" module --- platformio/__main__.py | 4 +-- platformio/app.py | 6 ++-- platformio/builder/tools/piolib.py | 7 ++--- platformio/builder/tools/pioplatform.py | 3 +- platformio/builder/tools/pioupload.py | 4 +-- platformio/builder/tools/piowinhooks.py | 6 ++-- platformio/builder/tools/platformio.py | 3 +- platformio/commands/upgrade.py | 3 +- platformio/compat.py | 42 +++++++++++++++++++++++++ platformio/downloader.py | 9 +++--- platformio/ide/projectgenerator.py | 6 ++-- platformio/managers/core.py | 7 +++-- platformio/managers/lib.py | 5 +-- platformio/managers/package.py | 10 +++--- platformio/managers/platform.py | 3 +- platformio/project/helpers.py | 6 ++-- platformio/util.py | 41 +++++++++--------------- 17 files changed, 98 insertions(+), 67 deletions(-) create mode 100644 platformio/compat.py diff --git a/platformio/__main__.py b/platformio/__main__.py index 9138fe3c..0dc0cca1 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -14,13 +14,13 @@ import os import sys -from platform import system from traceback import format_exc import click from platformio import __version__, exception, maintenance from platformio.commands import PlatformioCLI +from platformio.compat import CYGWIN @click.command( @@ -45,7 +45,7 @@ def process_result(ctx, result, force, caller): # pylint: disable=W0613 def configure(): - if "cygwin" in system().lower(): + if CYGWIN: raise exception.CygwinEnvDetected() # https://urllib3.readthedocs.org diff --git a/platformio/app.py b/platformio/app.py index 247fb65c..c8a071fc 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,6 +25,7 @@ from time import time import requests from platformio import exception, lockfile, util +from platformio.compat import PY2, WINDOWS def projects_dir_validate(projects_dir): @@ -173,7 +174,7 @@ class ContentCache(object): if not arg: continue arg = str(arg) - h.update(arg if util.PY2 else arg.encode()) + h.update(arg if PY2 else arg.encode()) return h.hexdigest() def get(self, key): @@ -361,7 +362,6 @@ def get_cid(): uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(str(uid).encode()).digest()) cid = str(cid) - if ("windows" in util.get_systype() - or os.getuid() > 0): # yapf: disable pylint: disable=no-member + if WINDOWS or os.getuid() > 0: # yapf: disable pylint: disable=no-member set_state_item("cid", cid) return cid diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index e1a0b50b..270f3f21 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -33,6 +33,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, util from platformio.builder.tools import platformio as piotool +from platformio.compat import PY2, WINDOWS from platformio.managers.lib import LibraryManager from platformio.managers.package import PackageManager @@ -91,8 +92,6 @@ class LibBuilderFactory(object): class LibBuilderBase(object): - IS_WINDOWS = "windows" in util.get_systype() - LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"] LDF_MODE_DEFAULT = "chain" @@ -135,7 +134,7 @@ class LibBuilderBase(object): def __contains__(self, path): p1 = self.path p2 = path - if self.IS_WINDOWS: + if WINDOWS: p1 = p1.lower() p2 = p2.lower() return commonprefix((p1 + sep, p2)) == p1 + sep @@ -191,7 +190,7 @@ class LibBuilderBase(object): @property def build_dir(self): lib_hash = hashlib.sha1( - self.path if util.PY2 else self.path.encode()).hexdigest()[:3] + self.path if PY2 else self.path.encode()).hexdigest()[:3] return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) @property diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 6427b33c..cc61a709 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -21,6 +21,7 @@ from os.path import isdir, isfile, join from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import exception, util +from platformio.compat import WINDOWS from platformio.managers.platform import PlatformFactory # pylint: disable=too-many-branches, too-many-locals @@ -79,7 +80,7 @@ def LoadPioPlatform(env, variables): env.PrependENVPath( "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir) - if ("windows" not in systype and isdir(join(pkg_dir, "lib")) + if (not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain"): env.PrependENVPath( "DYLD_LIBRARY_PATH" diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 0d5a55ea..a7b2685d 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -26,6 +26,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from serial import Serial, SerialException from platformio import exception, util +from platformio.compat import WINDOWS # pylint: disable=unused-argument @@ -134,8 +135,7 @@ def AutodetectUploadPort(*args, **kwargs): continue port = item['port'] if upload_protocol.startswith("blackmagic"): - if "windows" in util.get_systype() and \ - port.startswith("COM") and len(port) > 4: + if WINDOWS and port.startswith("COM") and len(port) > 4: port = "\\\\.\\%s" % port if "GDB" in item['description']: return port diff --git a/platformio/builder/tools/piowinhooks.py b/platformio/builder/tools/piowinhooks.py index c893ffd5..1105091f 100644 --- a/platformio/builder/tools/piowinhooks.py +++ b/platformio/builder/tools/piowinhooks.py @@ -18,7 +18,7 @@ from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join -from platformio import util +from platformio.compat import PY2, WINDOWS # Windows CLI has limit with command length to 8192 # Leave 2000 chars for flags and other options @@ -63,7 +63,7 @@ def _file_long_data(env, data): makedirs(build_dir) tmp_file = join( build_dir, - "longcmd-%s" % md5(data if util.PY2 else data.encode()).hexdigest()) + "longcmd-%s" % md5(data if PY2 else data.encode()).hexdigest()) if isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: @@ -76,7 +76,7 @@ def exists(_): def generate(env): - if "windows" not in util.get_systype(): + if not WINDOWS: return None env.Replace(_long_sources_hook=long_sources_hook) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 41f32700..a8272b83 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -27,7 +27,8 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Export # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error -from platformio.util import glob_escape, pioversion_to_intstr, string_types +from platformio.compat import string_types +from platformio.util import glob_escape, pioversion_to_intstr SRC_HEADER_EXT = ["h", "hpp"] SRC_C_EXT = ["c", "cc", "cpp"] diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index c47ff8dc..6acb83ae 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -20,6 +20,7 @@ import click import requests from platformio import VERSION, __version__, exception, util +from platformio.compat import WINDOWS from platformio.managers.core import shutdown_piohome_servers @@ -68,7 +69,7 @@ def cli(dev): raise exception.UpgradeError("\n".join([str(cmd), str(e)])) permission_errors = ("permission denied", "not permitted") if (any(m in r['err'].lower() for m in permission_errors) - and "windows" not in util.get_systype()): + and not WINDOWS): click.secho( """ ----------------- diff --git a/platformio/compat.py b/platformio/compat.py new file mode 100644 index 00000000..cd50d905 --- /dev/null +++ b/platformio/compat.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +PY2 = sys.version_info[0] == 2 +CYGWIN = sys.platform.startswith('cygwin') +WINDOWS = sys.platform.startswith('win') + + +def get_filesystem_encoding(): + return sys.getfilesystemencoding() or sys.getdefaultencoding() + + +if PY2: + # pylint: disable=undefined-variable + string_types = (str, unicode) + + def is_bytes(x): + return isinstance(x, (buffer, bytearray)) + + def path_to_unicode(path): + return path.decode(get_filesystem_encoding()).encode("utf-8") +else: + string_types = (str, ) + + def is_bytes(x): + return isinstance(x, (bytes, memoryview, bytearray)) + + def path_to_unicode(path): + return path diff --git a/platformio/downloader.py b/platformio/downloader.py index 0b0ef5df..f86b723d 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -15,13 +15,14 @@ from email.utils import parsedate_tz from math import ceil from os.path import getsize, join -from sys import getfilesystemencoding, version_info +from sys import version_info from time import mktime import click import requests from platformio import util +from platformio.compat import PY2, get_filesystem_encoding from platformio.exception import (FDSHASumMismatch, FDSizeMismatch, FDUnrecognizedStatusCode) @@ -45,15 +46,15 @@ class FileDownloader(object): if disposition and "filename=" in disposition: self._fname = disposition[disposition.index("filename=") + 9:].replace('"', "").replace("'", "") - if util.PY2: + if PY2: self._fname = self._fname.encode("utf8") else: self._fname = [p for p in url.split("/") if p][-1] self._destination = self._fname if dest_dir: - if util.PY2: - dest_dir = dest_dir.decode(getfilesystemencoding()) + if PY2: + dest_dir = dest_dir.decode(get_filesystem_encoding()) self.set_destination(join(dest_dir, self._fname)) def set_destination(self, destination): diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 9cbf3028..1216aaee 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -23,6 +23,7 @@ from click.testing import CliRunner from platformio import exception, util from platformio.commands.run import cli as cmd_run +from platformio.compat import PY2, WINDOWS from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_projectlib_dir, get_projectlibdeps_dir, get_projectsrc_dir) @@ -120,7 +121,7 @@ class ProjectGenerator(object): contents = self._render_tpl(tpl_path) self._merge_contents( join(dst_dir, file_name), - contents.encode("utf8") if util.PY2 else contents) + contents.encode("utf8") if PY2 else contents) def _render_tpl(self, tpl_path): return bottle.template( @@ -155,5 +156,4 @@ class ProjectGenerator(object): @staticmethod def _fix_os_path(path): - return (re.sub(r"[\\]+", '\\' * 4, path) - if "windows" in util.get_systype() else path) + return (re.sub(r"[\\]+", '\\' * 4, path) if WINDOWS else path) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 0dd84552..62492bcf 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,6 +21,7 @@ from time import sleep import requests from platformio import __version__, exception, util +from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager CORE_PACKAGES = { @@ -29,7 +30,7 @@ CORE_PACKAGES = { "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.2.0", "tool-unity": "~1.20403.0", - "tool-scons": "~2.20501.7" if util.PY2 else "~3.30005.0" + "tool-scons": "~2.20501.7" if PY2 else "~3.30005.0" } PIOPLUS_AUTO_UPDATES_MAX = 100 @@ -99,7 +100,7 @@ def update_core_packages(only_check=False, silent=False): if not silent or pm.outdated(pkg_dir, requirements): if name == "tool-pioplus" and not only_check: shutdown_piohome_servers() - if "windows" in util.get_systype(): + if WINDOWS: sleep(1) pm.update(name, requirements, only_check=only_check) return True @@ -117,7 +118,7 @@ def shutdown_piohome_servers(): def pioplus_call(args, **kwargs): - if "windows" in util.get_systype() and sys.version_info < (2, 7, 6): + if WINDOWS and sys.version_info < (2, 7, 6): raise exception.PlatformioException( "PlatformIO Core Plus v%s does not run under Python version %s.\n" "Minimum supported version is 2.7.6, please upgrade Python.\n" diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 4cbb34a7..b478f49c 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -23,6 +23,7 @@ from os.path import isdir, join import click from platformio import app, commands, exception, util +from platformio.compat import string_types from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager @@ -120,7 +121,7 @@ class LibraryManager(BasePkgManager): # convert listed items via comma to array for key in ("keywords", "frameworks", "platforms"): if key not in manifest or \ - not isinstance(manifest[key], util.string_types): + not isinstance(manifest[key], string_types): continue manifest[key] = [ i.strip() for i in manifest[key].split(",") if i.strip() @@ -147,7 +148,7 @@ class LibraryManager(BasePkgManager): continue if item[k] == "*": del item[k] - elif isinstance(item[k], util.string_types): + elif isinstance(item[k], string_types): item[k] = [ i.strip() for i in item[k].split(",") if i.strip() ] diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 516a951c..4d637716 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -588,9 +588,8 @@ class PkgInstallerMixin(object): if "__src_url" in cur_manifest: target_dirname = "%s@src-%s" % ( pkg_dirname, - hashlib.md5(cur_manifest['__src_url'] if util. - PY2 else cur_manifest['__src_url'].encode( - )).hexdigest()) + hashlib.md5( + cur_manifest['__src_url'].encode()).hexdigest()) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: @@ -599,9 +598,8 @@ class PkgInstallerMixin(object): if "__src_url" in tmp_manifest: target_dirname = "%s@src-%s" % ( pkg_dirname, - hashlib.md5(tmp_manifest['__src_url'] if util. - PY2 else tmp_manifest['__src_url'].encode( - )).hexdigest()) + hashlib.md5( + tmp_manifest['__src_url'].encode()).hexdigest()) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index bfcda3f2..8c0b7de9 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -23,6 +23,7 @@ import click import semantic_version from platformio import __version__, app, exception, util +from platformio.compat import PY2 from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.project.helpers import get_projectboards_dir @@ -389,7 +390,7 @@ class PlatformRunMixin(object): # encode and append variables for key, value in variables.items(): - if util.PY2: + if PY2: cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) else: cmd.append( diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index d7c19595..66a9c39b 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -13,16 +13,14 @@ # limitations under the License. import os -import sys from hashlib import sha1 from os import walk from os.path import abspath, dirname, expanduser, isdir, isfile, join from platformio import __version__ +from platformio.compat import PY2, WINDOWS from platformio.project.config import ProjectConfig -PY2 = sys.version_info[0] == 2 - def get_project_dir(): return os.getcwd() @@ -132,7 +130,7 @@ def calculate_project_hash(): if path.endswith(check_suffixes): chunks.append(path) chunks_to_str = ",".join(sorted(chunks)) - if sys.platform == "win32": + if WINDOWS: # Fix issue with useless project rebuilding for case insensitive FS. # A case of disk drive can differ... chunks_to_str = chunks_to_str.lower() diff --git a/platformio/util.py b/platformio/util.py index 8055cc4d..6e5db848 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -32,19 +32,15 @@ import click import requests from platformio import __apiurl__, __version__, exception +from platformio.compat import path_to_unicode # pylint: disable=unused-import +from platformio.compat import PY2, WINDOWS, string_types from platformio.project.config import ProjectConfig from platformio.project.helpers import ( # pylint: disable=unused-import get_project_dir, get_project_optional_dir, get_projectboards_dir, get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir, get_projectsrc_dir, get_projecttest_dir, is_platformio_project) -# FIXME: check platformio.project.helpers imports - -PY2 = sys.version_info[0] == 2 -if PY2: - string_types = basestring # pylint: disable=undefined-variable -else: - string_types = str +# FIXME: remove import of path_to_unicode class AsyncPipe(Thread): @@ -148,12 +144,6 @@ def singleton(cls): return get_instance -def path_to_unicode(path): - if not PY2: - return path - return path.decode(sys.getfilesystemencoding()).encode("utf-8") - - def load_json(file_path): try: with open(file_path, "r") as f: @@ -180,7 +170,7 @@ def get_home_dir(): home_dir = get_project_optional_dir("home_dir", join(expanduser("~"), ".platformio")) win_home_dir = None - if "windows" in get_systype(): + if WINDOWS: win_home_dir = splitdrive(home_dir)[0] + "\\.platformio" if isdir(win_home_dir): home_dir = win_home_dir @@ -281,7 +271,7 @@ def copy_pythonpath_to_osenv(): _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) for p in os.sys.path: conditions = [p not in _PYTHONPATH] - if "windows" not in get_systype(): + if not WINDOWS: conditions.append( isdir(join(p, "click")) or isdir(join(p, "platformio"))) if all(conditions): @@ -299,14 +289,12 @@ def get_serial_ports(filter_hwid=False): for p, d, h in comports(): if not p: continue - if "windows" in get_systype(): - if PY2: - try: - d = unicode( # pylint: disable=undefined-variable - d, - errors="ignore") - except TypeError: - pass + if WINDOWS and PY2: + try: + # pylint: disable=undefined-variable + d = unicode(d, errors="ignore") + except TypeError: + pass if not filter_hwid or "VID:PID" in h: result.append({"port": p, "description": d, "hwid": h}) @@ -326,7 +314,7 @@ get_serialports = get_serial_ports def get_logical_devices(): items = [] - if "windows" in get_systype(): + if WINDOWS: try: result = exec_command( ["wmic", "logicaldisk", "get", "name,VolumeName"]).get( @@ -586,9 +574,8 @@ def where_is_program(program, envpath=None): # try OS's built-in commands try: - result = exec_command( - ["where" if "windows" in get_systype() else "which", program], - env=env) + result = exec_command(["where" if WINDOWS else "which", program], + env=env) if result['returncode'] == 0 and isfile(result['out'].strip()): return result['out'].strip() except OSError: From 59848c3115c76a513dc8f779b4c5e143e7abe8cb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 15:49:01 +0300 Subject: [PATCH 139/333] Check unknown build environment passed by user --- platformio/commands/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index fe497f3b..a7327d34 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -80,7 +80,7 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, config = ProjectConfig.get_instance( project_conf or join(project_dir, "platformio.ini")) - config.validate() + config.validate(environment) results = [] start_time = time() From d2449762c23a01f03ea094221033066f5d218f16 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 17:26:10 +0300 Subject: [PATCH 140/333] Move Python or Platform dependent code to "compat" module --- platformio/__main__.py | 4 +-- platformio/app.py | 6 ++-- platformio/builder/tools/piolib.py | 7 ++--- platformio/builder/tools/pioplatform.py | 3 +- platformio/builder/tools/pioupload.py | 4 +-- platformio/builder/tools/piowinhooks.py | 6 ++-- platformio/builder/tools/platformio.py | 3 +- platformio/commands/upgrade.py | 3 +- platformio/compat.py | 42 +++++++++++++++++++++++++ platformio/downloader.py | 9 +++--- platformio/ide/projectgenerator.py | 6 ++-- platformio/managers/core.py | 7 +++-- platformio/managers/lib.py | 5 +-- platformio/managers/package.py | 10 +++--- platformio/managers/platform.py | 3 +- platformio/project/helpers.py | 6 ++-- platformio/util.py | 41 +++++++++--------------- 17 files changed, 98 insertions(+), 67 deletions(-) create mode 100644 platformio/compat.py diff --git a/platformio/__main__.py b/platformio/__main__.py index 9138fe3c..0dc0cca1 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -14,13 +14,13 @@ import os import sys -from platform import system from traceback import format_exc import click from platformio import __version__, exception, maintenance from platformio.commands import PlatformioCLI +from platformio.compat import CYGWIN @click.command( @@ -45,7 +45,7 @@ def process_result(ctx, result, force, caller): # pylint: disable=W0613 def configure(): - if "cygwin" in system().lower(): + if CYGWIN: raise exception.CygwinEnvDetected() # https://urllib3.readthedocs.org diff --git a/platformio/app.py b/platformio/app.py index 247fb65c..c8a071fc 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,6 +25,7 @@ from time import time import requests from platformio import exception, lockfile, util +from platformio.compat import PY2, WINDOWS def projects_dir_validate(projects_dir): @@ -173,7 +174,7 @@ class ContentCache(object): if not arg: continue arg = str(arg) - h.update(arg if util.PY2 else arg.encode()) + h.update(arg if PY2 else arg.encode()) return h.hexdigest() def get(self, key): @@ -361,7 +362,6 @@ def get_cid(): uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(str(uid).encode()).digest()) cid = str(cid) - if ("windows" in util.get_systype() - or os.getuid() > 0): # yapf: disable pylint: disable=no-member + if WINDOWS or os.getuid() > 0: # yapf: disable pylint: disable=no-member set_state_item("cid", cid) return cid diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index e1a0b50b..270f3f21 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -33,6 +33,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, util from platformio.builder.tools import platformio as piotool +from platformio.compat import PY2, WINDOWS from platformio.managers.lib import LibraryManager from platformio.managers.package import PackageManager @@ -91,8 +92,6 @@ class LibBuilderFactory(object): class LibBuilderBase(object): - IS_WINDOWS = "windows" in util.get_systype() - LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"] LDF_MODE_DEFAULT = "chain" @@ -135,7 +134,7 @@ class LibBuilderBase(object): def __contains__(self, path): p1 = self.path p2 = path - if self.IS_WINDOWS: + if WINDOWS: p1 = p1.lower() p2 = p2.lower() return commonprefix((p1 + sep, p2)) == p1 + sep @@ -191,7 +190,7 @@ class LibBuilderBase(object): @property def build_dir(self): lib_hash = hashlib.sha1( - self.path if util.PY2 else self.path.encode()).hexdigest()[:3] + self.path if PY2 else self.path.encode()).hexdigest()[:3] return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) @property diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 6427b33c..cc61a709 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -21,6 +21,7 @@ from os.path import isdir, isfile, join from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import exception, util +from platformio.compat import WINDOWS from platformio.managers.platform import PlatformFactory # pylint: disable=too-many-branches, too-many-locals @@ -79,7 +80,7 @@ def LoadPioPlatform(env, variables): env.PrependENVPath( "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir) - if ("windows" not in systype and isdir(join(pkg_dir, "lib")) + if (not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain"): env.PrependENVPath( "DYLD_LIBRARY_PATH" diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 0d5a55ea..a7b2685d 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -26,6 +26,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from serial import Serial, SerialException from platformio import exception, util +from platformio.compat import WINDOWS # pylint: disable=unused-argument @@ -134,8 +135,7 @@ def AutodetectUploadPort(*args, **kwargs): continue port = item['port'] if upload_protocol.startswith("blackmagic"): - if "windows" in util.get_systype() and \ - port.startswith("COM") and len(port) > 4: + if WINDOWS and port.startswith("COM") and len(port) > 4: port = "\\\\.\\%s" % port if "GDB" in item['description']: return port diff --git a/platformio/builder/tools/piowinhooks.py b/platformio/builder/tools/piowinhooks.py index c893ffd5..1105091f 100644 --- a/platformio/builder/tools/piowinhooks.py +++ b/platformio/builder/tools/piowinhooks.py @@ -18,7 +18,7 @@ from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join -from platformio import util +from platformio.compat import PY2, WINDOWS # Windows CLI has limit with command length to 8192 # Leave 2000 chars for flags and other options @@ -63,7 +63,7 @@ def _file_long_data(env, data): makedirs(build_dir) tmp_file = join( build_dir, - "longcmd-%s" % md5(data if util.PY2 else data.encode()).hexdigest()) + "longcmd-%s" % md5(data if PY2 else data.encode()).hexdigest()) if isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: @@ -76,7 +76,7 @@ def exists(_): def generate(env): - if "windows" not in util.get_systype(): + if not WINDOWS: return None env.Replace(_long_sources_hook=long_sources_hook) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 41f32700..a8272b83 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -27,7 +27,8 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Export # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error -from platformio.util import glob_escape, pioversion_to_intstr, string_types +from platformio.compat import string_types +from platformio.util import glob_escape, pioversion_to_intstr SRC_HEADER_EXT = ["h", "hpp"] SRC_C_EXT = ["c", "cc", "cpp"] diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index c47ff8dc..6acb83ae 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -20,6 +20,7 @@ import click import requests from platformio import VERSION, __version__, exception, util +from platformio.compat import WINDOWS from platformio.managers.core import shutdown_piohome_servers @@ -68,7 +69,7 @@ def cli(dev): raise exception.UpgradeError("\n".join([str(cmd), str(e)])) permission_errors = ("permission denied", "not permitted") if (any(m in r['err'].lower() for m in permission_errors) - and "windows" not in util.get_systype()): + and not WINDOWS): click.secho( """ ----------------- diff --git a/platformio/compat.py b/platformio/compat.py new file mode 100644 index 00000000..cd50d905 --- /dev/null +++ b/platformio/compat.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +PY2 = sys.version_info[0] == 2 +CYGWIN = sys.platform.startswith('cygwin') +WINDOWS = sys.platform.startswith('win') + + +def get_filesystem_encoding(): + return sys.getfilesystemencoding() or sys.getdefaultencoding() + + +if PY2: + # pylint: disable=undefined-variable + string_types = (str, unicode) + + def is_bytes(x): + return isinstance(x, (buffer, bytearray)) + + def path_to_unicode(path): + return path.decode(get_filesystem_encoding()).encode("utf-8") +else: + string_types = (str, ) + + def is_bytes(x): + return isinstance(x, (bytes, memoryview, bytearray)) + + def path_to_unicode(path): + return path diff --git a/platformio/downloader.py b/platformio/downloader.py index 0b0ef5df..f86b723d 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -15,13 +15,14 @@ from email.utils import parsedate_tz from math import ceil from os.path import getsize, join -from sys import getfilesystemencoding, version_info +from sys import version_info from time import mktime import click import requests from platformio import util +from platformio.compat import PY2, get_filesystem_encoding from platformio.exception import (FDSHASumMismatch, FDSizeMismatch, FDUnrecognizedStatusCode) @@ -45,15 +46,15 @@ class FileDownloader(object): if disposition and "filename=" in disposition: self._fname = disposition[disposition.index("filename=") + 9:].replace('"', "").replace("'", "") - if util.PY2: + if PY2: self._fname = self._fname.encode("utf8") else: self._fname = [p for p in url.split("/") if p][-1] self._destination = self._fname if dest_dir: - if util.PY2: - dest_dir = dest_dir.decode(getfilesystemencoding()) + if PY2: + dest_dir = dest_dir.decode(get_filesystem_encoding()) self.set_destination(join(dest_dir, self._fname)) def set_destination(self, destination): diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 9cbf3028..1216aaee 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -23,6 +23,7 @@ from click.testing import CliRunner from platformio import exception, util from platformio.commands.run import cli as cmd_run +from platformio.compat import PY2, WINDOWS from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_projectlib_dir, get_projectlibdeps_dir, get_projectsrc_dir) @@ -120,7 +121,7 @@ class ProjectGenerator(object): contents = self._render_tpl(tpl_path) self._merge_contents( join(dst_dir, file_name), - contents.encode("utf8") if util.PY2 else contents) + contents.encode("utf8") if PY2 else contents) def _render_tpl(self, tpl_path): return bottle.template( @@ -155,5 +156,4 @@ class ProjectGenerator(object): @staticmethod def _fix_os_path(path): - return (re.sub(r"[\\]+", '\\' * 4, path) - if "windows" in util.get_systype() else path) + return (re.sub(r"[\\]+", '\\' * 4, path) if WINDOWS else path) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 1e469a03..04369712 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,6 +21,7 @@ from time import sleep import requests from platformio import __version__, exception, util +from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager CORE_PACKAGES = { @@ -29,7 +30,7 @@ CORE_PACKAGES = { "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), "tool-pioplus": "^2.2.0", "tool-unity": "~1.20403.0", - "tool-scons": "~2.20501.7" if util.PY2 else "~3.30005.0" + "tool-scons": "~2.20501.7" if PY2 else "~3.30005.0" } PIOPLUS_AUTO_UPDATES_MAX = 100 @@ -99,7 +100,7 @@ def update_core_packages(only_check=False, silent=False): if not silent or pm.outdated(pkg_dir, requirements): if name == "tool-pioplus" and not only_check: shutdown_piohome_servers() - if "windows" in util.get_systype(): + if WINDOWS: sleep(1) pm.update(name, requirements, only_check=only_check) return True @@ -126,7 +127,7 @@ def inject_contrib_pysite(): def pioplus_call(args, **kwargs): - if "windows" in util.get_systype() and sys.version_info < (2, 7, 6): + if WINDOWS and sys.version_info < (2, 7, 6): raise exception.PlatformioException( "PlatformIO Core Plus v%s does not run under Python version %s.\n" "Minimum supported version is 2.7.6, please upgrade Python.\n" diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 4cbb34a7..b478f49c 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -23,6 +23,7 @@ from os.path import isdir, join import click from platformio import app, commands, exception, util +from platformio.compat import string_types from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager @@ -120,7 +121,7 @@ class LibraryManager(BasePkgManager): # convert listed items via comma to array for key in ("keywords", "frameworks", "platforms"): if key not in manifest or \ - not isinstance(manifest[key], util.string_types): + not isinstance(manifest[key], string_types): continue manifest[key] = [ i.strip() for i in manifest[key].split(",") if i.strip() @@ -147,7 +148,7 @@ class LibraryManager(BasePkgManager): continue if item[k] == "*": del item[k] - elif isinstance(item[k], util.string_types): + elif isinstance(item[k], string_types): item[k] = [ i.strip() for i in item[k].split(",") if i.strip() ] diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 516a951c..4d637716 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -588,9 +588,8 @@ class PkgInstallerMixin(object): if "__src_url" in cur_manifest: target_dirname = "%s@src-%s" % ( pkg_dirname, - hashlib.md5(cur_manifest['__src_url'] if util. - PY2 else cur_manifest['__src_url'].encode( - )).hexdigest()) + hashlib.md5( + cur_manifest['__src_url'].encode()).hexdigest()) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: @@ -599,9 +598,8 @@ class PkgInstallerMixin(object): if "__src_url" in tmp_manifest: target_dirname = "%s@src-%s" % ( pkg_dirname, - hashlib.md5(tmp_manifest['__src_url'] if util. - PY2 else tmp_manifest['__src_url'].encode( - )).hexdigest()) + hashlib.md5( + tmp_manifest['__src_url'].encode()).hexdigest()) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index bfcda3f2..8c0b7de9 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -23,6 +23,7 @@ import click import semantic_version from platformio import __version__, app, exception, util +from platformio.compat import PY2 from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.project.helpers import get_projectboards_dir @@ -389,7 +390,7 @@ class PlatformRunMixin(object): # encode and append variables for key, value in variables.items(): - if util.PY2: + if PY2: cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) else: cmd.append( diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index d7c19595..66a9c39b 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -13,16 +13,14 @@ # limitations under the License. import os -import sys from hashlib import sha1 from os import walk from os.path import abspath, dirname, expanduser, isdir, isfile, join from platformio import __version__ +from platformio.compat import PY2, WINDOWS from platformio.project.config import ProjectConfig -PY2 = sys.version_info[0] == 2 - def get_project_dir(): return os.getcwd() @@ -132,7 +130,7 @@ def calculate_project_hash(): if path.endswith(check_suffixes): chunks.append(path) chunks_to_str = ",".join(sorted(chunks)) - if sys.platform == "win32": + if WINDOWS: # Fix issue with useless project rebuilding for case insensitive FS. # A case of disk drive can differ... chunks_to_str = chunks_to_str.lower() diff --git a/platformio/util.py b/platformio/util.py index bf8292d7..56db49e5 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -32,19 +32,15 @@ import click import requests from platformio import __apiurl__, __version__, exception +from platformio.compat import path_to_unicode # pylint: disable=unused-import +from platformio.compat import PY2, WINDOWS, string_types from platformio.project.config import ProjectConfig from platformio.project.helpers import ( # pylint: disable=unused-import get_project_dir, get_project_optional_dir, get_projectboards_dir, get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir, get_projectsrc_dir, get_projecttest_dir, is_platformio_project) -# FIXME: check platformio.project.helpers imports - -PY2 = sys.version_info[0] == 2 -if PY2: - string_types = basestring # pylint: disable=undefined-variable -else: - string_types = str +# FIXME: remove import of path_to_unicode class AsyncPipe(Thread): @@ -148,12 +144,6 @@ def singleton(cls): return get_instance -def path_to_unicode(path): - if not PY2: - return path - return path.decode(sys.getfilesystemencoding()).encode("utf-8") - - def load_json(file_path): try: with open(file_path, "r") as f: @@ -180,7 +170,7 @@ def get_home_dir(): home_dir = get_project_optional_dir("home_dir", join(expanduser("~"), ".platformio")) win_home_dir = None - if "windows" in get_systype(): + if WINDOWS: win_home_dir = splitdrive(home_dir)[0] + "\\.platformio" if isdir(win_home_dir): home_dir = win_home_dir @@ -281,7 +271,7 @@ def copy_pythonpath_to_osenv(): _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) for p in os.sys.path: conditions = [p not in _PYTHONPATH] - if "windows" not in get_systype(): + if not WINDOWS: conditions.append( isdir(join(p, "click")) or isdir(join(p, "platformio"))) if all(conditions): @@ -299,14 +289,12 @@ def get_serial_ports(filter_hwid=False): for p, d, h in comports(): if not p: continue - if "windows" in get_systype(): - if PY2: - try: - d = unicode( # pylint: disable=undefined-variable - d, - errors="ignore") - except TypeError: - pass + if WINDOWS and PY2: + try: + # pylint: disable=undefined-variable + d = unicode(d, errors="ignore") + except TypeError: + pass if not filter_hwid or "VID:PID" in h: result.append({"port": p, "description": d, "hwid": h}) @@ -326,7 +314,7 @@ get_serialports = get_serial_ports def get_logical_devices(): items = [] - if "windows" in get_systype(): + if WINDOWS: try: result = exec_command( ["wmic", "logicaldisk", "get", "name,VolumeName"]).get( @@ -583,9 +571,8 @@ def where_is_program(program, envpath=None): # try OS's built-in commands try: - result = exec_command( - ["where" if "windows" in get_systype() else "which", program], - env=env) + result = exec_command(["where" if WINDOWS else "which", program], + env=env) if result['returncode'] == 0 and isfile(result['out'].strip()): return result['out'].strip() except OSError: From abfee8308e85567467b5546eff9bb0400ddbd3c2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 17:47:02 +0300 Subject: [PATCH 141/333] Switch Python or Platform dependent code to "compat" module --- platformio/commands/debug/client.py | 3 ++- platformio/commands/debug/process.py | 5 +++-- platformio/commands/home/rpc/handlers/app.py | 10 +++++----- platformio/commands/home/rpc/handlers/os.py | 6 +++--- platformio/commands/home/rpc/handlers/piocore.py | 10 +++++----- platformio/commands/home/rpc/handlers/project.py | 5 ++--- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index ac97243f..25a57db1 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -29,6 +29,7 @@ from platformio import app, exception, util from platformio.commands.debug import helpers, initcfgs from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer +from platformio.compat import PY2 from platformio.telemetry import MeasurementProtocol LOG_FILE = None @@ -60,7 +61,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes def spawn(self, gdb_path, prog_path): session_hash = gdb_path + prog_path self._session_id = sha1( - session_hash if util.PY2 else session_hash.encode()).hexdigest() + session_hash if PY2 else session_hash.encode()).hexdigest() self._kill_previous_session() patterns = { diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index 0079e18c..0d70abe1 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -19,6 +19,7 @@ from twisted.internet import protocol # pylint: disable=import-error from platformio import util from platformio.commands.debug import helpers +from platformio.compat import string_types LOG_FILE = None @@ -42,13 +43,13 @@ class BaseProcess(protocol.ProcessProtocol, object): text = text.replace(pattern, value or "") return text - if isinstance(source, util.string_types): + if isinstance(source, string_types): source = _replace(source) elif isinstance(source, (list, dict)): items = enumerate(source) if isinstance(source, list) else source.items() for key, value in items: - if isinstance(value, util.string_types): + if isinstance(value, string_types): source[key] = _replace(value) elif isinstance(value, (list, dict)): source[key] = self.apply_patterns(value, patterns) diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index ce95a218..8241b03a 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -18,6 +18,7 @@ import json from os.path import expanduser, isfile, join from platformio import __version__, app, exception, util +from platformio.compat import path_to_unicode class AppRPC(object): @@ -55,12 +56,11 @@ class AppRPC(object): for key in storage['coreSettings']: if not key.endswith("dir"): continue - storage['coreSettings'][key][ - 'default_value'] = util.path_to_unicode( - storage['coreSettings'][key]['default_value']) - storage['coreSettings'][key]['value'] = util.path_to_unicode( + storage['coreSettings'][key]['default_value'] = path_to_unicode( + storage['coreSettings'][key]['default_value']) + storage['coreSettings'][key]['value'] = path_to_unicode( storage['coreSettings'][key]['value']) - storage['homeDir'] = util.path_to_unicode(expanduser("~")) + storage['homeDir'] = path_to_unicode(expanduser("~")) storage['projectsDir'] = storage['coreSettings']['projects_dir'][ 'value'] diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 780dbec8..8b263632 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -17,7 +17,6 @@ from __future__ import absolute_import import glob import os import shutil -import sys from functools import cmp_to_key from os.path import expanduser, isdir, isfile, join @@ -26,6 +25,7 @@ from twisted.internet import defer # pylint: disable=import-error from platformio import app, util from platformio.commands.home import helpers +from platformio.compat import PY2, get_filesystem_encoding, path_to_unicode class OSRPC(object): @@ -81,7 +81,7 @@ class OSRPC(object): @staticmethod def reveal_file(path): return click.launch( - path.encode(sys.getfilesystemencoding()) if util.PY2 else path, + path.encode(get_filesystem_encoding()) if PY2 else path, locate=True) @staticmethod @@ -148,6 +148,6 @@ class OSRPC(object): items = [] for item in util.get_logical_devices(): if item['name']: - item['name'] = util.path_to_unicode(item['name']) + item['name'] = path_to_unicode(item['name']) items.append(item) return items diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 80a642cb..b651b498 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -17,13 +17,13 @@ from __future__ import absolute_import import json import os import re -import sys import jsonrpc # pylint: disable=import-error from twisted.internet import utils # pylint: disable=import-error -from platformio import __version__, util +from platformio import __version__ from platformio.commands.home import helpers +from platformio.compat import get_filesystem_encoding, string_types class PIOCoreRPC(object): @@ -33,8 +33,8 @@ class PIOCoreRPC(object): json_output = "--json-output" in args try: args = [ - arg.encode(sys.getfilesystemencoding()) if isinstance( - arg, util.string_types) else str(arg) for arg in args + arg.encode(get_filesystem_encoding()) if isinstance( + arg, string_types) else str(arg) for arg in args ] except UnicodeError: raise jsonrpc.exceptions.JSONRPCDispatchException( @@ -54,7 +54,7 @@ class PIOCoreRPC(object): result = list(result) assert len(result) == 3 for i in (0, 1): - result[i] = result[i].decode(sys.getfilesystemencoding()).strip() + result[i] = result[i].decode(get_filesystem_encoding()).strip() out, err, code = result text = ("%s\n\n%s" % (out, err)).strip() if code != 0: diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index c3f48fcc..ae2eaa07 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -16,7 +16,6 @@ from __future__ import absolute_import import os import shutil -import sys import time from os.path import (basename, expanduser, getmtime, isdir, isfile, join, realpath, sep) @@ -26,6 +25,7 @@ import jsonrpc # pylint: disable=import-error from platformio import exception, util from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.compat import get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig @@ -215,8 +215,7 @@ class ProjectRPC(object): if isdir(src_dir): util.rmtree_(src_dir) shutil.copytree( - arduino_project_dir.encode(sys.getfilesystemencoding()), - src_dir) + arduino_project_dir.encode(get_filesystem_encoding()), src_dir) return project_dir @staticmethod From 5e1a931145d05503de4bffbecdf70d02477ba609 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 10 May 2019 17:50:08 +0300 Subject: [PATCH 142/333] Switch Python or Platform dependent code to "compat" module --- platformio/builder/main.py | 5 +++-- platformio/managers/package.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index c656da0b..95c4d4ca 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -29,6 +29,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error from platformio import util +from platformio.compat import PY2, path_to_unicode from platformio.project.helpers import ( get_project_dir, get_project_optional_dir, get_projectbuild_dir, get_projectdata_dir, get_projectinclude_dir, get_projectlib_dir, @@ -174,7 +175,7 @@ env.LoadPioPlatform(commonvars) env.SConscriptChdir(0) env.SConsignFile( join("$PROJECTBUILD_DIR", - ".sconsign.dblite" if util.PY2 else ".sconsign3.dblite")) + ".sconsign.dblite" if PY2 else ".sconsign3.dblite")) for item in env.GetExtraScripts("pre"): env.SConscript(item, exports="env") @@ -216,7 +217,7 @@ if "envdump" in COMMAND_LINE_TARGETS: if "idedata" in COMMAND_LINE_TARGETS: try: - print("\n%s\n" % util.path_to_unicode( + print("\n%s\n" % path_to_unicode( json.dumps(env.DumpIDEData(), ensure_ascii=False))) env.Exit(0) except UnicodeDecodeError: diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 4d637716..a089cc5d 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -26,6 +26,7 @@ import requests import semantic_version from platformio import __version__, app, exception, telemetry, util +from platformio.compat import path_to_unicode from platformio.downloader import FileDownloader from platformio.lockfile import LockFile from platformio.unpacker import FileUnpacker @@ -392,7 +393,7 @@ class PkgInstallerMixin(object): if "version" not in manifest: manifest['version'] = "0.0.0" - manifest['__pkg_dir'] = util.path_to_unicode(pkg_dir) + manifest['__pkg_dir'] = path_to_unicode(pkg_dir) self.cache_set(cache_key, manifest) return manifest @@ -449,7 +450,7 @@ class PkgInstallerMixin(object): def get_package_by_dir(self, pkg_dir): for manifest in self.get_installed(): - if manifest['__pkg_dir'] == util.path_to_unicode(abspath(pkg_dir)): + if manifest['__pkg_dir'] == path_to_unicode(abspath(pkg_dir)): return manifest return None From 8b93ad00a2ee1cbf75a00b386e06fef95c5ed635 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 11 May 2019 22:18:35 +0300 Subject: [PATCH 143/333] Docs: ESP32: Implement "espota" protocol --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index c5a33bc0..1f6e64fe 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c5a33bc006dfedfa2d1a2ab1a340615467b885fd +Subproject commit 1f6e64fe3e32b78dcd38755c4d96a79b8874ef60 From f26553b45154d17568065fd444b416cbd33f75ef Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 13 May 2019 22:37:24 +0300 Subject: [PATCH 144/333] ESP8266, docs for "SSL Support" --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 1f6e64fe..700a8ac4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1f6e64fe3e32b78dcd38755c4d96a79b8874ef60 +Subproject commit 700a8ac46791bf06ce43d2efe5e22147e0178d38 From b14abeff48ae46c849848bec15052b448a24634a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 13 May 2019 22:37:45 +0300 Subject: [PATCH 145/333] Bump version to 4.0.0a13 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 232433f9..c68c1fe5 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a12") +VERSION = (4, 0, "0a13") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From aaf61082c1c570c6780eb3b82febe89f31c88012 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 May 2019 20:02:45 +0300 Subject: [PATCH 146/333] Replace "--only-check" CLI option by "--dry-run" --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/lib.py | 10 ++++++++-- platformio/commands/platform.py | 11 +++++++++-- platformio/commands/remote.py | 8 ++++++-- platformio/commands/update.py | 11 +++++++++-- platformio/maintenance.py | 2 +- 7 files changed, 35 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f5a953f6..5f0dd973 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ PlatformIO 4.0 * Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields * Added support for the latest Python "Click" package (CLI Builder) (`issue #349 `_) +* Deprecated ``--only-check`` CLI option for "update" sub-commands, please use ``--dry-run`` instead PlatformIO 3.0 -------------- diff --git a/docs b/docs index 700a8ac4..6e000475 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 700a8ac46791bf06ce43d2efe5e22147e0178d38 +Subproject commit 6e0004755ac1849b66cf45ec8e0fca47a35226e0 diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index c52350b1..dde2aa0b 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -123,13 +123,19 @@ def lib_uninstall(lm, libraries): "-c", "--only-check", is_flag=True, - help="Do not update, only check for new version") + help="DEPRECATED. Please use `--dry-run` instead") +@click.option( + "--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) @click.pass_obj -def lib_update(lm, libraries, only_check, json_output): +def lib_update(lm, libraries, only_check, dry_run, json_output): if not libraries: libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()] + only_check = dry_run or only_check + if only_check and json_output: result = [] for library in libraries: diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index bf7ecf75..db0149f3 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -329,9 +329,14 @@ def platform_uninstall(platforms): "-c", "--only-check", is_flag=True, - help="Do not update, only check for a new version") + help="DEPRECATED. Please use `--dry-run` instead") +@click.option( + "--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) -def platform_update(platforms, only_packages, only_check, json_output): +def platform_update(platforms, only_packages, only_check, dry_run, + json_output): pm = PlatformManager() pkg_dir_to_name = {} if not platforms: @@ -341,6 +346,8 @@ def platform_update(platforms, only_packages, only_check, json_output): pkg_dir_to_name[manifest['__pkg_dir']] = manifest.get( "title", manifest['name']) + only_check = dry_run or only_check + if only_check and json_output: result = [] for platform in platforms: diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index d506d5a5..b37e843c 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -68,8 +68,12 @@ def remote_agent_list(): "-c", "--only-check", is_flag=True, - help="Do not update, only check for new version") -def remote_update(only_check): + help="DEPRECATED. Please use `--dry-run` instead") +@click.option( + "--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") +def remote_update(only_check, dry_run): pioplus_call(sys.argv[1:]) diff --git a/platformio/commands/update.py b/platformio/commands/update.py index 924fb290..d72a35d4 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -29,12 +29,19 @@ from platformio.managers.lib import LibraryManager "-c", "--only-check", is_flag=True, - help="Do not update, only check for new version") + help="DEPRECATED. Please use `--dry-run` instead") +@click.option( + "--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") +@click.option("--json-output", is_flag=True) @click.pass_context -def cli(ctx, core_packages, only_check): +def cli(ctx, core_packages, only_check, dry_run): # cleanup lib search results, cached board and platform lists app.clean_cache() + only_check = dry_run or only_check + update_core_packages(only_check) if core_packages: diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 8841b7f5..4bebcb7b 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -329,7 +329,7 @@ def check_internal_updates(ctx, what): fg="yellow", nl=False) click.secho( - "`platformio %s update --only-check`" % + "`platformio %s update --dry-run`" % ("lib --global" if what == "libraries" else "platform"), fg="cyan", nl=False) From 971049b41cb137cb092d165364f32331f8096a49 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 May 2019 21:03:15 +0300 Subject: [PATCH 147/333] Move process related helpers to "proc" module --- platformio/app.py | 3 +- platformio/builder/main.py | 3 +- platformio/builder/tools/pioide.py | 3 +- platformio/builder/tools/piomisc.py | 3 +- platformio/builder/tools/pioupload.py | 3 +- platformio/commands/lib.py | 3 +- platformio/commands/platform.py | 4 +- platformio/commands/upgrade.py | 13 +-- platformio/downloader.py | 6 +- platformio/maintenance.py | 5 +- platformio/managers/core.py | 5 +- platformio/managers/platform.py | 8 +- platformio/proc.py | 143 ++++++++++++++++++++++++++ platformio/telemetry.py | 5 +- platformio/util.py | 136 ++---------------------- platformio/vcsclient.py | 4 +- 16 files changed, 190 insertions(+), 157 deletions(-) create mode 100644 platformio/proc.py diff --git a/platformio/app.py b/platformio/app.py index c8a071fc..b8e3b691 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -26,6 +26,7 @@ import requests from platformio import exception, lockfile, util from platformio.compat import PY2, WINDOWS +from platformio.proc import is_ci def projects_dir_validate(projects_dir): @@ -339,7 +340,7 @@ def set_session_var(name, value): def is_disabled_progressbar(): return any([ get_session_var("force_option"), - util.is_ci(), + is_ci(), getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true" ]) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 95c4d4ca..287c0e84 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -30,6 +30,7 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import util from platformio.compat import PY2, path_to_unicode +from platformio.proc import get_pythonexe_path from platformio.project.helpers import ( get_project_dir, get_project_optional_dir, get_projectbuild_dir, get_projectdata_dir, get_projectinclude_dir, get_projectlib_dir, @@ -123,7 +124,7 @@ DEFAULT_ENV_OPTIONS = dict( ], PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), - PYTHONEXE=util.get_pythonexe_path()) + PYTHONEXE=get_pythonexe_path()) if not int(ARGUMENTS.get("PIOVERBOSE", 0)): DEFAULT_ENV_OPTIONS['ARCOMSTR'] = "Archiving $TARGET" diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index db496741..311b0203 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -22,6 +22,7 @@ from SCons.Defaults import processDefines # pylint: disable=import-error from platformio import util from platformio.managers.core import get_core_package_dir +from platformio.proc import exec_command def _dump_includes(env): @@ -71,7 +72,7 @@ def _get_gcc_defines(env): try: sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) - result = util.exec_command( + result = exec_command( "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True) except OSError: return items diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 01a797ea..19768146 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -26,6 +26,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import util from platformio.managers.core import get_core_package_dir +from platformio.proc import exec_command class InoToCPPConverter(object): @@ -211,7 +212,7 @@ def _get_compiler_type(env): try: sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) - result = util.exec_command([env.subst("$CC"), "-v"], env=sysenv) + result = exec_command([env.subst("$CC"), "-v"], env=sysenv) except OSError: return None if result['returncode'] != 0: diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index a7b2685d..6db94bfc 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -27,6 +27,7 @@ from serial import Serial, SerialException from platformio import exception, util from platformio.compat import WINDOWS +from platformio.proc import exec_command # pylint: disable=unused-argument @@ -211,7 +212,7 @@ def CheckUploadSize(_, target, source, env): cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg] sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) - result = util.exec_command(env.subst(cmd), env=sysenv) + result = exec_command(env.subst(cmd), env=sysenv) if result['returncode'] != 0: return None return result['out'].strip() diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index dde2aa0b..81a4640a 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -23,6 +23,7 @@ import click from platformio import exception, util from platformio.commands import PlatformioCLI from platformio.managers.lib import LibraryManager, get_builtin_libs +from platformio.proc import is_ci from platformio.project.helpers import ( get_project_dir, get_projectlibdeps_dir, is_platformio_project) @@ -62,7 +63,7 @@ def cli(ctx, **options): storage_dir = join(util.get_home_dir(), "lib") elif is_platformio_project(): storage_dir = get_projectlibdeps_dir() - elif util.is_ci(): + elif is_ci(): storage_dir = join(util.get_home_dir(), "lib") click.secho( "Warning! Global library storage is used automatically. " diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index db0149f3..70612f51 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -335,8 +335,8 @@ def platform_uninstall(platforms): is_flag=True, help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) -def platform_update(platforms, only_packages, only_check, dry_run, - json_output): +def platform_update( # pylint: disable=too-many-locals + platforms, only_packages, only_check, dry_run, json_output): pm = PlatformManager() pkg_dir_to_name = {} if not platforms: diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 6acb83ae..18e49cb3 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -22,6 +22,7 @@ import requests from platformio import VERSION, __version__, exception, util from platformio.compat import WINDOWS from platformio.managers.core import shutdown_piohome_servers +from platformio.proc import exec_command, get_pythonexe_path @click.command( @@ -47,13 +48,13 @@ def cli(dev): r = {} try: for cmd in cmds: - cmd = [util.get_pythonexe_path(), "-m"] + cmd - r = util.exec_command(cmd) + cmd = [get_pythonexe_path(), "-m"] + cmd + r = exec_command(cmd) # try pip with disabled cache if r['returncode'] != 0 and cmd[2] == "pip": cmd.insert(3, "--no-cache-dir") - r = util.exec_command(cmd) + r = exec_command(cmd) assert r['returncode'] == 0 assert "version" in r['out'] @@ -100,9 +101,9 @@ def get_pip_package(to_develop): pkg_name = os.path.join(cache_dir, "piocoredevelop.zip") try: with open(pkg_name, "w") as fp: - r = util.exec_command(["curl", "-fsSL", dl_url], - stdout=fp, - universal_newlines=True) + r = exec_command(["curl", "-fsSL", dl_url], + stdout=fp, + universal_newlines=True) assert r['returncode'] == 0 # check ZIP structure with ZipFile(pkg_name) as zp: diff --git a/platformio/downloader.py b/platformio/downloader.py index f86b723d..ab84e518 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -25,6 +25,7 @@ from platformio import util from platformio.compat import PY2, get_filesystem_encoding from platformio.exception import (FDSHASumMismatch, FDSizeMismatch, FDUnrecognizedStatusCode) +from platformio.proc import exec_command class FileDownloader(object): @@ -105,12 +106,11 @@ class FileDownloader(object): dlsha1 = None try: - result = util.exec_command(["sha1sum", self._destination]) + result = exec_command(["sha1sum", self._destination]) dlsha1 = result['out'] except (OSError, ValueError): try: - result = util.exec_command( - ["shasum", "-a", "1", self._destination]) + result = exec_command(["shasum", "-a", "1", self._destination]) dlsha1 = result['out'] except (OSError, ValueError): pass diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 4bebcb7b..4b208385 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -33,6 +33,7 @@ from platformio.commands.upgrade import get_latest_version from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.proc import is_ci, is_container def on_platformio_start(ctx, force, caller): @@ -79,7 +80,7 @@ def set_caller(caller=None): caller = getenv("PLATFORMIO_CALLER") elif getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"): caller = "vscode" - elif util.is_container(): + elif is_container(): if getenv("C9_UID"): caller = "C9" elif getenv("USER") == "cabox": @@ -222,7 +223,7 @@ def after_upgrade(ctx): "- %s PlatformIO IDE for IoT development > %s" % (click.style("try", fg="cyan"), click.style("https://platformio.org/platformio-ide", fg="cyan"))) - if not util.is_ci(): + if not is_ci(): click.echo("- %s us with PlatformIO Plus > %s" % (click.style( "support", fg="cyan"), click.style( "https://pioplus.com", fg="cyan"))) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 62492bcf..3c66592c 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -23,6 +23,7 @@ import requests from platformio import __version__, exception, util from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager +from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path CORE_PACKAGES = { "contrib-piohome": "^2.0.1", @@ -125,13 +126,13 @@ def pioplus_call(args, **kwargs): "Python 3 is not yet supported.\n" % (__version__, sys.version)) pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus") - pythonexe_path = util.get_pythonexe_path() + pythonexe_path = get_pythonexe_path() os.environ['PYTHONEXEPATH'] = pythonexe_path os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite") os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "") os.environ['PATH'] = (os.pathsep).join( [dirname(pythonexe_path), os.environ['PATH']]) - util.copy_pythonpath_to_osenv() + copy_pythonpath_to_osenv() code = subprocess.call([pioplus_path] + args, **kwargs) # handle remote update request diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8c0b7de9..f6dac646 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -26,6 +26,8 @@ from platformio import __version__, app, exception, util from platformio.compat import PY2 from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager +from platformio.proc import (copy_pythonpath_to_osenv, exec_command, + get_pythonexe_path) from platformio.project.helpers import get_projectboards_dir try: @@ -379,7 +381,7 @@ class PlatformRunMixin(object): def _run_scons(self, variables, targets): cmd = [ - util.get_pythonexe_path(), + get_pythonexe_path(), join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q", "-j %d" % self.get_job_nums(), "--warn=no-no-parallel-support", "-f", @@ -397,8 +399,8 @@ class PlatformRunMixin(object): "%s=%s" % (key.upper(), base64.b64encode( value.encode()).decode())) - util.copy_pythonpath_to_osenv() - result = util.exec_command( + copy_pythonpath_to_osenv() + result = exec_command( cmd, stdout=util.AsyncPipe(self.on_run_out), stderr=util.AsyncPipe(self.on_run_err)) diff --git a/platformio/proc.py b/platformio/proc.py new file mode 100644 index 00000000..a0432458 --- /dev/null +++ b/platformio/proc.py @@ -0,0 +1,143 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys +from os.path import isdir, isfile, join, normpath +from threading import Thread + +from platformio import exception +from platformio.compat import PY2, WINDOWS, string_types + + +class AsyncPipe(Thread): + + def __init__(self, outcallback=None): + super(AsyncPipe, self).__init__() + self.outcallback = outcallback + + self._fd_read, self._fd_write = os.pipe() + self._pipe_reader = os.fdopen(self._fd_read) + self._buffer = [] + + self.start() + + def get_buffer(self): + return self._buffer + + def fileno(self): + return self._fd_write + + def run(self): + for line in iter(self._pipe_reader.readline, ""): + line = line.strip() + self._buffer.append(line) + if self.outcallback: + self.outcallback(line) + else: + print(line) + self._pipe_reader.close() + + def close(self): + os.close(self._fd_write) + self.join() + + +def exec_command(*args, **kwargs): + result = {"out": None, "err": None, "returncode": None} + + default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) + default.update(kwargs) + kwargs = default + + p = subprocess.Popen(*args, **kwargs) + try: + result['out'], result['err'] = p.communicate() + result['returncode'] = p.returncode + except KeyboardInterrupt: + raise exception.AbortedByUser() + finally: + for s in ("stdout", "stderr"): + if isinstance(kwargs[s], AsyncPipe): + kwargs[s].close() + + for s in ("stdout", "stderr"): + if isinstance(kwargs[s], AsyncPipe): + result[s[3:]] = "\n".join(kwargs[s].get_buffer()) + + for k, v in result.items(): + if not PY2 and isinstance(result[k], bytes): + result[k] = result[k].decode() + if v and isinstance(v, string_types): + result[k] = result[k].strip() + + return result + + +def is_ci(): + return os.getenv("CI", "").lower() == "true" + + +def is_container(): + if not isfile("/proc/1/cgroup"): + return False + with open("/proc/1/cgroup") as fp: + for line in fp: + line = line.strip() + if ":" in line and not line.endswith(":/"): + return True + return False + + +def get_pythonexe_path(): + return os.environ.get("PYTHONEXEPATH", normpath(sys.executable)) + + +def copy_pythonpath_to_osenv(): + _PYTHONPATH = [] + if "PYTHONPATH" in os.environ: + _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) + for p in os.sys.path: + conditions = [p not in _PYTHONPATH] + if not WINDOWS: + conditions.append( + isdir(join(p, "click")) or isdir(join(p, "platformio"))) + if all(conditions): + _PYTHONPATH.append(p) + os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH) + + +def where_is_program(program, envpath=None): + env = os.environ + if envpath: + env['PATH'] = envpath + + # try OS's built-in commands + try: + result = exec_command(["where" if WINDOWS else "which", program], + env=env) + if result['returncode'] == 0 and isfile(result['out'].strip()): + return result['out'].strip() + except OSError: + pass + + # look up in $PATH + for bin_dir in env.get("PATH", "").split(os.pathsep): + if isfile(join(bin_dir, program)): + return join(bin_dir, program) + if isfile(join(bin_dir, "%s.exe" % program)): + return join(bin_dir, "%s.exe" % program) + + return program diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 987d5899..6bdb48a1 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -28,6 +28,7 @@ import requests from platformio import __version__, app, exception, util from platformio.commands import PlatformioCLI +from platformio.proc import is_ci, is_container try: import queue @@ -122,7 +123,7 @@ class MeasurementProtocol(TelemetryBase): platform.platform()) # self['cd3'] = " ".join(_filter_args(sys.argv[1:])) self['cd4'] = 1 if (not util.is_ci() and - (caller_id or not util.is_container())) else 0 + (caller_id or not is_container())) else 0 if caller_id: self['cd5'] = caller_id.lower() @@ -273,7 +274,7 @@ def on_command(): mp = MeasurementProtocol() mp.send("screenview") - if util.is_ci(): + if is_ci(): measure_ci() diff --git a/platformio/util.py b/platformio/util.py index 6e5db848..4e0979fc 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,69 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. +# FIXME: Remove line below before 4.0 release +# pylint: disable=unused-import + import json import os import platform import re import socket import stat -import subprocess import sys import time from functools import wraps from glob import glob from os.path import (abspath, basename, dirname, expanduser, isdir, isfile, - join, normpath, splitdrive) + join, splitdrive) from shutil import rmtree -from threading import Thread import click import requests from platformio import __apiurl__, __version__, exception -from platformio.compat import path_to_unicode # pylint: disable=unused-import -from platformio.compat import PY2, WINDOWS, string_types +from platformio.compat import PY2, WINDOWS, path_to_unicode +from platformio.proc import AsyncPipe, exec_command, is_ci, where_is_program from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( # pylint: disable=unused-import +from platformio.project.helpers import ( get_project_dir, get_project_optional_dir, get_projectboards_dir, get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir, get_projectsrc_dir, get_projecttest_dir, is_platformio_project) -# FIXME: remove import of path_to_unicode - - -class AsyncPipe(Thread): - - def __init__(self, outcallback=None): - super(AsyncPipe, self).__init__() - self.outcallback = outcallback - - self._fd_read, self._fd_write = os.pipe() - self._pipe_reader = os.fdopen(self._fd_read) - self._buffer = [] - - self.start() - - def get_buffer(self): - return self._buffer - - def fileno(self): - return self._fd_write - - def run(self): - for line in iter(self._pipe_reader.readline, ""): - line = line.strip() - self._buffer.append(line) - if self.outcallback: - self.outcallback(line) - else: - print(line) - self._pipe_reader.close() - - def close(self): - os.close(self._fd_write) - self.join() - class cd(object): @@ -219,66 +185,6 @@ def change_filemtime(path, mtime): os.utime(path, (mtime, mtime)) -def is_ci(): - return os.getenv("CI", "").lower() == "true" - - -def is_container(): - if not isfile("/proc/1/cgroup"): - return False - with open("/proc/1/cgroup") as fp: - for line in fp: - line = line.strip() - if ":" in line and not line.endswith(":/"): - return True - return False - - -def exec_command(*args, **kwargs): - result = {"out": None, "err": None, "returncode": None} - - default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) - default.update(kwargs) - kwargs = default - - p = subprocess.Popen(*args, **kwargs) - try: - result['out'], result['err'] = p.communicate() - result['returncode'] = p.returncode - except KeyboardInterrupt: - raise exception.AbortedByUser() - finally: - for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipe): - kwargs[s].close() - - for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipe): - result[s[3:]] = "\n".join(kwargs[s].get_buffer()) - - for k, v in result.items(): - if not PY2 and isinstance(result[k], bytes): - result[k] = result[k].decode() - if v and isinstance(v, string_types): - result[k] = result[k].strip() - - return result - - -def copy_pythonpath_to_osenv(): - _PYTHONPATH = [] - if "PYTHONPATH" in os.environ: - _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) - for p in os.sys.path: - conditions = [p not in _PYTHONPATH] - if not WINDOWS: - conditions.append( - isdir(join(p, "click")) or isdir(join(p, "platformio"))) - if all(conditions): - _PYTHONPATH.append(p) - os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH) - - def get_serial_ports(filter_hwid=False): try: from serial.tools.list_ports import comports @@ -563,34 +469,6 @@ def internet_on(raise_exception=False): return result -def get_pythonexe_path(): - return os.environ.get("PYTHONEXEPATH", normpath(sys.executable)) - - -def where_is_program(program, envpath=None): - env = os.environ - if envpath: - env['PATH'] = envpath - - # try OS's built-in commands - try: - result = exec_command(["where" if WINDOWS else "which", program], - env=env) - if result['returncode'] == 0 and isfile(result['out'].strip()): - return result['out'].strip() - except OSError: - pass - - # look up in $PATH - for bin_dir in env.get("PATH", "").split(os.pathsep): - if isfile(join(bin_dir, program)): - return join(bin_dir, program) - if isfile(join(bin_dir, "%s.exe" % program)): - return join(bin_dir, "%s.exe" % program) - - return program - - def pepver_to_semver(pepver): return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) diff --git a/platformio/vcsclient.py b/platformio/vcsclient.py index 67ef29d2..222ba0e5 100644 --- a/platformio/vcsclient.py +++ b/platformio/vcsclient.py @@ -17,8 +17,8 @@ from os.path import join from subprocess import CalledProcessError, check_call from sys import modules -from platformio import util from platformio.exception import PlatformioException, UserSideException +from platformio.proc import exec_command try: from urllib.parse import urlparse @@ -109,7 +109,7 @@ class VCSClientBase(object): args = [self.command] + args if "cwd" not in kwargs: kwargs['cwd'] = self.src_dir - result = util.exec_command(args, **kwargs) + result = exec_command(args, **kwargs) if result['returncode'] == 0: return result['out'].strip() raise PlatformioException( From 899de600e4557c28eb66f9058f11dc5da574d230 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 May 2019 21:11:21 +0300 Subject: [PATCH 148/333] Fix broken "util.string_types" --- platformio/builder/tools/piolib.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 270f3f21..a5b277cd 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -33,7 +33,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, util from platformio.builder.tools import platformio as piotool -from platformio.compat import PY2, WINDOWS +from platformio.compat import PY2, WINDOWS, string_types from platformio.managers.lib import LibraryManager from platformio.managers.package import PackageManager @@ -233,7 +233,7 @@ class LibBuilderBase(object): @staticmethod def validate_ldf_mode(mode): - if isinstance(mode, util.string_types): + if isinstance(mode, string_types): mode = mode.strip().lower() if mode in LibBuilderBase.LDF_MODES: return mode @@ -245,7 +245,7 @@ class LibBuilderBase(object): @staticmethod def validate_compat_mode(mode): - if isinstance(mode, util.string_types): + if isinstance(mode, string_types): mode = mode.strip().lower() if mode in LibBuilderBase.COMPAT_MODES: return mode From f94fbb951adac7ab62cec0f145c414360b9a937a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 May 2019 21:27:32 +0300 Subject: [PATCH 149/333] Typo fix --- platformio/commands/update.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/commands/update.py b/platformio/commands/update.py index d72a35d4..d4217217 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -34,7 +34,6 @@ from platformio.managers.lib import LibraryManager "--dry-run", is_flag=True, help="Do not update, only check for the new versions") -@click.option("--json-output", is_flag=True) @click.pass_context def cli(ctx, core_packages, only_check, dry_run): # cleanup lib search results, cached board and platform lists From f844d9cb473bda79ec88feff473486e1019bd2c4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 17 May 2019 12:53:51 +0300 Subject: [PATCH 150/333] Remove line-buffering from "platformio run" command which was leading to omitting progress bar from upload tools // Resolve #856, Resolve #857 --- HISTORY.rst | 4 +- platformio/managers/platform.py | 23 ++++++--- platformio/proc.py | 86 +++++++++++++++++++++++++-------- platformio/util.py | 3 +- 4 files changed, 87 insertions(+), 29 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5f0dd973..65b23388 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,7 +18,9 @@ PlatformIO 4.0 * Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields * Added support for the latest Python "Click" package (CLI Builder) (`issue #349 `_) -* Deprecated ``--only-check`` CLI option for "update" sub-commands, please use ``--dry-run`` instead +* Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead +* Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools + (`issue #856 `_) PlatformIO 3.0 -------------- diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index f6dac646..b5537937 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -15,6 +15,7 @@ import base64 import os import re +import sys from imp import load_source from multiprocessing import cpu_count from os.path import basename, dirname, isdir, isfile, join @@ -26,8 +27,8 @@ from platformio import __version__, app, exception, util from platformio.compat import PY2 from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager -from platformio.proc import (copy_pythonpath_to_osenv, exec_command, - get_pythonexe_path) +from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, + exec_command, get_pythonexe_path) from platformio.project.helpers import get_projectboards_dir try: @@ -399,19 +400,27 @@ class PlatformRunMixin(object): "%s=%s" % (key.upper(), base64.b64encode( value.encode()).decode())) + def _write_and_flush(stream, data): + stream.write(data) + stream.flush() + copy_pythonpath_to_osenv() result = exec_command( cmd, - stdout=util.AsyncPipe(self.on_run_out), - stderr=util.AsyncPipe(self.on_run_err)) + stdout=BuildAsyncPipe( + line_callback=self._on_stdout_line, + data_callback=lambda data: _write_and_flush(sys.stdout, data)), + stderr=BuildAsyncPipe( + line_callback=self._on_stderr_line, + data_callback=lambda data: _write_and_flush(sys.stderr, data))) return result - def on_run_out(self, line): + def _on_stdout_line(self, line): if "`buildprog' is up to date." in line: return self._echo_line(line, level=1) - def on_run_err(self, line): + def _on_stderr_line(self, line): is_error = self.LINE_ERROR_RE.search(line) is not None self._echo_line(line, level=3 if is_error else 2) @@ -430,7 +439,7 @@ class PlatformRunMixin(object): fg = (None, "yellow", "red")[level - 1] if level == 1 and "is up to date" in line: fg = "green" - click.secho(line, fg=fg, err=level > 1) + click.secho(line, fg=fg, err=level > 1, nl=False) @staticmethod def _echo_missed_dependency(filename): diff --git a/platformio/proc.py b/platformio/proc.py index a0432458..7cc3d8a7 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -22,17 +22,14 @@ from platformio import exception from platformio.compat import PY2, WINDOWS, string_types -class AsyncPipe(Thread): - - def __init__(self, outcallback=None): - super(AsyncPipe, self).__init__() - self.outcallback = outcallback +class AsyncPipeBase(object): + def __init__(self): self._fd_read, self._fd_write = os.pipe() self._pipe_reader = os.fdopen(self._fd_read) - self._buffer = [] - - self.start() + self._buffer = "" + self._thread = Thread(target=self.run) + self._thread.start() def get_buffer(self): return self._buffer @@ -41,18 +38,67 @@ class AsyncPipe(Thread): return self._fd_write def run(self): - for line in iter(self._pipe_reader.readline, ""): - line = line.strip() - self._buffer.append(line) - if self.outcallback: - self.outcallback(line) - else: - print(line) - self._pipe_reader.close() + try: + self.do_reading() + except (KeyboardInterrupt, SystemExit, IOError): + self.close() + + def do_reading(self): + raise NotImplementedError() def close(self): + self._buffer = "" os.close(self._fd_write) - self.join() + self._thread.join() + + +class BuildAsyncPipe(AsyncPipeBase): + + def __init__(self, line_callback, data_callback): + self.line_callback = line_callback + self.data_callback = data_callback + super(BuildAsyncPipe, self).__init__() + + def do_reading(self): + line = "" + print_immediately = False + + for byte in iter(lambda: self._pipe_reader.read(1), ""): + self._buffer += byte + + if line and line[-3:] == (line[-1] * 3): + print_immediately = True + + if print_immediately: + # leftover bytes + if line: + self.data_callback(line) + line = "" + self.data_callback(byte) + if byte == "\n": + print_immediately = False + else: + line += byte + if byte != "\n": + continue + self.line_callback(line) + line = "" + + self._pipe_reader.close() + + +class LineBufferedAsyncPipe(AsyncPipeBase): + + def __init__(self, line_callback): + self.line_callback = line_callback + super(LineBufferedAsyncPipe, self).__init__() + + def do_reading(self): + for line in iter(self._pipe_reader.readline, ""): + self._buffer += line + # FIXME: Remove striping + self.line_callback(line.strip()) + self._pipe_reader.close() def exec_command(*args, **kwargs): @@ -70,12 +116,12 @@ def exec_command(*args, **kwargs): raise exception.AbortedByUser() finally: for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipe): + if isinstance(kwargs[s], AsyncPipeBase): kwargs[s].close() for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipe): - result[s[3:]] = "\n".join(kwargs[s].get_buffer()) + if isinstance(kwargs[s], AsyncPipeBase): + result[s[3:]] = kwargs[s].get_buffer() for k, v in result.items(): if not PY2 and isinstance(result[k], bytes): diff --git a/platformio/util.py b/platformio/util.py index 4e0979fc..83552d6b 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -34,7 +34,8 @@ import requests from platformio import __apiurl__, __version__, exception from platformio.compat import PY2, WINDOWS, path_to_unicode -from platformio.proc import AsyncPipe, exec_command, is_ci, where_is_program +from platformio.proc import LineBufferedAsyncPipe as AsyncPipe +from platformio.proc import exec_command, is_ci, where_is_program from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_project_dir, get_project_optional_dir, get_projectboards_dir, From 8643f0454e12bb43f14a8043f130bcca16f38ead Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 17 May 2019 13:18:15 +0300 Subject: [PATCH 151/333] Move "glob_escape" and "get_file_contents" helpers to "compat" module --- platformio/builder/tools/pioide.py | 3 ++- platformio/builder/tools/piolib.py | 4 +-- platformio/builder/tools/piomisc.py | 7 +++--- platformio/builder/tools/platformio.py | 4 +-- platformio/commands/ci.py | 3 ++- platformio/commands/remote.py | 3 ++- platformio/compat.py | 34 +++++++++++++++++++++++++ platformio/ide/projectgenerator.py | 5 ++-- platformio/managers/lib.py | 4 +-- platformio/util.py | 35 +------------------------- 10 files changed, 53 insertions(+), 49 deletions(-) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 311b0203..c6d8fbc0 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -21,6 +21,7 @@ from os.path import abspath, isfile, join from SCons.Defaults import processDefines # pylint: disable=import-error from platformio import util +from platformio.compat import glob_escape from platformio.managers.core import get_core_package_dir from platformio.proc import exec_command @@ -40,7 +41,7 @@ def _dump_includes(env): for name in p.get_installed_packages(): if p.get_package_type(name) != "toolchain": continue - toolchain_dir = util.glob_escape(p.get_package_dir(name)) + toolchain_dir = glob_escape(p.get_package_dir(name)) toolchain_incglobs = [ join(toolchain_dir, "*", "include*"), join(toolchain_dir, "*", "include", "c++", "*"), diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index a5b277cd..2f38e649 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -33,7 +33,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, util from platformio.builder.tools import platformio as piotool -from platformio.compat import PY2, WINDOWS, string_types +from platformio.compat import PY2, WINDOWS, get_file_contents, string_types from platformio.managers.lib import LibraryManager from platformio.managers.package import PackageManager @@ -80,7 +80,7 @@ class LibBuilderFactory(object): if not env.IsFileWithExt( fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT): continue - content = util.get_file_contents(join(root, fname)) + content = get_file_contents(join(root, fname)) if not content: continue if "Arduino.h" in content and include_re.search(content): diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 19768146..bfc41a76 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -25,6 +25,7 @@ from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import util +from platformio.compat import get_file_contents, glob_escape from platformio.managers.core import get_core_package_dir from platformio.proc import exec_command @@ -59,7 +60,7 @@ class InoToCPPConverter(object): assert nodes lines = [] for node in nodes: - contents = util.get_file_contents(node.get_path()) + contents = get_file_contents(node.get_path()) _lines = [ '# 1 "%s"' % node.get_path().replace("\\", "/"), contents ] @@ -77,7 +78,7 @@ class InoToCPPConverter(object): def process(self, contents): out_file = self._main_ino + ".cpp" assert self._gcc_preprocess(contents, out_file) - contents = util.get_file_contents(out_file) + contents = get_file_contents(out_file) contents = self._join_multiline_strings(contents) with open(out_file, "w") as fp: fp.write(self.append_prototypes(contents)) @@ -188,7 +189,7 @@ class InoToCPPConverter(object): def ConvertInoToCpp(env): - src_dir = util.glob_escape(env.subst("$PROJECTSRC_DIR")) + src_dir = glob_escape(env.subst("$PROJECTSRC_DIR")) ino_nodes = ( env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde"))) if not ino_nodes: diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index a8272b83..3a0347c4 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -27,8 +27,8 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Export # pylint: disable=import-error from SCons.Script import SConscript # pylint: disable=import-error -from platformio.compat import string_types -from platformio.util import glob_escape, pioversion_to_intstr +from platformio.compat import glob_escape, string_types +from platformio.util import pioversion_to_intstr SRC_HEADER_EXT = ["h", "hpp"] SRC_C_EXT = ["c", "cc", "cpp"] diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 9fd157e3..7d97eb48 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -24,6 +24,7 @@ from platformio import app, util from platformio.commands.init import cli as cmd_init from platformio.commands.init import validate_boards from platformio.commands.run import cli as cmd_run +from platformio.compat import glob_escape from platformio.exception import CIBuildEnvsEmpty from platformio.project.config import ProjectConfig @@ -152,7 +153,7 @@ def _copy_contents(dst_dir, contents): def _exclude_contents(dst_dir, patterns): contents = [] for p in patterns: - contents += glob(join(util.glob_escape(dst_dir), p)) + contents += glob(join(glob_escape(dst_dir), p)) for path in contents: path = abspath(path) if isdir(path): diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index b37e843c..ded45eda 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -23,6 +23,7 @@ import click from platformio import exception, util from platformio.commands.device import device_monitor as cmd_device_monitor +from platformio.compat import get_file_contents from platformio.managers.core import pioplus_call # pylint: disable=unused-argument @@ -206,7 +207,7 @@ def device_monitor(ctx, **kwargs): sleep(0.1) if not t.is_alive(): return - kwargs['port'] = util.get_file_contents(sock_file) + kwargs['port'] = get_file_contents(sock_file) ctx.invoke(cmd_device_monitor, **kwargs) t.join(2) finally: diff --git a/platformio/compat.py b/platformio/compat.py index cd50d905..3c3afc52 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=unused-import + +import os +import re import sys PY2 = sys.version_info[0] == 2 @@ -32,7 +36,29 @@ if PY2: def path_to_unicode(path): return path.decode(get_filesystem_encoding()).encode("utf-8") + + def get_file_contents(path): + with open(path) as f: + return f.read() + + _magic_check = re.compile('([*?[])') + _magic_check_bytes = re.compile(b'([*?[])') + + def glob_escape(pathname): + """Escape all special characters.""" + # https://github.com/python/cpython/blob/master/Lib/glob.py#L161 + # Escaping is done by wrapping any of "*?[" between square brackets. + # Metacharacters do not work in the drive part and shouldn't be + # escaped. + drive, pathname = os.path.splitdrive(pathname) + if isinstance(pathname, bytes): + pathname = _magic_check_bytes.sub(br'[\1]', pathname) + else: + pathname = _magic_check.sub(r'[\1]', pathname) + return drive + pathname else: + from glob import escape as glob_escape # pylint: disable=no-name-in-module + string_types = (str, ) def is_bytes(x): @@ -40,3 +66,11 @@ else: def path_to_unicode(path): return path + + def get_file_contents(path): + try: + with open(path) as f: + return f.read() + except UnicodeDecodeError: + with open(path, encoding="latin-1") as f: + return f.read() diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 1216aaee..37d0db53 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -23,7 +23,7 @@ from click.testing import CliRunner from platformio import exception, util from platformio.commands.run import cli as cmd_run -from platformio.compat import PY2, WINDOWS +from platformio.compat import PY2, WINDOWS, get_file_contents from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_projectlib_dir, get_projectlibdeps_dir, get_projectsrc_dir) @@ -124,8 +124,7 @@ class ProjectGenerator(object): contents.encode("utf8") if PY2 else contents) def _render_tpl(self, tpl_path): - return bottle.template( - util.get_file_contents(tpl_path), **self._tplvars) + return bottle.template(get_file_contents(tpl_path), **self._tplvars) @staticmethod def _merge_contents(dst_path, contents): diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index b478f49c..ca143b58 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -23,7 +23,7 @@ from os.path import isdir, join import click from platformio import app, commands, exception, util -from platformio.compat import string_types +from platformio.compat import glob_escape, string_types from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager @@ -48,7 +48,7 @@ class LibraryManager(BasePkgManager): return path # if library without manifest, returns first source file - src_dir = join(util.glob_escape(pkg_dir)) + src_dir = join(glob_escape(pkg_dir)) if isdir(join(pkg_dir, "src")): src_dir = join(src_dir, "src") chs_files = glob(join(src_dir, "*.[chS]")) diff --git a/platformio/util.py b/platformio/util.py index 83552d6b..5de68256 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -33,7 +33,7 @@ import click import requests from platformio import __apiurl__, __version__, exception -from platformio.compat import PY2, WINDOWS, path_to_unicode +from platformio.compat import PY2, WINDOWS, get_file_contents, path_to_unicode from platformio.proc import LineBufferedAsyncPipe as AsyncPipe from platformio.proc import exec_command, is_ci, where_is_program from platformio.project.config import ProjectConfig @@ -523,15 +523,6 @@ def merge_dicts(d1, d2, path=None): return d1 -def get_file_contents(path): - try: - with open(path) as f: - return f.read() - except UnicodeDecodeError: - with open(path, encoding="latin-1") as f: - return f.read() - - def ensure_udev_rules(): def _rules_to_set(rules_path): @@ -588,27 +579,3 @@ def rmtree_(path): err=True) return rmtree(path, onerror=_onerror) - - -# -# Glob.Escape from Python 3.4 -# https://github.com/python/cpython/blob/master/Lib/glob.py#L161 -# - -try: - from glob import escape as glob_escape # pylint: disable=unused-import -except ImportError: - magic_check = re.compile('([*?[])') - magic_check_bytes = re.compile(b'([*?[])') - - def glob_escape(pathname): - """Escape all special characters.""" - # Escaping is done by wrapping any of "*?[" between square brackets. - # Metacharacters do not work in the drive part and shouldn't be - # escaped. - drive, pathname = os.path.splitdrive(pathname) - if isinstance(pathname, bytes): - pathname = magic_check_bytes.sub(br'[\1]', pathname) - else: - pathname = magic_check.sub(r'[\1]', pathname) - return drive + pathname From 774380c2ef6ef873847361a1dd889f6c042b1c86 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 May 2019 12:38:49 +0300 Subject: [PATCH 152/333] Bump version to 4.0.0a14 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index c68c1fe5..a78b131c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a13") +VERSION = (4, 0, "0a14") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 5fe4de626bb7755bd1d62da88d11916f54359ea6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 May 2019 17:01:54 +0300 Subject: [PATCH 153/333] Implement unified project workspace storage ".pio" // Resolve #1778 --- HISTORY.rst | 32 +++++++++++-------- docs | 2 +- platformio/builder/main.py | 4 ++- platformio/commands/init.py | 2 +- platformio/commands/run.py | 5 +++ platformio/ide/tpls/atom/.gitignore.tpl | 1 - platformio/ide/tpls/clion/.gitignore.tpl | 1 - platformio/ide/tpls/clion/.idea/misc.xml.tpl | 3 -- platformio/ide/tpls/emacs/.gitignore.tpl | 1 - .../netbeans/nbproject/configurations.xml.tpl | 2 +- platformio/ide/tpls/vim/.gitignore.tpl | 1 - platformio/ide/tpls/vscode/.gitignore.tpl | 1 - platformio/project/config.py | 1 + platformio/project/helpers.py | 29 ++++++++++------- 14 files changed, 47 insertions(+), 38 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 65b23388..e9936d1d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,20 +7,24 @@ PlatformIO 4.0 4.0.0 (2019-??-??) ~~~~~~~~~~~~~~~~~~ -* Python 3 support - (`issue #895 `_) -* Share common (global) options between declared build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) `__ - (`issue #1643 `_) -* Include external configuration files in `"platformio.ini" (Project Configuration File) `__ with `extra_configs `__ option - (`issue #1590 `_) -* Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands - (`issue #1913 `_) -* Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields -* Added support for the latest Python "Click" package (CLI Builder) - (`issue #349 `_) -* Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead -* Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools - (`issue #856 `_) +* **PlatformIO-based Project** + + - Implemented unified project workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Dependency Finder, and other internal services (`issue #1778 `_) + - Moved ``.pioenvs`` build directory to workspace storage ``.pio/build`` + - Share common (global) options between build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) `__ (`issue #1643 `_) + - Include external configuration files in `"platformio.ini" (Project Configuration File) `__ with `extra_configs `__ option (`issue #1590 `_) + - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) + +* **Infrastructure** + + - Python 3 support (`issue #895 `_) + - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) + - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) + +* **Miscellaneous** + + - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields + - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead PlatformIO 3.0 -------------- diff --git a/docs b/docs index 6e000475..c7591d26 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6e0004755ac1849b66cf45ec8e0fca47a35226e0 +Subproject commit c7591d260bea5cb04aaab1fd93ca008586f333ea diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 287c0e84..5a5c8676 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -34,7 +34,8 @@ from platformio.proc import get_pythonexe_path from platformio.project.helpers import ( get_project_dir, get_project_optional_dir, get_projectbuild_dir, get_projectdata_dir, get_projectinclude_dir, get_projectlib_dir, - get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir) + get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir, + get_projectworkspace_dir) AllowSubstExceptions(NameError) @@ -108,6 +109,7 @@ DEFAULT_ENV_OPTIONS = dict( UNIX_TIME=int(time()), PIOHOME_DIR=util.get_home_dir(), PROJECT_DIR=get_project_dir(), + PROJECTWORKSPACE_DIR=get_projectworkspace_dir(), PROJECTINCLUDE_DIR=get_projectinclude_dir(), PROJECTSRC_DIR=get_projectsrc_dir(), PROJECTTEST_DIR=get_projecttest_dir(), diff --git a/platformio/commands/init.py b/platformio/commands/init.py index 70f8a8e4..b8f293d2 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -359,7 +359,7 @@ def init_cvs_ignore(project_dir): if isfile(conf_path): return with open(conf_path, "w") as fp: - fp.writelines([".pio\n", ".pioenvs\n", ".piolibdeps\n"]) + fp.writelines([".pio\n", ".piolibdeps\n"]) def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, diff --git a/platformio/commands/run.py b/platformio/commands/run.py index a7327d34..00c2622a 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -263,6 +263,11 @@ def _autoinstall_libdeps(ctx, libraries, verbose=False): def _clean_build_dir(build_dir): + # remove legacy ".pioenvs" folder + legacy_build_dir = join(get_project_dir(), ".pioenvs") + if isdir(legacy_build_dir) and legacy_build_dir != build_dir: + util.rmtree_(legacy_build_dir) + structhash_file = join(build_dir, "structure.hash") proj_hash = calculate_project_hash() diff --git a/platformio/ide/tpls/atom/.gitignore.tpl b/platformio/ide/tpls/atom/.gitignore.tpl index f1520281..04de5dbc 100644 --- a/platformio/ide/tpls/atom/.gitignore.tpl +++ b/platformio/ide/tpls/atom/.gitignore.tpl @@ -1,5 +1,4 @@ .pio -.pioenvs .piolibdeps .clang_complete .gcc-flags.json diff --git a/platformio/ide/tpls/clion/.gitignore.tpl b/platformio/ide/tpls/clion/.gitignore.tpl index 4c5921e1..ff5b1094 100644 --- a/platformio/ide/tpls/clion/.gitignore.tpl +++ b/platformio/ide/tpls/clion/.gitignore.tpl @@ -1,4 +1,3 @@ .pio -.pioenvs .piolibdeps CMakeListsPrivate.txt diff --git a/platformio/ide/tpls/clion/.idea/misc.xml.tpl b/platformio/ide/tpls/clion/.idea/misc.xml.tpl index 9b483ee6..8ba8c6a0 100644 --- a/platformio/ide/tpls/clion/.idea/misc.xml.tpl +++ b/platformio/ide/tpls/clion/.idea/misc.xml.tpl @@ -12,8 +12,5 @@ - - - \ No newline at end of file diff --git a/platformio/ide/tpls/emacs/.gitignore.tpl b/platformio/ide/tpls/emacs/.gitignore.tpl index 2c3fccfb..c086bafd 100644 --- a/platformio/ide/tpls/emacs/.gitignore.tpl +++ b/platformio/ide/tpls/emacs/.gitignore.tpl @@ -1,4 +1,3 @@ .pio -.pioenvs .piolibdeps .clang_complete diff --git a/platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl b/platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl index 7c43f565..691c6558 100644 --- a/platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl +++ b/platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl @@ -11,7 +11,7 @@ nbproject/private/launcher.properties - ^(nbproject|.pio|.pioenvs)$ + ^(nbproject|.pio)$ . diff --git a/platformio/ide/tpls/vim/.gitignore.tpl b/platformio/ide/tpls/vim/.gitignore.tpl index f1520281..04de5dbc 100644 --- a/platformio/ide/tpls/vim/.gitignore.tpl +++ b/platformio/ide/tpls/vim/.gitignore.tpl @@ -1,5 +1,4 @@ .pio -.pioenvs .piolibdeps .clang_complete .gcc-flags.json diff --git a/platformio/ide/tpls/vscode/.gitignore.tpl b/platformio/ide/tpls/vscode/.gitignore.tpl index 2de98aba..e2fb3b9a 100644 --- a/platformio/ide/tpls/vscode/.gitignore.tpl +++ b/platformio/ide/tpls/vscode/.gitignore.tpl @@ -1,5 +1,4 @@ .pio -.pioenvs .piolibdeps .vscode/.browse.c_cpp.db* .vscode/c_cpp_properties.json diff --git a/platformio/project/config.py b/platformio/project/config.py index d4f82c1b..419ced01 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -46,6 +46,7 @@ KNOWN_PLATFORMIO_OPTIONS = [ # Dirs "home_dir", + "workspace_dir", "lib_dir", "libdeps_dir", "include_dir", diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 66a9c39b..4112f352 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -73,6 +73,23 @@ def get_project_optional_dir(name, default=None): return paths +def get_projectworkspace_dir(): + return get_project_optional_dir("workspace_dir", + join(get_project_dir(), ".pio")) + + +def get_projectbuild_dir(force=False): + path = get_project_optional_dir("build_dir", + join(get_projectworkspace_dir(), "build")) + try: + if not isdir(path): + os.makedirs(path) + except Exception as e: # pylint: disable=broad-except + if not force: + raise Exception(e) + return path + + def get_projectlib_dir(): return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) @@ -101,18 +118,6 @@ def get_projectboards_dir(): join(get_project_dir(), "boards")) -def get_projectbuild_dir(force=False): - path = get_project_optional_dir("build_dir", - join(get_project_dir(), ".pioenvs")) - try: - if not isdir(path): - os.makedirs(path) - except Exception as e: # pylint: disable=broad-except - if not force: - raise Exception(e) - return path - - def get_projectdata_dir(): return get_project_optional_dir("data_dir", join(get_project_dir(), "data")) From c46643f0fd4ac78c4d35fa29a0887ab56a919b87 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 May 2019 17:07:59 +0300 Subject: [PATCH 154/333] =?UTF-8?q?Custom=20project=20"***=5Fdir"=20option?= =?UTF-8?q?s=20declared=20in=20=E2=80=9Cplatformio=E2=80=9D=20section=20of?= =?UTF-8?q?=20=E2=80=9Cplatformio.ini=E2=80=9D=20have=20higher=20priority?= =?UTF-8?q?=20than=20Environment=20variables?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HISTORY.rst | 3 ++- platformio/project/helpers.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e9936d1d..a9397d32 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,10 +10,11 @@ PlatformIO 4.0 * **PlatformIO-based Project** - Implemented unified project workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Dependency Finder, and other internal services (`issue #1778 `_) - - Moved ``.pioenvs`` build directory to workspace storage ``.pio/build`` - Share common (global) options between build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) `__ (`issue #1643 `_) - Include external configuration files in `"platformio.ini" (Project Configuration File) `__ with `extra_configs `__ option (`issue #1590 `_) - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) + - Custom project ``***_dir`` options declared in "platformio" section of `"platformio.ini" (Project Configuration File) `__ have higher priority than `Environment variables `__ + - Moved ``.pioenvs`` build directory to workspace storage ``.pio/build`` * **Infrastructure** diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 4112f352..ec72b213 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -44,15 +44,18 @@ def find_project_dir_above(path): def get_project_optional_dir(name, default=None): paths = None + + # check for system environment variable var_name = "PLATFORMIO_%s" % name.upper() if var_name in os.environ: paths = os.getenv(var_name) - else: - config = ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini")) - if (config.has_section("platformio") - and config.has_option("platformio", name)): - paths = config.get("platformio", name) + + config = ProjectConfig.get_instance( + join(get_project_dir(), "platformio.ini")) + if (config.has_section("platformio") + and config.has_option("platformio", name)): + paths = config.get("platformio", name) + if not paths: return default From f29a74042f1841d25d772e1d5ac970a05c532ff3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 20 May 2019 21:12:45 +0300 Subject: [PATCH 155/333] Drop support for "lib_extra_dirs" in "platformio" section --- docs | 2 +- platformio/builder/main.py | 19 +++++++-------- platformio/commands/run.py | 6 ++--- platformio/project/config.py | 46 ++++++++++++++++++++++++++---------- 4 files changed, 46 insertions(+), 27 deletions(-) diff --git a/docs b/docs index c7591d26..374f5e11 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c7591d260bea5cb04aaab1fd93ca008586f333ea +Subproject commit 374f5e110f01eebad6065ca9774b831380e0cfb2 diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 5a5c8676..e6b5d727 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -31,11 +31,11 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import util from platformio.compat import PY2, path_to_unicode from platformio.proc import get_pythonexe_path +from platformio.project.config import ProjectConfig from platformio.project.helpers import ( - get_project_dir, get_project_optional_dir, get_projectbuild_dir, - get_projectdata_dir, get_projectinclude_dir, get_projectlib_dir, - get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir, - get_projectworkspace_dir) + get_project_dir, get_projectbuild_dir, get_projectdata_dir, + get_projectinclude_dir, get_projectlib_dir, get_projectlibdeps_dir, + get_projectsrc_dir, get_projecttest_dir, get_projectworkspace_dir) AllowSubstExceptions(NameError) @@ -144,7 +144,7 @@ for k in list(commonvars.keys()): if isinstance(env[k], bytes): env[k] = env[k].decode() if k in MULTILINE_VARS: - env[k] = util.parse_conf_multi_values(env[k]) + env[k] = ProjectConfig.parse_multi_values(env[k]) if env.GetOption('clean'): env.PioClean(env.subst("$BUILD_DIR")) @@ -161,18 +161,15 @@ for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPTS", if var in ("UPLOAD_PORT", ): env[var] = environ.get(k) continue - env.Append(**{var: util.parse_conf_multi_values(environ.get(k))}) + env.Append(**{var: ProjectConfig.parse_multi_values(environ.get(k))}) -# Configure extra library source directories for LDF -if get_project_optional_dir("lib_extra_dirs"): - env.Prepend( - LIBSOURCE_DIRS=util.parse_conf_multi_values( - get_project_optional_dir("lib_extra_dirs"))) env.Prepend(LIBSOURCE_DIRS=env.get("LIB_EXTRA_DIRS", [])) env['LIBSOURCE_DIRS'] = [ expanduser(d) if d.startswith("~") else d for d in env['LIBSOURCE_DIRS'] ] +print(env['LIBSOURCE_DIRS']) + env.LoadPioPlatform(commonvars) env.SConscriptChdir(0) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 00c2622a..e3085626 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -161,8 +161,8 @@ class EnvironmentProcessor(object): for k, v in self.options.items(): self.options[k] = self.options[k].strip() if self.verbose or k in self.DEFAULT_DUMP_OPTIONS: - env_dump.append( - "%s: %s" % (k, ", ".join(util.parse_conf_multi_values(v)))) + env_dump.append("%s: %s" % (k, ", ".join( + ProjectConfig.parse_multi_values(v)))) if not self.silent: click.echo("Processing %s (%s)" % (click.style( @@ -230,7 +230,7 @@ class EnvironmentProcessor(object): if "lib_deps" in self.options: _autoinstall_libdeps( self.cmd_ctx, - util.parse_conf_multi_values(self.options['lib_deps']), + ProjectConfig.parse_multi_values(self.options['lib_deps']), self.verbose) try: diff --git a/platformio/project/config.py b/platformio/project/config.py index 419ced01..19842922 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -54,8 +54,7 @@ KNOWN_PLATFORMIO_OPTIONS = [ "build_dir", "data_dir", "test_dir", - "boards_dir", - "lib_extra_dirs" + "boards_dir" ] KNOWN_ENV_OPTIONS = [ @@ -275,17 +274,40 @@ class ProjectConfig(object): return self.validate_options() if validate_options else True def validate_options(self): - warnings = set() - # check [platformio] section - if self._parser.has_section("platformio"): - unknown = set(k for k, _ in self.items("platformio")) - set( - KNOWN_PLATFORMIO_OPTIONS) - if unknown: - warnings.add( - "Ignore unknown `%s` options in section `[platformio]`" % - ", ".join(unknown)) + return (self._validate_platformio_options() + and self._validate_env_options()) + + def _validate_platformio_options(self): + if not self._parser.has_section("platformio"): + return True + warnings = set() + + # legacy `lib_extra_dirs` + if self._parser.has_option("platformio", "lib_extra_dirs"): + if not self._parser.has_section("env"): + self._parser.add_section("env") + self._parser.set("env", "lib_extra_dirs", + self._parser.get("platformio", "lib_extra_dirs")) + self._parser.remove_option("platformio", "lib_extra_dirs") + warnings.add( + "`lib_extra_dirs` option is deprecated in section " + "`platformio`! Please move it to global `env` section") + + unknown = set(k for k, _ in self.items("platformio")) - set( + KNOWN_PLATFORMIO_OPTIONS) + if unknown: + warnings.add( + "Ignore unknown `%s` options in section `[platformio]`" % + ", ".join(unknown)) + + for warning in warnings: + click.secho("Warning! %s" % warning, fg="yellow") + + return True + + def _validate_env_options(self): + warnings = set() - # check [env:*] sections for section in self._parser.sections(): if section != "env" and not section.startswith("env:"): continue From a30b79c5fc31116204c1f11dbeb4c63fa8e4e4d8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 21 May 2019 12:01:14 +0300 Subject: [PATCH 156/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 374f5e11..b96d5511 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 374f5e110f01eebad6065ca9774b831380e0cfb2 +Subproject commit b96d5511cb06b411e60165a23b321cf878a38977 From 4386dc56ea8f795d2cd394ce18bd9bf886ccd6cf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 21 May 2019 13:18:11 +0300 Subject: [PATCH 157/333] Move "in_silence" to PlatformioCLI --- docs | 2 +- platformio/commands/__init__.py | 8 ++++++++ platformio/commands/lib.py | 2 +- platformio/maintenance.py | 12 ++---------- platformio/util.py | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs b/docs index b96d5511..103ed844 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b96d5511cb06b411e60165a23b321cf878a38977 +Subproject commit 103ed8445c5993ff305cc14b0fb9929d5fa47053 diff --git a/platformio/commands/__init__.py b/platformio/commands/__init__.py index 83c88e11..7dc69d58 100644 --- a/platformio/commands/__init__.py +++ b/platformio/commands/__init__.py @@ -22,6 +22,14 @@ class PlatformioCLI(click.MultiCommand): leftover_args = [] + @staticmethod + def in_silence(): + args = PlatformioCLI.leftover_args + return args and any([ + args[0] == "debug" and "--interpreter" in " ".join(args), + args[0] == "upgrade", "--json-output" in args, "--version" in args + ]) + def invoke(self, ctx): PlatformioCLI.leftover_args = ctx.args if hasattr(ctx, "protected_args"): diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 81a4640a..9defcc51 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -80,7 +80,7 @@ def cli(ctx, **options): ctx.invoked_subcommand) ctx.obj = LibraryManager(storage_dir) - if "--json-output" not in PlatformioCLI.leftover_args: + if not PlatformioCLI.in_silence(): click.echo("Library Storage: " + storage_dir) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 4b208385..cf563289 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -42,12 +42,12 @@ def on_platformio_start(ctx, force, caller): set_caller(caller) telemetry.on_command() - if not in_silence(): + if not PlatformioCLI.in_silence(): after_upgrade(ctx) def on_platformio_end(ctx, result): # pylint: disable=unused-argument - if in_silence(): + if PlatformioCLI.in_silence(): return try: @@ -66,14 +66,6 @@ def on_platformio_exception(e): telemetry.on_exception(e) -def in_silence(): - args = PlatformioCLI.leftover_args - return args and any([ - args[0] == "debug" and "--interpreter" in " ".join(args), - args[0] == "upgrade", "--json-output" in args, "--version" in args - ]) - - def set_caller(caller=None): if not caller: if getenv("PLATFORMIO_CALLER"): diff --git a/platformio/util.py b/platformio/util.py index 5de68256..a043abd9 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -33,6 +33,7 @@ import click import requests from platformio import __apiurl__, __version__, exception +from platformio.commands import PlatformioCLI from platformio.compat import PY2, WINDOWS, get_file_contents, path_to_unicode from platformio.proc import LineBufferedAsyncPipe as AsyncPipe from platformio.proc import exec_command, is_ci, where_is_program @@ -425,9 +426,8 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): return json.loads(result) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: - from platformio.maintenance import in_silence total += 1 - if not in_silence(): + if not PlatformioCLI.in_silence(): click.secho( "[API] ConnectionError: {0} (incremented retry: max={1}, " "total={2})".format(e, max_retries, total), From e7d75d1412189d1a26d8f9c8e82e4e01f6ee6b8a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 21 May 2019 21:47:20 +0300 Subject: [PATCH 158/333] Remove debug code --- platformio/builder/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e6b5d727..b9937453 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -168,8 +168,6 @@ env['LIBSOURCE_DIRS'] = [ expanduser(d) if d.startswith("~") else d for d in env['LIBSOURCE_DIRS'] ] -print(env['LIBSOURCE_DIRS']) - env.LoadPioPlatform(commonvars) env.SConscriptChdir(0) From 21e2ac6695288ad0b1dae41b731a7b74acab47f9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 May 2019 00:23:24 +0300 Subject: [PATCH 159/333] Use isolated library dependency storage per project build environment // Resolve #1696 --- HISTORY.rst | 13 +- docs | 2 +- platformio/builder/main.py | 3 +- platformio/commands/lib.py | 248 +++++++++++++++++------------ platformio/commands/run.py | 39 +++-- platformio/commands/update.py | 2 +- platformio/ide/projectgenerator.py | 3 +- platformio/maintenance.py | 2 +- platformio/managers/core.py | 4 +- platformio/project/config.py | 5 + platformio/project/helpers.py | 10 +- platformio/util.py | 3 +- 12 files changed, 207 insertions(+), 127 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a9397d32..c757e11b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,14 +7,20 @@ PlatformIO 4.0 4.0.0 (2019-??-??) ~~~~~~~~~~~~~~~~~~ -* **PlatformIO-based Project** +* **Project Management** - - Implemented unified project workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Dependency Finder, and other internal services (`issue #1778 `_) + - Unified workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 `_) - Share common (global) options between build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) `__ (`issue #1643 `_) - Include external configuration files in `"platformio.ini" (Project Configuration File) `__ with `extra_configs `__ option (`issue #1590 `_) - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - Custom project ``***_dir`` options declared in "platformio" section of `"platformio.ini" (Project Configuration File) `__ have higher priority than `Environment variables `__ - - Moved ``.pioenvs`` build directory to workspace storage ``.pio/build`` + - Use workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` + +* **Library Management** + + - Use isolated library dependency storage per project build environment (`issue #1696 `_) + - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields + - Use workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` * **Infrastructure** @@ -24,7 +30,6 @@ PlatformIO 4.0 * **Miscellaneous** - - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead PlatformIO 3.0 diff --git a/docs b/docs index 103ed844..bbf0f91e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 103ed8445c5993ff305cc14b0fb9929d5fa47053 +Subproject commit bbf0f91e9fc5096edcd9e0656f051da3dbbabf05 diff --git a/platformio/builder/main.py b/platformio/builder/main.py index b9937453..4ccbd0f9 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -110,6 +110,7 @@ DEFAULT_ENV_OPTIONS = dict( PIOHOME_DIR=util.get_home_dir(), PROJECT_DIR=get_project_dir(), PROJECTWORKSPACE_DIR=get_projectworkspace_dir(), + PROJECTLIBDEPS_DIR=get_projectlibdeps_dir(), PROJECTINCLUDE_DIR=get_projectinclude_dir(), PROJECTSRC_DIR=get_projectsrc_dir(), PROJECTTEST_DIR=get_projecttest_dir(), @@ -121,7 +122,7 @@ DEFAULT_ENV_OPTIONS = dict( LIBPATH=["$BUILD_DIR"], LIBSOURCE_DIRS=[ get_projectlib_dir(), - get_projectlibdeps_dir(), + join("$PROJECTLIBDEPS_DIR", "$PIOENV"), join("$PIOHOME_DIR", "lib") ], PROGNAME="program", diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 9defcc51..bcba1935 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -21,9 +21,9 @@ from os.path import isdir, join import click from platformio import exception, util -from platformio.commands import PlatformioCLI from platformio.managers.lib import LibraryManager, get_builtin_libs from platformio.proc import is_ci +from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_project_dir, get_projectlibdeps_dir, is_platformio_project) @@ -34,14 +34,10 @@ except ImportError: @click.group(short_help="Library Manager") -@click.option( - "-g", - "--global", - is_flag=True, - help="Manage global PlatformIO library storage") @click.option( "-d", "--storage-dir", + multiple=True, default=None, type=click.Path( exists=True, @@ -50,38 +46,56 @@ except ImportError: writable=True, resolve_path=True), help="Manage custom library storage") +@click.option( + "-g", + "--global", + is_flag=True, + help="Manage global PlatformIO library storage") +@click.option( + "-e", + "--environment", + multiple=True, + help=("Manage libraries for the specific project build environments " + "declared in `platformio.ini`")) @click.pass_context def cli(ctx, **options): - non_storage_cmds = ("search", "show", "register", "stats", "builtin") + storage_cmds = ("install", "uninstall", "update", "list") # skip commands that don't need storage folder - if ctx.invoked_subcommand in non_storage_cmds or \ + if ctx.invoked_subcommand not in storage_cmds or \ (len(ctx.args) == 2 and ctx.args[1] in ("-h", "--help")): return - storage_dir = options['storage_dir'] - if not storage_dir: - if options['global']: - storage_dir = join(util.get_home_dir(), "lib") - elif is_platformio_project(): - storage_dir = get_projectlibdeps_dir() + storage_dirs = list(options['storage_dir']) + if options['global']: + storage_dirs.append(join(util.get_home_dir(), "lib")) + if not storage_dirs: + if is_platformio_project(): + storage_dirs = [get_project_dir()] elif is_ci(): - storage_dir = join(util.get_home_dir(), "lib") + storage_dirs = [join(util.get_home_dir(), "lib")] click.secho( "Warning! Global library storage is used automatically. " "Please use `platformio lib --global %s` command to remove " "this warning." % ctx.invoked_subcommand, fg="yellow") - elif is_platformio_project(storage_dir): - with util.cd(storage_dir): - storage_dir = get_projectlibdeps_dir() - if not storage_dir and not is_platformio_project(): + if not storage_dirs: raise exception.NotGlobalLibDir(get_project_dir(), join(util.get_home_dir(), "lib"), ctx.invoked_subcommand) - - ctx.obj = LibraryManager(storage_dir) - if not PlatformioCLI.in_silence(): - click.echo("Library Storage: " + storage_dir) + ctx.obj = [] + for storage_dir in storage_dirs: + if is_platformio_project(storage_dir): + with util.cd(storage_dir): + config = ProjectConfig.get_instance( + join(storage_dir, "platformio.ini")) + config.validate(options['environment']) + libdeps_dir = get_projectlibdeps_dir() + for env in config.envs(): + if (not options['environment'] + or env in options['environment']): + ctx.obj.append(join(libdeps_dir, env)) + else: + ctx.obj.append(storage_dir) @cli.command("install", short_help="Install library") @@ -103,19 +117,24 @@ def cli(ctx, **options): is_flag=True, help="Reinstall/redownload library if exists") @click.pass_obj -def lib_install(lm, libraries, silent, interactive, force): - # @TODO: "save" option - for library in libraries: - lm.install( - library, silent=silent, interactive=interactive, force=force) +def lib_install(storage_dirs, libraries, silent, interactive, force): + for storage_dir in storage_dirs: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) + for library in libraries: + lm.install( + library, silent=silent, interactive=interactive, force=force) @cli.command("uninstall", short_help="Uninstall libraries") @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]") @click.pass_obj -def lib_uninstall(lm, libraries): - for library in libraries: - lm.uninstall(library) +def lib_uninstall(storage_dirs, libraries): + for storage_dir in storage_dirs: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) + for library in libraries: + lm.uninstall(library) @cli.command("update", short_help="Update installed libraries") @@ -131,68 +150,72 @@ def lib_uninstall(lm, libraries): help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) @click.pass_obj -def lib_update(lm, libraries, only_check, dry_run, json_output): - if not libraries: - libraries = [manifest['__pkg_dir'] for manifest in lm.get_installed()] - +def lib_update(storage_dirs, libraries, only_check, dry_run, json_output): only_check = dry_run or only_check + json_result = {} + for storage_dir in storage_dirs: + if not json_output: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) - if only_check and json_output: - result = [] - for library in libraries: - pkg_dir = library if isdir(library) else None - requirements = None - url = None - if not pkg_dir: - name, requirements, url = lm.parse_pkg_uri(library) - pkg_dir = lm.get_package_dir(name, requirements, url) - if not pkg_dir: - continue - latest = lm.outdated(pkg_dir, requirements) - if not latest: - continue - manifest = lm.load_manifest(pkg_dir) - manifest['versionLatest'] = latest - result.append(manifest) - return click.echo(json.dumps(result)) + _libraries = libraries + if not _libraries: + _libraries = [ + manifest['__pkg_dir'] for manifest in lm.get_installed() + ] - for library in libraries: - lm.update(library, only_check=only_check) + if only_check and json_output: + result = [] + for library in _libraries: + pkg_dir = library if isdir(library) else None + requirements = None + url = None + if not pkg_dir: + name, requirements, url = lm.parse_pkg_uri(library) + pkg_dir = lm.get_package_dir(name, requirements, url) + if not pkg_dir: + continue + latest = lm.outdated(pkg_dir, requirements) + if not latest: + continue + manifest = lm.load_manifest(pkg_dir) + manifest['versionLatest'] = latest + result.append(manifest) + json_result[storage_dir] = result + else: + for library in _libraries: + lm.update(library, only_check=only_check) + + if json_output: + return click.echo( + json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) == + 1 else json_result)) return True -def print_lib_item(item): - click.secho(item['name'], fg="cyan") - click.echo("=" * len(item['name'])) - if "id" in item: - click.secho("#ID: %d" % item['id'], bold=True) - if "description" in item or "url" in item: - click.echo(item.get("description", item.get("url", ""))) - click.echo() - - for key in ("version", "homepage", "license", "keywords"): - if key not in item or not item[key]: - continue - if isinstance(item[key], list): - click.echo("%s: %s" % (key.title(), ", ".join(item[key]))) +@cli.command("list", short_help="List installed libraries") +@click.option("--json-output", is_flag=True) +@click.pass_obj +def lib_list(storage_dirs, json_output): + json_result = {} + for storage_dir in storage_dirs: + if not json_output: + print_storage_header(storage_dirs, storage_dir) + lm = LibraryManager(storage_dir) + items = lm.get_installed() + if json_output: + json_result[storage_dir] = items else: - click.echo("%s: %s" % (key.title(), item[key])) + for item in sorted(items, key=lambda i: i['name']): + print_lib_item(item) - for key in ("frameworks", "platforms"): - if key not in item: - continue - click.echo("Compatible %s: %s" % (key, ", ".join( - [i['title'] if isinstance(i, dict) else i for i in item[key]]))) + if json_output: + return click.echo( + json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) == + 1 else json_result)) - if "authors" in item or "authornames" in item: - click.echo("Authors: %s" % ", ".join( - item.get("authornames", - [a.get("name", "") for a in item.get("authors", [])]))) - - if "__src_url" in item: - click.secho("Source: %s" % item['__src_url']) - click.echo() + return True @cli.command("search", short_help="Search for a library") @@ -275,24 +298,6 @@ def lib_search(query, json_output, page, noninteractive, **filters): cache_valid="1d") -@cli.command("list", short_help="List installed libraries") -@click.option("--json-output", is_flag=True) -@click.pass_obj -def lib_list(lm, json_output): - items = lm.get_installed() - - if json_output: - return click.echo(json.dumps(items)) - - if not items: - return None - - for item in sorted(items, key=lambda i: i['name']): - print_lib_item(item) - - return True - - @cli.command("builtin", short_help="List built-in libraries") @click.option("--storage", multiple=True) @click.option("--json-output", is_flag=True) @@ -484,3 +489,44 @@ def lib_stats(json_output): click.echo() return True + + +def print_storage_header(storage_dirs, storage_dir): + if storage_dirs and storage_dirs[0] != storage_dir: + click.echo("") + click.echo( + click.style("Library Storage: ", bold=True) + + click.style(storage_dir, fg="blue")) + + +def print_lib_item(item): + click.secho(item['name'], fg="cyan") + click.echo("=" * len(item['name'])) + if "id" in item: + click.secho("#ID: %d" % item['id'], bold=True) + if "description" in item or "url" in item: + click.echo(item.get("description", item.get("url", ""))) + click.echo() + + for key in ("version", "homepage", "license", "keywords"): + if key not in item or not item[key]: + continue + if isinstance(item[key], list): + click.echo("%s: %s" % (key.title(), ", ".join(item[key]))) + else: + click.echo("%s: %s" % (key.title(), item[key])) + + for key in ("frameworks", "platforms"): + if key not in item: + continue + click.echo("Compatible %s: %s" % (key, ", ".join( + [i['title'] if isinstance(i, dict) else i for i in item[key]]))) + + if "authors" in item or "authornames" in item: + click.echo("Authors: %s" % ", ".join( + item.get("authornames", + [a.get("name", "") for a in item.get("authors", [])]))) + + if "__src_url" in item: + click.secho("Source: %s" % item['__src_url']) + click.echo() diff --git a/platformio/commands/run.py b/platformio/commands/run.py index e3085626..f643eb40 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -23,7 +23,7 @@ from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.commands.lib import lib_install as cmd_lib_install from platformio.commands.platform import \ platform_install as cmd_platform_install -from platformio.managers.lib import LibraryManager, is_builtin_lib +from platformio.managers.lib import is_builtin_lib from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.helpers import ( @@ -82,6 +82,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, project_conf or join(project_dir, "platformio.ini")) config.validate(environment) + _handle_legacy_libdeps(project_dir, config) + results = [] start_time = time() default_envs = config.default_envs() @@ -94,7 +96,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, results.append((envname, None)) continue - if not silent and results: + if not silent and any( + status is not None for (_, status) in results): click.echo() options = config.items(env=envname, as_dict=True) @@ -222,14 +225,14 @@ class EnvironmentProcessor(object): if "nobuild" not in build_targets: # install dependent libraries if "lib_install" in self.options: - _autoinstall_libdeps(self.cmd_ctx, [ + _autoinstall_libdeps(self.cmd_ctx, self.name, [ int(d.strip()) for d in self.options['lib_install'].split(",") if d.strip() ], self.verbose) if "lib_deps" in self.options: _autoinstall_libdeps( - self.cmd_ctx, + self.cmd_ctx, self.name, ProjectConfig.parse_multi_values(self.options['lib_deps']), self.verbose) @@ -245,13 +248,31 @@ class EnvironmentProcessor(object): return p.run(build_vars, build_targets, self.silent, self.verbose) -def _autoinstall_libdeps(ctx, libraries, verbose=False): +def _handle_legacy_libdeps(project_dir, config): + legacy_libdeps_dir = join(project_dir, ".piolibdeps") + if (not isdir(legacy_libdeps_dir) + or legacy_libdeps_dir == get_projectlibdeps_dir()): + return + if not config.has_section("env"): + config.add_section("env") + lib_extra_dirs = [] + if config.has_option("env", "lib_extra_dirs"): + lib_extra_dirs = config.getlist("env", "lib_extra_dirs") + lib_extra_dirs.append(legacy_libdeps_dir) + config.set("env", "lib_extra_dirs", lib_extra_dirs) + click.secho( + "DEPRECATED! A legacy library storage `{0}` has been found in a " + "project. \nPlease declare project dependencies in `platformio.ini`" + " file using `lib_deps` option and remove `{0}` folder." + "\nMore details -> http://docs.platformio.org/page/projectconf/" + "section_env_library.html#lib-deps".format(legacy_libdeps_dir), + fg="yellow") + + +def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): if not libraries: return - storage_dir = get_projectlibdeps_dir() - ctx.obj = LibraryManager(storage_dir) - if verbose: - click.echo("Library Storage: " + storage_dir) + ctx.obj = [join(get_projectlibdeps_dir(), envname)] for lib in libraries: try: ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose) diff --git a/platformio/commands/update.py b/platformio/commands/update.py index d4217217..c4ee9a58 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -54,5 +54,5 @@ def cli(ctx, core_packages, only_check, dry_run): click.echo() click.echo("Library Manager") click.echo("===============") - ctx.obj = LibraryManager() + ctx.obj = [LibraryManager().package_dir] ctx.invoke(cmd_lib_update, only_check=only_check) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 37d0db53..58b87cf2 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -144,7 +144,8 @@ class ProjectGenerator(object): "project_dir": self.project_dir, "project_src_dir": get_projectsrc_dir(), "project_lib_dir": get_projectlib_dir(), - "project_libdeps_dir": get_projectlibdeps_dir(), + "project_libdeps_dir": join( + get_projectlibdeps_dir(), self.env_name), "systype": util.get_systype(), "platformio_path": self._fix_os_path( sys.argv[0] if isfile(sys.argv[0]) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index cf563289..3a963038 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -332,7 +332,7 @@ def check_internal_updates(ctx, what): if what == "platforms": ctx.invoke(cmd_platform_update, platforms=outdated_items) elif what == "libraries": - ctx.obj = pm + ctx.obj = [pm.package_dir] ctx.invoke(cmd_lib_update, libraries=outdated_items) click.echo() diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 3c66592c..700a928d 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -26,10 +26,10 @@ from platformio.managers.package import PackageManager from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path CORE_PACKAGES = { - "contrib-piohome": "^2.0.1", + "contrib-piohome": "^2.1.0", "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.2.0", + "tool-pioplus": "^2.3.0", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30005.0" } diff --git a/platformio/project/config.py b/platformio/project/config.py index 19842922..a7f9b7b8 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -227,6 +227,11 @@ class ProjectConfig(object): return [(option, self.get(section, option)) for option in self.options(section)] + def set(self, section, option, value): + if isinstance(value, (list, tuple)): + value = "\n".join(value) + self._parser.set(section, option, value) + def get(self, section, option): if not self.expand_interpolations: return self._parser.get(section, option) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index ec72b213..f215a511 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -93,15 +93,15 @@ def get_projectbuild_dir(force=False): return path +def get_projectlibdeps_dir(): + return get_project_optional_dir( + "libdeps_dir", join(get_projectworkspace_dir(), "libdeps")) + + def get_projectlib_dir(): return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) -def get_projectlibdeps_dir(): - return get_project_optional_dir("libdeps_dir", - join(get_project_dir(), ".piolibdeps")) - - def get_projectsrc_dir(): return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) diff --git a/platformio/util.py b/platformio/util.py index a043abd9..304d3a14 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -41,7 +41,8 @@ from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_project_dir, get_project_optional_dir, get_projectboards_dir, get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir, - get_projectsrc_dir, get_projecttest_dir, is_platformio_project) + get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir, + is_platformio_project) class cd(object): From 0fda79a07577bf52f8ba3943e29e1f8765df938f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 May 2019 13:05:44 +0300 Subject: [PATCH 160/333] Switch to Click meta context for lib CLI --- platformio/commands/lib.py | 30 +++++++++++++++++++----------- platformio/commands/run.py | 5 ++++- platformio/commands/update.py | 3 ++- platformio/maintenance.py | 3 ++- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index bcba1935..5fcd6ab9 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -32,6 +32,8 @@ try: except ImportError: from urllib import quote +CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs" + @click.group(short_help="Library Manager") @click.option( @@ -82,7 +84,8 @@ def cli(ctx, **options): raise exception.NotGlobalLibDir(get_project_dir(), join(util.get_home_dir(), "lib"), ctx.invoked_subcommand) - ctx.obj = [] + + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [] for storage_dir in storage_dirs: if is_platformio_project(storage_dir): with util.cd(storage_dir): @@ -93,9 +96,10 @@ def cli(ctx, **options): for env in config.envs(): if (not options['environment'] or env in options['environment']): - ctx.obj.append(join(libdeps_dir, env)) + ctx.meta[CTX_META_STORAGE_DIRS_KEY].append( + join(libdeps_dir, env)) else: - ctx.obj.append(storage_dir) + ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) @cli.command("install", short_help="Install library") @@ -116,8 +120,9 @@ def cli(ctx, **options): "--force", is_flag=True, help="Reinstall/redownload library if exists") -@click.pass_obj -def lib_install(storage_dirs, libraries, silent, interactive, force): +@click.pass_context +def lib_install(ctx, libraries, silent, interactive, force): + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] for storage_dir in storage_dirs: print_storage_header(storage_dirs, storage_dir) lm = LibraryManager(storage_dir) @@ -128,8 +133,9 @@ def lib_install(storage_dirs, libraries, silent, interactive, force): @cli.command("uninstall", short_help="Uninstall libraries") @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]") -@click.pass_obj -def lib_uninstall(storage_dirs, libraries): +@click.pass_context +def lib_uninstall(ctx, libraries): + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] for storage_dir in storage_dirs: print_storage_header(storage_dirs, storage_dir) lm = LibraryManager(storage_dir) @@ -149,8 +155,9 @@ def lib_uninstall(storage_dirs, libraries): is_flag=True, help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) -@click.pass_obj -def lib_update(storage_dirs, libraries, only_check, dry_run, json_output): +@click.pass_context +def lib_update(ctx, libraries, only_check, dry_run, json_output): + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] only_check = dry_run or only_check json_result = {} for storage_dir in storage_dirs: @@ -196,8 +203,9 @@ def lib_update(storage_dirs, libraries, only_check, dry_run, json_output): @cli.command("list", short_help="List installed libraries") @click.option("--json-output", is_flag=True) -@click.pass_obj -def lib_list(storage_dirs, json_output): +@click.pass_context +def lib_list(ctx, json_output): + storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] json_result = {} for storage_dir in storage_dirs: if not json_output: diff --git a/platformio/commands/run.py b/platformio/commands/run.py index f643eb40..47a804db 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -20,6 +20,7 @@ import click from platformio import exception, telemetry, util from platformio.commands.device import device_monitor as cmd_device_monitor +from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib import lib_install as cmd_lib_install from platformio.commands.platform import \ platform_install as cmd_platform_install @@ -272,7 +273,9 @@ def _handle_legacy_libdeps(project_dir, config): def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): if not libraries: return - ctx.obj = [join(get_projectlibdeps_dir(), envname)] + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [ + join(get_projectlibdeps_dir(), envname) + ] for lib in libraries: try: ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose) diff --git a/platformio/commands/update.py b/platformio/commands/update.py index c4ee9a58..78d2f8bd 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -15,6 +15,7 @@ import click from platformio import app +from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.managers.core import update_core_packages @@ -54,5 +55,5 @@ def cli(ctx, core_packages, only_check, dry_run): click.echo() click.echo("Library Manager") click.echo("===============") - ctx.obj = [LibraryManager().package_dir] + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [LibraryManager().package_dir] ctx.invoke(cmd_lib_update, only_check=only_check) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 3a963038..23fdb6c2 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -23,6 +23,7 @@ import semantic_version from platformio import __version__, app, exception, telemetry, util from platformio.commands import PlatformioCLI +from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib import lib_update as cmd_lib_update from platformio.commands.platform import \ platform_install as cmd_platform_install @@ -332,7 +333,7 @@ def check_internal_updates(ctx, what): if what == "platforms": ctx.invoke(cmd_platform_update, platforms=outdated_items) elif what == "libraries": - ctx.obj = [pm.package_dir] + ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [pm.package_dir] ctx.invoke(cmd_lib_update, libraries=outdated_items) click.echo() From c2a549b0c274b691248c9eb708c411b31452fbf5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 May 2019 18:37:08 +0300 Subject: [PATCH 161/333] Install all project dependencies declared via "lib_deps" option using "platformio lib install" command // Resolve #2147 --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/lib.py | 57 ++++++++++++++++++++++++++------------ platformio/commands/run.py | 26 ++++++++--------- 4 files changed, 55 insertions(+), 31 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c757e11b..2c946a5b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ PlatformIO 4.0 * **Library Management** + - Install all project dependencies declared via `lib_deps `__ option using `platformio lib install `__ command (`issue #2147 `_) - Use isolated library dependency storage per project build environment (`issue #1696 `_) - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - Use workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` diff --git a/docs b/docs index bbf0f91e..51593ac3 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit bbf0f91e9fc5096edcd9e0656f051da3dbbabf05 +Subproject commit 51593ac34b0a5db3dbff589e7b2672d940ce635e diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 5fcd6ab9..777ab237 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -21,7 +21,8 @@ from os.path import isdir, join import click from platformio import exception, util -from platformio.managers.lib import LibraryManager, get_builtin_libs +from platformio.managers.lib import (LibraryManager, get_builtin_libs, + is_builtin_lib) from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import ( @@ -33,6 +34,7 @@ except ImportError: from urllib import quote CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs" +CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps" @click.group(short_help="Library Manager") @@ -86,20 +88,24 @@ def cli(ctx, **options): ctx.invoked_subcommand) ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [] + ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] = {} for storage_dir in storage_dirs: - if is_platformio_project(storage_dir): - with util.cd(storage_dir): - config = ProjectConfig.get_instance( - join(storage_dir, "platformio.ini")) - config.validate(options['environment']) - libdeps_dir = get_projectlibdeps_dir() - for env in config.envs(): - if (not options['environment'] - or env in options['environment']): - ctx.meta[CTX_META_STORAGE_DIRS_KEY].append( - join(libdeps_dir, env)) - else: + if not is_platformio_project(storage_dir): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) + continue + with util.cd(storage_dir): + libdeps_dir = get_projectlibdeps_dir() + config = ProjectConfig.get_instance( + join(storage_dir, "platformio.ini")) + config.validate(options['environment']) + for env in config.envs(): + if options['environment'] and env not in options['environment']: + continue + storage_dir = join(libdeps_dir, env) + ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) + if config.has_option("env:" + env, "lib_deps"): + ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][ + storage_dir] = config.getlist("env:" + env, "lib_deps") @cli.command("install", short_help="Install library") @@ -122,13 +128,30 @@ def cli(ctx, **options): help="Reinstall/redownload library if exists") @click.pass_context def lib_install(ctx, libraries, silent, interactive, force): + storage_libdeps = ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] for storage_dir in storage_dirs: - print_storage_header(storage_dirs, storage_dir) + if not silent and (libraries or storage_dir in storage_libdeps): + print_storage_header(storage_dirs, storage_dir) lm = LibraryManager(storage_dir) - for library in libraries: - lm.install( - library, silent=silent, interactive=interactive, force=force) + if libraries: + for library in libraries: + lm.install( + library, + silent=silent, + interactive=interactive, + force=force) + elif storage_dir in storage_libdeps: + for library in storage_libdeps[storage_dir]: + try: + lm.install( + library, + silent=silent, + interactive=interactive, + force=force) + except exception.LibNotFound as e: + if not silent or not is_builtin_lib(library): + click.secho("Warning! %s" % e, fg="yellow") @cli.command("uninstall", short_help="Uninstall libraries") diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 47a804db..27a1a93b 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -20,11 +20,11 @@ import click from platformio import exception, telemetry, util from platformio.commands.device import device_monitor as cmd_device_monitor -from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY +from platformio.commands.lib import (CTX_META_STORAGE_DIRS_KEY, + CTX_META_STORAGE_LIBDEPS_KEY) from platformio.commands.lib import lib_install as cmd_lib_install from platformio.commands.platform import \ platform_install as cmd_platform_install -from platformio.managers.lib import is_builtin_lib from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.helpers import ( @@ -273,17 +273,17 @@ def _handle_legacy_libdeps(project_dir, config): def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): if not libraries: return - ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [ - join(get_projectlibdeps_dir(), envname) - ] - for lib in libraries: - try: - ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose) - except exception.LibNotFound as e: - if verbose or not is_builtin_lib(lib): - click.secho("Warning! %s" % e, fg="yellow") - except exception.InternetIsOffline as e: - click.secho(str(e), fg="yellow") + libdeps_dir = join(get_projectlibdeps_dir(), envname) + ctx.meta.update({ + CTX_META_STORAGE_DIRS_KEY: [libdeps_dir], + CTX_META_STORAGE_LIBDEPS_KEY: { + libdeps_dir: libraries + } + }) + try: + ctx.invoke(cmd_lib_install, silent=not verbose) + except exception.InternetIsOffline as e: + click.secho(str(e), fg="yellow") def _clean_build_dir(build_dir): From 70a5d3292598a27e058c8d6061f71bc6bddf46f7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 23 May 2019 19:39:04 +0300 Subject: [PATCH 162/333] Add "--save" flag to "platformio lib install" command // Resolve #1028 --- HISTORY.rst | 3 ++- docs | 2 +- platformio/commands/lib.py | 37 +++++++++++++++++++++++++++++------- platformio/project/config.py | 2 ++ platformio/telemetry.py | 2 +- 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2c946a5b..5dbb1ba7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,7 +18,8 @@ PlatformIO 4.0 * **Library Management** - - Install all project dependencies declared via `lib_deps `__ option using `platformio lib install `__ command (`issue #2147 `_) + - Save libraries passed to `platformio lib install `__ command into the project dependency list (`lib_deps `__) with a new ``--save`` flag (`issue #1028 `_) + - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - Use isolated library dependency storage per project build environment (`issue #1696 `_) - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - Use workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` diff --git a/docs b/docs index 51593ac3..2ed6a490 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 51593ac34b0a5db3dbff589e7b2672d940ce635e +Subproject commit 2ed6a490e5853bd71afaa08bc34750970293d43d diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 777ab237..af0e2105 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -33,6 +33,8 @@ try: except ImportError: from urllib import quote +CTX_META_INPUT_DIRS_KEY = __name__ + ".input_dirs" +CTX_META_PROJECT_ENVIRONMENTS_KEY = __name__ + ".project_environments" CTX_META_STORAGE_DIRS_KEY = __name__ + ".storage_dirs" CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps" @@ -87,6 +89,8 @@ def cli(ctx, **options): join(util.get_home_dir(), "lib"), ctx.invoked_subcommand) + ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options['environment'] + ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [] ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] = {} for storage_dir in storage_dirs: @@ -110,11 +114,10 @@ def cli(ctx, **options): @cli.command("install", short_help="Install library") @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") -# @click.option( -# "--save", -# is_flag=True, -# help="Save installed libraries into the project's platformio.ini " -# "library dependencies") +@click.option( + "--save", + is_flag=True, + help="Save installed libraries into the `platformio.ini` dependency list") @click.option( "-s", "--silent", is_flag=True, help="Suppress progress reporting") @click.option( @@ -127,9 +130,29 @@ def cli(ctx, **options): is_flag=True, help="Reinstall/redownload library if exists") @click.pass_context -def lib_install(ctx, libraries, silent, interactive, force): - storage_libdeps = ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY] +def lib_install( # pylint: disable=too-many-arguments + ctx, libraries, save, silent, interactive, force): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] + input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) + storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, []) + + if save and libraries: + project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] + for input_dir in input_dirs: + config = ProjectConfig.get_instance( + join(input_dir, "platformio.ini")) + config.validate(project_environments) + for env in config.envs(): + if project_environments and env not in project_environments: + continue + config.expand_interpolations = False + lib_deps = (config.getlist( + "env:" + env, "lib_deps") if config.has_option( + "env:" + env, "lib_deps") else []) + lib_deps.extend(l for l in libraries if l not in lib_deps) + config.set("env:" + env, "lib_deps", lib_deps) + config.save() + for storage_dir in storage_dirs: if not silent and (libraries or storage_dir in storage_libdeps): print_storage_header(storage_dirs, storage_dir) diff --git a/platformio/project/config.py b/platformio/project/config.py index a7f9b7b8..27557d56 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -230,6 +230,8 @@ class ProjectConfig(object): def set(self, section, option, value): if isinstance(value, (list, tuple)): value = "\n".join(value) + if value: + value = "\n" + value # start from a new line self._parser.set(section, option, value) def get(self, section, option): diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 6bdb48a1..7564a03d 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -391,7 +391,7 @@ def backup_reports(items): for params in items: # skip static options - for key in params.keys(): + for key in list(params.keys()): if key in ("v", "tid", "cid", "cd1", "cd2", "sr", "an"): del params[key] From 9a95b0df564fd0bee1d9e80e5a6dc19b0dd46206 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 May 2019 01:15:47 +0300 Subject: [PATCH 163/333] Fix handling custom includeDir and srcDir for library.json // Resolve #2518 --- platformio/builder/tools/piolib.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 2f38e649..a132c379 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -182,7 +182,7 @@ class LibBuilderBase(object): def get_include_dirs(self): items = [] include_dir = self.include_dir - if include_dir and include_dir not in items: + if include_dir: items.append(include_dir) items.append(self.src_dir) return items @@ -723,13 +723,15 @@ class PlatformIOLibBuilder(LibBuilderBase): @property def include_dir(self): if "includeDir" in self._manifest.get("build", {}): - return self._manifest.get("build").get("includeDir") + with util.cd(self.path): + return realpath(self._manifest.get("build").get("includeDir")) return LibBuilderBase.include_dir.fget(self) @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): - return self._manifest.get("build").get("srcDir") + with util.cd(self.path): + return realpath(self._manifest.get("build").get("srcDir")) return LibBuilderBase.src_dir.fget(self) @property @@ -804,6 +806,7 @@ class PlatformIOLibBuilder(LibBuilderBase): for path in self.env.get("CPPPATH", []): if path not in self.envorigin.get("CPPPATH", []): include_dirs.append(self.env.subst(path)) + return include_dirs From e6d1805f0b3c8ca7e27d084d4991ab81ca06619c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 May 2019 14:09:25 +0300 Subject: [PATCH 164/333] Save library requirements when using `--save` option // Issue #1028 --- docs | 2 +- platformio/commands/lib.py | 53 ++++++++++++++++++++++++-------------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/docs b/docs index 2ed6a490..82316b22 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2ed6a490e5853bd71afaa08bc34750970293d43d +Subproject commit 82316b22ff38d4e62dd38dd8e465afa38bd062b5 diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index af0e2105..bf3458b1 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -19,6 +19,7 @@ import time from os.path import isdir, join import click +import semantic_version from platformio import exception, util from platformio.managers.lib import (LibraryManager, get_builtin_libs, @@ -133,49 +134,61 @@ def cli(ctx, **options): def lib_install( # pylint: disable=too-many-arguments ctx, libraries, save, silent, interactive, force): storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] - input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) storage_libdeps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, []) - if save and libraries: - project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] - for input_dir in input_dirs: - config = ProjectConfig.get_instance( - join(input_dir, "platformio.ini")) - config.validate(project_environments) - for env in config.envs(): - if project_environments and env not in project_environments: - continue - config.expand_interpolations = False - lib_deps = (config.getlist( - "env:" + env, "lib_deps") if config.has_option( - "env:" + env, "lib_deps") else []) - lib_deps.extend(l for l in libraries if l not in lib_deps) - config.set("env:" + env, "lib_deps", lib_deps) - config.save() - + installed_manifests = {} for storage_dir in storage_dirs: if not silent and (libraries or storage_dir in storage_libdeps): print_storage_header(storage_dirs, storage_dir) lm = LibraryManager(storage_dir) if libraries: for library in libraries: - lm.install( + pkg_dir = lm.install( library, silent=silent, interactive=interactive, force=force) + installed_manifests[library] = lm.load_manifest(pkg_dir) elif storage_dir in storage_libdeps: for library in storage_libdeps[storage_dir]: try: - lm.install( + pkg_dir = lm.install( library, silent=silent, interactive=interactive, force=force) + installed_manifests[library] = lm.load_manifest(pkg_dir) except exception.LibNotFound as e: if not silent or not is_builtin_lib(library): click.secho("Warning! %s" % e, fg="yellow") + if not save or not libraries: + return + + input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, []) + project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] + for input_dir in input_dirs: + config = ProjectConfig.get_instance(join(input_dir, "platformio.ini")) + config.validate(project_environments) + for env in config.envs(): + if project_environments and env not in project_environments: + continue + config.expand_interpolations = False + lib_deps = (config.getlist("env:" + env, "lib_deps") + if config.has_option("env:" + env, "lib_deps") else []) + for library in libraries: + if library in lib_deps: + continue + manifest = installed_manifests[library] + try: + assert library.lower() == manifest['name'].lower() + assert semantic_version.Version(manifest['version']) + lib_deps.append("{name}@^{version}".format(**manifest)) + except (AssertionError, ValueError): + lib_deps.append(library) + config.set("env:" + env, "lib_deps", lib_deps) + config.save() + @cli.command("uninstall", short_help="Uninstall libraries") @click.argument("libraries", nargs=-1, metavar="[LIBRARY...]") From b38c57bcf9ab8a45f6b6d9b3891276a1b6dab6ae Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 May 2019 14:57:59 +0300 Subject: [PATCH 165/333] Fix an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag // Resolve #2508 --- HISTORY.rst | 1 + platformio/builder/tools/platformio.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 5dbb1ba7..4481d4d6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -33,6 +33,7 @@ PlatformIO 4.0 * **Miscellaneous** - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead + - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) PlatformIO 3.0 -------------- diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 3a0347c4..2d1fc0fd 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -198,6 +198,8 @@ def ProcessFlags(env, flags): # pylint: disable=too-many-branches if undefines: for undef in undefines: env['CCFLAGS'].remove(undef) + if undef[2:] in env['CPPDEFINES']: + env['CPPDEFINES'].remove(undef[2:]) env.Append(_CPPDEFFLAGS=" %s" % " ".join(undefines)) From 2baea815fe089b6c1d0c7f76a166845b405b6145 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 May 2019 15:06:33 +0300 Subject: [PATCH 166/333] Update history --- HISTORY.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4481d4d6..2add298a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -33,7 +33,6 @@ PlatformIO 4.0 * **Miscellaneous** - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead - - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) PlatformIO 3.0 -------------- @@ -41,8 +40,8 @@ PlatformIO 3.0 3.6.8 (2019-??-??) ~~~~~~~~~~~~~~~~~~ -* Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ - (`issue #2442 `_) +* Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) +* Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) 3.6.7 (2019-04-23) ~~~~~~~~~~~~~~~~~~ From a6e5a0c7f514451a5d573b77f72e988c55e66e5a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 May 2019 16:06:27 +0300 Subject: [PATCH 167/333] Fix an issue for Project Generator when include path search order is inconsistent to what passed to the compiler // Resolve #2509 --- HISTORY.rst | 1 + platformio/builder/main.py | 6 +++++- platformio/builder/tools/pioide.py | 10 +++++----- platformio/builder/tools/piolib.py | 4 +++- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2add298a..650412d3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -42,6 +42,7 @@ PlatformIO 3.0 * Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) * Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) +* Fixed an issue for Project Generator when include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) 3.6.7 (2019-04-23) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 4ccbd0f9..4604fc9c 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -26,6 +26,7 @@ from SCons.Script import AllowSubstExceptions # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from SCons.Script import Default # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error +from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error from platformio import util @@ -216,8 +217,11 @@ if "envdump" in COMMAND_LINE_TARGETS: if "idedata" in COMMAND_LINE_TARGETS: try: + Import("projenv") print("\n%s\n" % path_to_unicode( - json.dumps(env.DumpIDEData(), ensure_ascii=False))) + json.dumps( + env.DumpIDEData(projenv), # pylint: disable=undefined-variable + ensure_ascii=False))) env.Exit(0) except UnicodeDecodeError: sys.stderr.write( diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index c6d8fbc0..8efcd60d 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -26,11 +26,11 @@ from platformio.managers.core import get_core_package_dir from platformio.proc import exec_command -def _dump_includes(env): +def _dump_includes(env, projenv): includes = [] - for item in env.get("CPPPATH", []): - includes.append(env.subst(item)) + for item in projenv.get("CPPPATH", []): + includes.append(projenv.subst(item)) # installed libs for lb in env.GetLibBuilders(): @@ -135,7 +135,7 @@ def _get_svd_path(env): return None -def DumpIDEData(env): +def DumpIDEData(env, projenv): LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS" LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS" @@ -145,7 +145,7 @@ def DumpIDEData(env): "defines": _dump_defines(env), "includes": - _dump_includes(env), + _dump_includes(env, projenv), "cc_flags": env.subst(LINTCCOM), "cxx_flags": diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index a132c379..b63dd0f9 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -832,7 +832,9 @@ class ProjectAsLibBuilder(LibBuilderBase): project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR") if isdir(project_include_dir): include_dirs.append(project_include_dir) - include_dirs.extend(LibBuilderBase.get_include_dirs(self)) + for include_dir in LibBuilderBase.get_include_dirs(self): + if include_dir not in include_dirs: + include_dirs.append(include_dir) return include_dirs def get_search_files(self): From 19725fec04302387af15c3942790bb26c98a7bfa Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 24 May 2019 20:49:05 +0300 Subject: [PATCH 168/333] Add options to override default locations used by PlatformIO Core // Resolve #1615 --- HISTORY.rst | 1 + docs | 2 +- platformio/app.py | 5 +- platformio/builder/main.py | 27 +++++------ platformio/commands/init.py | 12 ++--- platformio/commands/lib.py | 11 +++-- platformio/commands/run.py | 12 ++--- platformio/ide/projectgenerator.py | 10 ++-- platformio/maintenance.py | 3 +- platformio/managers/core.py | 12 ++--- platformio/managers/lib.py | 3 +- platformio/managers/platform.py | 19 ++++---- platformio/project/config.py | 14 ++++-- platformio/project/helpers.py | 76 +++++++++++++++++++++++------- platformio/util.py | 53 ++++++++------------- 15 files changed, 149 insertions(+), 111 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 650412d3..379224a7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,6 +28,7 @@ PlatformIO 4.0 - Python 3 support (`issue #895 `_) - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) + - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) * **Miscellaneous** diff --git a/docs b/docs index 82316b22..dde6b061 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 82316b22ff38d4e62dd38dd8e465afa38bd062b5 +Subproject commit dde6b061a61872e35a920775c20a64063b445065 diff --git a/platformio/app.py b/platformio/app.py index b8e3b691..798fecab 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -27,6 +27,7 @@ import requests from platformio import exception, lockfile, util from platformio.compat import PY2, WINDOWS from platformio.proc import is_ci +from platformio.project.helpers import get_project_core_dir def projects_dir_validate(projects_dir): @@ -90,7 +91,7 @@ class State(object): self.path = path self.lock = lock if not self.path: - self.path = join(util.get_home_dir(), "appstate.json") + self.path = join(get_project_core_dir(), "appstate.json") self._state = {} self._prev_state = {} self._lockfile = None @@ -111,7 +112,7 @@ class State(object): with codecs.open(self.path, "w", encoding="utf8") as fp: json.dump(self._state, fp) except IOError: - raise exception.HomeDirPermissionsError(util.get_home_dir()) + raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() def _lock_state_file(self): diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 4604fc9c..8358d52e 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -32,11 +32,8 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import util from platformio.compat import PY2, path_to_unicode from platformio.proc import get_pythonexe_path +from platformio.project import helpers as project_helpers from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_dir, get_projectbuild_dir, get_projectdata_dir, - get_projectinclude_dir, get_projectlib_dir, get_projectlibdeps_dir, - get_projectsrc_dir, get_projecttest_dir, get_projectworkspace_dir) AllowSubstExceptions(NameError) @@ -108,23 +105,23 @@ DEFAULT_ENV_OPTIONS = dict( PIOVARIABLES=list(commonvars.keys()), ENV=environ, UNIX_TIME=int(time()), - PIOHOME_DIR=util.get_home_dir(), - PROJECT_DIR=get_project_dir(), - PROJECTWORKSPACE_DIR=get_projectworkspace_dir(), - PROJECTLIBDEPS_DIR=get_projectlibdeps_dir(), - PROJECTINCLUDE_DIR=get_projectinclude_dir(), - PROJECTSRC_DIR=get_projectsrc_dir(), - PROJECTTEST_DIR=get_projecttest_dir(), - PROJECTDATA_DIR=get_projectdata_dir(), - PROJECTBUILD_DIR=get_projectbuild_dir(), + PROJECT_DIR=project_helpers.get_project_dir(), + PROJECTCORE_DIR=project_helpers.get_project_core_dir(), + PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(), + PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(), + PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(), + PROJECTSRC_DIR=project_helpers.get_project_src_dir(), + PROJECTTEST_DIR=project_helpers.get_project_test_dir(), + PROJECTDATA_DIR=project_helpers.get_project_data_dir(), + PROJECTBUILD_DIR=project_helpers.get_project_build_dir(), BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"), BUILDSRC_DIR=join("$BUILD_DIR", "src"), BUILDTEST_DIR=join("$BUILD_DIR", "test"), LIBPATH=["$BUILD_DIR"], LIBSOURCE_DIRS=[ - get_projectlib_dir(), + project_helpers.get_project_lib_dir(), join("$PROJECTLIBDEPS_DIR", "$PIOENV"), - join("$PIOHOME_DIR", "lib") + project_helpers.get_project_global_lib_dir() ], PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), diff --git a/platformio/commands/init.py b/platformio/commands/init.py index b8f293d2..c002cc74 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -26,8 +26,8 @@ from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig from platformio.project.helpers import ( - get_projectinclude_dir, get_projectlib_dir, get_projectsrc_dir, - get_projecttest_dir, is_platformio_project) + get_project_include_dir, get_project_lib_dir, get_project_src_dir, + get_project_test_dir, is_platformio_project) def validate_boards(ctx, param, value): # pylint: disable=W0613 @@ -155,10 +155,10 @@ def init_base_project(project_dir): ProjectConfig(join(project_dir, "platformio.ini")).save() with util.cd(project_dir): dir_to_readme = [ - (get_projectsrc_dir(), None), - (get_projectinclude_dir(), init_include_readme), - (get_projectlib_dir(), init_lib_readme), - (get_projecttest_dir(), init_test_readme), + (get_project_src_dir(), None), + (get_project_include_dir(), init_include_readme), + (get_project_lib_dir(), init_lib_readme), + (get_project_test_dir(), init_test_readme), ] for (path, cb) in dir_to_readme: if isdir(path): diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index bf3458b1..6ad3252b 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -27,7 +27,8 @@ from platformio.managers.lib import (LibraryManager, get_builtin_libs, from platformio.proc import is_ci from platformio.project.config import ProjectConfig from platformio.project.helpers import ( - get_project_dir, get_projectlibdeps_dir, is_platformio_project) + get_project_dir, get_project_global_lib_dir, get_project_libdeps_dir, + is_platformio_project) try: from urllib.parse import quote @@ -73,12 +74,12 @@ def cli(ctx, **options): return storage_dirs = list(options['storage_dir']) if options['global']: - storage_dirs.append(join(util.get_home_dir(), "lib")) + storage_dirs.append(get_project_global_lib_dir()) if not storage_dirs: if is_platformio_project(): storage_dirs = [get_project_dir()] elif is_ci(): - storage_dirs = [join(util.get_home_dir(), "lib")] + storage_dirs = [get_project_global_lib_dir()] click.secho( "Warning! Global library storage is used automatically. " "Please use `platformio lib --global %s` command to remove " @@ -87,7 +88,7 @@ def cli(ctx, **options): if not storage_dirs: raise exception.NotGlobalLibDir(get_project_dir(), - join(util.get_home_dir(), "lib"), + get_project_global_lib_dir(), ctx.invoked_subcommand) ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options['environment'] @@ -99,7 +100,7 @@ def cli(ctx, **options): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue with util.cd(storage_dir): - libdeps_dir = get_projectlibdeps_dir() + libdeps_dir = get_project_libdeps_dir() config = ProjectConfig.get_instance( join(storage_dir, "platformio.ini")) config.validate(options['environment']) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 27a1a93b..5ee46c8f 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -28,8 +28,8 @@ from platformio.commands.platform import \ from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.helpers import ( - calculate_project_hash, find_project_dir_above, get_project_dir, - get_projectbuild_dir, get_projectlibdeps_dir) + calculate_project_hash, find_project_dir_above, get_project_build_dir, + get_project_dir, get_project_libdeps_dir) # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @@ -71,12 +71,12 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, # clean obsolete build dir if not disable_auto_clean: try: - _clean_build_dir(get_projectbuild_dir()) + _clean_build_dir(get_project_build_dir()) except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " "it manually to avoid build issues" % - get_projectbuild_dir(force=True), + get_project_build_dir(force=True), fg="yellow") config = ProjectConfig.get_instance( @@ -252,7 +252,7 @@ class EnvironmentProcessor(object): def _handle_legacy_libdeps(project_dir, config): legacy_libdeps_dir = join(project_dir, ".piolibdeps") if (not isdir(legacy_libdeps_dir) - or legacy_libdeps_dir == get_projectlibdeps_dir()): + or legacy_libdeps_dir == get_project_libdeps_dir()): return if not config.has_section("env"): config.add_section("env") @@ -273,7 +273,7 @@ def _handle_legacy_libdeps(project_dir, config): def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): if not libraries: return - libdeps_dir = join(get_projectlibdeps_dir(), envname) + libdeps_dir = join(get_project_libdeps_dir(), envname) ctx.meta.update({ CTX_META_STORAGE_DIRS_KEY: [libdeps_dir], CTX_META_STORAGE_LIBDEPS_KEY: { diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 58b87cf2..9898dd88 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -26,7 +26,7 @@ from platformio.commands.run import cli as cmd_run from platformio.compat import PY2, WINDOWS, get_file_contents from platformio.project.config import ProjectConfig from platformio.project.helpers import ( - get_projectlib_dir, get_projectlibdeps_dir, get_projectsrc_dir) + get_project_lib_dir, get_project_libdeps_dir, get_project_src_dir) class ProjectGenerator(object): @@ -91,7 +91,7 @@ class ProjectGenerator(object): def get_src_files(self): result = [] with util.cd(self.project_dir): - for root, _, files in os.walk(get_projectsrc_dir()): + for root, _, files in os.walk(get_project_src_dir()): for f in files: result.append(relpath(join(root, f))) return result @@ -142,10 +142,10 @@ class ProjectGenerator(object): "src_files": self.get_src_files(), "user_home_dir": abspath(expanduser("~")), "project_dir": self.project_dir, - "project_src_dir": get_projectsrc_dir(), - "project_lib_dir": get_projectlib_dir(), + "project_src_dir": get_project_src_dir(), + "project_lib_dir": get_project_lib_dir(), "project_libdeps_dir": join( - get_projectlibdeps_dir(), self.env_name), + get_project_libdeps_dir(), self.env_name), "systype": util.get_systype(), "platformio_path": self._fix_os_path( sys.argv[0] if isfile(sys.argv[0]) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 23fdb6c2..ea7c75f2 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -35,6 +35,7 @@ from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager from platformio.managers.platform import PlatformFactory, PlatformManager from platformio.proc import is_ci, is_container +from platformio.project.helpers import get_project_core_dir def on_platformio_start(ctx, force, caller): @@ -113,7 +114,7 @@ class Upgrader(object): @staticmethod def _upgrade_to_3_0_0(ctx): # convert custom board configuration - boards_dir = join(util.get_home_dir(), "boards") + boards_dir = join(get_project_core_dir(), "boards") if isdir(boards_dir): for item in os.listdir(boards_dir): if not item.endswith(".json"): diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 700a928d..bda6c795 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -24,6 +24,7 @@ from platformio import __version__, exception, util from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path +from platformio.project.helpers import get_project_packages_dir CORE_PACKAGES = { "contrib-piohome": "^2.1.0", @@ -42,12 +43,11 @@ PIOPLUS_AUTO_UPDATES_MAX = 100 class CorePackageManager(PackageManager): def __init__(self): - super(CorePackageManager, self).__init__( - join(util.get_home_dir(), "packages"), [ - "https://dl.bintray.com/platformio/dl-packages/manifest.json", - "http%s://dl.platformio.org/packages/manifest.json" % - ("" if sys.version_info < (2, 7, 9) else "s") - ]) + super(CorePackageManager, self).__init__(get_project_packages_dir(), [ + "https://dl.bintray.com/platformio/dl-packages/manifest.json", + "http%s://dl.platformio.org/packages/manifest.json" % + ("" if sys.version_info < (2, 7, 9) else "s") + ]) def install( # pylint: disable=keyword-arg-before-vararg self, diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index ca143b58..e26a6cb8 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -26,13 +26,14 @@ from platformio import app, commands, exception, util from platformio.compat import glob_escape, string_types from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.project.helpers import get_project_global_lib_dir class LibraryManager(BasePkgManager): def __init__(self, package_dir=None): if not package_dir: - package_dir = join(util.get_home_dir(), "lib") + package_dir = get_project_global_lib_dir() super(LibraryManager, self).__init__(package_dir) @property diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index b5537937..56625efe 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -29,7 +29,9 @@ from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, exec_command, get_pythonexe_path) -from platformio.project.helpers import get_projectboards_dir +from platformio.project.helpers import ( + get_project_boards_dir, get_project_core_dir, get_project_packages_dir, + get_project_platforms_dir) try: from urllib.parse import quote @@ -48,9 +50,8 @@ class PlatformManager(BasePkgManager): "{0}://dl.platformio.org/platforms/manifest.json".format( "https" if app.get_setting("enable_ssl") else "http") ] - BasePkgManager.__init__( - self, package_dir or join(util.get_home_dir(), "platforms"), - repositories) + BasePkgManager.__init__(self, package_dir + or get_project_platforms_dir(), repositories) @property def manifest_names(self): @@ -156,7 +157,7 @@ class PlatformManager(BasePkgManager): deppkgs[pkgname] = set() deppkgs[pkgname].add(pkgmanifest['version']) - pm = PackageManager(join(util.get_home_dir(), "packages")) + pm = PackageManager(get_project_packages_dir()) for manifest in pm.get_installed(): if manifest['name'] not in names: continue @@ -481,8 +482,8 @@ class PlatformBase( # pylint: disable=too-many-public-methods self.manifest_path = manifest_path self._manifest = util.load_json(manifest_path) - self.pm = PackageManager( - join(util.get_home_dir(), "packages"), self.package_repositories) + self.pm = PackageManager(get_project_packages_dir(), + self.package_repositories) self.silent = False self.verbose = False @@ -579,8 +580,8 @@ class PlatformBase( # pylint: disable=too-many-public-methods self._BOARDS_CACHE[board_id] = config bdirs = [ - get_projectboards_dir(), - join(util.get_home_dir(), "boards"), + get_project_boards_dir(), + join(get_project_core_dir(), "boards"), join(self.get_dir(), "boards"), ] diff --git a/platformio/project/config.py b/platformio/project/config.py index 27557d56..7b5baac0 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -45,16 +45,20 @@ KNOWN_PLATFORMIO_OPTIONS = [ "extra_configs", # Dirs - "home_dir", + "core_dir", + "globallib_dir", + "platforms_dir", + "packages_dir", + "cache_dir", "workspace_dir", - "lib_dir", + "build_dir", "libdeps_dir", + "lib_dir", "include_dir", "src_dir", - "build_dir", - "data_dir", "test_dir", - "boards_dir" + "boards_dir", + "data_dir" ] KNOWN_ENV_OPTIONS = [ diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index f215a511..a2acce4d 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -15,7 +15,8 @@ import os from hashlib import sha1 from os import walk -from os.path import abspath, dirname, expanduser, isdir, isfile, join +from os.path import (abspath, dirname, expanduser, isdir, isfile, join, + splitdrive) from platformio import __version__ from platformio.compat import PY2, WINDOWS @@ -76,14 +77,57 @@ def get_project_optional_dir(name, default=None): return paths -def get_projectworkspace_dir(): +def get_project_core_dir(): + core_dir = get_project_optional_dir( + "core_dir", + get_project_optional_dir("home_dir", + join(expanduser("~"), ".platformio"))) + win_core_dir = None + if WINDOWS: + win_core_dir = splitdrive(core_dir)[0] + "\\.platformio" + if isdir(win_core_dir): + core_dir = win_core_dir + + if not isdir(core_dir): + try: + os.makedirs(core_dir) + except: # pylint: disable=bare-except + if win_core_dir: + os.makedirs(win_core_dir) + core_dir = win_core_dir + + assert isdir(core_dir) + return core_dir + + +def get_project_global_lib_dir(): + return get_project_optional_dir("globallib_dir", + join(get_project_core_dir(), "lib")) + + +def get_project_platforms_dir(): + return get_project_optional_dir("platforms_dir", + join(get_project_core_dir(), "platforms")) + + +def get_project_packages_dir(): + return get_project_optional_dir("packages_dir", + join(get_project_core_dir(), "packages")) + + +def get_project_cache_dir(): + return get_project_optional_dir("cache_dir", + join(get_project_core_dir(), ".cache")) + + +def get_project_workspace_dir(): return get_project_optional_dir("workspace_dir", join(get_project_dir(), ".pio")) -def get_projectbuild_dir(force=False): +def get_project_build_dir(force=False): path = get_project_optional_dir("build_dir", - join(get_projectworkspace_dir(), "build")) + join(get_project_workspace_dir(), "build")) try: if not isdir(path): os.makedirs(path) @@ -93,35 +137,35 @@ def get_projectbuild_dir(force=False): return path -def get_projectlibdeps_dir(): +def get_project_libdeps_dir(): return get_project_optional_dir( - "libdeps_dir", join(get_projectworkspace_dir(), "libdeps")) + "libdeps_dir", join(get_project_workspace_dir(), "libdeps")) -def get_projectlib_dir(): +def get_project_lib_dir(): return get_project_optional_dir("lib_dir", join(get_project_dir(), "lib")) -def get_projectsrc_dir(): - return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) - - -def get_projectinclude_dir(): +def get_project_include_dir(): return get_project_optional_dir("include_dir", join(get_project_dir(), "include")) -def get_projecttest_dir(): +def get_project_src_dir(): + return get_project_optional_dir("src_dir", join(get_project_dir(), "src")) + + +def get_project_test_dir(): return get_project_optional_dir("test_dir", join(get_project_dir(), "test")) -def get_projectboards_dir(): +def get_project_boards_dir(): return get_project_optional_dir("boards_dir", join(get_project_dir(), "boards")) -def get_projectdata_dir(): +def get_project_data_dir(): return get_project_optional_dir("data_dir", join(get_project_dir(), "data")) @@ -129,7 +173,7 @@ def get_projectdata_dir(): def calculate_project_hash(): check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") chunks = [__version__] - for d in (get_projectsrc_dir(), get_projectlib_dir()): + for d in (get_project_src_dir(), get_project_lib_dir()): if not isdir(d): continue for root, _, files in walk(d): diff --git a/platformio/util.py b/platformio/util.py index 304d3a14..edfd8e60 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -25,8 +25,7 @@ import sys import time from functools import wraps from glob import glob -from os.path import (abspath, basename, dirname, expanduser, isdir, isfile, - join, splitdrive) +from os.path import abspath, basename, dirname, isdir, isfile, join from shutil import rmtree import click @@ -38,11 +37,25 @@ from platformio.compat import PY2, WINDOWS, get_file_contents, path_to_unicode from platformio.proc import LineBufferedAsyncPipe as AsyncPipe from platformio.proc import exec_command, is_ci, where_is_program from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_dir, get_project_optional_dir, get_projectboards_dir, - get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir, - get_projectlibdeps_dir, get_projectsrc_dir, get_projecttest_dir, - is_platformio_project) +from platformio.project.helpers import \ + get_project_boards_dir as get_projectboards_dir +from platformio.project.helpers import \ + get_project_build_dir as get_projectbuild_dir +from platformio.project.helpers import get_project_cache_dir as get_cache_dir +from platformio.project.helpers import get_project_core_dir as get_home_dir +from platformio.project.helpers import \ + get_project_data_dir as get_projectdata_dir +from platformio.project.helpers import get_project_dir +from platformio.project.helpers import \ + get_project_lib_dir as get_projectlib_dir +from platformio.project.helpers import \ + get_project_libdeps_dir as get_projectlibdeps_dir +from platformio.project.helpers import get_project_optional_dir +from platformio.project.helpers import \ + get_project_src_dir as get_projectsrc_dir +from platformio.project.helpers import \ + get_project_test_dir as get_projecttest_dir +from platformio.project.helpers import is_platformio_project class cd(object): @@ -135,32 +148,6 @@ def pioversion_to_intstr(): return [int(i) for i in vermatch.group(1).split(".")[:3]] -def get_home_dir(): - home_dir = get_project_optional_dir("home_dir", - join(expanduser("~"), ".platformio")) - win_home_dir = None - if WINDOWS: - win_home_dir = splitdrive(home_dir)[0] + "\\.platformio" - if isdir(win_home_dir): - home_dir = win_home_dir - - if not isdir(home_dir): - try: - os.makedirs(home_dir) - except: # pylint: disable=bare-except - if win_home_dir: - os.makedirs(win_home_dir) - home_dir = win_home_dir - - assert isdir(home_dir) - return home_dir - - -def get_cache_dir(): - return get_project_optional_dir("cache_dir", join(get_home_dir(), - ".cache")) - - def get_source_dir(): curpath = abspath(__file__) if not isfile(curpath): From 4266cba53b821f9a7a68eaf3caf4a9a02cd22cd4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 May 2019 01:10:35 +0300 Subject: [PATCH 169/333] Cleanup ".piolibdeps" --- platformio/commands/init.py | 2 +- platformio/ide/tpls/atom/.gitignore.tpl | 1 - platformio/ide/tpls/clion/.gitignore.tpl | 1 - platformio/ide/tpls/clion/.idea/misc.xml.tpl | 2 +- platformio/ide/tpls/emacs/.gitignore.tpl | 1 - platformio/ide/tpls/vim/.gitignore.tpl | 1 - platformio/ide/tpls/vscode/.gitignore.tpl | 1 - 7 files changed, 2 insertions(+), 7 deletions(-) diff --git a/platformio/commands/init.py b/platformio/commands/init.py index c002cc74..7232a5ef 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -359,7 +359,7 @@ def init_cvs_ignore(project_dir): if isfile(conf_path): return with open(conf_path, "w") as fp: - fp.writelines([".pio\n", ".piolibdeps\n"]) + fp.write(".pio\n") def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, diff --git a/platformio/ide/tpls/atom/.gitignore.tpl b/platformio/ide/tpls/atom/.gitignore.tpl index 04de5dbc..bbdd36c7 100644 --- a/platformio/ide/tpls/atom/.gitignore.tpl +++ b/platformio/ide/tpls/atom/.gitignore.tpl @@ -1,4 +1,3 @@ .pio -.piolibdeps .clang_complete .gcc-flags.json diff --git a/platformio/ide/tpls/clion/.gitignore.tpl b/platformio/ide/tpls/clion/.gitignore.tpl index ff5b1094..ff1a5181 100644 --- a/platformio/ide/tpls/clion/.gitignore.tpl +++ b/platformio/ide/tpls/clion/.gitignore.tpl @@ -1,3 +1,2 @@ .pio -.piolibdeps CMakeListsPrivate.txt diff --git a/platformio/ide/tpls/clion/.idea/misc.xml.tpl b/platformio/ide/tpls/clion/.idea/misc.xml.tpl index 8ba8c6a0..3463fba1 100644 --- a/platformio/ide/tpls/clion/.idea/misc.xml.tpl +++ b/platformio/ide/tpls/clion/.idea/misc.xml.tpl @@ -7,7 +7,7 @@ - + diff --git a/platformio/ide/tpls/emacs/.gitignore.tpl b/platformio/ide/tpls/emacs/.gitignore.tpl index c086bafd..b8e379fa 100644 --- a/platformio/ide/tpls/emacs/.gitignore.tpl +++ b/platformio/ide/tpls/emacs/.gitignore.tpl @@ -1,3 +1,2 @@ .pio -.piolibdeps .clang_complete diff --git a/platformio/ide/tpls/vim/.gitignore.tpl b/platformio/ide/tpls/vim/.gitignore.tpl index 04de5dbc..bbdd36c7 100644 --- a/platformio/ide/tpls/vim/.gitignore.tpl +++ b/platformio/ide/tpls/vim/.gitignore.tpl @@ -1,4 +1,3 @@ .pio -.piolibdeps .clang_complete .gcc-flags.json diff --git a/platformio/ide/tpls/vscode/.gitignore.tpl b/platformio/ide/tpls/vscode/.gitignore.tpl index e2fb3b9a..18453b74 100644 --- a/platformio/ide/tpls/vscode/.gitignore.tpl +++ b/platformio/ide/tpls/vscode/.gitignore.tpl @@ -1,5 +1,4 @@ .pio -.piolibdeps .vscode/.browse.c_cpp.db* .vscode/c_cpp_properties.json .vscode/launch.json From 67f7b6cda39b607e3c093bbb254443ef93cd3a68 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 May 2019 01:18:28 +0300 Subject: [PATCH 170/333] Bump version to 4.0.0a15 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index a78b131c..245edd47 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a14") +VERSION = (4, 0, "0a15") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 7b2c1f27fce57bba873e7a802aee1ed6073a1554 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 May 2019 20:46:23 +0300 Subject: [PATCH 171/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index dde6b061..401b1447 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit dde6b061a61872e35a920775c20a64063b445065 +Subproject commit 401b1447a3667bf256d6d13718bc8454c7f76888 From 7beb332b3170141331b1c58837d3380872b1adcc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 May 2019 20:46:56 +0300 Subject: [PATCH 172/333] Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file --- HISTORY.rst | 1 + platformio/ide/tpls/clion/CMakeLists.txt.tpl | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 379224a7..8fe10b5e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -41,6 +41,7 @@ PlatformIO 3.0 3.6.8 (2019-??-??) ~~~~~~~~~~~~~~~~~~ +* Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file * Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) * Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) * Fixed an issue for Project Generator when include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 7b17a983..526aeafe 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -1,8 +1,19 @@ +# !!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE +# https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags +# +# If you need to override existing CMake configuration or add extra, +# please create `CMakeListsUser.txt` in the root of project. +# The `CMakeListsUser.txt` will not be overwritten by PlatformIO. + cmake_minimum_required(VERSION 3.2) project({{project_name}}) include(CMakeListsPrivate.txt) +if(EXISTS CMakeListsUser.txt) +include(CMakeListsUser.txt) +endif() + add_custom_target( PLATFORMIO_BUILD ALL COMMAND ${PLATFORMIO_CMD} -f -c clion run From fea7e971128fb72d91c62af6600d5467aec11748 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 May 2019 20:47:39 +0300 Subject: [PATCH 173/333] Fix an issue with hardcoded C stadard version when generating project for CLion IDE // Resolve #2527 --- HISTORY.rst | 1 + .../ide/tpls/clion/CMakeListsPrivate.txt.tpl | 23 +++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8fe10b5e..2b10f752 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -42,6 +42,7 @@ PlatformIO 3.0 ~~~~~~~~~~~~~~~~~~ * Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file +* Fixed an issue with hardcoded C stadard version when generating project for CLion IDE (`issue #2527 `_) * Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) * Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) * Fixed an issue for Project Generator when include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index 0f7c053e..b3d84a75 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -1,7 +1,12 @@ -# !!! WARNING !!! -# PLEASE DO NOT MODIFY THIS FILE! -# USE https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags +# !!! WARNING !!! AUTO-GENERATED FILE, PLEASE DO NOT MODIFY IT AND USE +# https://docs.platformio.org/page/projectconf/section_env_build.html#build-flags +# +# If you need to override existing CMake configuration or add extra, +# please create `CMakeListsUser.txt` in the root of project. +# The `CMakeListsUser.txt` will not be overwritten by PlatformIO. +% import re +% % def _normalize_path(path): % if project_dir in path: % path = path.replace(project_dir, "${CMAKE_CURRENT_LIST_DIR}") @@ -21,9 +26,17 @@ SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}") SET(CMAKE_CXX_COMPILER "{{ _normalize_path(cxx_path) }}") SET(CMAKE_CXX_FLAGS_DISTRIBUTION "{{cxx_flags}}") SET(CMAKE_C_FLAGS_DISTRIBUTION "{{cc_flags}}") -set(CMAKE_CXX_STANDARD 11) -% import re +% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)") +% cc_stds = STD_RE.findall(cc_flags) +% cxx_stds = STD_RE.findall(cxx_flags) +% if cc_stds: +SET(CMAKE_C_STANDARD {{ cc_stds[-1] }}) +% end +% if cxx_stds: +set(CMAKE_CXX_STANDARD {{ cxx_stds[-1] }}) +% end + % for define in defines: add_definitions(-D'{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}') % end From 70a0bd72c0b837362b66532f9cc810588c584bc9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 May 2019 21:06:08 +0300 Subject: [PATCH 174/333] Sync "include" directory for PIO Remote // Resolve #2210 --- platformio/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/util.py b/platformio/util.py index edfd8e60..98fbae93 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -46,6 +46,8 @@ from platformio.project.helpers import get_project_core_dir as get_home_dir from platformio.project.helpers import \ get_project_data_dir as get_projectdata_dir from platformio.project.helpers import get_project_dir +from platformio.project.helpers import \ + get_project_include_dir as get_projectinclude_dir from platformio.project.helpers import \ get_project_lib_dir as get_projectlib_dir from platformio.project.helpers import \ From a84195bb5a54e8c13439acbcd31716186ec1fae9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 May 2019 21:49:51 +0300 Subject: [PATCH 175/333] Add user-definable monitor options to platformio.ini // Resolve #2165 --- HISTORY.rst | 7 ++++--- docs | 2 +- platformio/commands/device.py | 20 ++++++++++++++------ platformio/project/config.py | 1 + 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2b10f752..9a9b8145 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,11 +10,12 @@ PlatformIO 4.0 * **Project Management** - Unified workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 `_) + - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` - Share common (global) options between build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) `__ (`issue #1643 `_) - Include external configuration files in `"platformio.ini" (Project Configuration File) `__ with `extra_configs `__ option (`issue #1590 `_) - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - Custom project ``***_dir`` options declared in "platformio" section of `"platformio.ini" (Project Configuration File) `__ have higher priority than `Environment variables `__ - - Use workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` + - Added new `monitor_flags `__ option to `"platformio.ini" (Project Configuration File) `__ which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) * **Library Management** @@ -22,7 +23,7 @@ PlatformIO 4.0 - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - Use isolated library dependency storage per project build environment (`issue #1696 `_) - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - - Use workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` + - Switched to workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` * **Infrastructure** @@ -43,7 +44,7 @@ PlatformIO 3.0 * Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file * Fixed an issue with hardcoded C stadard version when generating project for CLion IDE (`issue #2527 `_) -* Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) +* Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) * Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) * Fixed an issue for Project Generator when include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) diff --git a/docs b/docs index 401b1447..158fb3c0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 401b1447a3667bf256d6d13718bc8454c7f76888 +Subproject commit 158fb3c0484586ef4b87dbd6ef7d780cbe74786c diff --git a/platformio/commands/device.py b/platformio/commands/device.py index e75660ea..9f80062e 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -162,16 +162,20 @@ def device_list( # pylint: disable=too-many-branches "--environment", help="Load configuration from `platformio.ini` and specified environment") def device_monitor(**kwargs): # pylint: disable=too-many-branches + custom_monitor_flags = [] try: - monitor_options = get_project_options(kwargs['project_dir'], - kwargs['environment']) - if monitor_options: + env_options = get_project_options(kwargs['project_dir'], + kwargs['environment']) + if "monitor_flags" in env_options: + custom_monitor_flags = ProjectConfig.parse_multi_values( + env_options['monitor_flags']) + if env_options: for k in ("port", "speed", "rts", "dtr"): k2 = "monitor_%s" % k if k == "speed": k = "baud" - if kwargs[k] is None and k2 in monitor_options: - kwargs[k] = monitor_options[k2] + if kwargs[k] is None and k2 in env_options: + kwargs[k] = env_options[k2] if k != "port": kwargs[k] = int(kwargs[k]) except exception.NotPlatformIOProject: @@ -182,11 +186,13 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches if len(ports) == 1: kwargs['port'] = ports[0]['port'] - sys.argv = ["monitor"] + sys.argv = ["monitor"] + custom_monitor_flags for k, v in kwargs.items(): if k in ("port", "baud", "rts", "dtr", "environment", "project_dir"): continue k = "--" + k.replace("_", "-") + if k in custom_monitor_flags: + continue if isinstance(v, bool): if v: sys.argv.append(k) @@ -196,6 +202,8 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches else: sys.argv.extend([k, str(v)]) + print(sys.argv) + try: miniterm.main( default_port=kwargs['port'], diff --git a/platformio/project/config.py b/platformio/project/config.py index 7b5baac0..d67df78b 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -86,6 +86,7 @@ KNOWN_ENV_OPTIONS = [ "monitor_speed", "monitor_rts", "monitor_dtr", + "monitor_flags", # Library "lib_deps", From 6e87089ded56f68ce2088317a9235a7ff7b444e8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 May 2019 22:14:38 +0300 Subject: [PATCH 176/333] Add support for Unix shell-style wildcards for "monitor_port" option // Resolve #2541 --- HISTORY.rst | 11 ++++++----- docs | 2 +- platformio/commands/device.py | 7 ++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9a9b8145..44a6cee6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,15 +7,16 @@ PlatformIO 4.0 4.0.0 (2019-??-??) ~~~~~~~~~~~~~~~~~~ -* **Project Management** +* **Project Configuration** - Unified workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 `_) - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` - - Share common (global) options between build environments using ``[env]`` section in `"platformio.ini" (Project Configuration File) `__ (`issue #1643 `_) - - Include external configuration files in `"platformio.ini" (Project Configuration File) `__ with `extra_configs `__ option (`issue #1590 `_) + - Share common (global) options between project environments using `[env] `__ section (`issue #1643 `_) + - Include external configuration files with `extra_configs `__ option (`issue #1590 `_) - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - - Custom project ``***_dir`` options declared in "platformio" section of `"platformio.ini" (Project Configuration File) `__ have higher priority than `Environment variables `__ - - Added new `monitor_flags `__ option to `"platformio.ini" (Project Configuration File) `__ which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) + - Custom project ``***_dir`` options declared in `platformio `__ section have higher priority than `Environment variables `__ + - Added support for Unix shell-style wildcards for `monitor_port `__ option (`issue #2541 `_) + - Added new `monitor_flags `__ option which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) * **Library Management** diff --git a/docs b/docs index 158fb3c0..abdccc98 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 158fb3c0484586ef4b87dbd6ef7d780cbe74786c +Subproject commit abdccc987a03354bb80863e98dc109dd6b990cbe diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 9f80062e..c5bde2bd 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -14,6 +14,7 @@ import json import sys +from fnmatch import fnmatch from os import getcwd from os.path import join @@ -202,7 +203,11 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches else: sys.argv.extend([k, str(v)]) - print(sys.argv) + if kwargs['port'] and (set(["*", "?", "[", "]"]) & set(kwargs['port'])): + for item in util.get_serial_ports(): + if fnmatch(item['port'], kwargs['port']): + kwargs['port'] = item['port'] + break try: miniterm.main( From e1aa29cb36bde3e15c75b7dcc8080fc8a4a5ec34 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 25 May 2019 22:18:23 +0300 Subject: [PATCH 177/333] Bump version to 4.0.0a16 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 245edd47..75e6e461 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a15") +VERSION = (4, 0, "0a16") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 3f96dc1432ca57eeb8170298672b49df2368c237 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 12:24:20 +0300 Subject: [PATCH 178/333] Fixed an issue when package cache (Library Manager) expires too fast // Resolve #2559 --- HISTORY.rst | 1 + docs | 2 +- platformio/managers/lib.py | 2 ++ platformio/managers/package.py | 6 ++---- platformio/managers/platform.py | 2 -- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 44a6cee6..0d7d0a7f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -48,6 +48,7 @@ PlatformIO 3.0 * Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) * Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) * Fixed an issue for Project Generator when include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) +* Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 `_) 3.6.7 (2019-04-23) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index abdccc98..4dd1a548 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit abdccc987a03354bb80863e98dc109dd6b990cbe +Subproject commit 4dd1a548609ffaa4c53999756f06e924e8de20f2 diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index e26a6cb8..9824afcd 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -31,6 +31,8 @@ from platformio.project.helpers import get_project_global_lib_dir class LibraryManager(BasePkgManager): + FILE_CACHE_VALID = "30d" # 1 month + def __init__(self, package_dir=None): if not package_dir: package_dir = get_project_global_lib_dir() diff --git a/platformio/managers/package.py b/platformio/managers/package.py index a089cc5d..cd0f0d1a 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -139,8 +139,8 @@ class PkgInstallerMixin(object): SRC_MANIFEST_NAME = ".piopkgmanager.json" TMP_FOLDER_PREFIX = "_tmp_installing-" - FILE_CACHE_VALID = "1m" # 1 month - FILE_CACHE_MAX_SIZE = 1024 * 1024 + FILE_CACHE_VALID = None # for example, 1 week = "7d" + FILE_CACHE_MAX_SIZE = 1024 * 1024 * 50 # 50 Mb MEMORY_CACHE = {} @@ -843,8 +843,6 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): class PackageManager(BasePkgManager): - FILE_CACHE_VALID = None # disable package caching - @property def manifest_names(self): return ["package.json"] diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 56625efe..8a5de2ef 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -41,8 +41,6 @@ except ImportError: class PlatformManager(BasePkgManager): - FILE_CACHE_VALID = None # disable platform download caching - def __init__(self, package_dir=None, repositories=None): if not repositories: repositories = [ From cb8af5add908485b5115893e5ef73b54001a76a3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 12:28:04 +0300 Subject: [PATCH 179/333] Print "No items found" when there are no packages for llisting --- platformio/commands/lib.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 6ad3252b..68bda15e 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -277,6 +277,8 @@ def lib_list(ctx, json_output): else: for item in sorted(items, key=lambda i: i['name']): print_lib_item(item) + else: + click.echo("No items found") if json_output: return click.echo( From 16864509afcbbe82bde0deddbec3746f4858a74f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 14:21:48 +0300 Subject: [PATCH 180/333] Document "Override package files" --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 4dd1a548..cc7e8727 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4dd1a548609ffaa4c53999756f06e924e8de20f2 +Subproject commit cc7e872743ca611c1a6f2381d0607cc64ba26f6b From 0a523fc06c7d6441d2944e8483e895111a00d4e2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 14:33:38 +0300 Subject: [PATCH 181/333] Docs: Minor tweak --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index cc7e8727..6c09ed44 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit cc7e872743ca611c1a6f2381d0607cc64ba26f6b +Subproject commit 6c09ed44621604986010dcd35946d4ca46bbf839 From a0fb88e28a1fcd36add93bca9245209cd2ac486e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 17:57:46 +0300 Subject: [PATCH 182/333] Implement "envLibdepsDirs" per project for PIO Home --- .../commands/home/rpc/handlers/project.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index ae2eaa07..7b492b8b 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -29,6 +29,7 @@ from platformio.compat import get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_libdeps_dir class ProjectRPC(object): @@ -37,9 +38,10 @@ class ProjectRPC(object): def _get_projects(project_dirs=None): def _get_project_data(project_dir): - data = {"boards": [], "libExtraDirs": []} + data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} config = ProjectConfig(join(project_dir, "platformio.ini")) config.validate(validate_options=False) + libdeps_dir = get_project_libdeps_dir() if config.has_section("platformio") and \ config.has_option("platformio", "lib_extra_dirs"): @@ -50,6 +52,7 @@ class ProjectRPC(object): for section in config.sections(): if not section.startswith("env:"): continue + data['envLibdepsDirs'].append(join(libdeps_dir, section[4:])) if config.has_option(section, "board"): data['boards'].append(config.get(section, "board")) if config.has_option(section, "lib_extra_dirs"): @@ -57,18 +60,13 @@ class ProjectRPC(object): util.parse_conf_multi_values( config.get(section, "lib_extra_dirs"))) - # resolve libExtraDirs paths - with util.cd(project_dir): - data['libExtraDirs'] = [ + # skip non existing folders and resolve full path + for key in ("envLibdepsDirs", "libExtraDirs"): + data[key] = [ expanduser(d) if d.startswith("~") else realpath(d) - for d in data['libExtraDirs'] + for d in data[key] if isdir(d) ] - # skip non existing folders - data['libExtraDirs'] = [ - d for d in data['libExtraDirs'] if isdir(d) - ] - return data def _path_to_name(path): @@ -83,7 +81,8 @@ class ProjectRPC(object): data = {} boards = [] try: - data = _get_project_data(project_dir) + with util.cd(project_dir): + data = _get_project_data(project_dir) except exception.PlatformIOProjectException: continue @@ -104,6 +103,10 @@ class ProjectRPC(object): int(getmtime(project_dir)), "boards": boards, + "envLibStorages": [{ + "name": basename(d), + "path": d + } for d in data.get("envLibdepsDirs", [])], "extraLibStorages": [{ "name": _path_to_name(d), "path": d From 078b0af3122e08a3a83e8a7aac27ca4a5a02175c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 18:17:57 +0300 Subject: [PATCH 183/333] Test only 1 project example per dev/platform for AppVeyor CI --- .appveyor.yml | 2 -- tests/test_examples.py | 15 ++++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index ee0986dc..fcaaeaaf 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,8 +11,6 @@ environment: install: - cmd: git submodule update --init --recursive - cmd: SET PATH=C:\MinGW\bin;%PATH% - - if %PLATFORM% == x64 SET PATH=C:\Python27-x64;C:\Python27-x64\Scripts;%PATH% - - if %PLATFORM% == x86 SET PATH=C:\Python27;C:\Python27\Scripts;%PATH% - cmd: pip install tox test_script: diff --git a/tests/test_examples.py b/tests/test_examples.py index 3f34433b..f2469f8f 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -14,7 +14,7 @@ import random from glob import glob -from os import listdir, walk +from os import getenv, listdir, walk from os.path import dirname, getsize, isdir, isfile, join, normpath import pytest @@ -54,11 +54,16 @@ def pytest_generate_tests(metafunc): continue platform_examples.append(root) - # test random 3 examples random.shuffle(platform_examples) - project_dirs.extend(platform_examples[:3]) - project_dirs.sort() - metafunc.parametrize("pioproject_dir", project_dirs) + + if getenv("APPVEYOR"): + # use only 1 example for AppVeyor CI + project_dirs.append(platform_examples[0]) + else: + # test random 3 examples + project_dirs.extend(platform_examples[:3]) + + metafunc.parametrize("pioproject_dir", sorted(project_dirs)) @pytest.mark.examples From 5999bcee3f336ff9fe80dbd9714d620b56cc644b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 18:45:24 +0300 Subject: [PATCH 184/333] Update history --- HISTORY.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0d7d0a7f..e3404124 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ PlatformIO 4.0 - Custom project ``***_dir`` options declared in `platformio `__ section have higher priority than `Environment variables `__ - Added support for Unix shell-style wildcards for `monitor_port `__ option (`issue #2541 `_) - Added new `monitor_flags `__ option which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) + - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) * **Library Management** @@ -33,23 +34,21 @@ PlatformIO 4.0 - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) +* **Integration** + + - Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file + - Fixed an issue with hardcoded C standard version when generating project for CLion IDE (`issue #2527 `_) + - Fixed an issue with Project Generator when include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) + * **Miscellaneous** - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead + - Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) + - Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 `_) PlatformIO 3.0 -------------- -3.6.8 (2019-??-??) -~~~~~~~~~~~~~~~~~~ - -* Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file -* Fixed an issue with hardcoded C stadard version when generating project for CLion IDE (`issue #2527 `_) -* Fixed "systemd-udevd" warnings in `99-platformio-udev.rules `__ (`issue #2442 `_) -* Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) -* Fixed an issue for Project Generator when include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) -* Fixed an issue when package cache (Library Manager) expires too fast (`issue #2559 `_) - 3.6.7 (2019-04-23) ~~~~~~~~~~~~~~~~~~ From b88c262a9dbdb8f4c47b12a9c0363b5c95b9959a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 18:49:35 +0300 Subject: [PATCH 185/333] Use the latest version of tox for AppVeyor CI --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index fcaaeaaf..ddee3c59 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,7 +11,7 @@ environment: install: - cmd: git submodule update --init --recursive - cmd: SET PATH=C:\MinGW\bin;%PATH% - - cmd: pip install tox + - cmd: pip install --force-reinstall tox test_script: - cmd: tox From 3adcf664536426316e8c43fa88f6703e4edd8e13 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 19:03:31 +0300 Subject: [PATCH 186/333] Docs: Use native Python ConfigParser for extra scripting examples --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 6c09ed44..56f8af55 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6c09ed44621604986010dcd35946d4ca46bbf839 +Subproject commit 56f8af5524a6a561795392d5a6077aaf7a11d6b4 From 3df01405a1f8dba6a10d727b0d2a6d6097fe1cbf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 22:25:22 +0300 Subject: [PATCH 187/333] Remove unused Python imports --- platformio/app.py | 5 +- platformio/builder/tools/pioide.py | 9 ++- platformio/commands/debug/client.py | 7 ++- platformio/commands/debug/command.py | 3 +- platformio/commands/debug/process.py | 5 +- platformio/commands/debug/server.py | 3 +- platformio/commands/home/helpers.py | 3 +- platformio/commands/home/rpc/handlers/app.py | 6 +- .../commands/home/rpc/handlers/project.py | 17 +++--- platformio/commands/run.py | 11 ---- platformio/commands/test/command.py | 18 +++--- platformio/commands/test/native.py | 11 ++-- platformio/commands/test/processor.py | 6 +- platformio/commands/upgrade.py | 3 +- platformio/exception.py | 2 +- platformio/ide/projectgenerator.py | 3 +- platformio/managers/core.py | 2 +- platformio/proc.py | 3 +- platformio/util.py | 45 +------------- tests/commands/test_init.py | 13 ++-- tests/test_examples.py | 7 +-- tests/test_managers.py | 60 +++++++++++-------- 22 files changed, 111 insertions(+), 131 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 798fecab..c0923bf1 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -27,7 +27,8 @@ import requests from platformio import exception, lockfile, util from platformio.compat import PY2, WINDOWS from platformio.proc import is_ci -from platformio.project.helpers import get_project_core_dir +from platformio.project.helpers import (get_project_cache_dir, + get_project_core_dir) def projects_dir_validate(projects_dir): @@ -139,7 +140,7 @@ class ContentCache(object): self._db_path = None self._lockfile = None - self.cache_dir = cache_dir or util.get_cache_dir() + self.cache_dir = cache_dir or get_project_cache_dir() self._db_path = join(self.cache_dir, "db.data") def __enter__(self): diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 8efcd60d..12ebefba 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -20,10 +20,9 @@ from os.path import abspath, isfile, join from SCons.Defaults import processDefines # pylint: disable=import-error -from platformio import util from platformio.compat import glob_escape from platformio.managers.core import get_core_package_dir -from platformio.proc import exec_command +from platformio.proc import exec_command, where_is_program def _dump_includes(env, projenv): @@ -151,11 +150,11 @@ def DumpIDEData(env, projenv): "cxx_flags": env.subst(LINTCXXCOM), "cc_path": - util.where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), + where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")), "cxx_path": - util.where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")), + where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")), "gdb_path": - util.where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), + where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), "prog_path": env.subst("$PROG_PATH"), "flash_extra_images": [{ diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 25a57db1..460682d9 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -30,6 +30,7 @@ from platformio.commands.debug import helpers, initcfgs from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer from platformio.compat import PY2 +from platformio.project.helpers import get_project_cache_dir from platformio.telemetry import MeasurementProtocol LOG_FILE = None @@ -49,10 +50,10 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes self._debug_server = DebugServer(debug_options, env_options) self._session_id = None - if not isdir(util.get_cache_dir()): - os.makedirs(util.get_cache_dir()) + if not isdir(get_project_cache_dir()): + os.makedirs(get_project_cache_dir()) self._gdbsrc_dir = mkdtemp( - dir=util.get_cache_dir(), prefix=".piodebug-") + dir=get_project_cache_dir(), prefix=".piodebug-") self._target_is_run = False self._last_server_activity = 0 diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 7e1bf62e..153f1e74 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -24,6 +24,7 @@ from platformio import exception, util from platformio.commands.debug import helpers from platformio.managers.core import inject_contrib_pysite from platformio.project.config import ProjectConfig +from platformio.project.helpers import is_platformio_project @click.command( @@ -66,7 +67,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") % line) - if not util.is_platformio_project(project_dir) and os.getenv("CWD"): + if not is_platformio_project(project_dir) and os.getenv("CWD"): project_dir = os.getenv("CWD") with util.cd(project_dir): diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index 0d70abe1..68d76d00 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -17,9 +17,9 @@ import os import click from twisted.internet import protocol # pylint: disable=import-error -from platformio import util from platformio.commands.debug import helpers from platformio.compat import string_types +from platformio.project.helpers import get_project_core_dir LOG_FILE = None @@ -29,7 +29,8 @@ class BaseProcess(protocol.ProcessProtocol, object): STDOUT_CHUNK_SIZE = 2048 COMMON_PATTERNS = { - "PLATFORMIO_HOME_DIR": helpers.escape_path(util.get_home_dir()), + "PLATFORMIO_HOME_DIR": helpers.escape_path(get_project_core_dir()), + "PLATFORMIO_CORE_DIR": helpers.escape_path(get_project_core_dir()), "PYTHONEXE": os.getenv("PYTHONEXEPATH", "") } diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index ca5c2a2e..8719b1f0 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -20,6 +20,7 @@ from twisted.internet import reactor # pylint: disable=import-error from platformio import exception, util from platformio.commands.debug import helpers from platformio.commands.debug.process import BaseProcess +from platformio.proc import where_is_program class DebugServer(BaseProcess): @@ -47,7 +48,7 @@ class DebugServer(BaseProcess): server_executable = server_executable + ".exe" if not isfile(server_executable): - server_executable = util.where_is_program(server_executable) + server_executable = where_is_program(server_executable) if not isfile(server_executable): raise exception.DebugInvalidOptions( "\nCould not launch Debug Server '%s'. Please check that it " diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index 20b9503c..a4dd142a 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -23,6 +23,7 @@ from twisted.internet import reactor # pylint: disable=import-error from twisted.internet import threads # pylint: disable=import-error from platformio import util +from platformio.proc import where_is_program class AsyncSession(requests.Session): @@ -49,7 +50,7 @@ def requests_session(): @util.memoized() def get_core_fullpath(): - return util.where_is_program( + return where_is_program( "platformio" + (".exe" if "windows" in util.get_systype() else "")) diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index 8241b03a..e356fa06 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -19,11 +19,13 @@ from os.path import expanduser, isfile, join from platformio import __version__, app, exception, util from platformio.compat import path_to_unicode +from platformio.project.helpers import (get_project_core_dir, + is_platformio_project) class AppRPC(object): - APPSTATE_PATH = join(util.get_home_dir(), "homestate.json") + APPSTATE_PATH = join(get_project_core_dir(), "homestate.json") @staticmethod def load_state(): @@ -67,7 +69,7 @@ class AppRPC(object): # skip non-existing recent projects storage['recentProjects'] = [ p for p in storage.get("recentProjects", []) - if util.is_platformio_project(p) + if is_platformio_project(p) ] state['storage'] = storage diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 7b492b8b..aee042e2 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -29,7 +29,8 @@ from platformio.compat import get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import get_project_libdeps_dir +from platformio.project.helpers import ( + get_project_libdeps_dir, get_project_src_dir, is_platformio_project) class ProjectRPC(object): @@ -46,8 +47,7 @@ class ProjectRPC(object): if config.has_section("platformio") and \ config.has_option("platformio", "lib_extra_dirs"): data['libExtraDirs'].extend( - util.parse_conf_multi_values( - config.get("platformio", "lib_extra_dirs"))) + config.getlist("platformio", "lib_extra_dirs")) for section in config.sections(): if not section.startswith("env:"): @@ -57,8 +57,7 @@ class ProjectRPC(object): data['boards'].append(config.get(section, "board")) if config.has_option(section, "lib_extra_dirs"): data['libExtraDirs'].extend( - util.parse_conf_multi_values( - config.get(section, "lib_extra_dirs"))) + config.getlist(section, "lib_extra_dirs")) # skip non existing folders and resolve full path for key in ("envLibdepsDirs", "libExtraDirs"): @@ -165,7 +164,7 @@ class ProjectRPC(object): if not main_content: return project_dir with util.cd(project_dir): - src_dir = util.get_projectsrc_dir() + src_dir = get_project_src_dir() main_path = join(src_dir, "main.cpp") if isfile(main_path): return project_dir @@ -177,7 +176,7 @@ class ProjectRPC(object): def import_arduino(self, board, use_arduino_libs, arduino_project_dir): # don't import PIO Project - if util.is_platformio_project(arduino_project_dir): + if is_platformio_project(arduino_project_dir): return arduino_project_dir is_arduino_project = any([ @@ -214,7 +213,7 @@ class ProjectRPC(object): @staticmethod def _finalize_arduino_import(_, project_dir, arduino_project_dir): with util.cd(project_dir): - src_dir = util.get_projectsrc_dir() + src_dir = get_project_src_dir() if isdir(src_dir): util.rmtree_(src_dir) shutil.copytree( @@ -261,7 +260,7 @@ class ProjectRPC(object): @staticmethod def import_pio(project_dir): - if not project_dir or not util.is_platformio_project(project_dir): + if not project_dir or not is_platformio_project(project_dir): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4001, message="Not an PlatformIO project: %s" % project_dir) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 5ee46c8f..b75c8962 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -349,14 +349,3 @@ def print_summary(results, start_time): (click.style("SUCCESS", fg="green", bold=True) if successed else click.style("ERROR", fg="red", bold=True)), time() - start_time), is_error=not successed) - - -def check_project_envs(config, environments=None): # FIXME: Remove - if not config.sections(): - raise exception.ProjectEnvsNotAvailable() - - known = set(s[4:] for s in config.sections() if s.startswith("env:")) - unknown = set(environments or []) - known - if unknown: - raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) - return True diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index 2fc2eca1..bd98d3f4 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -26,6 +26,7 @@ from platformio.commands.run import print_header from platformio.commands.test.embedded import EmbeddedTestProcessor from platformio.commands.test.native import NativeTestProcessor from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_test_dir @click.command("test", short_help="Unit Testing") @@ -84,7 +85,7 @@ def cli( # pylint: disable=redefined-builtin project_conf, without_building, without_uploading, without_testing, no_reset, monitor_rts, monitor_dtr, verbose): with util.cd(project_dir): - test_dir = util.get_projecttest_dir() + test_dir = get_project_test_dir() if not isdir(test_dir): raise exception.TestDirNotExists(test_dir) test_names = get_test_names(test_dir) @@ -147,10 +148,13 @@ def cli( # pylint: disable=redefined-builtin if without_testing: return - click.echo() - print_header("[%s]" % click.style("TEST SUMMARY")) - passed = True + testname_max_len = max([len(r[1]) for r in results]) + envname_max_len = max([len(click.style(r[2], fg="cyan")) for r in results]) + + print_header("[%s]" % click.style("TEST SUMMARY")) + click.echo() + for result in results: status, testname, envname = result status_str = click.style("PASSED", fg="green") @@ -161,9 +165,9 @@ def cli( # pylint: disable=redefined-builtin status_str = click.style("IGNORED", fg="yellow") click.echo( - "test/%s/env:%s\t[%s]" % (click.style(testname, fg="yellow"), - click.style(envname, fg="cyan"), - status_str), + ("test/{:<%d} > {:<%d}\t[{}]" % + (testname_max_len, envname_max_len)).format( + testname, click.style(envname, fg="cyan"), status_str), err=status is False) print_header( diff --git a/platformio/commands/test/native.py b/platformio/commands/test/native.py index 0945b10f..70505e7b 100644 --- a/platformio/commands/test/native.py +++ b/platformio/commands/test/native.py @@ -16,6 +16,8 @@ from os.path import join from platformio import util from platformio.commands.test.processor import TestProcessorBase +from platformio.proc import LineBufferedAsyncPipe +from platformio.project.helpers import get_project_build_dir class NativeTestProcessor(TestProcessorBase): @@ -31,9 +33,10 @@ class NativeTestProcessor(TestProcessorBase): def run(self): with util.cd(self.options['project_dir']): - build_dir = util.get_projectbuild_dir() - result = util.exec_command([join(build_dir, self.env_name, "program")], - stdout=util.AsyncPipe(self.on_run_out), - stderr=util.AsyncPipe(self.on_run_out)) + build_dir = get_project_build_dir() + result = util.exec_command( + [join(build_dir, self.env_name, "program")], + stdout=LineBufferedAsyncPipe(self.on_run_out), + stderr=LineBufferedAsyncPipe(self.on_run_out)) assert "returncode" in result return result['returncode'] == 0 and not self._run_failed diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 4a4bc9dd..e5a4e32d 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -19,9 +19,10 @@ from string import Template import click -from platformio import exception, util +from platformio import exception from platformio.commands.run import cli as cmd_run from platformio.commands.run import print_header +from platformio.project.helpers import get_project_test_dir TRANSPORT_OPTIONS = { "arduino": { @@ -113,7 +114,7 @@ class TestProcessorBase(object): def build_or_upload(self, target): if not self._outputcpp_generated: - self.generate_outputcpp(util.get_projecttest_dir()) + self.generate_outputcpp(get_project_test_dir()) self._outputcpp_generated = True if self.test_name != "*": @@ -138,6 +139,7 @@ class TestProcessorBase(object): raise NotImplementedError def on_run_out(self, line): + line = line.strip() if line.endswith(":PASS"): click.echo( "%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green"))) diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 18e49cb3..2c6b367e 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -23,6 +23,7 @@ from platformio import VERSION, __version__, exception, util from platformio.compat import WINDOWS from platformio.managers.core import shutdown_piohome_servers from platformio.proc import exec_command, get_pythonexe_path +from platformio.project.helpers import get_project_cache_dir @click.command( @@ -95,7 +96,7 @@ def get_pip_package(to_develop): return "platformio" dl_url = ("https://github.com/platformio/" "platformio-core/archive/develop.zip") - cache_dir = util.get_cache_dir() + cache_dir = get_project_cache_dir() if not os.path.isdir(cache_dir): os.makedirs(cache_dir) pkg_name = os.path.join(cache_dir, "piocoredevelop.zip") diff --git a/platformio/exception.py b/platformio/exception.py index 9af0f0fe..343dc03f 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -174,7 +174,7 @@ class ProjectEnvsNotAvailable(PlatformIOProjectException): MESSAGE = "Please setup environments in `platformio.ini` file" -class UnknownEnvNames(PlatformIOProjectException): # FIXME: UnknownProjectEnvs +class UnknownEnvNames(PlatformIOProjectException): MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 9898dd88..3482db28 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -24,6 +24,7 @@ from click.testing import CliRunner from platformio import exception, util from platformio.commands.run import cli as cmd_run from platformio.compat import PY2, WINDOWS, get_file_contents +from platformio.proc import where_is_program from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_project_lib_dir, get_project_libdeps_dir, get_project_src_dir) @@ -149,7 +150,7 @@ class ProjectGenerator(object): "systype": util.get_systype(), "platformio_path": self._fix_os_path( sys.argv[0] if isfile(sys.argv[0]) - else util.where_is_program("platformio")), + else where_is_program("platformio")), "env_pathsep": os.pathsep, "env_path": self._fix_os_path(os.getenv("PATH")) }) # yapf: disable diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 139f99d3..8e8afaaf 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -30,7 +30,7 @@ CORE_PACKAGES = { "contrib-piohome": "^2.1.0", "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.3.0", + "tool-pioplus": "^2.4.0", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30005.0" } diff --git a/platformio/proc.py b/platformio/proc.py index 7cc3d8a7..9ffad954 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -96,8 +96,7 @@ class LineBufferedAsyncPipe(AsyncPipeBase): def do_reading(self): for line in iter(self._pipe_reader.readline, ""): self._buffer += line - # FIXME: Remove striping - self.line_callback(line.strip()) + self.line_callback(line) self._pipe_reader.close() diff --git a/platformio/util.py b/platformio/util.py index 98fbae93..c4dbe537 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# FIXME: Remove line below before 4.0 release -# pylint: disable=unused-import - import json import os import platform @@ -25,7 +22,7 @@ import sys import time from functools import wraps from glob import glob -from os.path import abspath, basename, dirname, isdir, isfile, join +from os.path import abspath, basename, dirname, isfile, join from shutil import rmtree import click @@ -33,31 +30,8 @@ import requests from platformio import __apiurl__, __version__, exception from platformio.commands import PlatformioCLI -from platformio.compat import PY2, WINDOWS, get_file_contents, path_to_unicode -from platformio.proc import LineBufferedAsyncPipe as AsyncPipe -from platformio.proc import exec_command, is_ci, where_is_program -from platformio.project.config import ProjectConfig -from platformio.project.helpers import \ - get_project_boards_dir as get_projectboards_dir -from platformio.project.helpers import \ - get_project_build_dir as get_projectbuild_dir -from platformio.project.helpers import get_project_cache_dir as get_cache_dir -from platformio.project.helpers import get_project_core_dir as get_home_dir -from platformio.project.helpers import \ - get_project_data_dir as get_projectdata_dir -from platformio.project.helpers import get_project_dir -from platformio.project.helpers import \ - get_project_include_dir as get_projectinclude_dir -from platformio.project.helpers import \ - get_project_lib_dir as get_projectlib_dir -from platformio.project.helpers import \ - get_project_libdeps_dir as get_projectlibdeps_dir -from platformio.project.helpers import get_project_optional_dir -from platformio.project.helpers import \ - get_project_src_dir as get_projectsrc_dir -from platformio.project.helpers import \ - get_project_test_dir as get_projecttest_dir -from platformio.project.helpers import is_platformio_project +from platformio.compat import PY2, WINDOWS, get_file_contents +from platformio.proc import exec_command, is_ci class cd(object): @@ -160,19 +134,6 @@ def get_source_dir(): return dirname(curpath) -def load_project_config(path=None): # FIXME: Remove - if not path or isdir(path): - path = join(path or get_project_dir(), "platformio.ini") - if not isfile(path): - raise exception.NotPlatformIOProject( - dirname(path) if path.endswith("platformio.ini") else path) - return ProjectConfig(path) - - -def parse_conf_multi_values(items): # FIXME: Remove - return ProjectConfig.parse_multi_values(items) - - def change_filemtime(path, mtime): os.utime(path, (mtime, mtime)) diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 4e8eac09..800a0b47 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -19,6 +19,7 @@ from os.path import getsize, isdir, isfile, join from platformio import exception, util from platformio.commands.boards import cli as cmd_boards from platformio.commands.init import cli as cmd_init +from platformio.project.config import ProjectConfig def validate_pioproject(pioproject_dir): @@ -50,7 +51,8 @@ def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir): result = clirunner.invoke(cmd_init, ["-b", "uno", "-b", "uno"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) - config = util.load_project_config() + config = ProjectConfig(join(getcwd(), "platformio.ini")) + config.validate() assert set(config.sections()) == set(["env:uno"]) @@ -105,7 +107,8 @@ def test_init_special_board(clirunner, validate_cliresult): validate_cliresult(result) boards = json.loads(result.output) - config = util.load_project_config() + config = ProjectConfig(join(getcwd(), "platformio.ini")) + config.validate() expected_result = [("platform", str(boards[0]['platform'])), ("framework", str(boards[0]['frameworks'][0])), ("board", "uno")] @@ -121,7 +124,8 @@ def test_init_enable_auto_uploading(clirunner, validate_cliresult): cmd_init, ["-b", "uno", "--project-option", "targets=upload"]) validate_cliresult(result) validate_pioproject(getcwd()) - config = util.load_project_config() + config = ProjectConfig(join(getcwd(), "platformio.ini")) + config.validate() expected_result = [("platform", "atmelavr"), ("framework", "arduino"), ("board", "uno"), ("targets", "upload")] assert config.has_section("env:uno") @@ -135,7 +139,8 @@ def test_init_custom_framework(clirunner, validate_cliresult): cmd_init, ["-b", "teensy31", "--project-option", "framework=mbed"]) validate_cliresult(result) validate_pioproject(getcwd()) - config = util.load_project_config() + config = ProjectConfig(join(getcwd(), "platformio.ini")) + config.validate() expected_result = [("platform", "teensy"), ("framework", "mbed"), ("board", "teensy31")] assert config.has_section("env:teensy31") diff --git a/tests/test_examples.py b/tests/test_examples.py index f2469f8f..3da3f15d 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -21,6 +21,7 @@ import pytest from platformio import util from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.project.config import ProjectConfig def pytest_generate_tests(metafunc): @@ -73,11 +74,7 @@ def test_run(pioproject_dir): if isdir(build_dir): util.rmtree_(build_dir) - env_names = [] - for section in util.load_project_config().sections(): - if section.startswith("env:"): - env_names.append(section[4:]) - + env_names = ProjectConfig(join(pioproject_dir, "platformio.ini")).envs() result = util.exec_command( ["platformio", "run", "-e", random.choice(env_names)]) diff --git a/tests/test_managers.py b/tests/test_managers.py index ee2d5df1..f2946f12 100644 --- a/tests/test_managers.py +++ b/tests/test_managers.py @@ -15,8 +15,8 @@ import json from os.path import join -from platformio import util from platformio.managers.package import PackageManager +from platformio.project.helpers import get_project_core_dir def test_pkg_input_parser(): @@ -28,16 +28,16 @@ def test_pkg_input_parser(): ["id=13", ("id=13", None, None)], ["id=13@~1.2.3", ("id=13", "~1.2.3", None)], [ - util.get_home_dir(), - (".platformio", None, "file://" + util.get_home_dir()) + get_project_core_dir(), + (".platformio", None, "file://" + get_project_core_dir()) ], [ - "LocalName=" + util.get_home_dir(), - ("LocalName", None, "file://" + util.get_home_dir()) + "LocalName=" + get_project_core_dir(), + ("LocalName", None, "file://" + get_project_core_dir()) ], [ - "LocalName=%s@>2.3.0" % util.get_home_dir(), - ("LocalName", ">2.3.0", "file://" + util.get_home_dir()) + "LocalName=%s@>2.3.0" % get_project_core_dir(), + ("LocalName", ">2.3.0", "file://" + get_project_core_dir()) ], [ "https://github.com/user/package.git", @@ -130,7 +130,8 @@ def test_pkg_input_parser(): ], [ "LocalName=git@github.com:user/package.git#v1.2.0@~1.2.0", - ("LocalName", "~1.2.0", "git+git@github.com:user/package.git#v1.2.0") + ("LocalName", "~1.2.0", + "git+git@github.com:user/package.git#v1.2.0") ], [ "git+ssh://git@gitlab.private-server.com/user/package#1.2.0", @@ -164,15 +165,18 @@ def test_install_packages(isolated_pio_home, tmpdir): dict(id=1, name="name_1", version="1.2"), dict(id=1, name="name_1", version="1.0.0"), dict(name="name_2", version="1.0.0"), - dict(name="name_2", version="2.0.0", + dict(name="name_2", + version="2.0.0", __src_url="git+https://github.com"), - dict(name="name_2", version="3.0.0", + dict(name="name_2", + version="3.0.0", __src_url="git+https://github2.com"), - dict(name="name_2", version="4.0.0", + dict(name="name_2", + version="4.0.0", __src_url="git+https://github2.com") ] - pm = PackageManager(join(util.get_home_dir(), "packages")) + pm = PackageManager(join(get_project_core_dir(), "packages")) for package in packages: tmp_dir = tmpdir.mkdir("tmp-package") tmp_dir.join("package.json").write(json.dumps(package)) @@ -182,36 +186,44 @@ def test_install_packages(isolated_pio_home, tmpdir): assert len(pm.get_installed()) == len(packages) - 1 pkg_dirnames = [ - 'name_1_ID1', 'name_1_ID1@1.0.0', 'name_1_ID1@1.2', - 'name_1_ID1@2.0.0', 'name_1_ID1@shasum', 'name_2', + 'name_1_ID1', 'name_1_ID1@1.0.0', 'name_1_ID1@1.2', 'name_1_ID1@2.0.0', + 'name_1_ID1@shasum', 'name_2', 'name_2@src-177cbce1f0705580d17790fda1cc2ef5', 'name_2@src-f863b537ab00f4c7b5011fc44b120e1f' ] - assert set([p.basename for p in isolated_pio_home.join( - "packages").listdir()]) == set(pkg_dirnames) + assert set([ + p.basename for p in isolated_pio_home.join("packages").listdir() + ]) == set(pkg_dirnames) def test_get_package(): tests = [ [("unknown", ), None], [("1", ), None], - [("id=1", "shasum"), dict(id=1, name="name_1", version="shasum")], - [("id=1", "*"), dict(id=1, name="name_1", version="2.1.0")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("id=1", "^1"), dict(id=1, name="name_1", version="1.2")], - [("name_1", "<2"), dict(id=1, name="name_1", version="1.2")], + [("id=1", "shasum"), + dict(id=1, name="name_1", version="shasum")], + [("id=1", "*"), + dict(id=1, name="name_1", version="2.1.0")], + [("id=1", "^1"), + dict(id=1, name="name_1", version="1.2")], + [("id=1", "^1"), + dict(id=1, name="name_1", version="1.2")], + [("name_1", "<2"), + dict(id=1, name="name_1", version="1.2")], [("name_1", ">2"), None], [("name_1", "2-0-0"), None], [("name_2", ), dict(name="name_2", version="4.0.0")], [("url_has_higher_priority", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", + dict(name="name_2", + version="2.0.0", __src_url="git+https://github.com")], [("name_2", None, "git+https://github.com"), - dict(name="name_2", version="2.0.0", + dict(name="name_2", + version="2.0.0", __src_url="git+https://github.com")], ] - pm = PackageManager(join(util.get_home_dir(), "packages")) + pm = PackageManager(join(get_project_core_dir(), "packages")) for test in tests: manifest = pm.get_package(*test[0]) if test[1] is None: From 99377130ebebfa788b25d7fda0baf15aee6426c8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 27 May 2019 22:25:48 +0300 Subject: [PATCH 188/333] Enhance unit testing summary --- platformio/commands/run.py | 5 +++-- platformio/commands/test/command.py | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index b75c8962..41fca7e0 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -314,11 +314,12 @@ def _clean_build_dir(build_dir): f.write(proj_hash) -def print_header(label, is_error=False): +def print_header(label, is_error=False, fg=None): terminal_width, _ = click.get_terminal_size() width = len(click.unstyle(label)) half_line = "=" * int((terminal_width - width - 2) / 2) - click.echo("%s %s %s" % (half_line, label, half_line), err=is_error) + click.secho( + "%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) def print_summary(results, start_time): diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index bd98d3f4..5f41649b 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -148,7 +148,8 @@ def cli( # pylint: disable=redefined-builtin if without_testing: return - passed = True + passed_nums = 0 + failed_nums = 0 testname_max_len = max([len(r[1]) for r in results]) envname_max_len = max([len(click.style(r[2], fg="cyan")) for r in results]) @@ -157,12 +158,14 @@ def cli( # pylint: disable=redefined-builtin for result in results: status, testname, envname = result - status_str = click.style("PASSED", fg="green") if status is False: - passed = False + failed_nums += 1 status_str = click.style("FAILED", fg="red") elif status is None: status_str = click.style("IGNORED", fg="yellow") + else: + passed_nums += 1 + status_str = click.style("PASSED", fg="green") click.echo( ("test/{:<%d} > {:<%d}\t[{}]" % @@ -171,12 +174,13 @@ def cli( # pylint: disable=redefined-builtin err=status is False) print_header( - "[%s] Took %.2f seconds" % ( - (click.style("PASSED", fg="green", bold=True) if passed else - click.style("FAILED", fg="red", bold=True)), time() - start_time), - is_error=not passed) + "%s%d passed in %.2f seconds" + % ("%d failed, " % failed_nums if failed_nums else "", passed_nums, + time() - start_time), + is_error=failed_nums, + fg="red" if failed_nums else "green") - if not passed: + if failed_nums: raise exception.ReturnErrorCode(1) From dec43bec9da9b4eec5c55271cde1fc6429686027 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 28 May 2019 00:09:20 +0300 Subject: [PATCH 189/333] Fix test --- tests/test_examples.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 3da3f15d..8e655c14 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -22,6 +22,7 @@ import pytest from platformio import util from platformio.managers.platform import PlatformFactory, PlatformManager from platformio.project.config import ProjectConfig +from platformio.project.helpers import get_project_build_dir def pytest_generate_tests(metafunc): @@ -70,7 +71,7 @@ def pytest_generate_tests(metafunc): @pytest.mark.examples def test_run(pioproject_dir): with util.cd(pioproject_dir): - build_dir = util.get_projectbuild_dir() + build_dir = get_project_build_dir() if isdir(build_dir): util.rmtree_(build_dir) From 9ce9171a3612ee1507845879909475dcde3597be Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 28 May 2019 01:19:51 +0300 Subject: [PATCH 190/333] Fix typo --- platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl b/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl index b05b34a8..fcf8b23d 100644 --- a/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl +++ b/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl @@ -15,7 +15,7 @@ From ddc8a353cbdfb1facf06b8f2f72e259ef860524b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 28 May 2019 12:53:54 +0300 Subject: [PATCH 191/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 56f8af55..e5a6a44a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 56f8af5524a6a561795392d5a6077aaf7a11d6b4 +Subproject commit e5a6a44a781304a6edbae95e7fa7ee7d1e58f6d7 From 7f845ab943ce332df4b0de6d698727c0ec965d63 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 14:32:49 +0300 Subject: [PATCH 192/333] Sync docs and examples --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index e5a6a44a..e9107b7d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e5a6a44a781304a6edbae95e7fa7ee7d1e58f6d7 +Subproject commit e9107b7dc8280bbb25b79ca61f71815986869aa6 diff --git a/examples b/examples index 46bac491..45ec933e 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 46bac491dc59bfdc3baf8edafe091cd8b30b0d10 +Subproject commit 45ec933e2858ae117c31d7783d66b83d96a6e609 From 8d05903bf31d2b280be7333b254b90f2a2b10336 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 14:36:04 +0300 Subject: [PATCH 193/333] Log THE ONLY non sensitive data (used board, platform, and framework) --- platformio/telemetry.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 7564a03d..071db390 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -310,12 +310,15 @@ def measure_ci(): def on_run_environment(options, targets): - opts = [ - "%s=%s" % (opt, value.replace("\n", ", ") if "\n" in value else value) - for opt, value in sorted(options.items()) - ] + non_sensative_values = ["board", "platform", "framework"] + safe_options = [] + for key, value in sorted(options.items()): + if key in non_sensative_values: + safe_options.append("%s=%s" % (key, value)) + else: + safe_options.append(key) targets = [t.title() for t in targets or ["run"]] - on_event("Env", " ".join(targets), "&".join(opts)) + on_event("Env", " ".join(targets), "&".join(safe_options)) def on_event(category, action, label=None, value=None, screen_name=None): From 3cc4af1723e2acde841869f6b57a503ce205d21a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 16:38:04 +0300 Subject: [PATCH 194/333] Refactor project config options --- .vscode/settings.json | 15 -- Makefile | 2 +- platformio/commands/device.py | 25 +- .../commands/home/rpc/handlers/project.py | 17 +- platformio/commands/init.py | 1 - platformio/commands/lib.py | 8 +- platformio/commands/test/command.py | 5 +- platformio/commands/test/processor.py | 3 +- platformio/project/config.py | 218 ++++++------------ platformio/project/helpers.py | 36 +-- platformio/project/options.py | 199 ++++++++++++++++ tests/commands/test_ci.py | 2 +- tests/commands/test_init.py | 25 +- tests/test_projectconf.py | 103 +++++++-- 14 files changed, 401 insertions(+), 258 deletions(-) delete mode 100644 .vscode/settings.json create mode 100644 platformio/project/options.py diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index d1efd097..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "python.pythonPath": "${workspaceRoot}/.tox/develop/bin/python", - "python.formatting.provider": "yapf", - "files.exclude": { - "**/*.pyc": true, - "*.egg-info": true, - ".cache": true, - "build": true, - "dist": true - }, - "editor.rulers": [79], - "restructuredtext.builtDocumentationPath": "${workspaceRoot}/docs/_build/html", - "restructuredtext.confPath": "${workspaceRoot}/docs", - "restructuredtext.linter.executablePath": "${workspaceRoot}/.tox/docs/bin/restructuredtext-lint" -} diff --git a/Makefile b/Makefile index 6df42c42..bc8ba602 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ yapf: yapf --recursive --in-place platformio/ test: - py.test -v -s -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py + py.test --verbose --capture=no --exitfirst -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py before-commit: isort yapf lint test diff --git a/platformio/commands/device.py b/platformio/commands/device.py index c5bde2bd..594020eb 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -163,22 +163,17 @@ def device_list( # pylint: disable=too-many-branches "--environment", help="Load configuration from `platformio.ini` and specified environment") def device_monitor(**kwargs): # pylint: disable=too-many-branches - custom_monitor_flags = [] try: env_options = get_project_options(kwargs['project_dir'], kwargs['environment']) - if "monitor_flags" in env_options: - custom_monitor_flags = ProjectConfig.parse_multi_values( - env_options['monitor_flags']) - if env_options: - for k in ("port", "speed", "rts", "dtr"): - k2 = "monitor_%s" % k - if k == "speed": - k = "baud" - if kwargs[k] is None and k2 in env_options: - kwargs[k] = env_options[k2] - if k != "port": - kwargs[k] = int(kwargs[k]) + for k in ("port", "speed", "rts", "dtr"): + k2 = "monitor_%s" % k + if k == "speed": + k = "baud" + if kwargs[k] is None and k2 in env_options: + kwargs[k] = env_options[k2] + if k != "port": + kwargs[k] = int(kwargs[k]) except exception.NotPlatformIOProject: pass @@ -187,12 +182,12 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches if len(ports) == 1: kwargs['port'] = ports[0]['port'] - sys.argv = ["monitor"] + custom_monitor_flags + sys.argv = ["monitor"] + env_options.get("monitor_flags", []) for k, v in kwargs.items(): if k in ("port", "baud", "rts", "dtr", "environment", "project_dir"): continue k = "--" + k.replace("_", "-") - if k in custom_monitor_flags: + if k in env_options.get("monitor_flags", []): continue if isinstance(v, bool): if v: diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index aee042e2..73bf69a3 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -44,10 +44,8 @@ class ProjectRPC(object): config.validate(validate_options=False) libdeps_dir = get_project_libdeps_dir() - if config.has_section("platformio") and \ - config.has_option("platformio", "lib_extra_dirs"): - data['libExtraDirs'].extend( - config.getlist("platformio", "lib_extra_dirs")) + data['libExtraDirs'].extend( + config.get("platformio", "lib_extra_dirs", [])) for section in config.sections(): if not section.startswith("env:"): @@ -55,9 +53,8 @@ class ProjectRPC(object): data['envLibdepsDirs'].append(join(libdeps_dir, section[4:])) if config.has_option(section, "board"): data['boards'].append(config.get(section, "board")) - if config.has_option(section, "lib_extra_dirs"): - data['libExtraDirs'].extend( - config.getlist(section, "lib_extra_dirs")) + data['libExtraDirs'].extend( + config.get(section, "lib_extra_dirs", [])) # skip non existing folders and resolve full path for key in ("envLibdepsDirs", "libExtraDirs"): @@ -233,10 +230,8 @@ class ProjectRPC(object): try: config = ProjectConfig(join(project_dir, "platformio.ini")) config.validate(validate_options=False) - if config.has_section("platformio") and \ - config.has_option("platformio", "description"): - project_description = config.get( - "platformio", "description") + project_description = config.get("platformio", + "description") except exception.PlatformIOProjectException: continue diff --git a/platformio/commands/init.py b/platformio/commands/init.py index 7232a5ef..35e459bf 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -70,7 +70,6 @@ def cli( project_option, env_prefix, silent): - if not silent: if project_dir == getcwd(): click.secho( diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index ae00902f..63b89950 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -109,9 +109,8 @@ def cli(ctx, **options): continue storage_dir = join(libdeps_dir, env) ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) - if config.has_option("env:" + env, "lib_deps"): - ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][ - storage_dir] = config.getlist("env:" + env, "lib_deps") + ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( + "env:" + env, "lib_deps", []) @cli.command("install", short_help="Install library") @@ -175,8 +174,7 @@ def lib_install( # pylint: disable=too-many-arguments if project_environments and env not in project_environments: continue config.expand_interpolations = False - lib_deps = (config.getlist("env:" + env, "lib_deps") - if config.has_option("env:" + env, "lib_deps") else []) + lib_deps = config.get("env:" + env, "lib_deps", []) for library in libraries: if library in lib_deps: continue diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index 5f41649b..e69fc953 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -107,9 +107,8 @@ def cli( # pylint: disable=redefined-builtin # filter and ignore patterns patterns = dict(filter=list(filter), ignore=list(ignore)) for key in patterns: - if config.has_option(section, "test_%s" % key): - patterns[key].extend( - config.getlist(section, "test_%s" % key)) + patterns[key].extend( + config.get(section, "test_%s" % key, [])) skip_conditions = [ environment and envname not in environment, diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index e5a4e32d..f116d34e 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -92,9 +92,10 @@ class TestProcessorBase(object): self._outputcpp_generated = False def get_transport(self): - transport = self.env_options.get("framework") if self.env_options.get("platform") == "native": transport = "native" + elif "framework" in self.env_options: + transport = self.env_options.get("framework")[0] if "test_transport" in self.env_options: transport = self.env_options['test_transport'] if transport not in TRANSPORT_OPTIONS: diff --git a/platformio/project/config.py b/platformio/project/config.py index d67df78b..a1b3b428 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -21,6 +21,7 @@ from os.path import isfile import click from platformio import exception +from platformio.project.options import ProjectOptions try: import ConfigParser as ConfigParser @@ -39,98 +40,6 @@ CONFIG_HEADER = """;PlatformIO Project Configuration File """ -KNOWN_PLATFORMIO_OPTIONS = [ - "description", - "env_default", - "extra_configs", - - # Dirs - "core_dir", - "globallib_dir", - "platforms_dir", - "packages_dir", - "cache_dir", - "workspace_dir", - "build_dir", - "libdeps_dir", - "lib_dir", - "include_dir", - "src_dir", - "test_dir", - "boards_dir", - "data_dir" -] - -KNOWN_ENV_OPTIONS = [ - # Generic - "platform", - "framework", - "board", - "targets", - - # Build - "build_flags", - "src_build_flags", - "build_unflags", - "src_filter", - - # Upload - "upload_port", - "upload_protocol", - "upload_speed", - "upload_flags", - "upload_resetmethod", - - # Monitor - "monitor_port", - "monitor_speed", - "monitor_rts", - "monitor_dtr", - "monitor_flags", - - # Library - "lib_deps", - "lib_ignore", - "lib_extra_dirs", - "lib_ldf_mode", - "lib_compat_mode", - "lib_archive", - - # Test - "piotest", - "test_filter", - "test_ignore", - "test_port", - "test_speed", - "test_transport", - "test_build_project_src", - - # Debug - "debug_tool", - "debug_init_break", - "debug_init_cmds", - "debug_extra_cmds", - "debug_load_cmd", - "debug_load_mode", - "debug_server", - "debug_port", - "debug_svd_path", - - # Other - "extra_scripts" -] - -RENAMED_OPTIONS = { - "lib_use": "lib_deps", - "lib_force": "lib_deps", - "extra_script": "extra_scripts", - "monitor_baud": "monitor_speed", - "board_mcu": "board_build.mcu", - "board_f_cpu": "board_build.f_cpu", - "board_f_flash": "board_build.f_flash", - "board_flash_mode": "board_build.flash_mode" -} - class ProjectConfig(object): @@ -191,11 +100,7 @@ class ProjectConfig(object): return # load extra configs - if (not self._parser.has_section("platformio") - or not self._parser.has_option("platformio", "extra_configs")): - return - extra_configs = self.getlist("platformio", "extra_configs") - for pattern in extra_configs: + for pattern in self.get("platformio", "extra_configs", []): for item in glob.glob(pattern): self.read(item) @@ -212,6 +117,14 @@ class ProjectConfig(object): if option not in options: options.append(option) + # handle system environment variables + scope = section.split(":", 1)[0] + for option_meta in ProjectOptions.values(): + if option_meta.scope != scope or option_meta.name in options: + continue + if option_meta.sysenvvar and option_meta.sysenvvar in os.environ: + options.append(option_meta.name) + return options def has_option(self, section, option): @@ -239,37 +152,64 @@ class ProjectConfig(object): value = "\n" + value # start from a new line self._parser.set(section, option, value) - def get(self, section, option): + def getraw(self, section, option): if not self.expand_interpolations: return self._parser.get(section, option) try: value = self._parser.get(section, option) - except ConfigParser.NoOptionError: + except ConfigParser.NoOptionError as e: + if not section.startswith("env:"): + raise e value = self._parser.get("env", option) - except ConfigParser.Error as e: - raise exception.InvalidProjectConf(self.path, str(e)) if "${" not in value or "}" not in value: return value - return self.VARTPL_RE.sub(self._re_sub_handler, value) + return self.VARTPL_RE.sub(self._re_interpolation_handler, value) - def _re_sub_handler(self, match): + def _re_interpolation_handler(self, match): section, option = match.group(1), match.group(2) if section == "sysenv": return os.getenv(option) - return self.get(section, option) + return self.getraw(section, option) - def getlist(self, section, option): - return self.parse_multi_values(self.get(section, option)) + def get(self, section, option, default=None): + value = default + try: + value = self.getraw(section, option) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + pass # handle value from system environment + except ConfigParser.Error as e: + raise exception.InvalidProjectConf(self.path, str(e)) + + option_meta = ProjectOptions.get( + "%s.%s" % (section.split(":", 1)[0], option)) + if not option_meta: + return value + + if value and option_meta.multiple: + value = self.parse_multi_values(value) + + if option_meta.sysenvvar: + envvar_value = os.getenv(option_meta.sysenvvar) + if not envvar_value and option_meta.oldnames: + for oldoption in option_meta.oldnames: + envvar_value = os.getenv("PLATFORMIO_" + oldoption.upper()) + if envvar_value: + break + if envvar_value and option_meta.multiple: + value = value or [] + value.extend(self.parse_multi_values(envvar_value)) + elif envvar_value and not value: + value = envvar_value + + return value def envs(self): return [s[4:] for s in self._parser.sections() if s.startswith("env:")] def default_envs(self): - if not self._parser.has_option("platformio", "env_default"): - return [] - return self.getlist("platformio", "env_default") + return self.get("platformio", "env_default", []) def validate(self, envs=None, validate_options=True): if not isfile(self.path): @@ -286,66 +226,54 @@ class ProjectConfig(object): return self.validate_options() if validate_options else True def validate_options(self): - return (self._validate_platformio_options() - and self._validate_env_options()) - - def _validate_platformio_options(self): - if not self._parser.has_section("platformio"): - return True - warnings = set() - - # legacy `lib_extra_dirs` - if self._parser.has_option("platformio", "lib_extra_dirs"): + # legacy `lib_extra_dirs` in [platformio] + if (self._parser.has_section("platformio") + and self._parser.has_option("platformio", "lib_extra_dirs")): if not self._parser.has_section("env"): self._parser.add_section("env") self._parser.set("env", "lib_extra_dirs", self._parser.get("platformio", "lib_extra_dirs")) self._parser.remove_option("platformio", "lib_extra_dirs") - warnings.add( - "`lib_extra_dirs` option is deprecated in section " - "`platformio`! Please move it to global `env` section") + click.secho( + "Warning! `lib_extra_dirs` option is deprecated in section " + "[platformio]! Please move it to global `env` section", + fg="yellow") - unknown = set(k for k, _ in self.items("platformio")) - set( - KNOWN_PLATFORMIO_OPTIONS) - if unknown: - warnings.add( - "Ignore unknown `%s` options in section `[platformio]`" % - ", ".join(unknown)) + return self._validate_unknown_options() - for warning in warnings: - click.secho("Warning! %s" % warning, fg="yellow") - - return True - - def _validate_env_options(self): + def _validate_unknown_options(self): warnings = set() + renamed_options = {} + for option in ProjectOptions.values(): + if option.oldnames: + renamed_options.update( + {name: option.name + for name in option.oldnames}) for section in self._parser.sections(): - if section != "env" and not section.startswith("env:"): - continue for option in self._parser.options(section): # obsolete - if option in RENAMED_OPTIONS: + if option in renamed_options: warnings.add( "`%s` option in section `[%s]` is deprecated and will " "be removed in the next release! Please use `%s` " - "instead" % (option, section, RENAMED_OPTIONS[option])) + "instead" % (option, section, renamed_options[option])) # rename on-the-fly - self._parser.set(section, RENAMED_OPTIONS[option], + self._parser.set(section, renamed_options[option], self._parser.get(section, option)) self._parser.remove_option(section, option) continue # unknown + scope = section.split(":", 1)[0] unknown_conditions = [ - option not in KNOWN_ENV_OPTIONS, - not option.startswith("custom_"), - not option.startswith("board_") + ("%s.%s" % (scope, option)) not in ProjectOptions, + scope != "env" or + not option.startswith(("custom_", "board_")) ] # yapf: disable if all(unknown_conditions): - warnings.add( - "Detected non-PlatformIO `%s` option in `[%s]` section" - % (option, section)) + warnings.add("Ignore unknown option `%s` in section `[%s]`" + % (option, section)) for warning in warnings: click.secho("Warning! %s" % warning, fg="yellow") diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index a2acce4d..2746c4d7 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -15,7 +15,7 @@ import os from hashlib import sha1 from os import walk -from os.path import (abspath, dirname, expanduser, isdir, isfile, join, +from os.path import (dirname, expanduser, isdir, isfile, join, realpath, splitdrive) from platformio import __version__ @@ -44,37 +44,23 @@ def find_project_dir_above(path): def get_project_optional_dir(name, default=None): - paths = None + project_dir = get_project_dir() + config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) + optional_dir = config.get("platformio", name) - # check for system environment variable - var_name = "PLATFORMIO_%s" % name.upper() - if var_name in os.environ: - paths = os.getenv(var_name) - - config = ProjectConfig.get_instance( - join(get_project_dir(), "platformio.ini")) - if (config.has_section("platformio") - and config.has_option("platformio", name)): - paths = config.get("platformio", name) - - if not paths: + if not optional_dir: return default - items = [] - for item in paths.split(", "): - if item.startswith("~"): - item = expanduser(item) - items.append(abspath(item)) - paths = ", ".join(items) - - while "$PROJECT_HASH" in paths: - project_dir = get_project_dir() - paths = paths.replace( + if "$PROJECT_HASH" in optional_dir: + optional_dir = optional_dir.replace( "$PROJECT_HASH", sha1(project_dir if PY2 else project_dir.encode()).hexdigest() [:10]) - return paths + if optional_dir.startswith("~"): + optional_dir = expanduser(optional_dir) + + return realpath(optional_dir) def get_project_core_dir(): diff --git a/platformio/project/options.py b/platformio/project/options.py new file mode 100644 index 00000000..7c68a884 --- /dev/null +++ b/platformio/project/options.py @@ -0,0 +1,199 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=redefined-builtin, too-many-arguments + +from collections import OrderedDict, namedtuple + +ConfigOptionClass = namedtuple("ConfigOption", [ + "scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames" +]) + + +def ConfigOption(scope, + name, + type=str, + multiple=False, + sysenvvar=None, + buildenvvar=None, + oldnames=None): + return ConfigOptionClass(scope, name, type, multiple, sysenvvar, + buildenvvar, oldnames) + + +def ConfigPlatformioOption(*args, **kwargs): + return ConfigOption("platformio", *args, **kwargs) + + +def ConfigEnvOption(*args, **kwargs): + return ConfigOption("env", *args, **kwargs) + + +ProjectOptions = OrderedDict([ + ("%s.%s" % (option.scope, option.name), option) for option in [ + # + # [platformio] + # + ConfigPlatformioOption(name="description"), + ConfigPlatformioOption( + name="env_default", + multiple=True, + sysenvvar="PLATFORMIO_ENV_DEFAULT"), + ConfigPlatformioOption(name="extra_configs", multiple=True), + + # Dirs + ConfigPlatformioOption( + name="core_dir", + oldnames=["home_dir"], + sysenvvar="PLATFORMIO_CORE_DIR"), + ConfigPlatformioOption( + name="globallib_dir", sysenvvar="PLATFORMIO_GLOBALLIB_DIR"), + ConfigPlatformioOption( + name="platforms_dir", sysenvvar="PLATFORMIO_PLATFORMS_DIR"), + ConfigPlatformioOption( + name="packages_dir", sysenvvar="PLATFORMIO_PACKAGES_DIR"), + ConfigPlatformioOption( + name="cache_dir", sysenvvar="PLATFORMIO_CACHE_DIR"), + ConfigPlatformioOption( + name="workspace_dir", sysenvvar="PLATFORMIO_WORKSPACE_DIR"), + ConfigPlatformioOption( + name="build_dir", sysenvvar="PLATFORMIO_BUILD_DIR"), + ConfigPlatformioOption( + name="libdeps_dir", sysenvvar="PLATFORMIO_LIBDEPS_DIR"), + ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"), + ConfigPlatformioOption( + name="include_dir", sysenvvar="PLATFORMIO_INCLUDE_DIR"), + ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"), + ConfigPlatformioOption( + name="test_dir", sysenvvar="PLATFORMIO_TEST_DIR"), + ConfigPlatformioOption( + name="boards_dir", sysenvvar="PLATFORMIO_BOARDS_DIR"), + ConfigPlatformioOption( + name="data_dir", sysenvvar="PLATFORMIO_DATA_DIR"), + + # + # [env] + # + + # Generic + ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"), + ConfigEnvOption( + name="framework", multiple=True, buildenvvar="PIOFRAMEWORK"), + ConfigEnvOption(name="targets", multiple=True), + + # Board + ConfigEnvOption(name="board", buildenvvar="BOARD"), + ConfigEnvOption( + name="board_build.mcu", + oldnames=["board_mcu"], + buildenvvar="BOARD_MCU"), + ConfigEnvOption( + name="board_build.f_cpu", + oldnames=["board_f_cpu"], + buildenvvar="BOARD_F_CPU"), + ConfigEnvOption( + name="board_build.f_flash", + oldnames=["board_f_flash"], + buildenvvar="BOARD_F_FLASH"), + ConfigEnvOption( + name="board_build.flash_mode", + oldnames=["board_flash_mode"], + buildenvvar="BOARD_FLASH_MODE"), + + # Build + ConfigEnvOption( + name="build_flags", + multiple=True, + sysenvvar="PLATFORMIO_BUILD_FLAGS", + buildenvvar="BUILD_FLAGS"), + ConfigEnvOption( + name="src_build_flags", + multiple=True, + sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS", + buildenvvar="SRC_BUILD_FLAGS"), + ConfigEnvOption( + name="build_unflags", + multiple=True, + sysenvvar="PLATFORMIO_BUILD_UNFLAGS", + buildenvvar="BUILD_UNFLAGS"), + ConfigEnvOption( + name="src_filter", + multiple=True, + sysenvvar="PLATFORMIO_SRC_FILTER", + buildenvvar="SRC_FILTER"), + + # Upload + ConfigEnvOption( + name="upload_port", + sysenvvar="PLATFORMIO_UPLOAD_PORT", + buildenvvar="UPLOAD_PORT"), + ConfigEnvOption(name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL"), + ConfigEnvOption(name="upload_speed", buildenvvar="UPLOAD_SPEED"), + ConfigEnvOption( + name="upload_flags", + multiple=True, + sysenvvar="PLATFORMIO_UPLOAD_FLAGS", + buildenvvar="UPLOAD_FLAGS"), + ConfigEnvOption( + name="upload_resetmethod", buildenvvar="UPLOAD_RESETMETHOD"), + + # Monitor + ConfigEnvOption(name="monitor_port"), + ConfigEnvOption( + name="monitor_speed", oldnames=["monitor_baud"], type=int), + ConfigEnvOption(name="monitor_rts"), + ConfigEnvOption(name="monitor_dtr"), + ConfigEnvOption(name="monitor_flags", multiple=True), + + # Library + ConfigEnvOption( + name="lib_deps", + oldnames=["lib_use", "lib_force", "lib_install"], + multiple=True), + ConfigEnvOption(name="lib_ignore", multiple=True), + ConfigEnvOption( + name="lib_extra_dirs", + multiple=True, + sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"), + ConfigEnvOption(name="lib_ldf_mode"), + ConfigEnvOption(name="lib_compat_mode"), + ConfigEnvOption(name="lib_archive", type=bool), # FIXME: B + + # Test + ConfigEnvOption(name="test_filter", multiple=True), + ConfigEnvOption(name="test_ignore", multiple=True), + ConfigEnvOption(name="test_port"), + ConfigEnvOption(name="test_speed"), + ConfigEnvOption(name="test_transport"), + ConfigEnvOption(name="test_build_project_src"), + + # Debug + ConfigEnvOption(name="debug_tool"), + ConfigEnvOption(name="debug_init_break"), + ConfigEnvOption(name="debug_init_cmds", multiple=True), + ConfigEnvOption(name="debug_extra_cmds", multiple=True), + ConfigEnvOption(name="debug_load_cmd"), + ConfigEnvOption(name="debug_load_mode"), + ConfigEnvOption(name="debug_server"), + ConfigEnvOption(name="debug_port"), + ConfigEnvOption(name="debug_svd_path"), + + # Other + ConfigEnvOption( + name="extra_scripts", + oldnames=["extra_script"], + multiple=True, + sysenvvar="PLATFORMIO_EXTRA_SCRIPTS") + ] +]) diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 67ff4420..bce43700 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -20,7 +20,7 @@ from platformio.commands.lib import cli as cmd_lib def test_ci_empty(clirunner): result = clirunner.invoke(cmd_ci) - assert result.exit_code == 2 + assert result.exit_code != 0 assert "Invalid value: Missing argument 'src'" in result.output diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 800a0b47..e8ad5547 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -16,7 +16,7 @@ import json from os import getcwd, makedirs from os.path import getsize, isdir, isfile, join -from platformio import exception, util +from platformio import exception from platformio.commands.boards import cli as cmd_boards from platformio.commands.init import cli as cmd_init from platformio.project.config import ProjectConfig @@ -109,13 +109,12 @@ def test_init_special_board(clirunner, validate_cliresult): config = ProjectConfig(join(getcwd(), "platformio.ini")) config.validate() - expected_result = [("platform", str(boards[0]['platform'])), - ("framework", - str(boards[0]['frameworks'][0])), ("board", "uno")] + expected_result = [("platform", boards[0]['platform']), + ("board", "uno"), + ("framework", [boards[0]['frameworks'][0]])] assert config.has_section("env:uno") - assert not set(expected_result).symmetric_difference( - set(config.items("env:uno"))) + assert config.items("env:uno") == expected_result def test_init_enable_auto_uploading(clirunner, validate_cliresult): @@ -126,11 +125,10 @@ def test_init_enable_auto_uploading(clirunner, validate_cliresult): validate_pioproject(getcwd()) config = ProjectConfig(join(getcwd(), "platformio.ini")) config.validate() - expected_result = [("platform", "atmelavr"), ("framework", "arduino"), - ("board", "uno"), ("targets", "upload")] + expected_result = [("targets", ["upload"]), ("platform", "atmelavr"), + ("board", "uno"), ("framework", ["arduino"])] assert config.has_section("env:uno") - assert not set(expected_result).symmetric_difference( - set(config.items("env:uno"))) + assert config.items("env:uno") == expected_result def test_init_custom_framework(clirunner, validate_cliresult): @@ -141,11 +139,10 @@ def test_init_custom_framework(clirunner, validate_cliresult): validate_pioproject(getcwd()) config = ProjectConfig(join(getcwd(), "platformio.ini")) config.validate() - expected_result = [("platform", "teensy"), ("framework", "mbed"), - ("board", "teensy31")] + expected_result = [("platform", "teensy"), ("board", "teensy31"), + ("framework", ["mbed"])] assert config.has_section("env:teensy31") - assert not set(expected_result).symmetric_difference( - set(config.items("env:teensy31"))) + assert config.items("env:teensy31") == expected_result def test_init_incorrect_board(clirunner): diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index d58ee0ce..5f10b305 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -14,7 +14,9 @@ import os -from platformio.project.config import ProjectConfig +import pytest + +from platformio.project.config import ConfigParser, ProjectConfig BASE_CONFIG = """ [platformio] @@ -26,7 +28,9 @@ extra_configs = # global options per [env:*] [env] monitor_speed = 115200 -lib_deps = Lib1, Lib2 +lib_deps = + Lib1 + Lib2 lib_ignore = ${custom.lib_ignore} [custom] @@ -46,6 +50,7 @@ build_flags = ${custom.lib_flags} ${custom.debug_flags} [env:extra_2] build_flags = ${custom.debug_flags} ${custom.extra_flags} lib_ignore = ${env.lib_ignore}, Lib3 +upload_port = /dev/extra_2/port """ EXTRA_DEBUG_CONFIG = """ @@ -58,7 +63,7 @@ build_flags = -Og """ -def test_parser(tmpdir): +def test_real_config(tmpdir): tmpdir.join("platformio.ini").write(BASE_CONFIG) tmpdir.join("extra_envs.ini").write(EXTRA_ENVS_CONFIG) tmpdir.join("extra_debug.ini").write(EXTRA_DEBUG_CONFIG) @@ -68,6 +73,16 @@ def test_parser(tmpdir): config = ProjectConfig(tmpdir.join("platformio.ini").strpath) assert config + # unknown section + with pytest.raises(ConfigParser.NoSectionError): + config.getraw("unknown_section", "unknown_option") + # unknown option + with pytest.raises(ConfigParser.NoOptionError): + config.getraw("custom", "unknown_option") + # unknown option even if exists in [env] + with pytest.raises(ConfigParser.NoOptionError): + config.getraw("platformio", "monitor_speed") + # sections assert config.sections() == [ "platformio", "env", "custom", "env:base", "env:extra_1", "env:extra_2" @@ -88,29 +103,75 @@ def test_parser(tmpdir): # sysenv assert config.get("custom", "extra_flags") == "" + assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] + assert config.get("env:base", "upload_port") is None + assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" + os.environ["PLATFORMIO_BUILD_FLAGS"] = "-DSYSENVDEPS1 -DSYSENVDEPS2" + os.environ["PLATFORMIO_UPLOAD_PORT"] = "/dev/sysenv/port" os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] = "-L /usr/local/lib" assert config.get("custom", "extra_flags") == "-L /usr/local/lib" + assert config.get("env:base", "build_flags") == [ + "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2" + ] + assert config.get("env:base", "upload_port") == "/dev/sysenv/port" + assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" + + # getraw + assert config.getraw("env:extra_1", "lib_deps") == "\nLib1\nLib2" + assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1" # get assert config.get("custom", "debug_flags") == "-D DEBUG=1" - assert config.get("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1" - assert config.get("env:extra_2", "build_flags") == "-Og" + assert config.get("env:extra_1", "build_flags") == [ + "-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2" + ] + assert config.get("env:extra_2", "build_flags") == [ + "-Og", "-DSYSENVDEPS1 -DSYSENVDEPS2"] assert config.get("env:extra_2", "monitor_speed") == "115200" - assert config.get("env:base", - "build_flags") == ("-D DEBUG=1 -L /usr/local/lib") + assert config.get("env:base", "build_flags") == ([ + "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2" + ]) # items - assert config.items("custom") == [("debug_flags", "-D DEBUG=1"), - ("lib_flags", "-lc -lm"), - ("extra_flags", "-L /usr/local/lib"), - ("lib_ignore", "LibIgnoreCustom")] - assert config.items(env="extra_1") == [("build_flags", - "-lc -lm -D DEBUG=1"), - ("monitor_speed", "115200"), - ("lib_deps", "Lib1, Lib2"), - ("lib_ignore", "LibIgnoreCustom")] - assert config.items(env="extra_2") == [("build_flags", "-Og"), - ("lib_ignore", - "LibIgnoreCustom, Lib3"), - ("monitor_speed", "115200"), - ("lib_deps", "Lib1, Lib2")] + assert config.items("custom") == [ + ("debug_flags", "-D DEBUG=1"), + ("lib_flags", "-lc -lm"), + ("extra_flags", "-L /usr/local/lib"), + ("lib_ignore", "LibIgnoreCustom") + ] # yapf: disable + assert config.items(env="extra_1") == [ + ("build_flags", ["-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), + ("monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["LibIgnoreCustom"]), + ("upload_port", "/dev/sysenv/port") + ] # yapf: disable + assert config.items(env="extra_2") == [ + ("build_flags", ["-Og", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), + ("lib_ignore", ["LibIgnoreCustom", "Lib3"]), + ("upload_port", "/dev/extra_2/port"), + ("monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]) + ] # yapf: disable + + # cleanup system environment variables + del os.environ["PLATFORMIO_BUILD_FLAGS"] + del os.environ["PLATFORMIO_UPLOAD_PORT"] + del os.environ["__PIO_TEST_CNF_EXTRA_FLAGS"] + + +def test_empty_config(): + config = ProjectConfig("/non/existing/platformio.ini") + + # unknown section + with pytest.raises(ConfigParser.NoSectionError): + config.getraw("unknown_section", "unknown_option") + + assert config.sections() == [] + assert config.get("section", "option") is None + assert config.get("section", "option", 13) == 13 + + # sysenv + os.environ["PLATFORMIO_CORE_DIR"] = "/custom/core/dir" + assert config.get("platformio", "core_dir") == "/custom/core/dir" + del os.environ["PLATFORMIO_CORE_DIR"] From d5e277b7cc6ef7cf45aaa9ad707672b6e618e2f4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 16:39:17 +0300 Subject: [PATCH 195/333] Minor improvements to unit testing engine --- platformio/commands/test/command.py | 9 ++++---- platformio/commands/test/embedded.py | 6 ++++-- platformio/commands/test/native.py | 3 ++- platformio/commands/test/processor.py | 30 +++++++++++++++------------ 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index e69fc953..87a84424 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -22,7 +22,7 @@ from time import time import click from platformio import exception, util -from platformio.commands.run import print_header +from platformio.commands.run.helpers import print_header from platformio.commands.test.embedded import EmbeddedTestProcessor from platformio.commands.test.native import NativeTestProcessor from platformio.project.config import ProjectConfig @@ -166,10 +166,11 @@ def cli( # pylint: disable=redefined-builtin passed_nums += 1 status_str = click.style("PASSED", fg="green") + format_str = "test/{:<%d} > {:<%d}\t[{}]" % (testname_max_len, + envname_max_len) click.echo( - ("test/{:<%d} > {:<%d}\t[{}]" % - (testname_max_len, envname_max_len)).format( - testname, click.style(envname, fg="cyan"), status_str), + format_str.format(testname, click.style(envname, fg="cyan"), + status_str), err=status is False) print_header( diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index 39e8555d..449763db 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -32,7 +32,8 @@ class EmbeddedTestProcessor(TestProcessorBase): target = ["__test"] if self.options['without_uploading']: target.append("checkprogsize") - self.build_or_upload(target) + if not self.build_or_upload(target): + return False if not self.options['without_uploading']: self.print_progress("Uploading... (2/3)") @@ -41,7 +42,8 @@ class EmbeddedTestProcessor(TestProcessorBase): target.append("nobuild") else: target.append("__test") - self.build_or_upload(target) + if not self.build_or_upload(target): + return False if self.options['without_testing']: return None diff --git a/platformio/commands/test/native.py b/platformio/commands/test/native.py index 70505e7b..7367094f 100644 --- a/platformio/commands/test/native.py +++ b/platformio/commands/test/native.py @@ -25,7 +25,8 @@ class NativeTestProcessor(TestProcessorBase): def process(self): if not self.options['without_building']: self.print_progress("Building... (1/2)") - self.build_or_upload(["__test"]) + if not self.build_or_upload(["__test"]): + return False if self.options['without_testing']: return None self.print_progress("Testing... (2/2)") diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index f116d34e..7b22f5ee 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -21,7 +21,7 @@ import click from platformio import exception from platformio.commands.run import cli as cmd_run -from platformio.commands.run import print_header +from platformio.commands.run.helpers import print_header from platformio.project.helpers import get_project_test_dir TRANSPORT_OPTIONS = { @@ -82,7 +82,7 @@ class TestProcessorBase(object): def __init__(self, cmd_ctx, testname, envname, options): self.cmd_ctx = cmd_ctx - self.cmd_ctx.meta['piotest_processor'] = True + self.cmd_ctx.meta['piotest_processor'] = True # FIXME self.test_name = testname self.options = options self.env_name = envname @@ -109,8 +109,9 @@ class TestProcessorBase(object): def print_progress(self, text, is_error=False): click.echo() print_header( - "[test/%s] %s" % (click.style( - self.test_name, fg="yellow", bold=True), text), + "[test/%s > %s] %s" % (click.style(self.test_name, fg="yellow"), + click.style(self.env_name, fg="cyan"), + text), is_error=is_error) def build_or_upload(self, target): @@ -119,19 +120,22 @@ class TestProcessorBase(object): self._outputcpp_generated = True if self.test_name != "*": - self.cmd_ctx.meta['piotest'] = self.test_name + self.cmd_ctx.meta['piotest'] = self.test_name # FIXME if not self.options['verbose']: click.echo("Please wait...") - return self.cmd_ctx.invoke( - cmd_run, - project_dir=self.options['project_dir'], - upload_port=self.options['upload_port'], - silent=not self.options['verbose'], - environment=[self.env_name], - disable_auto_clean="nobuild" in target, - target=target) + try: + return self.cmd_ctx.invoke( + cmd_run, + project_dir=self.options['project_dir'], + upload_port=self.options['upload_port'], + silent=not self.options['verbose'], + environment=[self.env_name], + disable_auto_clean="nobuild" in target, + target=target) + except exception.ReturnErrorCode: + return False def process(self): raise NotImplementedError From 0ce23438368e4d9fa323649c0ad7f9027ce26938 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 17:08:00 +0300 Subject: [PATCH 196/333] Do not pass project settings as SCons arguments // Resolve #1637 --- platformio/builder/main.py | 98 ++----- platformio/builder/tools/pioide.py | 5 +- platformio/builder/tools/piolib.py | 26 +- platformio/builder/tools/piomisc.py | 2 +- platformio/builder/tools/pioplatform.py | 53 ++-- platformio/builder/tools/pioproject.py | 49 ++++ platformio/commands/run.py | 352 ------------------------ platformio/commands/run/__init__.py | 15 + platformio/commands/run/command.py | 119 ++++++++ platformio/commands/run/helpers.py | 127 +++++++++ platformio/commands/run/processor.py | 115 ++++++++ platformio/managers/platform.py | 15 +- platformio/project/config.py | 20 +- platformio/project/options.py | 4 +- 14 files changed, 513 insertions(+), 487 deletions(-) create mode 100644 platformio/builder/tools/pioproject.py delete mode 100644 platformio/commands/run.py create mode 100644 platformio/commands/run/__init__.py create mode 100644 platformio/commands/run/command.py create mode 100644 platformio/commands/run/helpers.py create mode 100644 platformio/commands/run/processor.py diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 8358d52e..225af059 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -16,7 +16,7 @@ import base64 import json import sys from os import environ -from os.path import expanduser, join +from os.path import join from time import time from SCons.Script import ARGUMENTS # pylint: disable=import-error @@ -33,76 +33,29 @@ from platformio import util from platformio.compat import PY2, path_to_unicode from platformio.proc import get_pythonexe_path from platformio.project import helpers as project_helpers -from platformio.project.config import ProjectConfig AllowSubstExceptions(NameError) -# allow common variables from INI file -commonvars = Variables(None) -commonvars.AddVariables( +# append CLI arguments to build environment +clivars = Variables(None) +clivars.AddVariables( ("PLATFORM_MANIFEST",), ("BUILD_SCRIPT",), - ("EXTRA_SCRIPTS",), + ("PROJECT_CONFIG",), ("PIOENV",), ("PIOTEST",), - ("PIOPLATFORM",), - ("PIOFRAMEWORK",), - - # build options - ("BUILD_FLAGS",), - ("SRC_BUILD_FLAGS",), - ("BUILD_UNFLAGS",), - ("SRC_FILTER",), - - # library options - ("LIB_LDF_MODE",), - ("LIB_COMPAT_MODE",), - ("LIB_DEPS",), - ("LIB_IGNORE",), - ("LIB_EXTRA_DIRS",), - ("LIB_ARCHIVE",), - - # board options - ("BOARD",), - # deprecated options, use board_{object.path} instead - ("BOARD_MCU",), - ("BOARD_F_CPU",), - ("BOARD_F_FLASH",), - ("BOARD_FLASH_MODE",), - # end of deprecated options - - # upload options - ("UPLOAD_PORT",), - ("UPLOAD_PROTOCOL",), - ("UPLOAD_SPEED",), - ("UPLOAD_FLAGS",), - ("UPLOAD_RESETMETHOD",), - - # test options - ("TEST_BUILD_PROJECT_SRC",), - - # debug options - ("DEBUG_TOOL",), - ("DEBUG_SVD_PATH",), - + ("UPLOAD_PORT",) ) # yapf: disable -MULTILINE_VARS = [ - "EXTRA_SCRIPTS", "PIOFRAMEWORK", "BUILD_FLAGS", "SRC_BUILD_FLAGS", - "BUILD_UNFLAGS", "UPLOAD_FLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE", - "LIB_EXTRA_DIRS" -] - DEFAULT_ENV_OPTIONS = dict( tools=[ "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform", - "piowinhooks", "piolib", "pioupload", "piomisc", "pioide" - ], # yapf: disable + "pioproject", "piowinhooks", "piolib", "pioupload", "piomisc", "pioide" + ], toolpath=[join(util.get_source_dir(), "builder", "tools")], - variables=commonvars, + variables=clivars, # Propagating External Environment - PIOVARIABLES=list(commonvars.keys()), ENV=environ, UNIX_TIME=int(time()), PROJECT_DIR=project_helpers.get_project_dir(), @@ -136,38 +89,21 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)): env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) -# decode common variables -for k in list(commonvars.keys()): - if k in env: - env[k] = base64.b64decode(env[k]) - if isinstance(env[k], bytes): - env[k] = env[k].decode() - if k in MULTILINE_VARS: - env[k] = ProjectConfig.parse_multi_values(env[k]) - if env.GetOption('clean'): env.PioClean(env.subst("$BUILD_DIR")) env.Exit(0) elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): print("Verbose mode can be enabled via `-v, --verbose` option") -# Handle custom variables from system environment -for var in ("BUILD_FLAGS", "SRC_BUILD_FLAGS", "SRC_FILTER", "EXTRA_SCRIPTS", - "UPLOAD_PORT", "UPLOAD_FLAGS", "LIB_EXTRA_DIRS"): - k = "PLATFORMIO_%s" % var - if k not in environ: - continue - if var in ("UPLOAD_PORT", ): - env[var] = environ.get(k) - continue - env.Append(**{var: ProjectConfig.parse_multi_values(environ.get(k))}) +# Load variables from CLI +for key in list(clivars.keys()): + if key in env: + env[key] = base64.b64decode(env[key]) + if isinstance(env[key], bytes): + env[key] = env[key].decode() -env.Prepend(LIBSOURCE_DIRS=env.get("LIB_EXTRA_DIRS", [])) -env['LIBSOURCE_DIRS'] = [ - expanduser(d) if d.startswith("~") else d for d in env['LIBSOURCE_DIRS'] -] - -env.LoadPioPlatform(commonvars) +env.LoadProjectOptions() +env.LoadPioPlatform() env.SConscriptChdir(0) env.SConsignFile( diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 12ebefba..4c42ee88 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -113,7 +113,7 @@ def _dump_defines(env): def _get_svd_path(env): - svd_path = env.subst("$DEBUG_SVD_PATH") + svd_path = env.GetProjectOption("debug_svd_path") if svd_path: return abspath(svd_path) @@ -139,8 +139,7 @@ def DumpIDEData(env, projenv): LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS" data = { - "libsource_dirs": - [env.subst(l) for l in env.get("LIBSOURCE_DIRS", [])], + "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()], "defines": _dump_defines(env), "includes": diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index b63dd0f9..dba7d108 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -23,8 +23,8 @@ import os import re import sys from glob import glob -from os.path import (basename, commonprefix, dirname, isdir, isfile, join, - realpath, sep) +from os.path import (basename, commonprefix, dirname, expanduser, isdir, + isfile, join, realpath, sep) import SCons.Scanner # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error @@ -207,17 +207,18 @@ class LibBuilderBase(object): @property def lib_archive(self): - return self.env.get("LIB_ARCHIVE", "") != "false" + return self.env.GetProjectOption("lib_archive", True) @property def lib_ldf_mode(self): return self.validate_ldf_mode( - self.env.get("LIB_LDF_MODE", self.LDF_MODE_DEFAULT)) + self.env.GetProjectOption("lib_ldf_mode", self.LDF_MODE_DEFAULT)) @property def lib_compat_mode(self): return self.validate_compat_mode( - self.env.get("LIB_COMPAT_MODE", self.COMPAT_MODE_DEFAULT)) + self.env.GetProjectOption("lib_compat_mode", + self.COMPAT_MODE_DEFAULT)) @property def depbuilders(self): @@ -867,7 +868,7 @@ class ProjectAsLibBuilder(LibBuilderBase): pass def process_dependencies(self): # pylint: disable=too-many-branches - uris = self.env.get("LIB_DEPS", []) + uris = self.env.GetProjectOption("lib_deps", []) if not uris: return storage_dirs = [] @@ -907,6 +908,14 @@ class ProjectAsLibBuilder(LibBuilderBase): return result +def GetLibSourceDirs(env): + items = env.GetProjectOption("lib_extra_dirs", []) + items.extend(env['LIBSOURCE_DIRS']) + return [ + expanduser(item) if item.startswith("~") else item for item in items + ] + + def GetLibBuilders(env): # pylint: disable=too-many-branches if "__PIO_LIB_BUILDERS" in DefaultEnvironment(): @@ -920,7 +929,7 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches def _check_lib_builder(lb): compat_mode = lb.lib_compat_mode - if lb.name in env.get("LIB_IGNORE", []): + if lb.name in env.GetProjectOption("lib_ignore", []): if verbose: sys.stderr.write("Ignored library %s\n" % lb.path) return None @@ -939,7 +948,7 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches return True found_incompat = False - for libs_dir in env['LIBSOURCE_DIRS']: + for libs_dir in env.GetLibSourceDirs(): libs_dir = env.subst(libs_dir) if not isdir(libs_dir): continue @@ -1038,6 +1047,7 @@ def exists(_): def generate(env): + env.AddMethod(GetLibSourceDirs) env.AddMethod(GetLibBuilders) env.AddMethod(ConfigureProjectLibBuilder) return env diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index bfc41a76..3d434955 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -322,7 +322,7 @@ def ProcessTest(env): def GetExtraScripts(env, scope): items = [] - for item in env.get("EXTRA_SCRIPTS", []): + for item in env.GetProjectOption("extra_scripts", []): if scope == "post" and ":" not in item: items.append(item) elif item.startswith("%s:" % scope): diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index cc61a709..144f1f4b 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -14,7 +14,6 @@ from __future__ import absolute_import -import base64 import sys from os.path import isdir, isfile, join @@ -23,6 +22,7 @@ from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import exception, util from platformio.compat import WINDOWS from platformio.managers.platform import PlatformFactory +from platformio.project.config import ProjectOptions # pylint: disable=too-many-branches, too-many-locals @@ -33,10 +33,10 @@ def initPioPlatform(name): def PioPlatform(env): - variables = {} - for name in env['PIOVARIABLES']: - if name in env: - variables[name.lower()] = env[name] + variables = env.GetProjectOptions(as_dict=True) + if "framework" in variables: + # support PIO Core 3.0 dev/platforms + variables['pioframework'] = variables['framework'] p = initPioPlatform(env['PLATFORM_MANIFEST']) p.configure_default_packages(variables, COMMAND_LINE_TARGETS) return p @@ -63,7 +63,7 @@ def GetFrameworkScript(env, framework): return script_path -def LoadPioPlatform(env, variables): +def LoadPioPlatform(env): p = env.PioPlatform() installed_packages = p.get_installed_packages() @@ -92,36 +92,25 @@ def LoadPioPlatform(env, variables): env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")]) if "BOARD" not in env: - # handle _MCU and _F_CPU variables for AVR native - for key, value in variables.UnknownVariables().items(): - if not key.startswith("BOARD_"): - continue - value = base64.b64decode(value) - if isinstance(value, bytes): - value = value.decode() - env.Replace(**{key.upper().replace("BUILD.", ""): value}) return - # update board manifest with a custom data + # update board manifest with overridden data from INI config board_config = env.BoardConfig() - for key, value in variables.UnknownVariables().items(): - if not key.startswith("BOARD_"): - continue - value = base64.b64decode(value) - if isinstance(value, bytes): - value = value.decode() - board_config.update(key.lower()[6:], value) + for option, value in env.GetProjectOptions(): + if option.startswith("board_"): + board_config.update(option.lower()[6:], value) - # update default environment variables - for key in list(variables.keys()): - if key in env or \ - not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]): + # load default variables from board config + for option_meta in ProjectOptions.values(): + if not option_meta.buildenvvar or option_meta.buildenvvar in env: continue - _opt, _val = key.lower().split("_", 1) - if _opt == "board": - _opt = "build" - if _val in board_config.get(_opt): - env.Replace(**{key: board_config.get("%s.%s" % (_opt, _val))}) + data_path = (option_meta.name[6:] + if option_meta.name.startswith("board_") else + option_meta.name.replace("_", ".")) + try: + env[option_meta.buildenvvar] = board_config.get(data_path) + except KeyError: + pass if "build.ldscript" in board_config: env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript")) @@ -165,7 +154,7 @@ def PrintConfiguration(env): data = [ "CURRENT(%s)" % board_config.get_debug_tool_name( - env.subst("$DEBUG_TOOL")) + env.GetProjectOption("debug_tool")) ] onboard = [] external = [] diff --git a/platformio/builder/tools/pioproject.py b/platformio/builder/tools/pioproject.py new file mode 100644 index 00000000..5797755d --- /dev/null +++ b/platformio/builder/tools/pioproject.py @@ -0,0 +1,49 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +from platformio.project.config import ProjectConfig, ProjectOptions + + +def GetProjectConfig(env): + return ProjectConfig.get_instance(env['PROJECT_CONFIG']) + + +def GetProjectOptions(env, as_dict=False): + return env.GetProjectConfig().items(env=env['PIOENV'], as_dict=as_dict) + + +def GetProjectOption(env, option, default=None): + return env.GetProjectConfig().get("env:" + env['PIOENV'], option, default) + + +def LoadProjectOptions(env): + for option, value in env.GetProjectOptions(): + option_meta = ProjectOptions.get("env." + option) + if not option_meta or not option_meta.buildenvvar: + continue + env[option_meta.buildenvvar] = value + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(GetProjectConfig) + env.AddMethod(GetProjectOptions) + env.AddMethod(GetProjectOption) + env.AddMethod(LoadProjectOptions) + return env diff --git a/platformio/commands/run.py b/platformio/commands/run.py deleted file mode 100644 index 41fca7e0..00000000 --- a/platformio/commands/run.py +++ /dev/null @@ -1,352 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os import getcwd, makedirs -from os.path import getmtime, isdir, isfile, join -from time import time - -import click - -from platformio import exception, telemetry, util -from platformio.commands.device import device_monitor as cmd_device_monitor -from platformio.commands.lib import (CTX_META_STORAGE_DIRS_KEY, - CTX_META_STORAGE_LIBDEPS_KEY) -from platformio.commands.lib import lib_install as cmd_lib_install -from platformio.commands.platform import \ - platform_install as cmd_platform_install -from platformio.managers.platform import PlatformFactory -from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - calculate_project_hash, find_project_dir_above, get_project_build_dir, - get_project_dir, get_project_libdeps_dir) - -# pylint: disable=too-many-arguments,too-many-locals,too-many-branches - - -@click.command("run", short_help="Process project environments") -@click.option("-e", "--environment", multiple=True) -@click.option("-t", "--target", multiple=True) -@click.option("--upload-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=True, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option( - "-c", - "--project-conf", - type=click.Path( - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) -@click.option("-s", "--silent", is_flag=True) -@click.option("-v", "--verbose", is_flag=True) -@click.option("--disable-auto-clean", is_flag=True) -@click.pass_context -def cli(ctx, environment, target, upload_port, project_dir, project_conf, - silent, verbose, disable_auto_clean): - # find project directory on upper level - if isfile(project_dir): - project_dir = find_project_dir_above(project_dir) - - with util.cd(project_dir): - # clean obsolete build dir - if not disable_auto_clean: - try: - _clean_build_dir(get_project_build_dir()) - except: # pylint: disable=bare-except - click.secho( - "Can not remove temporary directory `%s`. Please remove " - "it manually to avoid build issues" % - get_project_build_dir(force=True), - fg="yellow") - - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini")) - config.validate(environment) - - _handle_legacy_libdeps(project_dir, config) - - results = [] - start_time = time() - default_envs = config.default_envs() - for envname in config.envs(): - skipenv = any([ - environment and envname not in environment, not environment - and default_envs and envname not in default_envs - ]) - if skipenv: - results.append((envname, None)) - continue - - if not silent and any( - status is not None for (_, status) in results): - click.echo() - - options = config.items(env=envname, as_dict=True) - if "piotest" not in options and "piotest" in ctx.meta: - options['piotest'] = ctx.meta['piotest'] - - ep = EnvironmentProcessor(ctx, envname, options, target, - upload_port, silent, verbose) - result = (envname, ep.process()) - results.append(result) - if result[1] and "monitor" in ep.get_build_targets() and \ - "nobuild" not in ep.get_build_targets(): - ctx.invoke( - cmd_device_monitor, - environment=environment[0] if environment else None) - - found_error = any(status is False for (_, status) in results) - - if (found_error or not silent) and len(results) > 1: - click.echo() - print_summary(results, start_time) - - if found_error: - raise exception.ReturnErrorCode(1) - return True - - -class EnvironmentProcessor(object): - - DEFAULT_DUMP_OPTIONS = ("platform", "framework", "board") - - IGNORE_BUILD_OPTIONS = [ - "test_transport", "test_filter", "test_ignore", "test_port", - "test_speed", "debug_port", "debug_init_cmds", "debug_extra_cmds", - "debug_server", "debug_init_break", "debug_load_cmd", - "debug_load_mode", "monitor_port", "monitor_speed", "monitor_rts", - "monitor_dtr" - ] - - REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"} - - def __init__( - self, # pylint: disable=R0913 - cmd_ctx, - name, - options, - targets, - upload_port, - silent, - verbose): - self.cmd_ctx = cmd_ctx - self.name = name - self.options = options - self.targets = targets - self.upload_port = upload_port - self.silent = silent - self.verbose = verbose - - def process(self): - terminal_width, _ = click.get_terminal_size() - start_time = time() - env_dump = [] - - for k, v in self.options.items(): - self.options[k] = self.options[k].strip() - if self.verbose or k in self.DEFAULT_DUMP_OPTIONS: - env_dump.append("%s: %s" % (k, ", ".join( - ProjectConfig.parse_multi_values(v)))) - - if not self.silent: - click.echo("Processing %s (%s)" % (click.style( - self.name, fg="cyan", bold=True), "; ".join(env_dump))) - click.secho("-" * terminal_width, bold=True) - - result = self._run() - is_error = result['returncode'] != 0 - - if self.silent and not is_error: - return True - - if is_error or "piotest_processor" not in self.cmd_ctx.meta: - print_header( - "[%s] Took %.2f seconds" % ( - (click.style("ERROR", fg="red", bold=True) if is_error else - click.style("SUCCESS", fg="green", bold=True)), - time() - start_time), - is_error=is_error) - - return not is_error - - def get_build_variables(self): - variables = {"pioenv": self.name} - if self.upload_port: - variables['upload_port'] = self.upload_port - for k, v in self.options.items(): - if k in self.REMAPED_OPTIONS: - k = self.REMAPED_OPTIONS[k] - if k in self.IGNORE_BUILD_OPTIONS: - continue - if k == "targets" or (k == "upload_port" and self.upload_port): - continue - variables[k] = v - return variables - - def get_build_targets(self): - targets = [] - if self.targets: - targets = [t for t in self.targets] - elif "targets" in self.options: - targets = self.options['targets'].split(", ") - return targets - - def _run(self): - if "platform" not in self.options: - raise exception.UndefinedEnvPlatform(self.name) - - build_vars = self.get_build_variables() - build_targets = self.get_build_targets() - - telemetry.on_run_environment(self.options, build_targets) - - # skip monitor target, we call it above - if "monitor" in build_targets: - build_targets.remove("monitor") - if "nobuild" not in build_targets: - # install dependent libraries - if "lib_install" in self.options: - _autoinstall_libdeps(self.cmd_ctx, self.name, [ - int(d.strip()) - for d in self.options['lib_install'].split(",") - if d.strip() - ], self.verbose) - if "lib_deps" in self.options: - _autoinstall_libdeps( - self.cmd_ctx, self.name, - ProjectConfig.parse_multi_values(self.options['lib_deps']), - self.verbose) - - try: - p = PlatformFactory.newPlatform(self.options['platform']) - except exception.UnknownPlatform: - self.cmd_ctx.invoke( - cmd_platform_install, - platforms=[self.options['platform']], - skip_default_package=True) - p = PlatformFactory.newPlatform(self.options['platform']) - - return p.run(build_vars, build_targets, self.silent, self.verbose) - - -def _handle_legacy_libdeps(project_dir, config): - legacy_libdeps_dir = join(project_dir, ".piolibdeps") - if (not isdir(legacy_libdeps_dir) - or legacy_libdeps_dir == get_project_libdeps_dir()): - return - if not config.has_section("env"): - config.add_section("env") - lib_extra_dirs = [] - if config.has_option("env", "lib_extra_dirs"): - lib_extra_dirs = config.getlist("env", "lib_extra_dirs") - lib_extra_dirs.append(legacy_libdeps_dir) - config.set("env", "lib_extra_dirs", lib_extra_dirs) - click.secho( - "DEPRECATED! A legacy library storage `{0}` has been found in a " - "project. \nPlease declare project dependencies in `platformio.ini`" - " file using `lib_deps` option and remove `{0}` folder." - "\nMore details -> http://docs.platformio.org/page/projectconf/" - "section_env_library.html#lib-deps".format(legacy_libdeps_dir), - fg="yellow") - - -def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): - if not libraries: - return - libdeps_dir = join(get_project_libdeps_dir(), envname) - ctx.meta.update({ - CTX_META_STORAGE_DIRS_KEY: [libdeps_dir], - CTX_META_STORAGE_LIBDEPS_KEY: { - libdeps_dir: libraries - } - }) - try: - ctx.invoke(cmd_lib_install, silent=not verbose) - except exception.InternetIsOffline as e: - click.secho(str(e), fg="yellow") - - -def _clean_build_dir(build_dir): - # remove legacy ".pioenvs" folder - legacy_build_dir = join(get_project_dir(), ".pioenvs") - if isdir(legacy_build_dir) and legacy_build_dir != build_dir: - util.rmtree_(legacy_build_dir) - - structhash_file = join(build_dir, "structure.hash") - proj_hash = calculate_project_hash() - - # if project's config is modified - if (isdir(build_dir) and getmtime( - join(get_project_dir(), "platformio.ini")) > getmtime(build_dir)): - util.rmtree_(build_dir) - - # check project structure - if isdir(build_dir) and isfile(structhash_file): - with open(structhash_file) as f: - if f.read() == proj_hash: - return - util.rmtree_(build_dir) - - if not isdir(build_dir): - makedirs(build_dir) - - with open(structhash_file, "w") as f: - f.write(proj_hash) - - -def print_header(label, is_error=False, fg=None): - terminal_width, _ = click.get_terminal_size() - width = len(click.unstyle(label)) - half_line = "=" * int((terminal_width - width - 2) / 2) - click.secho( - "%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) - - -def print_summary(results, start_time): - print_header("[%s]" % click.style("SUMMARY")) - - envname_max_len = 0 - for (envname, _) in results: - if len(envname) > envname_max_len: - envname_max_len = len(envname) - - successed = True - for (envname, status) in results: - status_str = click.style("SUCCESS", fg="green") - if status is False: - successed = False - status_str = click.style("ERROR", fg="red") - elif status is None: - status_str = click.style("SKIP", fg="yellow") - - format_str = ( - "Environment {0:<" + str(envname_max_len + 9) + "}\t[{1}]") - click.echo( - format_str.format(click.style(envname, fg="cyan"), status_str), - err=status is False) - - print_header( - "[%s] Took %.2f seconds" % ( - (click.style("SUCCESS", fg="green", bold=True) if successed else - click.style("ERROR", fg="red", bold=True)), time() - start_time), - is_error=not successed) diff --git a/platformio/commands/run/__init__.py b/platformio/commands/run/__init__.py new file mode 100644 index 00000000..05d4d370 --- /dev/null +++ b/platformio/commands/run/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.run.command import cli diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py new file mode 100644 index 00000000..1913314b --- /dev/null +++ b/platformio/commands/run/command.py @@ -0,0 +1,119 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import getcwd +from os.path import isfile, join +from time import time + +import click + +from platformio import exception, util +from platformio.commands.device import device_monitor as cmd_device_monitor +from platformio.commands.run.helpers import ( + _clean_build_dir, _handle_legacy_libdeps, print_summary) +from platformio.commands.run.processor import EnvironmentProcessor +from platformio.project.config import ProjectConfig +from platformio.project.helpers import (find_project_dir_above, + get_project_build_dir) + +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches + + +@click.command("run", short_help="Process project environments") +@click.option("-e", "--environment", multiple=True) +@click.option("-t", "--target", multiple=True) +@click.option("--upload-port") +@click.option( + "-d", + "--project-dir", + default=getcwd, + type=click.Path( + exists=True, + file_okay=True, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option( + "-c", + "--project-conf", + type=click.Path( + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) +@click.option("-s", "--silent", is_flag=True) +@click.option("-v", "--verbose", is_flag=True) +@click.option("--disable-auto-clean", is_flag=True) +@click.pass_context +def cli(ctx, environment, target, upload_port, project_dir, project_conf, + silent, verbose, disable_auto_clean): + # find project directory on upper level + if isfile(project_dir): + project_dir = find_project_dir_above(project_dir) + + with util.cd(project_dir): + # clean obsolete build dir + if not disable_auto_clean: + try: + _clean_build_dir(get_project_build_dir()) + except: # pylint: disable=bare-except + click.secho( + "Can not remove temporary directory `%s`. Please remove " + "it manually to avoid build issues" % + get_project_build_dir(force=True), + fg="yellow") + + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(environment) + + _handle_legacy_libdeps(project_dir, config) + + results = [] + start_time = time() + default_envs = config.default_envs() + for envname in config.envs(): + skipenv = any([ + environment and envname not in environment, not environment + and default_envs and envname not in default_envs + ]) + if skipenv: + results.append((envname, None)) + continue + + if not silent and any( + status is not None for (_, status) in results): + click.echo() + + ep = EnvironmentProcessor(ctx, envname, config, target, + upload_port, silent, verbose) + result = (envname, ep.process()) + results.append(result) + + if result[1] and "monitor" in ep.get_build_targets() and \ + "nobuild" not in ep.get_build_targets(): + ctx.invoke( + cmd_device_monitor, + environment=environment[0] if environment else None) + + found_error = any(status is False for (_, status) in results) + + if (found_error or not silent) and len(results) > 1: + click.echo() + print_summary(results, start_time) + + if found_error: + raise exception.ReturnErrorCode(1) + return True diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py new file mode 100644 index 00000000..fd1ac7e3 --- /dev/null +++ b/platformio/commands/run/helpers.py @@ -0,0 +1,127 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import makedirs +from os.path import getmtime, isdir, isfile, join +from time import time + +import click + +from platformio import exception, util +from platformio.commands.lib import (CTX_META_STORAGE_DIRS_KEY, + CTX_META_STORAGE_LIBDEPS_KEY) +from platformio.commands.lib import lib_install as cmd_lib_install +from platformio.project.helpers import ( + calculate_project_hash, get_project_dir, get_project_libdeps_dir) + + +def _handle_legacy_libdeps(project_dir, config): + legacy_libdeps_dir = join(project_dir, ".piolibdeps") + if (not isdir(legacy_libdeps_dir) + or legacy_libdeps_dir == get_project_libdeps_dir()): + return + if not config.has_section("env"): + config.add_section("env") + lib_extra_dirs = config.get("env", "lib_extra_dirs", []) + lib_extra_dirs.append(legacy_libdeps_dir) + config.set("env", "lib_extra_dirs", lib_extra_dirs) + click.secho( + "DEPRECATED! A legacy library storage `{0}` has been found in a " + "project. \nPlease declare project dependencies in `platformio.ini`" + " file using `lib_deps` option and remove `{0}` folder." + "\nMore details -> http://docs.platformio.org/page/projectconf/" + "section_env_library.html#lib-deps".format(legacy_libdeps_dir), + fg="yellow") + + +def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): + if not libraries: + return + libdeps_dir = join(get_project_libdeps_dir(), envname) + ctx.meta.update({ + CTX_META_STORAGE_DIRS_KEY: [libdeps_dir], + CTX_META_STORAGE_LIBDEPS_KEY: { + libdeps_dir: libraries + } + }) + try: + ctx.invoke(cmd_lib_install, silent=not verbose) + except exception.InternetIsOffline as e: + click.secho(str(e), fg="yellow") + + +def _clean_build_dir(build_dir): + # remove legacy ".pioenvs" folder + legacy_build_dir = join(get_project_dir(), ".pioenvs") + if isdir(legacy_build_dir) and legacy_build_dir != build_dir: + util.rmtree_(legacy_build_dir) + + structhash_file = join(build_dir, "structure.hash") + proj_hash = calculate_project_hash() + + # if project's config is modified + if (isdir(build_dir) and getmtime( + join(get_project_dir(), "platformio.ini")) > getmtime(build_dir)): + util.rmtree_(build_dir) + + # check project structure + if isdir(build_dir) and isfile(structhash_file): + with open(structhash_file) as f: + if f.read() == proj_hash: + return + util.rmtree_(build_dir) + + if not isdir(build_dir): + makedirs(build_dir) + + with open(structhash_file, "w") as f: + f.write(proj_hash) + + +def print_header(label, is_error=False, fg=None): + terminal_width, _ = click.get_terminal_size() + width = len(click.unstyle(label)) + half_line = "=" * int((terminal_width - width - 2) / 2) + click.secho( + "%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) + + +def print_summary(results, start_time): + print_header("[%s]" % click.style("SUMMARY")) + + succeeded_nums = 0 + failed_nums = 0 + envname_max_len = max( + [len(click.style(envname, fg="cyan")) for (envname, _) in results]) + for (envname, status) in results: + if status is False: + failed_nums += 1 + status_str = click.style("FAILED", fg="red") + elif status is None: + status_str = click.style("IGNORED", fg="yellow") + else: + succeeded_nums += 1 + status_str = click.style("SUCCESS", fg="green") + + format_str = "Environment {0:<%d}\t[{1}]" % envname_max_len + click.echo( + format_str.format(click.style(envname, fg="cyan"), status_str), + err=status is False) + + print_header( + "%s%d succeeded in %.2f seconds" % + ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums, + time() - start_time), + is_error=failed_nums, + fg="red" if failed_nums else "green") diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py new file mode 100644 index 00000000..6d57f257 --- /dev/null +++ b/platformio/commands/run/processor.py @@ -0,0 +1,115 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from time import time + +import click + +from platformio import exception, telemetry +from platformio.commands.platform import \ + platform_install as cmd_platform_install +from platformio.commands.run.helpers import _autoinstall_libdeps, print_header +from platformio.managers.platform import PlatformFactory + +# pylint: disable=too-many-instance-attributes + + +class EnvironmentProcessor(object): + + DEFAULT_PRINT_OPTIONS = ("platform", "framework", "board") + + def __init__( # pylint: disable=too-many-arguments + self, cmd_ctx, name, config, targets, upload_port, silent, + verbose): + self.cmd_ctx = cmd_ctx + self.name = name + self.config = config + self.targets = targets + self.upload_port = upload_port + self.silent = silent + self.verbose = verbose + self.options = config.items(env=name, as_dict=True) + + def process(self): + terminal_width, _ = click.get_terminal_size() + start_time = time() + env_dump = [] + + for k, v in self.options.items(): + if self.verbose or k in self.DEFAULT_PRINT_OPTIONS: + env_dump.append( + "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v)) + + if not self.silent: + click.echo("Processing %s (%s)" % (click.style( + self.name, fg="cyan", bold=True), "; ".join(env_dump))) + click.secho("-" * terminal_width, bold=True) + + result = self._run_platform() + is_error = result['returncode'] != 0 + + if self.silent and not is_error: + return True + + if is_error or "piotest_processor" not in self.cmd_ctx.meta: + print_header( + "[%s] Took %.2f seconds" % ( + (click.style("ERROR", fg="red", bold=True) if is_error else + click.style("SUCCESS", fg="green", bold=True)), + time() - start_time), + is_error=is_error) + + return not is_error + + def get_build_variables(self): + variables = {"pioenv": self.name, "project_config": self.config.path} + if "piotest" in self.cmd_ctx.meta: + variables['piotest'] = self.cmd_ctx.meta['piotest'] + if self.upload_port: + # override upload port with a custom from CLI + variables['upload_port'] = self.upload_port + return variables + + def get_build_targets(self): + if self.targets: + return [t for t in self.targets] + return self.config.get("env:" + self.name, "targets", []) + + def _run_platform(self): + if "platform" not in self.options: + raise exception.UndefinedEnvPlatform(self.name) + + build_vars = self.get_build_variables() + build_targets = self.get_build_targets() + + telemetry.on_run_environment(self.options, build_targets) + + # skip monitor target, we call it above + if "monitor" in build_targets: + build_targets.remove("monitor") + if "nobuild" not in build_targets and "lib_deps" in self.options: + _autoinstall_libdeps( + self.cmd_ctx, self.name, + self.config.get("env:" + self.name, "lib_deps"), self.verbose) + + try: + p = PlatformFactory.newPlatform(self.options['platform']) + except exception.UnknownPlatform: + self.cmd_ctx.invoke( + cmd_platform_install, + platforms=[self.options['platform']], + skip_default_package=True) + p = PlatformFactory.newPlatform(self.options['platform']) + + return p.run(build_vars, build_targets, self.silent, self.verbose) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8a5de2ef..81982219 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -29,6 +29,7 @@ from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, exec_command, get_pythonexe_path) +from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_project_boards_dir, get_project_core_dir, get_project_packages_dir, get_project_platforms_dir) @@ -358,7 +359,12 @@ class PlatformRunMixin(object): assert isinstance(variables, dict) assert isinstance(targets, list) - self.configure_default_packages(variables, targets) + config = ProjectConfig.get_instance(variables['project_config']) + options = config.items(env=variables['pioenv'], as_dict=True) + if "framework" in options: + # support PIO Core 3.0 dev/platforms + options['pioframework'] = options['framework'] + self.configure_default_packages(options, targets) self.install_packages(silent=True) self.silent = silent @@ -611,12 +617,9 @@ class PlatformBase( # pylint: disable=too-many-public-methods def get_package_type(self, name): return self.packages[name].get("type") - def configure_default_packages(self, variables, targets): + def configure_default_packages(self, options, targets): # enable used frameworks - frameworks = variables.get("pioframework", []) - if not isinstance(frameworks, list): - frameworks = frameworks.split(", ") - for framework in frameworks: + for framework in options.get("framework", []): if not self.frameworks: continue framework = framework.lower().strip() diff --git a/platformio/project/config.py b/platformio/project/config.py index a1b3b428..decbab72 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -174,7 +174,7 @@ class ProjectConfig(object): return self.getraw(section, option) def get(self, section, option, default=None): - value = default + value = None try: value = self.getraw(section, option) except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): @@ -185,7 +185,7 @@ class ProjectConfig(object): option_meta = ProjectOptions.get( "%s.%s" % (section.split(":", 1)[0], option)) if not option_meta: - return value + return default if value and option_meta.multiple: value = self.parse_multi_values(value) @@ -203,6 +203,22 @@ class ProjectConfig(object): elif envvar_value and not value: value = envvar_value + # option is not specified by user + if value is None: + return default + + # cast types + if not isinstance(value, (list, tuple)): + value = [value] + for i, v in enumerate(value): + if option_meta.type == bool: + value[i] = v in ("1", "true", "yes") + elif option_meta.type == int: + value[i] = int(v) + elif option_meta.type == float: + value[i] = float(v) + value = value if option_meta.multiple else value[0] + return value def envs(self): diff --git a/platformio/project/options.py b/platformio/project/options.py index 7c68a884..a5dd5615 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -151,7 +151,7 @@ ProjectOptions = OrderedDict([ # Monitor ConfigEnvOption(name="monitor_port"), ConfigEnvOption( - name="monitor_speed", oldnames=["monitor_baud"], type=int), + name="monitor_speed", oldnames=["monitor_baud"]), ConfigEnvOption(name="monitor_rts"), ConfigEnvOption(name="monitor_dtr"), ConfigEnvOption(name="monitor_flags", multiple=True), @@ -168,7 +168,7 @@ ProjectOptions = OrderedDict([ sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"), ConfigEnvOption(name="lib_ldf_mode"), ConfigEnvOption(name="lib_compat_mode"), - ConfigEnvOption(name="lib_archive", type=bool), # FIXME: B + ConfigEnvOption(name="lib_archive", type=bool), # Test ConfigEnvOption(name="test_filter", multiple=True), From 346579b93c16448eb50e24be2e49374c870d0f82 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 17:34:44 +0300 Subject: [PATCH 197/333] Improve type converting for config options --- platformio/project/config.py | 28 +++++++++++++++------------- platformio/project/options.py | 3 +-- tests/test_projectconf.py | 2 +- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index decbab72..181a6944 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -185,7 +185,7 @@ class ProjectConfig(object): option_meta = ProjectOptions.get( "%s.%s" % (section.split(":", 1)[0], option)) if not option_meta: - return default + return value or default if value and option_meta.multiple: value = self.parse_multi_values(value) @@ -207,19 +207,21 @@ class ProjectConfig(object): if value is None: return default - # cast types - if not isinstance(value, (list, tuple)): - value = [value] - for i, v in enumerate(value): - if option_meta.type == bool: - value[i] = v in ("1", "true", "yes") - elif option_meta.type == int: - value[i] = int(v) - elif option_meta.type == float: - value[i] = float(v) - value = value if option_meta.multiple else value[0] + return self._covert_value(value, option_meta.type) - return value + @staticmethod + def _covert_value(value, to_type): + items = value + if not isinstance(value, (list, tuple)): + items = [value] + for i, v in enumerate(items): + if to_type == bool: + items[i] = v in ("1", "true", "yes", "y") + elif to_type == int: + items[i] = int(v) + elif to_type == float: + items[i] = float(v) + return items if isinstance(value, (list, tuple)) else items[0] def envs(self): return [s[4:] for s in self._parser.sections() if s.startswith("env:")] diff --git a/platformio/project/options.py b/platformio/project/options.py index a5dd5615..b4bd705a 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -150,8 +150,7 @@ ProjectOptions = OrderedDict([ # Monitor ConfigEnvOption(name="monitor_port"), - ConfigEnvOption( - name="monitor_speed", oldnames=["monitor_baud"]), + ConfigEnvOption(name="monitor_speed", oldnames=["monitor_baud"]), ConfigEnvOption(name="monitor_rts"), ConfigEnvOption(name="monitor_dtr"), ConfigEnvOption(name="monitor_flags", multiple=True), diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 5f10b305..69b6308b 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -102,7 +102,7 @@ def test_real_config(tmpdir): assert not config.has_option("custom", "monitor_speed") # sysenv - assert config.get("custom", "extra_flags") == "" + assert config.get("custom", "extra_flags") is None assert config.get("env:base", "build_flags") == ["-D DEBUG=1"] assert config.get("env:base", "upload_port") is None assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" From 19c15749930525be213956635a6ca07121fecddf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 17:37:48 +0300 Subject: [PATCH 198/333] Use the latest TOX for Travis.CI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 07226606..5d9c8e3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: install: - git submodule update --init --recursive - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl -fsSL https://bootstrap.pypa.io/get-pip.py | sudo python; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox==3.0.0"; else pip install -U tox; fi + - pip install -U tox # ChipKIT issue: install 32-bit support for GCC PIC32 - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi From 1dbaed5beb22066c6cc15a16f6fd2fba3f86e7d5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 21:26:51 +0300 Subject: [PATCH 199/333] Implement "silent" mode for config.validate() --- platformio/builder/main.py | 1 + .../commands/home/rpc/handlers/project.py | 4 +- platformio/project/config.py | 43 ++++++++++--------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 225af059..e25d4af3 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -102,6 +102,7 @@ for key in list(clivars.keys()): if isinstance(env[key], bytes): env[key] = env[key].decode() +env.GetProjectConfig().validate([env['PIOENV']], silent=True) env.LoadProjectOptions() env.LoadPioPlatform() diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 73bf69a3..2b396094 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -41,7 +41,7 @@ class ProjectRPC(object): def _get_project_data(project_dir): data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} config = ProjectConfig(join(project_dir, "platformio.ini")) - config.validate(validate_options=False) + config.validate(silent=True) libdeps_dir = get_project_libdeps_dir() data['libExtraDirs'].extend( @@ -229,7 +229,7 @@ class ProjectRPC(object): project_description = None try: config = ProjectConfig(join(project_dir, "platformio.ini")) - config.validate(validate_options=False) + config.validate(silent=True) project_description = config.get("platformio", "description") except exception.PlatformIOProjectException: diff --git a/platformio/project/config.py b/platformio/project/config.py index 181a6944..45e9380c 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -229,7 +229,7 @@ class ProjectConfig(object): def default_envs(self): return self.get("platformio", "env_default", []) - def validate(self, envs=None, validate_options=True): + def validate(self, envs=None, silent=False): if not isfile(self.path): raise exception.NotPlatformIOProject(self.path) # check envs @@ -241,9 +241,10 @@ class ProjectConfig(object): if unknown: raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) - return self.validate_options() if validate_options else True + return self.validate_options(silent) - def validate_options(self): + def validate_options(self, silent=False): + warnings = [] # legacy `lib_extra_dirs` in [platformio] if (self._parser.has_section("platformio") and self._parser.has_option("platformio", "lib_extra_dirs")): @@ -252,15 +253,20 @@ class ProjectConfig(object): self._parser.set("env", "lib_extra_dirs", self._parser.get("platformio", "lib_extra_dirs")) self._parser.remove_option("platformio", "lib_extra_dirs") - click.secho( - "Warning! `lib_extra_dirs` option is deprecated in section " - "[platformio]! Please move it to global `env` section", - fg="yellow") + warnings.append( + "`lib_extra_dirs` configuration option is deprecated in " + "section [platformio]! Please move it to global `env` section") - return self._validate_unknown_options() + warnings.extend(self._validate_unknown_options()) + + if not silent: + for warning in warnings: + click.secho("Warning! %s" % warning, fg="yellow") + + return warnings def _validate_unknown_options(self): - warnings = set() + warnings = [] renamed_options = {} for option in ProjectOptions.values(): if option.oldnames: @@ -272,10 +278,11 @@ class ProjectConfig(object): for option in self._parser.options(section): # obsolete if option in renamed_options: - warnings.add( - "`%s` option in section `[%s]` is deprecated and will " - "be removed in the next release! Please use `%s` " - "instead" % (option, section, renamed_options[option])) + warnings.append( + "`%s` configuration option in section [%s] is " + "deprecated and will be removed in the next release! " + "Please use `%s` instead" % (option, section, + renamed_options[option])) # rename on-the-fly self._parser.set(section, renamed_options[option], self._parser.get(section, option)) @@ -290,13 +297,9 @@ class ProjectConfig(object): not option.startswith(("custom_", "board_")) ] # yapf: disable if all(unknown_conditions): - warnings.add("Ignore unknown option `%s` in section `[%s]`" - % (option, section)) - - for warning in warnings: - click.secho("Warning! %s" % warning, fg="yellow") - - return True + warnings.append("Ignore unknown configuration option `%s` " + "in section [%s]" % (option, section)) + return warnings def to_json(self): result = {} From f3c8277572c1c705b7682bfcb1b21de0daec6aea Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 21:27:12 +0300 Subject: [PATCH 200/333] Fix broken tests --- platformio/commands/device.py | 1 + platformio/commands/run/__init__.py | 1 + tests/commands/test_init.py | 27 +++++++++++++++++---------- tests/test_projectconf.py | 4 ++-- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 594020eb..992ad9b2 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -163,6 +163,7 @@ def device_list( # pylint: disable=too-many-branches "--environment", help="Load configuration from `platformio.ini` and specified environment") def device_monitor(**kwargs): # pylint: disable=too-many-branches + env_options = {} try: env_options = get_project_options(kwargs['project_dir'], kwargs['environment']) diff --git a/platformio/commands/run/__init__.py b/platformio/commands/run/__init__.py index 05d4d370..b9e69521 100644 --- a/platformio/commands/run/__init__.py +++ b/platformio/commands/run/__init__.py @@ -13,3 +13,4 @@ # limitations under the License. from platformio.commands.run.command import cli +from platformio.commands.run.helpers import print_header diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index e8ad5547..7c2e4a48 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -110,11 +110,12 @@ def test_init_special_board(clirunner, validate_cliresult): config = ProjectConfig(join(getcwd(), "platformio.ini")) config.validate() - expected_result = [("platform", boards[0]['platform']), - ("board", "uno"), - ("framework", [boards[0]['frameworks'][0]])] + expected_result = dict(platform=str(boards[0]['platform']), + board="uno", + framework=[str(boards[0]['frameworks'][0])]) assert config.has_section("env:uno") - assert config.items("env:uno") == expected_result + assert sorted(config.items(env="uno", as_dict=True).items()) == sorted( + expected_result.items()) def test_init_enable_auto_uploading(clirunner, validate_cliresult): @@ -125,10 +126,13 @@ def test_init_enable_auto_uploading(clirunner, validate_cliresult): validate_pioproject(getcwd()) config = ProjectConfig(join(getcwd(), "platformio.ini")) config.validate() - expected_result = [("targets", ["upload"]), ("platform", "atmelavr"), - ("board", "uno"), ("framework", ["arduino"])] + expected_result = dict(targets=["upload"], + platform="atmelavr", + board="uno", + framework=["arduino"]) assert config.has_section("env:uno") - assert config.items("env:uno") == expected_result + assert sorted(config.items(env="uno", as_dict=True).items()) == sorted( + expected_result.items()) def test_init_custom_framework(clirunner, validate_cliresult): @@ -139,10 +143,13 @@ def test_init_custom_framework(clirunner, validate_cliresult): validate_pioproject(getcwd()) config = ProjectConfig(join(getcwd(), "platformio.ini")) config.validate() - expected_result = [("platform", "teensy"), ("board", "teensy31"), - ("framework", ["mbed"])] + expected_result = dict(platform="teensy", + board="teensy31", + framework=["mbed"]) assert config.has_section("env:teensy31") - assert config.items("env:teensy31") == expected_result + assert sorted(config.items(env="teensy31", + as_dict=True).items()) == sorted( + expected_result.items()) def test_init_incorrect_board(clirunner): diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 69b6308b..8717e8da 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -172,6 +172,6 @@ def test_empty_config(): assert config.get("section", "option", 13) == 13 # sysenv - os.environ["PLATFORMIO_CORE_DIR"] = "/custom/core/dir" + os.environ["PLATFORMIO_HOME_DIR"] = "/custom/core/dir" assert config.get("platformio", "core_dir") == "/custom/core/dir" - del os.environ["PLATFORMIO_CORE_DIR"] + del os.environ["PLATFORMIO_HOME_DIR"] From 0570fc6c4889f0f9e4002078861d39f27e145c3a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 21:56:55 +0300 Subject: [PATCH 201/333] Don't override custom "core_dir" on Windows platform --- platformio/project/helpers.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 2746c4d7..6106b0a1 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -64,12 +64,11 @@ def get_project_optional_dir(name, default=None): def get_project_core_dir(): + default = join(expanduser("~"), ".platformio") core_dir = get_project_optional_dir( - "core_dir", - get_project_optional_dir("home_dir", - join(expanduser("~"), ".platformio"))) + "core_dir", get_project_optional_dir("home_dir", default)) win_core_dir = None - if WINDOWS: + if WINDOWS and core_dir == default: win_core_dir = splitdrive(core_dir)[0] + "\\.platformio" if isdir(win_core_dir): core_dir = win_core_dir @@ -77,10 +76,12 @@ def get_project_core_dir(): if not isdir(core_dir): try: os.makedirs(core_dir) - except: # pylint: disable=bare-except + except OSError as e: if win_core_dir: os.makedirs(win_core_dir) core_dir = win_core_dir + else: + raise e assert isdir(core_dir) return core_dir From 5c60d922ca4442789715c9f44d7e6fbba5817559 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 22:15:06 +0300 Subject: [PATCH 202/333] Skip "arduino-mock" from CI --- examples | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples b/examples index 45ec933e..89963fed 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 45ec933e2858ae117c31d7783d66b83d96a6e609 +Subproject commit 89963fed9ebd0e7123d58dad5f7fc5020769ca63 From 01db26f2048f2357bbb84d260d3ce583e2d477c5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 22:26:42 +0300 Subject: [PATCH 203/333] Add a link to PIO Unified Debugger options --- platformio/commands/debug/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 460682d9..4a950a7a 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -151,7 +151,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes ] # yapf: disable banner = [ - "echo PlatformIO Unified Debugger > http://bit.ly/pio-debug\\n", + "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n", "echo PlatformIO: Initializing remote target...\\n" ] footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER] @@ -228,6 +228,8 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes self.console_log( "PlatformIO: Resume the execution to `debug_init_break = %s`" % self.debug_options['init_break']) + self.console_log("PlatformIO: More configuration options -> " + "http://bit.ly/pio-debug") self.transport.write(b"0-exec-continue\n" if helpers. is_mi_mode(self.args) else b"continue\n") self._target_is_run = True From 1598c8197e4ef7612e1865857d091e7a4a26197b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 23:33:57 +0300 Subject: [PATCH 204/333] Fix "clean" target --- platformio/builder/main.py | 12 ++++++------ platformio/commands/debug/process.py | 5 ++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e25d4af3..a6d71fb6 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -89,12 +89,6 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)): env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) -if env.GetOption('clean'): - env.PioClean(env.subst("$BUILD_DIR")) - env.Exit(0) -elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): - print("Verbose mode can be enabled via `-v, --verbose` option") - # Load variables from CLI for key in list(clivars.keys()): if key in env: @@ -102,6 +96,12 @@ for key in list(clivars.keys()): if isinstance(env[key], bytes): env[key] = env[key].decode() +if env.GetOption('clean'): + env.PioClean(env.subst("$BUILD_DIR")) + env.Exit(0) +elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): + print("Verbose mode can be enabled via `-v, --verbose` option") + env.GetProjectConfig().validate([env['PIOENV']], silent=True) env.LoadProjectOptions() env.LoadPioPlatform() diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index 68d76d00..a9064492 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - import click from twisted.internet import protocol # pylint: disable=import-error from platformio.commands.debug import helpers from platformio.compat import string_types +from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_core_dir LOG_FILE = None @@ -31,7 +30,7 @@ class BaseProcess(protocol.ProcessProtocol, object): COMMON_PATTERNS = { "PLATFORMIO_HOME_DIR": helpers.escape_path(get_project_core_dir()), "PLATFORMIO_CORE_DIR": helpers.escape_path(get_project_core_dir()), - "PYTHONEXE": os.getenv("PYTHONEXEPATH", "") + "PYTHONEXE": get_pythonexe_path() } def apply_patterns(self, source, patterns=None): From 61db0f1d6a89a621281b12e1a913635905d0e035 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 30 May 2019 23:42:15 +0300 Subject: [PATCH 205/333] YAPF 0.27.0 --- platformio/__main__.py | 14 +- platformio/builder/tools/pioide.py | 5 +- platformio/builder/tools/piolib.py | 68 +++---- platformio/builder/tools/piomisc.py | 17 +- platformio/builder/tools/pioplatform.py | 8 +- platformio/builder/tools/pioupload.py | 15 +- platformio/builder/tools/platformio.py | 4 +- platformio/commands/boards.py | 26 ++- platformio/commands/ci.py | 50 +++--- platformio/commands/debug/client.py | 15 +- platformio/commands/debug/command.py | 41 ++--- platformio/commands/debug/helpers.py | 38 ++-- platformio/commands/device.py | 113 ++++++------ platformio/commands/home/helpers.py | 5 +- platformio/commands/home/rpc/handlers/misc.py | 23 +-- platformio/commands/home/rpc/handlers/os.py | 6 +- .../commands/home/rpc/handlers/project.py | 5 +- platformio/commands/home/rpc/server.py | 4 +- platformio/commands/init.py | 71 ++++---- platformio/commands/lib.py | 160 ++++++++--------- platformio/commands/platform.py | 146 ++++++++------- platformio/commands/remote.py | 167 +++++++++--------- platformio/commands/run/command.py | 48 +++-- platformio/commands/run/helpers.py | 31 ++-- platformio/commands/run/processor.py | 15 +- platformio/commands/settings.py | 16 +- platformio/commands/test/command.py | 115 ++++++------ platformio/commands/test/embedded.py | 4 +- platformio/commands/test/processor.py | 32 ++-- platformio/commands/update.py | 25 ++- platformio/commands/upgrade.py | 26 ++- platformio/downloader.py | 9 +- platformio/ide/projectgenerator.py | 10 +- platformio/maintenance.py | 95 +++++----- platformio/managers/core.py | 4 +- platformio/managers/lib.py | 114 ++++++------ platformio/managers/package.py | 80 ++++----- platformio/managers/platform.py | 46 ++--- platformio/project/config.py | 8 +- platformio/project/options.py | 165 ++++++++--------- platformio/telemetry.py | 9 +- platformio/unpacker.py | 5 +- platformio/util.py | 49 +++-- platformio/vcsclient.py | 13 +- 44 files changed, 920 insertions(+), 1000 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index 0dc0cca1..f8358867 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -23,15 +23,13 @@ from platformio.commands import PlatformioCLI from platformio.compat import CYGWIN -@click.command( - cls=PlatformioCLI, - context_settings=dict(help_option_names=["-h", "--help"])) +@click.command(cls=PlatformioCLI, + context_settings=dict(help_option_names=["-h", "--help"])) @click.version_option(__version__, prog_name="PlatformIO") -@click.option( - "--force", - "-f", - is_flag=True, - help="Force to accept any confirmation prompts.") +@click.option("--force", + "-f", + is_flag=True, + help="Force to accept any confirmation prompts.") @click.option("--caller", "-c", help="Caller ID (service).") @click.pass_context def cli(ctx, force, caller): diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 4c42ee88..1814f1b9 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -72,8 +72,9 @@ def _get_gcc_defines(env): try: sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) - result = exec_command( - "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True) + result = exec_command("echo | %s -dM -E -" % env.subst("$CC"), + env=sysenv, + shell=True) except OSError: return items if result['returncode'] != 0: diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index dba7d108..5b2c2e2b 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -54,8 +54,9 @@ class LibBuilderFactory(object): elif used_frameworks: clsname = "%sLibBuilder" % used_frameworks[0].title() - obj = getattr(sys.modules[__name__], clsname)( - env, path, verbose=verbose) + obj = getattr(sys.modules[__name__], clsname)(env, + path, + verbose=verbose) assert isinstance(obj, LibBuilderBase) return obj @@ -69,8 +70,8 @@ class LibBuilderFactory(object): if isfile(join(path, "module.json")): return ["mbed"] - include_re = re.compile( - r'^#include\s+(<|")(Arduino|mbed)\.h(<|")', flags=re.MULTILINE) + include_re = re.compile(r'^#include\s+(<|")(Arduino|mbed)\.h(<|")', + flags=re.MULTILINE) # check source files for root, _, files in os.walk(path, followlinks=True): @@ -270,12 +271,11 @@ class LibBuilderBase(object): self.env.ProcessFlags(self.build_flags) if self.extra_script: self.env.SConscriptChdir(1) - self.env.SConscript( - realpath(self.extra_script), - exports={ - "env": self.env, - "pio_lib_builder": self - }) + self.env.SConscript(realpath(self.extra_script), + exports={ + "env": self.env, + "pio_lib_builder": self + }) self.env.ProcessUnFlags(self.build_unflags) def process_dependencies(self): @@ -290,8 +290,9 @@ class LibBuilderBase(object): if (key in item and not util.items_in_list(self.env[env_key], item[key])): if self.verbose: - sys.stderr.write("Skip %s incompatible dependency %s\n" - % (key[:-1], item)) + sys.stderr.write( + "Skip %s incompatible dependency %s\n" % + (key[:-1], item)) skip = True if skip: continue @@ -404,9 +405,9 @@ class LibBuilderBase(object): if self != lb: if _already_depends(lb): if self.verbose: - sys.stderr.write( - "Warning! Circular dependencies detected " - "between `%s` and `%s`\n" % (self.path, lb.path)) + sys.stderr.write("Warning! Circular dependencies detected " + "between `%s` and `%s`\n" % + (self.path, lb.path)) self._circular_deps.append(lb) elif lb not in self._depbuilders: self._depbuilders.append(lb) @@ -648,8 +649,8 @@ class MbedLibBuilder(LibBuilderBase): for key, options in manifest.get("config", {}).items(): if "value" not in options: continue - macros[key] = dict( - name=options.get("macro_name"), value=options.get("value")) + macros[key] = dict(name=options.get("macro_name"), + value=options.get("value")) # overrode items per target for target, options in manifest.get("target_overrides", {}).items(): @@ -669,8 +670,10 @@ class MbedLibBuilder(LibBuilderBase): if "." not in macro['name']: macro['name'] = "%s.%s" % (manifest.get("name"), macro['name']) - macro['name'] = re.sub( - r"[^a-z\d]+", "_", macro['name'], flags=re.I).upper() + macro['name'] = re.sub(r"[^a-z\d]+", + "_", + macro['name'], + flags=re.I).upper() macro['name'] = "MBED_CONF_" + macro['name'] if isinstance(macro['value'], bool): macro['value'] = 1 if macro['value'] else 0 @@ -686,8 +689,8 @@ class MbedLibBuilder(LibBuilderBase): lines.append( "// PlatformIO Library Dependency Finder (LDF)") lines.extend([ - "#define %s %s" % (name, - value if value is not None else "") + "#define %s %s" % + (name, value if value is not None else "") for name, value in macros.items() ]) lines.append("") @@ -919,9 +922,8 @@ def GetLibSourceDirs(env): def GetLibBuilders(env): # pylint: disable=too-many-branches if "__PIO_LIB_BUILDERS" in DefaultEnvironment(): - return sorted( - DefaultEnvironment()['__PIO_LIB_BUILDERS'], - key=lambda lb: 0 if lb.dependent else 1) + return sorted(DefaultEnvironment()['__PIO_LIB_BUILDERS'], + key=lambda lb: 0 if lb.dependent else 1) items = [] verbose = int(ARGUMENTS.get("PIOVERBOSE", @@ -936,14 +938,14 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches if compat_mode == "strict" and not lb.is_platforms_compatible( env['PIOPLATFORM']): if verbose: - sys.stderr.write( - "Platform incompatible library %s\n" % lb.path) + sys.stderr.write("Platform incompatible library %s\n" % + lb.path) return False if compat_mode == "soft" and "PIOFRAMEWORK" in env and \ not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])): if verbose: - sys.stderr.write( - "Framework incompatible library %s\n" % lb.path) + sys.stderr.write("Framework incompatible library %s\n" % + lb.path) return False return True @@ -956,12 +958,14 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches if item == "__cores__" or not isdir(join(libs_dir, item)): continue try: - lb = LibBuilderFactory.new( - env, join(libs_dir, item), verbose=verbose) + lb = LibBuilderFactory.new(env, + join(libs_dir, item), + verbose=verbose) except exception.InvalidJSONFile: if verbose: - sys.stderr.write("Skip library with broken manifest: %s\n" - % join(libs_dir, item)) + sys.stderr.write( + "Skip library with broken manifest: %s\n" % + join(libs_dir, item)) continue if _check_lib_builder(lb): items.append(lb) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 3d434955..cf538812 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -190,8 +190,8 @@ class InoToCPPConverter(object): def ConvertInoToCpp(env): src_dir = glob_escape(env.subst("$PROJECTSRC_DIR")) - ino_nodes = ( - env.Glob(join(src_dir, "*.ino")) + env.Glob(join(src_dir, "*.pde"))) + ino_nodes = (env.Glob(join(src_dir, "*.ino")) + + env.Glob(join(src_dir, "*.pde"))) if not ino_nodes: return c = InoToCPPConverter(env) @@ -297,8 +297,8 @@ def PioClean(env, clean_dir): def ProcessDebug(env): if not env.subst("$PIODEBUGFLAGS"): env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"]) - env.Append( - BUILD_FLAGS=list(env['PIODEBUGFLAGS']) + ["-D__PLATFORMIO_DEBUG__"]) + env.Append(BUILD_FLAGS=list(env['PIODEBUGFLAGS']) + + ["-D__PLATFORMIO_DEBUG__"]) unflags = ["-Os"] for level in [0, 1, 2]: for flag in ("O", "g", "ggdb"): @@ -307,11 +307,10 @@ def ProcessDebug(env): def ProcessTest(env): - env.Append( - CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], - CPPPATH=[join("$BUILD_DIR", "UnityTestLib")]) - unitylib = env.BuildLibrary( - join("$BUILD_DIR", "UnityTestLib"), get_core_package_dir("tool-unity")) + env.Append(CPPDEFINES=["UNIT_TEST", "UNITY_INCLUDE_CONFIG_H"], + CPPPATH=[join("$BUILD_DIR", "UnityTestLib")]) + unitylib = env.BuildLibrary(join("$BUILD_DIR", "UnityTestLib"), + get_core_package_dir("tool-unity")) env.Prepend(LIBS=[unitylib]) src_filter = ["+<*.cpp>", "+<*.c>"] diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 144f1f4b..6d0d1c59 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -138,8 +138,8 @@ def PrintConfiguration(env): ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") hardware_data.append( - "%s RAM (%s Flash)" % (util.format_filesize(ram), - util.format_filesize(flash))) + "%s RAM (%s Flash)" % + (util.format_filesize(ram), util.format_filesize(flash))) configuration_data.append( "https://docs.platformio.org/page/boards/%s/%s.html" % (platform.name, board_config.id)) @@ -153,8 +153,8 @@ def PrintConfiguration(env): return data = [ - "CURRENT(%s)" % board_config.get_debug_tool_name( - env.GetProjectOption("debug_tool")) + "CURRENT(%s)" % + board_config.get_debug_tool_name(env.GetProjectOption("debug_tool")) ] onboard = [] external = [] diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 6db94bfc..dd03894a 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -198,10 +198,9 @@ def CheckUploadSize(_, target, source, env): return def _configure_defaults(): - env.Replace( - SIZECHECKCMD="$SIZETOOL -B -d $SOURCES", - SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s", - SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+") + env.Replace(SIZECHECKCMD="$SIZETOOL -B -d $SOURCES", + SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s", + SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+") def _get_size_output(): cmd = env.get("SIZECHECKCMD") @@ -251,8 +250,8 @@ def CheckUploadSize(_, target, source, env): if data_max_size and data_size > -1: print("DATA: %s" % _format_availale_bytes(data_size, data_max_size)) if program_size > -1: - print("PROGRAM: %s" % _format_availale_bytes(program_size, - program_max_size)) + print("PROGRAM: %s" % + _format_availale_bytes(program_size, program_max_size)) if int(ARGUMENTS.get("PIOVERBOSE", 0)): print(output) @@ -273,8 +272,8 @@ def PrintUploadInfo(env): configured = env.subst("$UPLOAD_PROTOCOL") available = [configured] if configured else [] if "BOARD" in env: - available.extend(env.BoardConfig().get("upload", {}).get( - "protocols", [])) + available.extend(env.BoardConfig().get("upload", + {}).get("protocols", [])) if available: print("AVAILABLE: %s" % ", ".join(sorted(set(available)))) if configured: diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 2d1fc0fd..f8f3656c 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -132,8 +132,8 @@ def BuildProgram(env): env.Prepend(_LIBFLAGS="-Wl,--start-group ") env.Append(_LIBFLAGS=" -Wl,--end-group") - program = env.Program( - join("$BUILD_DIR", env.subst("$PROGNAME")), env['PIOBUILDFILES']) + program = env.Program(join("$BUILD_DIR", env.subst("$PROGNAME")), + env['PIOBUILDFILES']) env.Replace(PIOMAINPROG=program) AlwaysBuild( diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 5764b9f8..b48be4df 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -51,24 +51,22 @@ def print_boards(boards): BOARDLIST_TPL = ("{type:<30} {mcu:<14} {frequency:<8} " " {flash:<7} {ram:<6} {name}") click.echo( - BOARDLIST_TPL.format( - type=click.style("ID", fg="cyan"), - mcu="MCU", - frequency="Frequency", - flash="Flash", - ram="RAM", - name="Name")) + BOARDLIST_TPL.format(type=click.style("ID", fg="cyan"), + mcu="MCU", + frequency="Frequency", + flash="Flash", + ram="RAM", + name="Name")) click.echo("-" * terminal_width) for board in boards: click.echo( - BOARDLIST_TPL.format( - type=click.style(board['id'], fg="cyan"), - mcu=board['mcu'], - frequency="%dMHz" % (board['fcpu'] / 1000000), - flash=util.format_filesize(board['rom']), - ram=util.format_filesize(board['ram']), - name=board['name'])) + BOARDLIST_TPL.format(type=click.style(board['id'], fg="cyan"), + mcu=board['mcu'], + frequency="%dMHz" % (board['fcpu'] / 1000000), + flash=util.format_filesize(board['rom']), + ram=util.format_filesize(board['ram']), + name=board['name'])) def _get_boards(installed=False): diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 7d97eb48..55ef07ad 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -48,26 +48,31 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument @click.command("ci", short_help="Continuous Integration") @click.argument("src", nargs=-1, callback=validate_path) -@click.option( - "-l", "--lib", multiple=True, callback=validate_path, metavar="DIRECTORY") +@click.option("-l", + "--lib", + multiple=True, + callback=validate_path, + metavar="DIRECTORY") @click.option("--exclude", multiple=True) -@click.option( - "-b", "--board", multiple=True, metavar="ID", callback=validate_boards) -@click.option( - "--build-dir", - default=mkdtemp, - type=click.Path( - file_okay=False, dir_okay=True, writable=True, resolve_path=True)) +@click.option("-b", + "--board", + multiple=True, + metavar="ID", + callback=validate_boards) +@click.option("--build-dir", + default=mkdtemp, + type=click.Path(file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) @click.option("--keep-build-dir", is_flag=True) -@click.option( - "-c", - "--project-conf", - type=click.Path( - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) +@click.option("-c", + "--project-conf", + type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) @click.option("-O", "--project-option", multiple=True) @click.option("-v", "--verbose", is_flag=True) @click.pass_context @@ -105,11 +110,10 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches _exclude_contents(build_dir, exclude) # initialise project - ctx.invoke( - cmd_init, - project_dir=build_dir, - board=board, - project_option=project_option) + ctx.invoke(cmd_init, + project_dir=build_dir, + board=board, + project_option=project_option) # process project ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 4a950a7a..2631621e 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -52,8 +52,8 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes if not isdir(get_project_cache_dir()): os.makedirs(get_project_cache_dir()) - self._gdbsrc_dir = mkdtemp( - dir=get_project_cache_dir(), prefix=".piodebug-") + self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), + prefix=".piodebug-") self._target_is_run = False self._last_server_activity = 0 @@ -98,8 +98,11 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes args.extend(["--data-directory", gdb_data_dir]) args.append(patterns['PROG_PATH']) - return reactor.spawnProcess( - self, gdb_path, args, path=self.project_dir, env=os.environ) + return reactor.spawnProcess(self, + gdb_path, + args, + path=self.project_dir, + env=os.environ) @staticmethod def _get_data_dir(gdb_path): @@ -109,8 +112,8 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes return gdb_data_dir if isdir(gdb_data_dir) else None def generate_pioinit(self, dst_dir, patterns): - server_exe = (self.debug_options.get("server") or {}).get( - "executable", "").lower() + server_exe = (self.debug_options.get("server") + or {}).get("executable", "").lower() if "jlink" in server_exe: cfg = initcfgs.GDB_JLINK_INIT_CONFIG elif "st-util" in server_exe: diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 153f1e74..129383d8 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -27,29 +27,24 @@ from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project -@click.command( - "debug", - context_settings=dict(ignore_unknown_options=True), - short_help="PIO Unified Debugger") -@click.option( - "-d", - "--project-dir", - default=os.getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option( - "-c", - "--project-conf", - type=click.Path( - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) +@click.command("debug", + context_settings=dict(ignore_unknown_options=True), + short_help="PIO Unified Debugger") +@click.option("-d", + "--project-dir", + default=os.getcwd, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("-c", + "--project-conf", + type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) @click.option("--environment", "-e", metavar="") @click.option("--verbose", "-v", is_flag=True) @click.option("--interface", type=click.Choice(["gdb"])) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 139411ea..88b4f64a 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -63,16 +63,15 @@ def validate_debug_options(cmd_ctx, env_options): try: platform = PlatformFactory.newPlatform(env_options['platform']) except exception.UnknownPlatform: - cmd_ctx.invoke( - cmd_platform_install, - platforms=[env_options['platform']], - skip_default_package=True) + cmd_ctx.invoke(cmd_platform_install, + platforms=[env_options['platform']], + skip_default_package=True) platform = PlatformFactory.newPlatform(env_options['platform']) board_config = platform.board_config(env_options['board']) tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) - tool_settings = board_config.get("debug", {}).get("tools", {}).get( - tool_name, {}) + tool_settings = board_config.get("debug", {}).get("tools", + {}).get(tool_name, {}) server_options = None # specific server per a system @@ -102,10 +101,9 @@ def validate_debug_options(cmd_ctx, env_options): server_package_dir = platform.get_package_dir( server_package) if server_package else None if server_package and not server_package_dir: - platform.install_packages( - with_packages=[server_package], - skip_default_package=True, - silent=True) + platform.install_packages(with_packages=[server_package], + skip_default_package=True, + silent=True) server_package_dir = platform.get_package_dir(server_package) server_options = dict( cwd=server_package_dir if server_package else None, @@ -143,12 +141,11 @@ def validate_debug_options(cmd_ctx, env_options): def predebug_project(ctx, project_dir, env_name, preload, verbose): - ctx.invoke( - cmd_run, - project_dir=project_dir, - environment=[env_name], - target=["__debug"] + (["upload"] if preload else []), - verbose=verbose) + ctx.invoke(cmd_run, + project_dir=project_dir, + environment=[env_name], + target=["__debug"] + (["upload"] if preload else []), + verbose=verbose) if preload: time.sleep(5) @@ -167,11 +164,10 @@ def capture_std_streams(stdout, stderr=None): def load_configuration(ctx, project_dir, env_name): output = BytesIO() with capture_std_streams(output): - ctx.invoke( - cmd_run, - project_dir=project_dir, - environment=[env_name], - target=["idedata"]) + ctx.invoke(cmd_run, + project_dir=project_dir, + environment=[env_name], + target=["idedata"]) result = output.getvalue().decode() output.close() if '"includes":' not in result: diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 992ad9b2..7876fae7 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -101,63 +101,57 @@ def device_list( # pylint: disable=too-many-branches @cli.command("monitor", short_help="Monitor device (Serial)") @click.option("--port", "-p", help="Port, a number or a device name") @click.option("--baud", "-b", type=int, help="Set baud rate, default=9600") -@click.option( - "--parity", - default="N", - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N") -@click.option( - "--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") -@click.option( - "--xonxoff", - is_flag=True, - help="Enable software flow control, default=Off") -@click.option( - "--rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state") -@click.option( - "--dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state") +@click.option("--parity", + default="N", + type=click.Choice(["N", "E", "O", "S", "M"]), + help="Set parity, default=N") +@click.option("--rtscts", + is_flag=True, + help="Enable RTS/CTS flow control, default=Off") +@click.option("--xonxoff", + is_flag=True, + help="Enable software flow control, default=Off") +@click.option("--rts", + default=None, + type=click.IntRange(0, 1), + help="Set initial RTS line state") +@click.option("--dtr", + default=None, + type=click.IntRange(0, 1), + help="Set initial DTR line state") @click.option("--echo", is_flag=True, help="Enable local echo, default=Off") -@click.option( - "--encoding", - default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8") +@click.option("--encoding", + default="UTF-8", + help="Set the encoding for the serial port (e.g. hexlify, " + "Latin1, UTF-8), default: UTF-8") @click.option("--filter", "-f", multiple=True, help="Add text transformation") -@click.option( - "--eol", - default="CRLF", - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF") -@click.option( - "--raw", is_flag=True, help="Do not apply any encodings/transformations") -@click.option( - "--exit-char", - type=int, - default=3, - help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)") -@click.option( - "--menu-char", - type=int, - default=20, - help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)") -@click.option( - "--quiet", - is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, resolve_path=True)) +@click.option("--eol", + default="CRLF", + type=click.Choice(["CR", "LF", "CRLF"]), + help="End of line mode, default=CRLF") +@click.option("--raw", + is_flag=True, + help="Do not apply any encodings/transformations") +@click.option("--exit-char", + type=int, + default=3, + help="ASCII code of special character that is used to exit " + "the application, default=3 (Ctrl+C)") +@click.option("--menu-char", + type=int, + default=20, + help="ASCII code of special character that is used to " + "control miniterm (menu), default=20 (DEC)") +@click.option("--quiet", + is_flag=True, + help="Diagnostics: suppress non-error messages, default=Off") +@click.option("-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True)) @click.option( "-e", "--environment", @@ -206,11 +200,10 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches break try: - miniterm.main( - default_port=kwargs['port'], - default_baudrate=kwargs['baud'] or 9600, - default_rts=kwargs['rts'], - default_dtr=kwargs['dtr']) + miniterm.main(default_port=kwargs['port'], + default_baudrate=kwargs['baud'] or 9600, + default_rts=kwargs['rts'], + default_dtr=kwargs['dtr']) except Exception as e: raise exception.MinitermException(e) diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index a4dd142a..a8ff898f 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -60,8 +60,9 @@ def is_twitter_blocked(): timeout = 2 try: if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): - requests.get( - "http://%s" % ip, allow_redirects=False, timeout=timeout) + requests.get("http://%s" % ip, + allow_redirects=False, + timeout=timeout) else: socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, 80)) return False diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index d3b53383..8de0e195 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -67,11 +67,11 @@ class MiscRPC(object): html_or_json = json.loads(html_or_json) assert "items_html" in html_or_json soup = BeautifulSoup(html_or_json['items_html'], "html.parser") - tweet_nodes = soup.find_all( - "div", attrs={ - "class": "tweet", - "data-tweet-id": True - }) + tweet_nodes = soup.find_all("div", + attrs={ + "class": "tweet", + "data-tweet-id": True + }) result = yield defer.DeferredList( [self._parse_tweet_node(node, username) for node in tweet_nodes], consumeErrors=True) @@ -97,13 +97,14 @@ class MiscRPC(object): node.get("data-expanded-url") for node in (quote_text_node or text_node).find_all( class_="twitter-timeline-link", - attrs={"data-expanded-url": True}) - ] + attrs={"data-expanded-url": True} + ) + ] # yapf: disable # fetch data from iframe card if (not photos or not urls) and tweet.get("data-card2-type"): - iframe_node = tweet.find( - "div", attrs={"data-full-card-iframe-url": True}) + iframe_node = tweet.find("div", + attrs={"data-full-card-iframe-url": True}) if iframe_node: iframe_card = yield self._fetch_iframe_card( twitter_url + iframe_node.get("data-full-card-iframe-url"), @@ -161,8 +162,8 @@ class MiscRPC(object): url_node = soup.find("a", class_="TwitterCard-container") text_node = soup.find("div", class_="SummaryCard-content") if text_node: - text_node.find( - "span", class_="SummaryCard-destination").decompose() + text_node.find("span", + class_="SummaryCard-destination").decompose() defer.returnValue({ "photo": photo_node.get("data-src") if photo_node else None, diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index 8b263632..c394d935 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -54,8 +54,10 @@ class OSRPC(object): session = helpers.requests_session() if data: - r = yield session.post( - uri, data=data, headers=headers, timeout=timeout) + r = yield session.post(uri, + data=data, + headers=headers, + timeout=timeout) else: r = yield session.get(uri, headers=headers, timeout=timeout) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 2b396094..e9435d35 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -29,8 +29,9 @@ from platformio.compat import get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_libdeps_dir, get_project_src_dir, is_platformio_project) +from platformio.project.helpers import (get_project_libdeps_dir, + get_project_src_dir, + is_platformio_project) class ProjectRPC(object): diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index bf339994..d62faa3d 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -48,8 +48,8 @@ class JSONRPCServerProtocol(WebSocketServerProtocol): if isinstance(failure.value, JSONRPCDispatchException): e = failure.value else: - e = JSONRPCDispatchException( - code=4999, message=failure.getErrorMessage()) + e = JSONRPCDispatchException(code=4999, + message=failure.getErrorMessage()) del response["result"] response['error'] = e.error._data # pylint: disable=protected-access print(response['error']) diff --git a/platformio/commands/init.py b/platformio/commands/init.py index 35e459bf..d3d8f52f 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -25,9 +25,11 @@ from platformio.commands.platform import \ from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_include_dir, get_project_lib_dir, get_project_src_dir, - get_project_test_dir, is_platformio_project) +from platformio.project.helpers import (get_project_include_dir, + get_project_lib_dir, + get_project_src_dir, + get_project_test_dir, + is_platformio_project) def validate_boards(ctx, param, value): # pylint: disable=W0613 @@ -42,22 +44,23 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 return value -@click.command( - "init", short_help="Initialize PlatformIO project or update existing") -@click.option( - "--project-dir", - "-d", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option( - "-b", "--board", multiple=True, metavar="ID", callback=validate_boards) -@click.option( - "--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) +@click.command("init", + short_help="Initialize PlatformIO project or update existing") +@click.option("--project-dir", + "-d", + default=getcwd, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("-b", + "--board", + multiple=True, + metavar="ID", + callback=validate_boards) +@click.option("--ide", + type=click.Choice(ProjectGenerator.get_supported_ides())) @click.option("-O", "--project-option", multiple=True) @click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) @@ -72,22 +75,23 @@ def cli( silent): if not silent: if project_dir == getcwd(): - click.secho( - "\nThe current working directory", fg="yellow", nl=False) + click.secho("\nThe current working directory", + fg="yellow", + nl=False) click.secho(" %s " % project_dir, fg="cyan", nl=False) click.secho("will be used for the project.", fg="yellow") click.echo("") click.echo("The next files/directories have been created in %s" % click.style(project_dir, fg="cyan")) - click.echo("%s - Put project header files here" % click.style( - "include", fg="cyan")) + click.echo("%s - Put project header files here" % + click.style("include", fg="cyan")) click.echo("%s - Put here project specific (private) libraries" % click.style("lib", fg="cyan")) - click.echo("%s - Put project source files here" % click.style( - "src", fg="cyan")) - click.echo("%s - Project Configuration File" % click.style( - "platformio.ini", fg="cyan")) + click.echo("%s - Put project source files here" % + click.style("src", fg="cyan")) + click.echo("%s - Project Configuration File" % + click.style("platformio.ini", fg="cyan")) is_new_project = not is_platformio_project(project_dir) if is_new_project: @@ -112,8 +116,8 @@ def cli( if ide: click.secho( "\nProject has been successfully %s including configuration files " - "for `%s` IDE." % ("initialized" if is_new_project else "updated", - ide), + "for `%s` IDE." % + ("initialized" if is_new_project else "updated", ide), fg="green") else: click.secho( @@ -363,8 +367,8 @@ def init_cvs_ignore(project_dir): def fill_project_envs(ctx, project_dir, board_ids, project_option, env_prefix, force_download): - config = ProjectConfig( - join(project_dir, "platformio.ini"), parse_extra=False) + config = ProjectConfig(join(project_dir, "platformio.ini"), + parse_extra=False) used_boards = [] for section in config.sections(): cond = [ @@ -417,6 +421,5 @@ def _install_dependent_platforms(ctx, platforms): ] if set(platforms) <= set(installed_platforms): return - ctx.invoke( - cli_platform_install, - platforms=list(set(platforms) - set(installed_platforms))) + ctx.invoke(cli_platform_install, + platforms=list(set(platforms) - set(installed_platforms))) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 63b89950..23094f43 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -26,9 +26,10 @@ from platformio.managers.lib import (LibraryManager, get_builtin_libs, is_builtin_lib) from platformio.proc import is_ci from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_dir, get_project_global_lib_dir, get_project_libdeps_dir, - is_platformio_project) +from platformio.project.helpers import (get_project_dir, + get_project_global_lib_dir, + get_project_libdeps_dir, + is_platformio_project) try: from urllib.parse import quote @@ -42,23 +43,20 @@ CTX_META_STORAGE_LIBDEPS_KEY = __name__ + ".storage_lib_deps" @click.group(short_help="Library Manager") -@click.option( - "-d", - "--storage-dir", - multiple=True, - default=None, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True), - help="Manage custom library storage") -@click.option( - "-g", - "--global", - is_flag=True, - help="Manage global PlatformIO library storage") +@click.option("-d", + "--storage-dir", + multiple=True, + default=None, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True), + help="Manage custom library storage") +@click.option("-g", + "--global", + is_flag=True, + help="Manage global PlatformIO library storage") @click.option( "-e", "--environment", @@ -101,8 +99,8 @@ def cli(ctx, **options): continue with util.cd(storage_dir): libdeps_dir = get_project_libdeps_dir() - config = ProjectConfig.get_instance( - join(storage_dir, "platformio.ini")) + config = ProjectConfig.get_instance(join(storage_dir, + "platformio.ini")) config.validate(options['environment']) for env in config.envs(): if options['environment'] and env not in options['environment']: @@ -119,17 +117,17 @@ def cli(ctx, **options): "--save", is_flag=True, help="Save installed libraries into the `platformio.ini` dependency list") -@click.option( - "-s", "--silent", is_flag=True, help="Suppress progress reporting") -@click.option( - "--interactive", - is_flag=True, - help="Allow to make a choice for all prompts") -@click.option( - "-f", - "--force", - is_flag=True, - help="Reinstall/redownload library if exists") +@click.option("-s", + "--silent", + is_flag=True, + help="Suppress progress reporting") +@click.option("--interactive", + is_flag=True, + help="Allow to make a choice for all prompts") +@click.option("-f", + "--force", + is_flag=True, + help="Reinstall/redownload library if exists") @click.pass_context def lib_install( # pylint: disable=too-many-arguments ctx, libraries, save, silent, interactive, force): @@ -143,20 +141,18 @@ def lib_install( # pylint: disable=too-many-arguments lm = LibraryManager(storage_dir) if libraries: for library in libraries: - pkg_dir = lm.install( - library, - silent=silent, - interactive=interactive, - force=force) + pkg_dir = lm.install(library, + silent=silent, + interactive=interactive, + force=force) installed_manifests[library] = lm.load_manifest(pkg_dir) elif storage_dir in storage_libdeps: for library in storage_libdeps[storage_dir]: try: - pkg_dir = lm.install( - library, - silent=silent, - interactive=interactive, - force=force) + pkg_dir = lm.install(library, + silent=silent, + interactive=interactive, + force=force) installed_manifests[library] = lm.load_manifest(pkg_dir) except exception.LibNotFound as e: if not silent or not is_builtin_lib(library): @@ -203,15 +199,13 @@ def lib_uninstall(ctx, libraries): @cli.command("update", short_help="Update installed libraries") @click.argument("libraries", required=False, nargs=-1, metavar="[LIBRARY...]") -@click.option( - "-c", - "--only-check", - is_flag=True, - help="DEPRECATED. Please use `--dry-run` instead") -@click.option( - "--dry-run", - is_flag=True, - help="Do not update, only check for the new versions") +@click.option("-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead") +@click.option("--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) @click.pass_context def lib_update(ctx, libraries, only_check, dry_run, json_output): @@ -297,10 +291,9 @@ def lib_list(ctx, json_output): @click.option("-f", "--framework", multiple=True) @click.option("-p", "--platform", multiple=True) @click.option("-i", "--header", multiple=True) -@click.option( - "--noninteractive", - is_flag=True, - help="Do not prompt, automatically paginate with delay") +@click.option("--noninteractive", + is_flag=True, + help="Do not prompt, automatically paginate with delay") def lib_search(query, json_output, page, noninteractive, **filters): if not query: query = [] @@ -311,10 +304,9 @@ def lib_search(query, json_output, page, noninteractive, **filters): for value in values: query.append('%s:"%s"' % (key, value)) - result = util.get_api_result( - "/v2/lib/search", - dict(query=" ".join(query), page=page), - cache_valid="1d") + result = util.get_api_result("/v2/lib/search", + dict(query=" ".join(query), page=page), + cache_valid="1d") if json_output: click.echo(json.dumps(result)) @@ -336,9 +328,8 @@ def lib_search(query, json_output, page, noninteractive, **filters): fg="cyan") return - click.secho( - "Found %d libraries:\n" % result['total'], - fg="green" if result['total'] else "yellow") + click.secho("Found %d libraries:\n" % result['total'], + fg="green" if result['total'] else "yellow") while True: for item in result['items']: @@ -350,20 +341,18 @@ def lib_search(query, json_output, page, noninteractive, **filters): if noninteractive: click.echo() - click.secho( - "Loading next %d libraries... Press Ctrl+C to stop!" % - result['perpage'], - fg="yellow") + click.secho("Loading next %d libraries... Press Ctrl+C to stop!" % + result['perpage'], + fg="yellow") click.echo() time.sleep(5) elif not click.confirm("Show next libraries?"): break - result = util.get_api_result( - "/v2/lib/search", { - "query": " ".join(query), - "page": int(result['page']) + 1 - }, - cache_valid="1d") + result = util.get_api_result("/v2/lib/search", { + "query": " ".join(query), + "page": int(result['page']) + 1 + }, + cache_valid="1d") @cli.command("builtin", short_help="List built-in libraries") @@ -475,13 +464,12 @@ def lib_register(config_url): and not config_url.startswith("https://")): raise exception.InvalidLibConfURL(config_url) - result = util.get_api_result( - "/lib/register", data=dict(config_url=config_url)) + result = util.get_api_result("/lib/register", + data=dict(config_url=config_url)) if "message" in result and result['message']: - click.secho( - result['message'], - fg="green" - if "successed" in result and result['successed'] else "red") + click.secho(result['message'], + fg="green" if "successed" in result and result['successed'] + else "red") @cli.command("stats", short_help="Library Registry Statistics") @@ -512,10 +500,9 @@ def lib_stats(json_output): date = str( time.strftime("%c", util.parse_date(item['date'])) if "date" in item else "") - url = click.style( - "https://platformio.org/lib/show/%s/%s" % (item['id'], - quote(item['name'])), - fg="blue") + url = click.style("https://platformio.org/lib/show/%s/%s" % + (item['id'], quote(item['name'])), + fg="blue") click.echo( (printitemdate_tpl if "date" in item else printitem_tpl).format( name=click.style(item['name'], fg="cyan"), date=date, url=url)) @@ -524,10 +511,9 @@ def lib_stats(json_output): click.echo( printitem_tpl.format( name=click.style(name, fg="cyan"), - url=click.style( - "https://platformio.org/lib/search?query=" + quote( - "keyword:%s" % name), - fg="blue"))) + url=click.style("https://platformio.org/lib/search?query=" + + quote("keyword:%s" % name), + fg="blue"))) for key in ("updated", "added"): _print_title("Recently " + key) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 70612f51..9f56f8d6 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -29,9 +29,9 @@ def cli(): def _print_platforms(platforms): for platform in platforms: - click.echo("{name} ~ {title}".format( - name=click.style(platform['name'], fg="cyan"), - title=platform['title'])) + click.echo("{name} ~ {title}".format(name=click.style(platform['name'], + fg="cyan"), + title=platform['title'])) click.echo("=" * (3 + len(platform['name'] + platform['title']))) click.echo(platform['description']) click.echo() @@ -65,19 +65,18 @@ def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): p = PlatformFactory.newPlatform(platform) - data = dict( - name=p.name, - title=p.title, - description=p.description, - version=p.version, - homepage=p.homepage, - repository=p.repository_url, - url=p.vendor_url, - docs=p.docs_url, - license=p.license, - forDesktop=not p.is_embedded(), - frameworks=sorted(list(p.frameworks) if p.frameworks else []), - packages=list(p.packages) if p.packages else []) + data = dict(name=p.name, + title=p.title, + description=p.description, + version=p.version, + homepage=p.homepage, + repository=p.repository_url, + url=p.vendor_url, + docs=p.docs_url, + license=p.license, + forDesktop=not p.is_embedded(), + frameworks=sorted(list(p.frameworks) if p.frameworks else []), + packages=list(p.packages) if p.packages else []) # if dump to API # del data['version'] @@ -99,11 +98,10 @@ def _get_installed_platform_data(platform, data['packages'] = [] installed_pkgs = p.get_installed_packages() for name, opts in p.packages.items(): - item = dict( - name=name, - type=p.get_package_type(name), - requirements=opts.get("version"), - optional=opts.get("optional") is True) + item = dict(name=name, + type=p.get_package_type(name), + requirements=opts.get("version"), + optional=opts.get("optional") is True) if name in installed_pkgs: for key, value in installed_pkgs[name].items(): if key not in ("url", "version", "description"): @@ -129,18 +127,17 @@ def _get_registry_platform_data( # pylint: disable=unused-argument if not _data: return None - data = dict( - name=_data['name'], - title=_data['title'], - description=_data['description'], - homepage=_data['homepage'], - repository=_data['repository'], - url=_data['url'], - license=_data['license'], - forDesktop=_data['forDesktop'], - frameworks=_data['frameworks'], - packages=_data['packages'], - versions=_data['versions']) + data = dict(name=_data['name'], + title=_data['title'], + description=_data['description'], + homepage=_data['homepage'], + repository=_data['repository'], + url=_data['url'], + license=_data['license'], + forDesktop=_data['forDesktop'], + frameworks=_data['frameworks'], + packages=_data['packages'], + versions=_data['versions']) if with_boards: data['boards'] = [ @@ -163,8 +160,9 @@ def platform_search(query, json_output): if query and query.lower() not in search_data.lower(): continue platforms.append( - _get_registry_platform_data( - platform['name'], with_boards=False, expose_packages=False)) + _get_registry_platform_data(platform['name'], + with_boards=False, + expose_packages=False)) if json_output: click.echo(json.dumps(platforms)) @@ -183,8 +181,8 @@ def platform_frameworks(query, json_output): search_data = json.dumps(framework) if query and query.lower() not in search_data.lower(): continue - framework['homepage'] = ( - "https://platformio.org/frameworks/" + framework['name']) + framework['homepage'] = ("https://platformio.org/frameworks/" + + framework['name']) framework['platforms'] = [ platform['name'] for platform in _get_registry_platforms() if framework['name'] in platform['frameworks'] @@ -205,10 +203,9 @@ def platform_list(json_output): pm = PlatformManager() for manifest in pm.get_installed(): platforms.append( - _get_installed_platform_data( - manifest['__pkg_dir'], - with_boards=False, - expose_packages=False)) + _get_installed_platform_data(manifest['__pkg_dir'], + with_boards=False, + expose_packages=False)) platforms = sorted(platforms, key=lambda manifest: manifest['name']) if json_output: @@ -227,8 +224,9 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches if json_output: return click.echo(json.dumps(data)) - click.echo("{name} ~ {title}".format( - name=click.style(data['name'], fg="cyan"), title=data['title'])) + click.echo("{name} ~ {title}".format(name=click.style(data['name'], + fg="cyan"), + title=data['title'])) click.echo("=" * (3 + len(data['name'] + data['title']))) click.echo(data['description']) click.echo() @@ -293,17 +291,15 @@ def platform_install(platforms, with_package, without_package, skip_default_package, force): pm = PlatformManager() for platform in platforms: - if pm.install( - name=platform, - with_packages=with_package, - without_packages=without_package, - skip_default_package=skip_default_package, - force=force): - click.secho( - "The platform '%s' has been successfully installed!\n" - "The rest of packages will be installed automatically " - "depending on your build environment." % platform, - fg="green") + if pm.install(name=platform, + with_packages=with_package, + without_packages=without_package, + skip_default_package=skip_default_package, + force=force): + click.secho("The platform '%s' has been successfully installed!\n" + "The rest of packages will be installed automatically " + "depending on your build environment." % platform, + fg="green") @cli.command("uninstall", short_help="Uninstall development platform") @@ -312,28 +308,24 @@ def platform_uninstall(platforms): pm = PlatformManager() for platform in platforms: if pm.uninstall(platform): - click.secho( - "The platform '%s' has been successfully " - "uninstalled!" % platform, - fg="green") + click.secho("The platform '%s' has been successfully " + "uninstalled!" % platform, + fg="green") @cli.command("update", short_help="Update installed development platforms") @click.argument("platforms", nargs=-1, required=False, metavar="[PLATFORM...]") -@click.option( - "-p", - "--only-packages", - is_flag=True, - help="Update only the platform packages") -@click.option( - "-c", - "--only-check", - is_flag=True, - help="DEPRECATED. Please use `--dry-run` instead") -@click.option( - "--dry-run", - is_flag=True, - help="Do not update, only check for the new versions") +@click.option("-p", + "--only-packages", + is_flag=True, + help="Update only the platform packages") +@click.option("-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead") +@click.option("--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) def platform_update( # pylint: disable=too-many-locals platforms, only_packages, only_check, dry_run, json_output): @@ -363,8 +355,9 @@ def platform_update( # pylint: disable=too-many-locals if (not latest and not PlatformFactory.newPlatform( pkg_dir).are_outdated_packages()): continue - data = _get_installed_platform_data( - pkg_dir, with_boards=False, expose_packages=False) + data = _get_installed_platform_data(pkg_dir, + with_boards=False, + expose_packages=False) if latest: data['versionLatest'] = latest result.append(data) @@ -373,8 +366,9 @@ def platform_update( # pylint: disable=too-many-locals # cleanup cached board and platform lists app.clean_cache() for platform in platforms: - click.echo("Platform %s" % click.style( - pkg_dir_to_name.get(platform, platform), fg="cyan")) + click.echo( + "Platform %s" % + click.style(pkg_dir_to_name.get(platform, platform), fg="cyan")) click.echo("--------") pm.update(platform, only_packages=only_packages, only_check=only_check) click.echo() diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index ded45eda..8dcdf9a2 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -43,12 +43,13 @@ def remote_agent(): @remote_agent.command("start", short_help="Start agent") @click.option("-n", "--name") @click.option("-s", "--share", multiple=True, metavar="E-MAIL") -@click.option( - "-d", - "--working-dir", - envvar="PLATFORMIO_REMOTE_AGENT_DIR", - type=click.Path( - file_okay=False, dir_okay=True, writable=True, resolve_path=True)) +@click.option("-d", + "--working-dir", + envvar="PLATFORMIO_REMOTE_AGENT_DIR", + type=click.Path(file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) def remote_agent_start(**kwargs): pioplus_call(sys.argv[1:]) @@ -63,17 +64,15 @@ def remote_agent_list(): pioplus_call(sys.argv[1:]) -@cli.command( - "update", short_help="Update installed Platforms, Packages and Libraries") -@click.option( - "-c", - "--only-check", - is_flag=True, - help="DEPRECATED. Please use `--dry-run` instead") -@click.option( - "--dry-run", - is_flag=True, - help="Do not update, only check for the new versions") +@cli.command("update", + short_help="Update installed Platforms, Packages and Libraries") +@click.option("-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead") +@click.option("--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") def remote_update(only_check, dry_run): pioplus_call(sys.argv[1:]) @@ -82,16 +81,14 @@ def remote_update(only_check, dry_run): @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=True, - dir_okay=True, - writable=True, - resolve_path=True)) +@click.option("-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, + file_okay=True, + dir_okay=True, + writable=True, + resolve_path=True)) @click.option("--disable-auto-clean", is_flag=True) @click.option("-r", "--force-remote", is_flag=True) @click.option("-s", "--silent", is_flag=True) @@ -105,16 +102,14 @@ def remote_run(**kwargs): @click.option("--ignore", "-i", multiple=True, metavar="") @click.option("--upload-port") @click.option("--test-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) +@click.option("-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) @click.option("-r", "--force-remote", is_flag=True) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @@ -136,59 +131,55 @@ def device_list(json_output): @remote_device.command("monitor", short_help="Monitor remote device") @click.option("--port", "-p", help="Port, a number or a device name") -@click.option( - "--baud", "-b", type=int, default=9600, help="Set baud rate, default=9600") -@click.option( - "--parity", - default="N", - type=click.Choice(["N", "E", "O", "S", "M"]), - help="Set parity, default=N") -@click.option( - "--rtscts", is_flag=True, help="Enable RTS/CTS flow control, default=Off") -@click.option( - "--xonxoff", - is_flag=True, - help="Enable software flow control, default=Off") -@click.option( - "--rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state") -@click.option( - "--dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state") +@click.option("--baud", + "-b", + type=int, + default=9600, + help="Set baud rate, default=9600") +@click.option("--parity", + default="N", + type=click.Choice(["N", "E", "O", "S", "M"]), + help="Set parity, default=N") +@click.option("--rtscts", + is_flag=True, + help="Enable RTS/CTS flow control, default=Off") +@click.option("--xonxoff", + is_flag=True, + help="Enable software flow control, default=Off") +@click.option("--rts", + default=None, + type=click.IntRange(0, 1), + help="Set initial RTS line state") +@click.option("--dtr", + default=None, + type=click.IntRange(0, 1), + help="Set initial DTR line state") @click.option("--echo", is_flag=True, help="Enable local echo, default=Off") -@click.option( - "--encoding", - default="UTF-8", - help="Set the encoding for the serial port (e.g. hexlify, " - "Latin1, UTF-8), default: UTF-8") +@click.option("--encoding", + default="UTF-8", + help="Set the encoding for the serial port (e.g. hexlify, " + "Latin1, UTF-8), default: UTF-8") @click.option("--filter", "-f", multiple=True, help="Add text transformation") -@click.option( - "--eol", - default="CRLF", - type=click.Choice(["CR", "LF", "CRLF"]), - help="End of line mode, default=CRLF") -@click.option( - "--raw", is_flag=True, help="Do not apply any encodings/transformations") -@click.option( - "--exit-char", - type=int, - default=3, - help="ASCII code of special character that is used to exit " - "the application, default=3 (Ctrl+C)") -@click.option( - "--menu-char", - type=int, - default=20, - help="ASCII code of special character that is used to " - "control miniterm (menu), default=20 (DEC)") -@click.option( - "--quiet", - is_flag=True, - help="Diagnostics: suppress non-error messages, default=Off") +@click.option("--eol", + default="CRLF", + type=click.Choice(["CR", "LF", "CRLF"]), + help="End of line mode, default=CRLF") +@click.option("--raw", + is_flag=True, + help="Do not apply any encodings/transformations") +@click.option("--exit-char", + type=int, + default=3, + help="ASCII code of special character that is used to exit " + "the application, default=3 (Ctrl+C)") +@click.option("--menu-char", + type=int, + default=20, + help="ASCII code of special character that is used to " + "control miniterm (menu), default=20 (DEC)") +@click.option("--quiet", + is_flag=True, + help="Diagnostics: suppress non-error messages, default=Off") @click.pass_context def device_monitor(ctx, **kwargs): diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 1913314b..23ca09d4 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -20,8 +20,9 @@ import click from platformio import exception, util from platformio.commands.device import device_monitor as cmd_device_monitor -from platformio.commands.run.helpers import ( - _clean_build_dir, _handle_legacy_libdeps, print_summary) +from platformio.commands.run.helpers import (_clean_build_dir, + _handle_legacy_libdeps, + print_summary) from platformio.commands.run.processor import EnvironmentProcessor from platformio.project.config import ProjectConfig from platformio.project.helpers import (find_project_dir_above, @@ -34,25 +35,21 @@ from platformio.project.helpers import (find_project_dir_above, @click.option("-e", "--environment", multiple=True) @click.option("-t", "--target", multiple=True) @click.option("--upload-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=True, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option( - "-c", - "--project-conf", - type=click.Path( - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) +@click.option("-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, + file_okay=True, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("-c", + "--project-conf", + type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--disable-auto-clean", is_flag=True) @@ -93,8 +90,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, results.append((envname, None)) continue - if not silent and any( - status is not None for (_, status) in results): + if not silent and any(status is not None + for (_, status) in results): click.echo() ep = EnvironmentProcessor(ctx, envname, config, target, @@ -104,9 +101,8 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, if result[1] and "monitor" in ep.get_build_targets() and \ "nobuild" not in ep.get_build_targets(): - ctx.invoke( - cmd_device_monitor, - environment=environment[0] if environment else None) + ctx.invoke(cmd_device_monitor, + environment=environment[0] if environment else None) found_error = any(status is False for (_, status) in results) diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index fd1ac7e3..d1e6b59f 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -22,8 +22,9 @@ from platformio import exception, util from platformio.commands.lib import (CTX_META_STORAGE_DIRS_KEY, CTX_META_STORAGE_LIBDEPS_KEY) from platformio.commands.lib import lib_install as cmd_lib_install -from platformio.project.helpers import ( - calculate_project_hash, get_project_dir, get_project_libdeps_dir) +from platformio.project.helpers import (calculate_project_hash, + get_project_dir, + get_project_libdeps_dir) def _handle_legacy_libdeps(project_dir, config): @@ -71,8 +72,8 @@ def _clean_build_dir(build_dir): proj_hash = calculate_project_hash() # if project's config is modified - if (isdir(build_dir) and getmtime( - join(get_project_dir(), "platformio.ini")) > getmtime(build_dir)): + if (isdir(build_dir) and getmtime(join( + get_project_dir(), "platformio.ini")) > getmtime(build_dir)): util.rmtree_(build_dir) # check project structure @@ -93,8 +94,9 @@ def print_header(label, is_error=False, fg=None): terminal_width, _ = click.get_terminal_size() width = len(click.unstyle(label)) half_line = "=" * int((terminal_width - width - 2) / 2) - click.secho( - "%s %s %s" % (half_line, label, half_line), fg=fg, err=is_error) + click.secho("%s %s %s" % (half_line, label, half_line), + fg=fg, + err=is_error) def print_summary(results, start_time): @@ -115,13 +117,12 @@ def print_summary(results, start_time): status_str = click.style("SUCCESS", fg="green") format_str = "Environment {0:<%d}\t[{1}]" % envname_max_len - click.echo( - format_str.format(click.style(envname, fg="cyan"), status_str), - err=status is False) + click.echo(format_str.format(click.style(envname, fg="cyan"), + status_str), + err=status is False) - print_header( - "%s%d succeeded in %.2f seconds" % - ("%d failed, " % failed_nums if failed_nums else "", succeeded_nums, - time() - start_time), - is_error=failed_nums, - fg="red" if failed_nums else "green") + print_header("%s%d succeeded in %.2f seconds" % + ("%d failed, " % failed_nums if failed_nums else "", + succeeded_nums, time() - start_time), + is_error=failed_nums, + fg="red" if failed_nums else "green") diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 6d57f257..f507780a 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -64,10 +64,10 @@ class EnvironmentProcessor(object): if is_error or "piotest_processor" not in self.cmd_ctx.meta: print_header( - "[%s] Took %.2f seconds" % ( - (click.style("ERROR", fg="red", bold=True) if is_error else - click.style("SUCCESS", fg="green", bold=True)), - time() - start_time), + "[%s] Took %.2f seconds" % + ((click.style("ERROR", fg="red", bold=True) if + is_error else click.style("SUCCESS", fg="green", bold=True)), + time() - start_time), is_error=is_error) return not is_error @@ -106,10 +106,9 @@ class EnvironmentProcessor(object): try: p = PlatformFactory.newPlatform(self.options['platform']) except exception.UnknownPlatform: - self.cmd_ctx.invoke( - cmd_platform_install, - platforms=[self.options['platform']], - skip_default_package=True) + self.cmd_ctx.invoke(cmd_platform_install, + platforms=[self.options['platform']], + skip_default_package=True) p = PlatformFactory.newPlatform(self.options['platform']) return p.run(build_vars, build_targets, self.silent, self.verbose) diff --git a/platformio/commands/settings.py b/platformio/commands/settings.py index a29d3997..d1d5ad59 100644 --- a/platformio/commands/settings.py +++ b/platformio/commands/settings.py @@ -30,11 +30,10 @@ def settings_get(name): terminal_width, _ = click.get_terminal_size() click.echo( - list_tpl.format( - name=click.style("Name", fg="cyan"), - value=(click.style("Value", fg="green") + click.style( - " [Default]", fg="yellow")), - description="Description")) + list_tpl.format(name=click.style("Name", fg="cyan"), + value=(click.style("Value", fg="green") + + click.style(" [Default]", fg="yellow")), + description="Description")) click.echo("-" * terminal_width) for _name, _data in sorted(app.DEFAULT_SETTINGS.items()): @@ -56,10 +55,9 @@ def settings_get(name): _value_str += click.style(" ", fg="yellow") click.echo( - list_tpl.format( - name=click.style(_name, fg="cyan"), - value=_value_str, - description=_data['description'])) + list_tpl.format(name=click.style(_name, fg="cyan"), + value=_value_str, + description=_data['description'])) @cli.command("set", short_help="Set new value for the setting") diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index 87a84424..d330a410 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -31,53 +31,45 @@ from platformio.project.helpers import get_project_test_dir @click.command("test", short_help="Unit Testing") @click.option("--environment", "-e", multiple=True, metavar="") -@click.option( - "--filter", - "-f", - multiple=True, - metavar="", - help="Filter tests by a pattern") -@click.option( - "--ignore", - "-i", - multiple=True, - metavar="", - help="Ignore tests by a pattern") +@click.option("--filter", + "-f", + multiple=True, + metavar="", + help="Filter tests by a pattern") +@click.option("--ignore", + "-i", + multiple=True, + metavar="", + help="Ignore tests by a pattern") @click.option("--upload-port") @click.option("--test-port") -@click.option( - "-d", - "--project-dir", - default=getcwd, - type=click.Path( - exists=True, - file_okay=False, - dir_okay=True, - writable=True, - resolve_path=True)) -@click.option( - "-c", - "--project-conf", - type=click.Path( - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - resolve_path=True)) +@click.option("-d", + "--project-dir", + default=getcwd, + type=click.Path(exists=True, + file_okay=False, + dir_okay=True, + writable=True, + resolve_path=True)) +@click.option("-c", + "--project-conf", + type=click.Path(exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True)) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @click.option("--without-testing", is_flag=True) @click.option("--no-reset", is_flag=True) -@click.option( - "--monitor-rts", - default=None, - type=click.IntRange(0, 1), - help="Set initial RTS line state for Serial Monitor") -@click.option( - "--monitor-dtr", - default=None, - type=click.IntRange(0, 1), - help="Set initial DTR line state for Serial Monitor") +@click.option("--monitor-rts", + default=None, + type=click.IntRange(0, 1), + help="Set initial RTS line state for Serial Monitor") +@click.option("--monitor-dtr", + default=None, + type=click.IntRange(0, 1), + help="Set initial DTR line state for Serial Monitor") @click.option("--verbose", "-v", is_flag=True) @click.pass_context def cli( # pylint: disable=redefined-builtin @@ -130,18 +122,17 @@ def cli( # pylint: disable=redefined-builtin EmbeddedTestProcessor) tp = cls( ctx, testname, envname, - dict( - project_config=config, - project_dir=project_dir, - upload_port=upload_port, - test_port=test_port, - without_building=without_building, - without_uploading=without_uploading, - without_testing=without_testing, - no_reset=no_reset, - monitor_rts=monitor_rts, - monitor_dtr=monitor_dtr, - verbose=verbose)) + dict(project_config=config, + project_dir=project_dir, + upload_port=upload_port, + test_port=test_port, + without_building=without_building, + without_uploading=without_uploading, + without_testing=without_testing, + no_reset=no_reset, + monitor_rts=monitor_rts, + monitor_dtr=monitor_dtr, + verbose=verbose)) results.append((tp.process(), testname, envname)) if without_testing: @@ -168,17 +159,15 @@ def cli( # pylint: disable=redefined-builtin format_str = "test/{:<%d} > {:<%d}\t[{}]" % (testname_max_len, envname_max_len) - click.echo( - format_str.format(testname, click.style(envname, fg="cyan"), - status_str), - err=status is False) + click.echo(format_str.format(testname, click.style(envname, fg="cyan"), + status_str), + err=status is False) - print_header( - "%s%d passed in %.2f seconds" - % ("%d failed, " % failed_nums if failed_nums else "", passed_nums, - time() - start_time), - is_error=failed_nums, - fg="red" if failed_nums else "green") + print_header("%s%d passed in %.2f seconds" % + ("%d failed, " % failed_nums if failed_nums else "", + passed_nums, time() - start_time), + is_error=failed_nums, + fg="red" if failed_nums else "green") if failed_nums: raise exception.ReturnErrorCode(1) diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index 449763db..681bfb44 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -57,8 +57,8 @@ class EmbeddedTestProcessor(TestProcessorBase): click.echo() try: - ser = serial.Serial( - baudrate=self.get_baudrate(), timeout=self.SERIAL_TIMEOUT) + ser = serial.Serial(baudrate=self.get_baudrate(), + timeout=self.SERIAL_TIMEOUT) ser.port = self.get_test_port() ser.rts = self.options['monitor_rts'] ser.dtr = self.options['monitor_dtr'] diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 7b22f5ee..973a3853 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -86,8 +86,8 @@ class TestProcessorBase(object): self.test_name = testname self.options = options self.env_name = envname - self.env_options = options['project_config'].items( - env=envname, as_dict=True) + self.env_options = options['project_config'].items(env=envname, + as_dict=True) self._run_failed = False self._outputcpp_generated = False @@ -108,11 +108,10 @@ class TestProcessorBase(object): def print_progress(self, text, is_error=False): click.echo() - print_header( - "[test/%s > %s] %s" % (click.style(self.test_name, fg="yellow"), - click.style(self.env_name, fg="cyan"), - text), - is_error=is_error) + print_header("[test/%s > %s] %s" % + (click.style(self.test_name, fg="yellow"), + click.style(self.env_name, fg="cyan"), text), + is_error=is_error) def build_or_upload(self, target): if not self._outputcpp_generated: @@ -126,14 +125,13 @@ class TestProcessorBase(object): click.echo("Please wait...") try: - return self.cmd_ctx.invoke( - cmd_run, - project_dir=self.options['project_dir'], - upload_port=self.options['upload_port'], - silent=not self.options['verbose'], - environment=[self.env_name], - disable_auto_clean="nobuild" in target, - target=target) + return self.cmd_ctx.invoke(cmd_run, + project_dir=self.options['project_dir'], + upload_port=self.options['upload_port'], + silent=not self.options['verbose'], + environment=[self.env_name], + disable_auto_clean="nobuild" in target, + target=target) except exception.ReturnErrorCode: return False @@ -146,8 +144,8 @@ class TestProcessorBase(object): def on_run_out(self, line): line = line.strip() if line.endswith(":PASS"): - click.echo( - "%s\t[%s]" % (line[:-5], click.style("PASSED", fg="green"))) + click.echo("%s\t[%s]" % + (line[:-5], click.style("PASSED", fg="green"))) elif ":FAIL" in line: self._run_failed = True click.echo("%s\t[%s]" % (line, click.style("FAILED", fg="red"))) diff --git a/platformio/commands/update.py b/platformio/commands/update.py index 78d2f8bd..dc6ed1c6 100644 --- a/platformio/commands/update.py +++ b/platformio/commands/update.py @@ -22,19 +22,18 @@ from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager -@click.command( - "update", short_help="Update installed platforms, packages and libraries") -@click.option( - "--core-packages", is_flag=True, help="Update only the core packages") -@click.option( - "-c", - "--only-check", - is_flag=True, - help="DEPRECATED. Please use `--dry-run` instead") -@click.option( - "--dry-run", - is_flag=True, - help="Do not update, only check for the new versions") +@click.command("update", + short_help="Update installed platforms, packages and libraries") +@click.option("--core-packages", + is_flag=True, + help="Update only the core packages") +@click.option("-c", + "--only-check", + is_flag=True, + help="DEPRECATED. Please use `--dry-run` instead") +@click.option("--dry-run", + is_flag=True, + help="Do not update, only check for the new versions") @click.pass_context def cli(ctx, core_packages, only_check, dry_run): # cleanup lib search results, cached board and platform lists diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 2c6b367e..91286230 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -26,8 +26,8 @@ from platformio.proc import exec_command, get_pythonexe_path from platformio.project.helpers import get_project_cache_dir -@click.command( - "upgrade", short_help="Upgrade PlatformIO to the latest version") +@click.command("upgrade", + short_help="Upgrade PlatformIO to the latest version") @click.option("--dev", is_flag=True, help="Use development branch") def cli(dev): if not dev and __version__ == get_latest_version(): @@ -60,20 +60,19 @@ def cli(dev): assert r['returncode'] == 0 assert "version" in r['out'] actual_version = r['out'].strip().split("version", 1)[1].strip() - click.secho( - "PlatformIO has been successfully upgraded to %s" % actual_version, - fg="green") + click.secho("PlatformIO has been successfully upgraded to %s" % + actual_version, + fg="green") click.echo("Release notes: ", nl=False) - click.secho( - "https://docs.platformio.org/en/latest/history.html", fg="cyan") + click.secho("https://docs.platformio.org/en/latest/history.html", + fg="cyan") except Exception as e: # pylint: disable=broad-except if not r: raise exception.UpgradeError("\n".join([str(cmd), str(e)])) permission_errors = ("permission denied", "not permitted") if (any(m in r['err'].lower() for m in permission_errors) and not WINDOWS): - click.secho( - """ + click.secho(""" ----------------- Permission denied ----------------- @@ -83,8 +82,8 @@ You need the `sudo` permission to install Python packages. Try WARNING! Don't use `sudo` for the rest PlatformIO commands. """, - fg="yellow", - err=True) + fg="yellow", + err=True) raise exception.ReturnErrorCode(1) raise exception.UpgradeError("\n".join([str(cmd), r['out'], r['err']])) @@ -150,8 +149,7 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - r = requests.get( - "https://pypi.org/pypi/platformio/json", - headers=util.get_request_defheaders()) + r = requests.get("https://pypi.org/pypi/platformio/json", + headers=util.get_request_defheaders()) r.raise_for_status() return r.json()['info']['version'] diff --git a/platformio/downloader.py b/platformio/downloader.py index ab84e518..be5e8888 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -35,11 +35,10 @@ class FileDownloader(object): def __init__(self, url, dest_dir=None): self._request = None # make connection - self._request = requests.get( - url, - stream=True, - headers=util.get_request_defheaders(), - verify=version_info >= (2, 7, 9)) + self._request = requests.get(url, + stream=True, + headers=util.get_request_defheaders(), + verify=version_info >= (2, 7, 9)) if self._request.status_code != 200: raise FDUnrecognizedStatusCode(self._request.status_code, url) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index 3482db28..e420802a 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -26,8 +26,9 @@ from platformio.commands.run import cli as cmd_run from platformio.compat import PY2, WINDOWS, get_file_contents from platformio.proc import where_is_program from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_lib_dir, get_project_libdeps_dir, get_project_src_dir) +from platformio.project.helpers import (get_project_lib_dir, + get_project_libdeps_dir, + get_project_src_dir) class ProjectGenerator(object): @@ -120,9 +121,8 @@ class ProjectGenerator(object): file_name = basename(tpl_path)[:-4] contents = self._render_tpl(tpl_path) - self._merge_contents( - join(dst_dir, file_name), - contents.encode("utf8") if PY2 else contents) + self._merge_contents(join(dst_dir, file_name), + contents.encode("utf8") if PY2 else contents) def _render_tpl(self, tpl_path): return bottle.template(get_file_contents(tpl_path), **self._tplvars) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index ea7c75f2..b1e2d7b9 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -92,12 +92,12 @@ class Upgrader(object): self.to_version = semantic_version.Version.coerce( util.pepver_to_semver(to_version)) - self._upgraders = [(semantic_version.Version("3.0.0-a.1"), - self._upgrade_to_3_0_0), - (semantic_version.Version("3.0.0-b.11"), - self._upgrade_to_3_0_0b11), - (semantic_version.Version("3.5.0-a.2"), - self._update_dev_platforms)] + self._upgraders = [ + (semantic_version.Version("3.0.0-a.1"), self._upgrade_to_3_0_0), + (semantic_version.Version("3.0.0-b.11"), + self._upgrade_to_3_0_0b11), + (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms) + ] def run(self, ctx): if self.from_version > self.to_version: @@ -166,12 +166,11 @@ def after_upgrade(ctx): last_version)) > semantic_version.Version.coerce( util.pepver_to_semver(__version__)): click.secho("*" * terminal_width, fg="yellow") - click.secho( - "Obsolete PIO Core v%s is used (previous was %s)" % (__version__, - last_version), - fg="yellow") - click.secho( - "Please remove multiple PIO Cores from a system:", fg="yellow") + click.secho("Obsolete PIO Core v%s is used (previous was %s)" % + (__version__, last_version), + fg="yellow") + click.secho("Please remove multiple PIO Cores from a system:", + fg="yellow") click.secho( "https://docs.platformio.org/page/faq.html" "#multiple-pio-cores-in-a-system", @@ -188,22 +187,20 @@ def after_upgrade(ctx): u = Upgrader(last_version, __version__) if u.run(ctx): app.set_state_item("last_version", __version__) - click.secho( - "PlatformIO has been successfully upgraded to %s!\n" % - __version__, - fg="green") - telemetry.on_event( - category="Auto", - action="Upgrade", - label="%s > %s" % (last_version, __version__)) + click.secho("PlatformIO has been successfully upgraded to %s!\n" % + __version__, + fg="green") + telemetry.on_event(category="Auto", + action="Upgrade", + label="%s > %s" % (last_version, __version__)) else: raise exception.UpgradeError("Auto upgrading...") click.echo("") # PlatformIO banner click.echo("*" * terminal_width) - click.echo( - "If you like %s, please:" % (click.style("PlatformIO", fg="cyan"))) + click.echo("If you like %s, please:" % + (click.style("PlatformIO", fg="cyan"))) click.echo("- %s us on Twitter to stay up-to-date " "on the latest project news > %s" % (click.style("follow", fg="cyan"), @@ -218,9 +215,9 @@ def after_upgrade(ctx): (click.style("try", fg="cyan"), click.style("https://platformio.org/platformio-ide", fg="cyan"))) if not is_ci(): - click.echo("- %s us with PlatformIO Plus > %s" % (click.style( - "support", fg="cyan"), click.style( - "https://pioplus.com", fg="cyan"))) + click.echo("- %s us with PlatformIO Plus > %s" % + (click.style("support", fg="cyan"), + click.style("https://pioplus.com", fg="cyan"))) click.echo("*" * terminal_width) click.echo("") @@ -250,14 +247,14 @@ def check_platformio_upgrade(): click.echo("") click.echo("*" * terminal_width) - click.secho( - "There is a new version %s of PlatformIO available.\n" - "Please upgrade it via `" % latest_version, - fg="yellow", - nl=False) + click.secho("There is a new version %s of PlatformIO available.\n" + "Please upgrade it via `" % latest_version, + fg="yellow", + nl=False) if getenv("PLATFORMIO_IDE"): - click.secho( - "PlatformIO IDE Menu: Upgrade PlatformIO", fg="cyan", nl=False) + click.secho("PlatformIO IDE Menu: Upgrade PlatformIO", + fg="cyan", + nl=False) click.secho("`.", fg="yellow") elif join("Cellar", "platformio") in util.get_source_dir(): click.secho("brew update && brew upgrade", fg="cyan", nl=False) @@ -268,8 +265,8 @@ def check_platformio_upgrade(): click.secho("pip install -U platformio", fg="cyan", nl=False) click.secho("` command.", fg="yellow") click.secho("Changes: ", fg="yellow", nl=False) - click.secho( - "https://docs.platformio.org/en/latest/history.html", fg="cyan") + click.secho("https://docs.platformio.org/en/latest/history.html", + fg="cyan") click.echo("*" * terminal_width) click.echo("") @@ -305,29 +302,26 @@ def check_internal_updates(ctx, what): click.echo("") click.echo("*" * terminal_width) - click.secho( - "There are the new updates for %s (%s)" % (what, - ", ".join(outdated_items)), - fg="yellow") + click.secho("There are the new updates for %s (%s)" % + (what, ", ".join(outdated_items)), + fg="yellow") if not app.get_setting("auto_update_" + what): click.secho("Please update them via ", fg="yellow", nl=False) - click.secho( - "`platformio %s update`" % - ("lib --global" if what == "libraries" else "platform"), - fg="cyan", - nl=False) + click.secho("`platformio %s update`" % + ("lib --global" if what == "libraries" else "platform"), + fg="cyan", + nl=False) click.secho(" command.\n", fg="yellow") click.secho( "If you want to manually check for the new versions " "without updating, please use ", fg="yellow", nl=False) - click.secho( - "`platformio %s update --dry-run`" % - ("lib --global" if what == "libraries" else "platform"), - fg="cyan", - nl=False) + click.secho("`platformio %s update --dry-run`" % + ("lib --global" if what == "libraries" else "platform"), + fg="cyan", + nl=False) click.secho(" command.", fg="yellow") else: click.secho("Please wait while updating %s ..." % what, fg="yellow") @@ -338,8 +332,9 @@ def check_internal_updates(ctx, what): ctx.invoke(cmd_lib_update, libraries=outdated_items) click.echo() - telemetry.on_event( - category="Auto", action="Update", label=what.title()) + telemetry.on_event(category="Auto", + action="Update", + label=what.title()) click.echo("*" * terminal_width) click.echo("") diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 8e8afaaf..6efc19f9 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -111,8 +111,8 @@ def shutdown_piohome_servers(): port = 8010 while port < 8050: try: - requests.get( - "http://127.0.0.1:%d?__shutdown__=1" % port, timeout=0.01) + requests.get("http://127.0.0.1:%d?__shutdown__=1" % port, + timeout=0.01) except: # pylint: disable=bare-except pass port += 1 diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 9824afcd..1c0d04dd 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -190,14 +190,12 @@ class LibraryManager(BasePkgManager): def get_latest_repo_version(self, name, requirements, silent=False): item = self.max_satisfying_repo_version( - util.get_api_result( - "/lib/info/%d" % self.search_lib_id( - { - "name": name, - "requirements": requirements - }, - silent=silent), - cache_valid="1h")['versions'], requirements) + util.get_api_result("/lib/info/%d" % self.search_lib_id( + { + "name": name, + "requirements": requirements + }, silent=silent), + cache_valid="1h")['versions'], requirements) return item['name'] if item else None def _install_from_piorepo(self, name, requirements): @@ -206,10 +204,9 @@ class LibraryManager(BasePkgManager): if not version: raise exception.UndefinedPackageVersion(requirements or "latest", util.get_systype()) - dl_data = util.get_api_result( - "/lib/download/" + str(name[3:]), - dict(version=version), - cache_valid="30d") + dl_data = util.get_api_result("/lib/download/" + str(name[3:]), + dict(version=version), + cache_valid="30d") assert dl_data return self._install_from_url( @@ -231,8 +228,8 @@ class LibraryManager(BasePkgManager): # looking in PIO Library Registry if not silent: - click.echo("Looking for %s library in registry" % click.style( - filters['name'], fg="cyan")) + click.echo("Looking for %s library in registry" % + click.style(filters['name'], fg="cyan")) query = [] for key in filters: if key not in ("name", "authors", "frameworks", "platforms"): @@ -245,19 +242,19 @@ class LibraryManager(BasePkgManager): (key[:-1] if key.endswith("s") else key, value)) lib_info = None - result = util.get_api_result( - "/v2/lib/search", dict(query=" ".join(query)), cache_valid="1h") + result = util.get_api_result("/v2/lib/search", + dict(query=" ".join(query)), + cache_valid="1h") if result['total'] == 1: lib_info = result['items'][0] elif result['total'] > 1: if silent and not interactive: lib_info = result['items'][0] else: - click.secho( - "Conflict: More than one library has been found " - "by request %s:" % json.dumps(filters), - fg="yellow", - err=True) + click.secho("Conflict: More than one library has been found " + "by request %s:" % json.dumps(filters), + fg="yellow", + err=True) for item in result['items']: commands.lib.print_lib_item(item) @@ -269,10 +266,11 @@ class LibraryManager(BasePkgManager): err=True) lib_info = result['items'][0] else: - deplib_id = click.prompt( - "Please choose library ID", - type=click.Choice( - [str(i['id']) for i in result['items']])) + deplib_id = click.prompt("Please choose library ID", + type=click.Choice([ + str(i['id']) + for i in result['items'] + ])) for item in result['items']: if item['id'] == int(deplib_id): lib_info = item @@ -306,9 +304,8 @@ class LibraryManager(BasePkgManager): continue if key not in manifest: return None - if not util.items_in_list( - util.items_to_list(filters[key]), - util.items_to_list(manifest[key])): + if not util.items_in_list(util.items_to_list(filters[key]), + util.items_to_list(manifest[key])): return None if "authors" in filters: @@ -339,20 +336,20 @@ class LibraryManager(BasePkgManager): force=False): _name, _requirements, _url = self.parse_pkg_uri(name, requirements) if not _url: - name = "id=%d" % self.search_lib_id({ - "name": _name, - "requirements": _requirements - }, - silent=silent, - interactive=interactive) + name = "id=%d" % self.search_lib_id( + { + "name": _name, + "requirements": _requirements + }, + silent=silent, + interactive=interactive) requirements = _requirements - pkg_dir = BasePkgManager.install( - self, - name, - requirements, - silent=silent, - after_update=after_update, - force=force) + pkg_dir = BasePkgManager.install(self, + name, + requirements, + silent=silent, + after_update=after_update, + force=force) if not pkg_dir: return None @@ -376,12 +373,11 @@ class LibraryManager(BasePkgManager): self.INSTALL_HISTORY.append(history_key) if any(s in filters.get("version", "") for s in ("\\", "/")): - self.install( - "{name}={version}".format(**filters), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force) + self.install("{name}={version}".format(**filters), + silent=silent, + after_update=after_update, + interactive=interactive, + force=force) else: try: lib_id = self.search_lib_id(filters, silent, interactive) @@ -391,20 +387,18 @@ class LibraryManager(BasePkgManager): continue if filters.get("version"): - self.install( - lib_id, - filters.get("version"), - silent=silent, - after_update=after_update, - interactive=interactive, - force=force) + self.install(lib_id, + filters.get("version"), + silent=silent, + after_update=after_update, + interactive=interactive, + force=force) else: - self.install( - lib_id, - silent=silent, - after_update=after_update, - interactive=interactive, - force=force) + self.install(lib_id, + silent=silent, + after_update=after_update, + interactive=interactive, + force=force) return pkg_dir diff --git a/platformio/managers/package.py b/platformio/managers/package.py index cd0f0d1a..905fb32f 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -92,8 +92,8 @@ class PkgRepoMixin(object): reqspec = None if requirements: try: - reqspec = self.parse_semver_spec( - requirements, raise_exception=True) + reqspec = self.parse_semver_spec(requirements, + raise_exception=True) except ValueError: pass @@ -430,8 +430,8 @@ class PkgInstallerMixin(object): try: if requirements and not self.parse_semver_spec( requirements, raise_exception=True).match( - self.parse_semver_version( - manifest['version'], raise_exception=True)): + self.parse_semver_version(manifest['version'], + raise_exception=True)): continue elif not best or (self.parse_semver_version( manifest['version'], raise_exception=True) > @@ -648,8 +648,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): if "__src_url" in manifest: try: - vcs = VCSClientFactory.newClient( - pkg_dir, manifest['__src_url'], silent=True) + vcs = VCSClientFactory.newClient(pkg_dir, + manifest['__src_url'], + silent=True) except (AttributeError, exception.PlatformioException): return None if not vcs.can_be_updated: @@ -658,8 +659,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): else: try: latest = self.get_latest_repo_version( - "id=%d" % manifest['id'] - if "id" in manifest else manifest['name'], + "id=%d" % + manifest['id'] if "id" in manifest else manifest['name'], requirements, silent=True) except (exception.PlatformioException, ValueError): @@ -671,10 +672,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): up_to_date = False try: assert "__src_url" not in manifest - up_to_date = (self.parse_semver_version( - manifest['version'], raise_exception=True) >= - self.parse_semver_version( - latest, raise_exception=True)) + up_to_date = (self.parse_semver_version(manifest['version'], + raise_exception=True) >= + self.parse_semver_version(latest, + raise_exception=True)) except (AssertionError, ValueError): up_to_date = latest == manifest['version'] @@ -720,8 +721,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): return package_dir if url: - pkg_dir = self._install_from_url( - name, url, requirements, track=True) + pkg_dir = self._install_from_url(name, + url, + requirements, + track=True) else: pkg_dir = self._install_from_piorepo(name, requirements) @@ -733,10 +736,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): assert manifest if not after_update: - telemetry.on_event( - category=self.__class__.__name__, - action="Install", - label=manifest['name']) + telemetry.on_event(category=self.__class__.__name__, + action="Install", + label=manifest['name']) if not silent: click.secho( @@ -759,14 +761,13 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): pkg_dir = self.get_package_dir(name, requirements, url) if not pkg_dir: - raise exception.UnknownPackage( - "%s @ %s" % (package, requirements or "*")) + raise exception.UnknownPackage("%s @ %s" % + (package, requirements or "*")) manifest = self.load_manifest(pkg_dir) - click.echo( - "Uninstalling %s @ %s: \t" % (click.style( - manifest['name'], fg="cyan"), manifest['version']), - nl=False) + click.echo("Uninstalling %s @ %s: \t" % (click.style( + manifest['name'], fg="cyan"), manifest['version']), + nl=False) if islink(pkg_dir): os.unlink(pkg_dir) @@ -785,10 +786,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): click.echo("[%s]" % click.style("OK", fg="green")) if not after_update: - telemetry.on_event( - category=self.__class__.__name__, - action="Uninstall", - label=manifest['name']) + telemetry.on_event(category=self.__class__.__name__, + action="Uninstall", + label=manifest['name']) return True @@ -799,17 +799,16 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): pkg_dir = self.get_package_dir(*self.parse_pkg_uri(package)) if not pkg_dir: - raise exception.UnknownPackage( - "%s @ %s" % (package, requirements or "*")) + raise exception.UnknownPackage("%s @ %s" % + (package, requirements or "*")) manifest = self.load_manifest(pkg_dir) name = manifest['name'] - click.echo( - "{} {:<40} @ {:<15}".format( - "Checking" if only_check else "Updating", - click.style(manifest['name'], fg="cyan"), manifest['version']), - nl=False) + click.echo("{} {:<40} @ {:<15}".format( + "Checking" if only_check else "Updating", + click.style(manifest['name'], fg="cyan"), manifest['version']), + nl=False) if not util.internet_on(): click.echo("[%s]" % (click.style("Off-line", fg="yellow"))) return None @@ -828,16 +827,15 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): if "__src_url" in manifest: vcs = VCSClientFactory.newClient(pkg_dir, manifest['__src_url']) assert vcs.update() - self._update_src_manifest( - dict(version=vcs.get_current_revision()), vcs.storage_dir) + self._update_src_manifest(dict(version=vcs.get_current_revision()), + vcs.storage_dir) else: self.uninstall(pkg_dir, after_update=True) self.install(name, latest, after_update=True) - telemetry.on_event( - category=self.__class__.__name__, - action="Update", - label=manifest['name']) + telemetry.on_event(category=self.__class__.__name__, + action="Update", + label=manifest['name']) return True diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 81982219..7aa153f8 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -30,9 +30,10 @@ from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, exec_command, get_pythonexe_path) from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( - get_project_boards_dir, get_project_core_dir, get_project_packages_dir, - get_project_platforms_dir) +from platformio.project.helpers import (get_project_boards_dir, + get_project_core_dir, + get_project_packages_dir, + get_project_platforms_dir) try: from urllib.parse import quote @@ -75,8 +76,11 @@ class PlatformManager(BasePkgManager): silent=False, force=False, **_): # pylint: disable=too-many-arguments, arguments-differ - platform_dir = BasePkgManager.install( - self, name, requirements, silent=silent, force=force) + platform_dir = BasePkgManager.install(self, + name, + requirements, + silent=silent, + force=force) p = PlatformFactory.newPlatform(platform_dir) # don't cleanup packages or install them after update @@ -84,12 +88,11 @@ class PlatformManager(BasePkgManager): if after_update: return True - p.install_packages( - with_packages, - without_packages, - skip_default_package, - silent=silent, - force=force) + p.install_packages(with_packages, + without_packages, + skip_default_package, + silent=silent, + force=force) return self.cleanup_packages(list(p.packages)) def uninstall(self, package, requirements=None, after_update=False): @@ -141,8 +144,8 @@ class PlatformManager(BasePkgManager): self.cleanup_packages(list(p.packages)) if missed_pkgs: - p.install_packages( - with_packages=list(missed_pkgs), skip_default_package=True) + p.install_packages(with_packages=list(missed_pkgs), + skip_default_package=True) return True @@ -253,8 +256,8 @@ class PlatformFactory(object): cls.load_module(name, join(platform_dir, "platform.py")), cls.get_clsname(name)) else: - platform_cls = type( - str(cls.get_clsname(name)), (PlatformBase, ), {}) + platform_cls = type(str(cls.get_clsname(name)), (PlatformBase, ), + {}) _instance = platform_cls(join(platform_dir, "platform.json")) assert isinstance(_instance, PlatformBase) @@ -285,8 +288,9 @@ class PlatformPackagesMixin(object): elif (name in with_packages or not (skip_default_package or opts.get("optional", False))): if ":" in version: - self.pm.install( - "%s=%s" % (name, version), silent=silent, force=force) + self.pm.install("%s=%s" % (name, version), + silent=silent, + force=force) else: self.pm.install(name, version, silent=silent, force=force) @@ -402,8 +406,8 @@ class PlatformRunMixin(object): cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) else: cmd.append( - "%s=%s" % (key.upper(), base64.b64encode( - value.encode()).decode())) + "%s=%s" % + (key.upper(), base64.b64encode(value.encode()).decode())) def _write_and_flush(stream, data): stream.write(data) @@ -461,8 +465,8 @@ class PlatformRunMixin(object): """.format(filename=filename, filename_styled=click.style(filename, fg="cyan"), link=click.style( - "https://platformio.org/lib/search?query=header:%s" % quote( - filename, safe=""), + "https://platformio.org/lib/search?query=header:%s" % + quote(filename, safe=""), fg="blue"), dots="*" * (56 + len(filename))) click.echo(banner, err=True) diff --git a/platformio/project/config.py b/platformio/project/config.py index 45e9380c..48a06bf6 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -182,8 +182,8 @@ class ProjectConfig(object): except ConfigParser.Error as e: raise exception.InvalidProjectConf(self.path, str(e)) - option_meta = ProjectOptions.get( - "%s.%s" % (section.split(":", 1)[0], option)) + option_meta = ProjectOptions.get("%s.%s" % + (section.split(":", 1)[0], option)) if not option_meta: return value or default @@ -281,8 +281,8 @@ class ProjectConfig(object): warnings.append( "`%s` configuration option in section [%s] is " "deprecated and will be removed in the next release! " - "Please use `%s` instead" % (option, section, - renamed_options[option])) + "Please use `%s` instead" % + (option, section, renamed_options[option])) # rename on-the-fly self._parser.set(section, renamed_options[option], self._parser.get(section, option)) diff --git a/platformio/project/options.py b/platformio/project/options.py index b4bd705a..fd7afd01 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -46,41 +46,39 @@ ProjectOptions = OrderedDict([ # [platformio] # ConfigPlatformioOption(name="description"), - ConfigPlatformioOption( - name="env_default", - multiple=True, - sysenvvar="PLATFORMIO_ENV_DEFAULT"), + ConfigPlatformioOption(name="env_default", + multiple=True, + sysenvvar="PLATFORMIO_ENV_DEFAULT"), ConfigPlatformioOption(name="extra_configs", multiple=True), # Dirs - ConfigPlatformioOption( - name="core_dir", - oldnames=["home_dir"], - sysenvvar="PLATFORMIO_CORE_DIR"), - ConfigPlatformioOption( - name="globallib_dir", sysenvvar="PLATFORMIO_GLOBALLIB_DIR"), - ConfigPlatformioOption( - name="platforms_dir", sysenvvar="PLATFORMIO_PLATFORMS_DIR"), - ConfigPlatformioOption( - name="packages_dir", sysenvvar="PLATFORMIO_PACKAGES_DIR"), - ConfigPlatformioOption( - name="cache_dir", sysenvvar="PLATFORMIO_CACHE_DIR"), - ConfigPlatformioOption( - name="workspace_dir", sysenvvar="PLATFORMIO_WORKSPACE_DIR"), - ConfigPlatformioOption( - name="build_dir", sysenvvar="PLATFORMIO_BUILD_DIR"), - ConfigPlatformioOption( - name="libdeps_dir", sysenvvar="PLATFORMIO_LIBDEPS_DIR"), + ConfigPlatformioOption(name="core_dir", + oldnames=["home_dir"], + sysenvvar="PLATFORMIO_CORE_DIR"), + ConfigPlatformioOption(name="globallib_dir", + sysenvvar="PLATFORMIO_GLOBALLIB_DIR"), + ConfigPlatformioOption(name="platforms_dir", + sysenvvar="PLATFORMIO_PLATFORMS_DIR"), + ConfigPlatformioOption(name="packages_dir", + sysenvvar="PLATFORMIO_PACKAGES_DIR"), + ConfigPlatformioOption(name="cache_dir", + sysenvvar="PLATFORMIO_CACHE_DIR"), + ConfigPlatformioOption(name="workspace_dir", + sysenvvar="PLATFORMIO_WORKSPACE_DIR"), + ConfigPlatformioOption(name="build_dir", + sysenvvar="PLATFORMIO_BUILD_DIR"), + ConfigPlatformioOption(name="libdeps_dir", + sysenvvar="PLATFORMIO_LIBDEPS_DIR"), ConfigPlatformioOption(name="lib_dir", sysenvvar="PLATFORMIO_LIB_DIR"), - ConfigPlatformioOption( - name="include_dir", sysenvvar="PLATFORMIO_INCLUDE_DIR"), + ConfigPlatformioOption(name="include_dir", + sysenvvar="PLATFORMIO_INCLUDE_DIR"), ConfigPlatformioOption(name="src_dir", sysenvvar="PLATFORMIO_SRC_DIR"), - ConfigPlatformioOption( - name="test_dir", sysenvvar="PLATFORMIO_TEST_DIR"), - ConfigPlatformioOption( - name="boards_dir", sysenvvar="PLATFORMIO_BOARDS_DIR"), - ConfigPlatformioOption( - name="data_dir", sysenvvar="PLATFORMIO_DATA_DIR"), + ConfigPlatformioOption(name="test_dir", + sysenvvar="PLATFORMIO_TEST_DIR"), + ConfigPlatformioOption(name="boards_dir", + sysenvvar="PLATFORMIO_BOARDS_DIR"), + ConfigPlatformioOption(name="data_dir", + sysenvvar="PLATFORMIO_DATA_DIR"), # # [env] @@ -94,59 +92,49 @@ ProjectOptions = OrderedDict([ # Board ConfigEnvOption(name="board", buildenvvar="BOARD"), - ConfigEnvOption( - name="board_build.mcu", - oldnames=["board_mcu"], - buildenvvar="BOARD_MCU"), - ConfigEnvOption( - name="board_build.f_cpu", - oldnames=["board_f_cpu"], - buildenvvar="BOARD_F_CPU"), - ConfigEnvOption( - name="board_build.f_flash", - oldnames=["board_f_flash"], - buildenvvar="BOARD_F_FLASH"), - ConfigEnvOption( - name="board_build.flash_mode", - oldnames=["board_flash_mode"], - buildenvvar="BOARD_FLASH_MODE"), + ConfigEnvOption(name="board_build.mcu", + oldnames=["board_mcu"], + buildenvvar="BOARD_MCU"), + ConfigEnvOption(name="board_build.f_cpu", + oldnames=["board_f_cpu"], + buildenvvar="BOARD_F_CPU"), + ConfigEnvOption(name="board_build.f_flash", + oldnames=["board_f_flash"], + buildenvvar="BOARD_F_FLASH"), + ConfigEnvOption(name="board_build.flash_mode", + oldnames=["board_flash_mode"], + buildenvvar="BOARD_FLASH_MODE"), # Build - ConfigEnvOption( - name="build_flags", - multiple=True, - sysenvvar="PLATFORMIO_BUILD_FLAGS", - buildenvvar="BUILD_FLAGS"), - ConfigEnvOption( - name="src_build_flags", - multiple=True, - sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS", - buildenvvar="SRC_BUILD_FLAGS"), - ConfigEnvOption( - name="build_unflags", - multiple=True, - sysenvvar="PLATFORMIO_BUILD_UNFLAGS", - buildenvvar="BUILD_UNFLAGS"), - ConfigEnvOption( - name="src_filter", - multiple=True, - sysenvvar="PLATFORMIO_SRC_FILTER", - buildenvvar="SRC_FILTER"), + ConfigEnvOption(name="build_flags", + multiple=True, + sysenvvar="PLATFORMIO_BUILD_FLAGS", + buildenvvar="BUILD_FLAGS"), + ConfigEnvOption(name="src_build_flags", + multiple=True, + sysenvvar="PLATFORMIO_SRC_BUILD_FLAGS", + buildenvvar="SRC_BUILD_FLAGS"), + ConfigEnvOption(name="build_unflags", + multiple=True, + sysenvvar="PLATFORMIO_BUILD_UNFLAGS", + buildenvvar="BUILD_UNFLAGS"), + ConfigEnvOption(name="src_filter", + multiple=True, + sysenvvar="PLATFORMIO_SRC_FILTER", + buildenvvar="SRC_FILTER"), # Upload - ConfigEnvOption( - name="upload_port", - sysenvvar="PLATFORMIO_UPLOAD_PORT", - buildenvvar="UPLOAD_PORT"), + ConfigEnvOption(name="upload_port", + sysenvvar="PLATFORMIO_UPLOAD_PORT", + buildenvvar="UPLOAD_PORT"), ConfigEnvOption(name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL"), ConfigEnvOption(name="upload_speed", buildenvvar="UPLOAD_SPEED"), - ConfigEnvOption( - name="upload_flags", - multiple=True, - sysenvvar="PLATFORMIO_UPLOAD_FLAGS", - buildenvvar="UPLOAD_FLAGS"), - ConfigEnvOption( - name="upload_resetmethod", buildenvvar="UPLOAD_RESETMETHOD"), + ConfigEnvOption(name="upload_flags", + multiple=True, + sysenvvar="PLATFORMIO_UPLOAD_FLAGS", + buildenvvar="UPLOAD_FLAGS"), + ConfigEnvOption(name="upload_resetmethod", + buildenvvar="UPLOAD_RESETMETHOD"), # Monitor ConfigEnvOption(name="monitor_port"), @@ -156,15 +144,13 @@ ProjectOptions = OrderedDict([ ConfigEnvOption(name="monitor_flags", multiple=True), # Library - ConfigEnvOption( - name="lib_deps", - oldnames=["lib_use", "lib_force", "lib_install"], - multiple=True), + ConfigEnvOption(name="lib_deps", + oldnames=["lib_use", "lib_force", "lib_install"], + multiple=True), ConfigEnvOption(name="lib_ignore", multiple=True), - ConfigEnvOption( - name="lib_extra_dirs", - multiple=True, - sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"), + ConfigEnvOption(name="lib_extra_dirs", + multiple=True, + sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"), ConfigEnvOption(name="lib_ldf_mode"), ConfigEnvOption(name="lib_compat_mode"), ConfigEnvOption(name="lib_archive", type=bool), @@ -189,10 +175,9 @@ ProjectOptions = OrderedDict([ ConfigEnvOption(name="debug_svd_path"), # Other - ConfigEnvOption( - name="extra_scripts", - oldnames=["extra_script"], - multiple=True, - sysenvvar="PLATFORMIO_EXTRA_SCRIPTS") + ConfigEnvOption(name="extra_scripts", + oldnames=["extra_script"], + multiple=True, + sysenvvar="PLATFORMIO_EXTRA_SCRIPTS") ] ]) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 071db390..f697a241 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -338,11 +338,10 @@ def on_exception(e): def _cleanup_description(text): text = text.replace("Traceback (most recent call last):", "") - text = re.sub( - r'File "([^"]+)"', - lambda m: join(*m.group(1).split(sep)[-2:]), - text, - flags=re.M) + text = re.sub(r'File "([^"]+)"', + lambda m: join(*m.group(1).split(sep)[-2:]), + text, + flags=re.M) text = re.sub(r"\s+", " ", text, flags=re.M) return text.strip() diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 58132716..271b4911 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -74,9 +74,8 @@ class ZIPArchive(ArchiveBase): @staticmethod def preserve_mtime(item, dest_dir): - util.change_filemtime( - join(dest_dir, item.filename), - mktime(tuple(item.date_time) + tuple([0, 0, 0]))) + util.change_filemtime(join(dest_dir, item.filename), + mktime(tuple(item.date_time) + tuple([0, 0, 0]))) def get_items(self): return self._afo.infolist() diff --git a/platformio/util.py b/platformio/util.py index c4dbe537..a7657526 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -176,8 +176,8 @@ def get_logical_devices(): if WINDOWS: try: result = exec_command( - ["wmic", "logicaldisk", "get", "name,VolumeName"]).get( - "out", "") + ["wmic", "logicaldisk", "get", + "name,VolumeName"]).get("out", "") devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?") for line in result.split("\n"): match = devicenamere.match(line.strip()) @@ -321,20 +321,18 @@ def _get_api_result( try: if data: - r = _api_request_session().post( - url, - params=params, - data=data, - headers=headers, - auth=auth, - verify=verify_ssl) + r = _api_request_session().post(url, + params=params, + data=data, + headers=headers, + auth=auth, + verify=verify_ssl) else: - r = _api_request_session().get( - url, - params=params, - headers=headers, - auth=auth, - verify=verify_ssl) + r = _api_request_session().get(url, + params=params, + headers=headers, + auth=auth, + verify=verify_ssl) result = r.json() r.raise_for_status() return r.text @@ -345,8 +343,8 @@ def _get_api_result( raise exception.APIRequestError(result['errors'][0]['title']) raise exception.APIRequestError(e) except ValueError: - raise exception.APIRequestError( - "Invalid response: %s" % r.text.encode("utf-8")) + raise exception.APIRequestError("Invalid response: %s" % + r.text.encode("utf-8")) finally: if r: r.close() @@ -403,11 +401,12 @@ def _internet_on(): for ip in PING_INTERNET_IPS: try: if os.getenv("HTTP_PROXY", os.getenv("HTTPS_PROXY")): - requests.get( - "http://%s" % ip, allow_redirects=False, timeout=timeout) + requests.get("http://%s" % ip, + allow_redirects=False, + timeout=timeout) else: - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((ip, - 80)) + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect( + (ip, 80)) return True except: # pylint: disable=bare-except pass @@ -524,9 +523,9 @@ def rmtree_(path): os.chmod(name, stat.S_IWRITE) os.remove(name) except Exception as e: # pylint: disable=broad-except - click.secho( - "%s \nPlease manually remove the file `%s`" % (str(e), name), - fg="red", - err=True) + click.secho("%s \nPlease manually remove the file `%s`" % + (str(e), name), + fg="red", + err=True) return rmtree(path, onerror=_onerror) diff --git a/platformio/vcsclient.py b/platformio/vcsclient.py index 222ba0e5..51f454d0 100644 --- a/platformio/vcsclient.py +++ b/platformio/vcsclient.py @@ -42,10 +42,11 @@ class VCSClientFactory(object): if "#" in remote_url: remote_url, tag = remote_url.rsplit("#", 1) if not type_: - raise PlatformioException( - "VCS: Unknown repository type %s" % remote_url) - obj = getattr(modules[__name__], "%sClient" % type_.title())( - src_dir, remote_url, tag, silent) + raise PlatformioException("VCS: Unknown repository type %s" % + remote_url) + obj = getattr(modules[__name__], + "%sClient" % type_.title())(src_dir, remote_url, tag, + silent) assert isinstance(obj, VCSClientBase) return obj @@ -102,8 +103,8 @@ class VCSClientBase(object): check_call(args, **kwargs) return True except CalledProcessError as e: - raise PlatformioException( - "VCS: Could not process command %s" % e.cmd) + raise PlatformioException("VCS: Could not process command %s" % + e.cmd) def get_cmd_output(self, args, **kwargs): args = [self.command] + args From 5f9fd9260eaccc691ca9e6b86b9e4857d380de78 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 31 May 2019 14:45:01 +0300 Subject: [PATCH 206/333] New project configuration parser with a strict options typing --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/debug/helpers.py | 6 ----- platformio/exception.py | 5 ++++ platformio/project/config.py | 25 +++++++++++--------- platformio/project/options.py | 35 ++++++++++++++++++---------- 6 files changed, 44 insertions(+), 30 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e3404124..7004f29e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ PlatformIO 4.0 * **Project Configuration** + - New project configuration parser with a strict options typing (`API `__) - Unified workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 `_) - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` - Share common (global) options between project environments using `[env] `__ section (`issue #1643 `_) diff --git a/docs b/docs index e9107b7d..683b3459 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e9107b7dc8280bbb25b79ca61f71815986869aa6 +Subproject commit 683b3459e2b558a36dd0e0d3c46a38926f85fa94 diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 88b4f64a..9b74f190 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -88,12 +88,6 @@ def validate_debug_options(cmd_ctx, env_options): "executable": None, "arguments": env_options.get("debug_server") } - if not isinstance(server_options['arguments'], list): - server_options['arguments'] = server_options['arguments'].split( - "\n") - server_options['arguments'] = [ - arg.strip() for arg in server_options['arguments'] if arg.strip() - ] server_options['executable'] = server_options['arguments'][0] server_options['arguments'] = server_options['arguments'][1:] elif "server" in tool_settings: diff --git a/platformio/exception.py b/platformio/exception.py index 343dc03f..09d7d2f9 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -179,6 +179,11 @@ class UnknownEnvNames(PlatformIOProjectException): MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" +class ProjectOptionValueError(PlatformIOProjectException): + + MESSAGE = "{0} for option `{1}` in section [{2}]" + + # # Library # diff --git a/platformio/project/config.py b/platformio/project/config.py index 48a06bf6..0f6326a7 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -43,6 +43,7 @@ CONFIG_HEADER = """;PlatformIO Project Configuration File class ProjectConfig(object): + INLINE_COMMENT_RE = re.compile(r"\s+;.*$") VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") expand_interpolations = True @@ -55,14 +56,15 @@ class ProjectConfig(object): result = [] if not items: return result - inline_comment_re = re.compile(r"\s+;.*$") - for item in items.split("\n" if "\n" in items else ", "): + if not isinstance(items, (list, tuple)): + items = items.split("\n" if "\n" in items else ", ") + for item in items: item = item.strip() # comment if not item or item.startswith((";", "#")): continue if ";" in item: - item = inline_comment_re.sub("", item).strip() + item = ProjectConfig.INLINE_COMMENT_RE.sub("", item).strip() result.append(item) return result @@ -207,20 +209,21 @@ class ProjectConfig(object): if value is None: return default - return self._covert_value(value, option_meta.type) + try: + return self._covert_value(value, option_meta.type) + except click.BadParameter as e: + raise exception.ProjectOptionValueError(e.format_message(), option, + section) @staticmethod def _covert_value(value, to_type): items = value if not isinstance(value, (list, tuple)): items = [value] - for i, v in enumerate(items): - if to_type == bool: - items[i] = v in ("1", "true", "yes", "y") - elif to_type == int: - items[i] = int(v) - elif to_type == float: - items[i] = float(v) + items = [ + to_type(item) if isinstance(to_type, click.ParamType) else item + for item in items + ] return items if isinstance(value, (list, tuple)) else items[0] def envs(self): diff --git a/platformio/project/options.py b/platformio/project/options.py index fd7afd01..e3e3c4ca 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -16,6 +16,8 @@ from collections import OrderedDict, namedtuple +import click + ConfigOptionClass = namedtuple("ConfigOption", [ "scope", "name", "type", "multiple", "sysenvvar", "buildenvvar", "oldnames" ]) @@ -128,7 +130,8 @@ ProjectOptions = OrderedDict([ sysenvvar="PLATFORMIO_UPLOAD_PORT", buildenvvar="UPLOAD_PORT"), ConfigEnvOption(name="upload_protocol", buildenvvar="UPLOAD_PROTOCOL"), - ConfigEnvOption(name="upload_speed", buildenvvar="UPLOAD_SPEED"), + ConfigEnvOption( + name="upload_speed", type=click.INT, buildenvvar="UPLOAD_SPEED"), ConfigEnvOption(name="upload_flags", multiple=True, sysenvvar="PLATFORMIO_UPLOAD_FLAGS", @@ -139,8 +142,8 @@ ProjectOptions = OrderedDict([ # Monitor ConfigEnvOption(name="monitor_port"), ConfigEnvOption(name="monitor_speed", oldnames=["monitor_baud"]), - ConfigEnvOption(name="monitor_rts"), - ConfigEnvOption(name="monitor_dtr"), + ConfigEnvOption(name="monitor_rts", type=click.IntRange(0, 1)), + ConfigEnvOption(name="monitor_dtr", type=click.IntRange(0, 1)), ConfigEnvOption(name="monitor_flags", multiple=True), # Library @@ -150,18 +153,23 @@ ProjectOptions = OrderedDict([ ConfigEnvOption(name="lib_ignore", multiple=True), ConfigEnvOption(name="lib_extra_dirs", multiple=True, - sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"), - ConfigEnvOption(name="lib_ldf_mode"), - ConfigEnvOption(name="lib_compat_mode"), - ConfigEnvOption(name="lib_archive", type=bool), + sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS", + type=click.Path( + exists=True, file_okay=False, dir_okay=True)), + ConfigEnvOption(name="lib_ldf_mode", + type=click.Choice( + ["off", "chain", "deep", "chain+", "deep+"])), + ConfigEnvOption(name="lib_compat_mode", + type=click.Choice(["off", "soft", "strict"])), + ConfigEnvOption(name="lib_archive", type=click.BOOL), # Test ConfigEnvOption(name="test_filter", multiple=True), ConfigEnvOption(name="test_ignore", multiple=True), ConfigEnvOption(name="test_port"), - ConfigEnvOption(name="test_speed"), + ConfigEnvOption(name="test_speed", type=click.INT), ConfigEnvOption(name="test_transport"), - ConfigEnvOption(name="test_build_project_src"), + ConfigEnvOption(name="test_build_project_src", type=click.BOOL), # Debug ConfigEnvOption(name="debug_tool"), @@ -169,10 +177,13 @@ ProjectOptions = OrderedDict([ ConfigEnvOption(name="debug_init_cmds", multiple=True), ConfigEnvOption(name="debug_extra_cmds", multiple=True), ConfigEnvOption(name="debug_load_cmd"), - ConfigEnvOption(name="debug_load_mode"), - ConfigEnvOption(name="debug_server"), + ConfigEnvOption(name="debug_load_mode", + type=click.Choice(["always", "modified", "manual"])), + ConfigEnvOption(name="debug_server", multiple=True), ConfigEnvOption(name="debug_port"), - ConfigEnvOption(name="debug_svd_path"), + ConfigEnvOption(name="debug_svd_path", + type=click.Path( + exists=True, file_okay=True, dir_okay=False)), # Other ConfigEnvOption(name="extra_scripts", From bae21f1cdd336d56325227b241a26dfbb37dd827 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 31 May 2019 14:45:48 +0300 Subject: [PATCH 207/333] Allow to pass multiple load commands to PIO Unified Debugger --- platformio/commands/debug/client.py | 2 +- platformio/commands/debug/command.py | 6 +++--- platformio/commands/debug/helpers.py | 27 +++++++++++++++------------ platformio/commands/debug/initcfgs.py | 10 +++++----- platformio/project/options.py | 4 +++- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 2631621e..e7945284 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -73,7 +73,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes "DEBUG_PORT": self.debug_options['port'], "UPLOAD_PROTOCOL": self.debug_options['upload_protocol'], "INIT_BREAK": self.debug_options['init_break'] or "", - "LOAD_CMD": self.debug_options['load_cmd'] or "", + "LOAD_CMDS": "\n".join(self.debug_options['load_cmds'] or []), } self._debug_server.spawn(patterns) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 129383d8..891363f1 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -93,11 +93,11 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, raise exception.PlatformioException("\n".join( [result['out'], result['err']])) - debug_options['load_cmd'] = helpers.configure_esp32_load_cmd( + debug_options['load_cmds'] = helpers.configure_esp32_load_cmds( debug_options, configuration) rebuild_prog = False - preload = debug_options['load_cmd'] == "preload" + preload = debug_options['load_cmds'] == ["preload"] load_mode = debug_options['load_mode'] if load_mode == "always": rebuild_prog = ( @@ -112,7 +112,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, if preload or (not rebuild_prog and load_mode != "always"): # don't load firmware through debug server - debug_options['load_cmd'] = None + debug_options['load_cmds'] = [] if rebuild_prog: if helpers.is_mi_mode(__unprocessed): diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 9b74f190..2d1f5205 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -26,6 +26,7 @@ from platformio.commands.platform import \ platform_install as cmd_platform_install from platformio.commands.run import cli as cmd_run from platformio.managers.platform import PlatformFactory +from platformio.project.config import ProjectConfig class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods @@ -53,12 +54,11 @@ def get_default_debug_env(config): def validate_debug_options(cmd_ctx, env_options): - def _cleanup_cmds(cmds): - if not cmds: - return [] - if not isinstance(cmds, list): - cmds = cmds.split("\n") - return [c.strip() for c in cmds if c.strip()] + def _cleanup_cmds(items): + items = ProjectConfig.parse_multi_values(items) + return [ + "$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items + ] try: platform = PlatformFactory.newPlatform(env_options['platform']) @@ -115,8 +115,11 @@ def validate_debug_options(cmd_ctx, env_options): upload_protocol=env_options.get( "upload_protocol", board_config.get("upload", {}).get("protocol")), - load_cmd=env_options.get("debug_load_cmd", - tool_settings.get("load_cmd", "load")), + load_cmds=_cleanup_cmds( + env_options.get( + "debug_load_cmds", + tool_settings.get("load_cmds", + tool_settings.get("load_cmd", "load")))), load_mode=env_options.get("debug_load_mode", tool_settings.get("load_mode", "always")), init_break=env_options.get( @@ -173,9 +176,9 @@ def load_configuration(ctx, project_dir, env_name): return None -def configure_esp32_load_cmd(debug_options, configuration): +def configure_esp32_load_cmds(debug_options, configuration): ignore_conds = [ - debug_options['load_cmd'] != "load", + debug_options['load_cmds'] != ["load"], "xtensa-esp32" not in configuration.get("cc_path", ""), not configuration.get("flash_extra_images"), not all([ isfile(item['path']) @@ -183,7 +186,7 @@ def configure_esp32_load_cmd(debug_options, configuration): ]) ] if any(ignore_conds): - return debug_options['load_cmd'] + return debug_options['load_cmds'] mon_cmds = [ 'monitor program_esp32 "{{{path}}}" {offset} verify'.format( @@ -192,7 +195,7 @@ def configure_esp32_load_cmd(debug_options, configuration): ] mon_cmds.append('monitor program_esp32 "{%s.bin}" 0x10000 verify' % escape_path(configuration['prog_path'][:-4])) - return "\n".join(mon_cmds) + return mon_cmds def has_debug_symbols(prog_path): diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index 62f36c0d..658418ae 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -24,7 +24,7 @@ end target extended-remote $DEBUG_PORT $INIT_BREAK pio_reset_halt_target -$LOAD_CMD +$LOAD_CMDS monitor init pio_reset_halt_target """ @@ -42,7 +42,7 @@ end target extended-remote $DEBUG_PORT $INIT_BREAK pio_reset_halt_target -$LOAD_CMD +$LOAD_CMDS pio_reset_halt_target """ @@ -59,7 +59,7 @@ end target extended-remote $DEBUG_PORT $INIT_BREAK pio_reset_halt_target -$LOAD_CMD +$LOAD_CMDS pio_reset_halt_target """ @@ -83,7 +83,7 @@ monitor swdp_scan attach 1 set mem inaccessible-by-default off $INIT_BREAK -$LOAD_CMD +$LOAD_CMDS set language c set *0xE000ED0C = 0x05FA0004 @@ -104,6 +104,6 @@ end target extended-remote $DEBUG_PORT $INIT_BREAK monitor erase -$LOAD_CMD +$LOAD_CMDS pio_reset_halt_target """ diff --git a/platformio/project/options.py b/platformio/project/options.py index e3e3c4ca..448b9188 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -176,7 +176,9 @@ ProjectOptions = OrderedDict([ ConfigEnvOption(name="debug_init_break"), ConfigEnvOption(name="debug_init_cmds", multiple=True), ConfigEnvOption(name="debug_extra_cmds", multiple=True), - ConfigEnvOption(name="debug_load_cmd"), + ConfigEnvOption(name="debug_load_cmds", + oldnames=["debug_load_cmd"], + multiple=True), ConfigEnvOption(name="debug_load_mode", type=click.Choice(["always", "modified", "manual"])), ConfigEnvOption(name="debug_server", multiple=True), From 33d16bfcf045832dad6341e68357e98304d39aad Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 31 May 2019 15:47:25 +0300 Subject: [PATCH 208/333] Use named context meta vars for unit testing --- platformio/builder/main.py | 2 +- platformio/builder/tools/piomisc.py | 4 ++-- platformio/commands/run/processor.py | 11 ++++++++--- platformio/commands/test/processor.py | 9 ++++++--- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index a6d71fb6..60dacf3b 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -43,7 +43,7 @@ clivars.AddVariables( ("BUILD_SCRIPT",), ("PROJECT_CONFIG",), ("PIOENV",), - ("PIOTEST",), + ("PIOTEST_RUNNING_NAME",), ("UPLOAD_PORT",) ) # yapf: disable diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index cf538812..a7fef99a 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -314,8 +314,8 @@ def ProcessTest(env): env.Prepend(LIBS=[unitylib]) src_filter = ["+<*.cpp>", "+<*.c>"] - if "PIOTEST" in env: - src_filter.append("+<%s%s>" % (env['PIOTEST'], sep)) + if "PIOTEST_RUNNING_NAME" in env: + src_filter.append("+<%s%s>" % (env['PIOTEST_RUNNING_NAME'], sep)) env.Replace(PIOTEST_SRC_FILTER=src_filter) diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index f507780a..66e65637 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -20,6 +20,8 @@ from platformio import exception, telemetry from platformio.commands.platform import \ platform_install as cmd_platform_install from platformio.commands.run.helpers import _autoinstall_libdeps, print_header +from platformio.commands.test.processor import (CTX_META_TEST_IS_RUNNING, + CTX_META_TEST_RUNNING_NAME) from platformio.managers.platform import PlatformFactory # pylint: disable=too-many-instance-attributes @@ -62,7 +64,7 @@ class EnvironmentProcessor(object): if self.silent and not is_error: return True - if is_error or "piotest_processor" not in self.cmd_ctx.meta: + if is_error or CTX_META_TEST_IS_RUNNING not in self.cmd_ctx.meta: print_header( "[%s] Took %.2f seconds" % ((click.style("ERROR", fg="red", bold=True) if @@ -74,8 +76,11 @@ class EnvironmentProcessor(object): def get_build_variables(self): variables = {"pioenv": self.name, "project_config": self.config.path} - if "piotest" in self.cmd_ctx.meta: - variables['piotest'] = self.cmd_ctx.meta['piotest'] + + if CTX_META_TEST_RUNNING_NAME in self.cmd_ctx.meta: + variables['piotest_running_name'] = self.cmd_ctx.meta[ + CTX_META_TEST_RUNNING_NAME] + if self.upload_port: # override upload port with a custom from CLI variables['upload_port'] = self.upload_port diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 973a3853..d3029a34 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -20,7 +20,6 @@ from string import Template import click from platformio import exception -from platformio.commands.run import cli as cmd_run from platformio.commands.run.helpers import print_header from platformio.project.helpers import get_project_test_dir @@ -75,6 +74,9 @@ TRANSPORT_OPTIONS = { } } +CTX_META_TEST_IS_RUNNING = __name__ + ".test_running" +CTX_META_TEST_RUNNING_NAME = __name__ + ".test_running_name" + class TestProcessorBase(object): @@ -82,7 +84,7 @@ class TestProcessorBase(object): def __init__(self, cmd_ctx, testname, envname, options): self.cmd_ctx = cmd_ctx - self.cmd_ctx.meta['piotest_processor'] = True # FIXME + self.cmd_ctx.meta[CTX_META_TEST_IS_RUNNING] = True self.test_name = testname self.options = options self.env_name = envname @@ -119,12 +121,13 @@ class TestProcessorBase(object): self._outputcpp_generated = True if self.test_name != "*": - self.cmd_ctx.meta['piotest'] = self.test_name # FIXME + self.cmd_ctx.meta[CTX_META_TEST_RUNNING_NAME] = self.test_name if not self.options['verbose']: click.echo("Please wait...") try: + from platformio.commands.run import cli as cmd_run return self.cmd_ctx.invoke(cmd_run, project_dir=self.options['project_dir'], upload_port=self.options['upload_port'], From 5011e47709af6929a3fb3752cbee795d26f7e3e7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 31 May 2019 21:18:37 +0300 Subject: [PATCH 209/333] Added support for "shared_dir" --- HISTORY.rst | 4 ++++ docs | 2 +- platformio/project/helpers.py | 5 +++++ platformio/project/options.py | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7004f29e..29fe6dac 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -41,6 +41,10 @@ PlatformIO 4.0 - Fixed an issue with hardcoded C standard version when generating project for CLion IDE (`issue #2527 `_) - Fixed an issue with Project Generator when include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) +* **PIO Remote** + + - Added support for `shared_dir `__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a remote machine + * **Miscellaneous** - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead diff --git a/docs b/docs index 683b3459..c9023fb4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 683b3459e2b558a36dd0e0d3c46a38926f85fa94 +Subproject commit c9023fb4dcaa44630c6aad854e47b2720ffd3022 diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 6106b0a1..c138cd8e 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -157,6 +157,11 @@ def get_project_data_dir(): "data")) +def get_project_shared_dir(): + return get_project_optional_dir("shared_dir", + join(get_project_dir(), "shared")) + + def calculate_project_hash(): check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") chunks = [__version__] diff --git a/platformio/project/options.py b/platformio/project/options.py index 448b9188..fbebff03 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -81,6 +81,8 @@ ProjectOptions = OrderedDict([ sysenvvar="PLATFORMIO_BOARDS_DIR"), ConfigPlatformioOption(name="data_dir", sysenvvar="PLATFORMIO_DATA_DIR"), + ConfigPlatformioOption(name="shared_dir", + sysenvvar="PLATFORMIO_SHARED_DIR"), # # [env] From d25f1ddc213e56bb666a43d133ff643d26f4e28f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 31 May 2019 21:47:50 +0300 Subject: [PATCH 210/333] Add project folder prefix to $PROJECT_HASH --- platformio/project/helpers.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index c138cd8e..bf63f8b0 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -15,8 +15,8 @@ import os from hashlib import sha1 from os import walk -from os.path import (dirname, expanduser, isdir, isfile, join, realpath, - splitdrive) +from os.path import (basename, dirname, expanduser, isdir, isfile, join, + realpath, splitdrive) from platformio import __version__ from platformio.compat import PY2, WINDOWS @@ -53,9 +53,10 @@ def get_project_optional_dir(name, default=None): if "$PROJECT_HASH" in optional_dir: optional_dir = optional_dir.replace( - "$PROJECT_HASH", - sha1(project_dir if PY2 else project_dir.encode()).hexdigest() - [:10]) + "$PROJECT_HASH", "%s-%s" % + (basename(project_dir), + sha1(project_dir if PY2 else project_dir.encode()).hexdigest() + [:10])) if optional_dir.startswith("~"): optional_dir = expanduser(optional_dir) From c0380744893323a622d9e0f494e9d2dba82f94db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Jun 2019 14:36:07 +0300 Subject: [PATCH 211/333] Override default development platform upload command with a custom // Resolve #2599 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/main.py | 3 +++ platformio/project/config.py | 2 ++ platformio/project/options.py | 1 + 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 29fe6dac..225b42bd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ PlatformIO 4.0 - Custom project ``***_dir`` options declared in `platformio `__ section have higher priority than `Environment variables `__ - Added support for Unix shell-style wildcards for `monitor_port `__ option (`issue #2541 `_) - Added new `monitor_flags `__ option which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) + - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) * **Library Management** diff --git a/docs b/docs index c9023fb4..c030b967 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c9023fb4dcaa44630c6aad854e47b2720ffd3022 +Subproject commit c030b967a5fa6d2e234f536d99ae2fe0b4d1a88d diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 60dacf3b..9c9da5f6 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -60,6 +60,7 @@ DEFAULT_ENV_OPTIONS = dict( UNIX_TIME=int(time()), PROJECT_DIR=project_helpers.get_project_dir(), PROJECTCORE_DIR=project_helpers.get_project_core_dir(), + PROJECTPACKAGES_DIR=project_helpers.get_project_packages_dir(), PROJECTWORKSPACE_DIR=project_helpers.get_project_workspace_dir(), PROJECTLIBDEPS_DIR=project_helpers.get_project_libdeps_dir(), PROJECTINCLUDE_DIR=project_helpers.get_project_include_dir(), @@ -118,6 +119,8 @@ env.SConscript("$BUILD_SCRIPT") if "UPLOAD_FLAGS" in env: env.Prepend(UPLOADERFLAGS=["$UPLOAD_FLAGS"]) +if env.GetProjectOption("upload_command"): + env.Replace(UPLOADCMD=env.GetProjectOption("upload_command")) for item in env.GetExtraScripts("post"): env.SConscript(item, exports="env") diff --git a/platformio/project/config.py b/platformio/project/config.py index 0f6326a7..f9b75fa0 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -278,6 +278,8 @@ class ProjectConfig(object): for name in option.oldnames}) for section in self._parser.sections(): + if not section.startswith("env:") or section != "platformio": + continue for option in self._parser.options(section): # obsolete if option in renamed_options: diff --git a/platformio/project/options.py b/platformio/project/options.py index fbebff03..1c5ff5e9 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -140,6 +140,7 @@ ProjectOptions = OrderedDict([ buildenvvar="UPLOAD_FLAGS"), ConfigEnvOption(name="upload_resetmethod", buildenvvar="UPLOAD_RESETMETHOD"), + ConfigEnvOption(name="upload_command", buildenvvar="UPLOADCMD"), # Monitor ConfigEnvOption(name="monitor_port"), From 6186b425d4ecd89a79c76c9d5d0e86e22c11a4ec Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Jun 2019 15:38:55 +0300 Subject: [PATCH 212/333] Typo fix --- platformio/project/config.py | 2 +- tests/test_projectconf.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index f9b75fa0..c9ba6b14 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -278,7 +278,7 @@ class ProjectConfig(object): for name in option.oldnames}) for section in self._parser.sections(): - if not section.startswith("env:") or section != "platformio": + if not section.startswith("env:") and section != "platformio": continue for option in self._parser.options(section): # obsolete diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 8717e8da..eeafb54e 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -16,6 +16,7 @@ import os import pytest +from platformio.exception import UnknownEnvNames from platformio.project.config import ConfigParser, ProjectConfig BASE_CONFIG = """ @@ -46,6 +47,7 @@ build_flags = ${custom.debug_flags} ${custom.extra_flags} EXTRA_ENVS_CONFIG = """ [env:extra_1] build_flags = ${custom.lib_flags} ${custom.debug_flags} +lib_install = 574 [env:extra_2] build_flags = ${custom.debug_flags} ${custom.extra_flags} @@ -73,6 +75,10 @@ def test_real_config(tmpdir): config = ProjectConfig(tmpdir.join("platformio.ini").strpath) assert config + len(config.validate(["extra_2", "base"], silent=True)) == 1 + with pytest.raises(UnknownEnvNames): + config.validate(["non-existing-env"]) + # unknown section with pytest.raises(ConfigParser.NoSectionError): config.getraw("unknown_section", "unknown_option") @@ -100,6 +106,7 @@ def test_real_config(tmpdir): # has_option assert config.has_option("env:base", "monitor_speed") assert not config.has_option("custom", "monitor_speed") + assert not config.has_option("env:extra_1", "lib_install") # sysenv assert config.get("custom", "extra_flags") is None @@ -117,7 +124,7 @@ def test_real_config(tmpdir): assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" # getraw - assert config.getraw("env:extra_1", "lib_deps") == "\nLib1\nLib2" + assert config.getraw("env:extra_1", "lib_deps") == "574" assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1" # get @@ -141,8 +148,8 @@ def test_real_config(tmpdir): ] # yapf: disable assert config.items(env="extra_1") == [ ("build_flags", ["-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), + ("lib_deps", ["574"]), ("monitor_speed", "115200"), - ("lib_deps", ["Lib1", "Lib2"]), ("lib_ignore", ["LibIgnoreCustom"]), ("upload_port", "/dev/sysenv/port") ] # yapf: disable From 4af615a49cbe53a1243ffe27664d05992ada7ca7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Jun 2019 16:58:14 +0300 Subject: [PATCH 213/333] Maintain renamed options when reading configuration file --- platformio/builder/main.py | 1 - .../commands/home/rpc/handlers/project.py | 1 - platformio/project/config.py | 116 +++++++++--------- tests/test_projectconf.py | 4 +- 4 files changed, 59 insertions(+), 63 deletions(-) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 9c9da5f6..cedea7ca 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -103,7 +103,6 @@ if env.GetOption('clean'): elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): print("Verbose mode can be enabled via `-v, --verbose` option") -env.GetProjectConfig().validate([env['PIOENV']], silent=True) env.LoadProjectOptions() env.LoadPioPlatform() diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index e9435d35..6e029aca 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -42,7 +42,6 @@ class ProjectRPC(object): def _get_project_data(project_dir): data = {"boards": [], "envLibdepsDirs": [], "libExtraDirs": []} config = ProjectConfig(join(project_dir, "platformio.ini")) - config.validate(silent=True) libdeps_dir = get_project_libdeps_dir() data['libExtraDirs'].extend( diff --git a/platformio/project/config.py b/platformio/project/config.py index c9ba6b14..a7001cf3 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -47,6 +47,8 @@ class ProjectConfig(object): VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") expand_interpolations = True + warnings = [] + _instances = {} _parser = None _parsed = [] @@ -81,6 +83,7 @@ class ProjectConfig(object): def __init__(self, path, parse_extra=True, expand_interpolations=True): self.path = path self.expand_interpolations = expand_interpolations + self.warnings = [] self._parsed = [] self._parser = ConfigParser.ConfigParser() if isfile(path): @@ -106,6 +109,57 @@ class ProjectConfig(object): for item in glob.glob(pattern): self.read(item) + self._maintain_renaimed_options() + + def _maintain_renaimed_options(self): + # legacy `lib_extra_dirs` in [platformio] + if (self._parser.has_section("platformio") + and self._parser.has_option("platformio", "lib_extra_dirs")): + if not self._parser.has_section("env"): + self._parser.add_section("env") + self._parser.set("env", "lib_extra_dirs", + self._parser.get("platformio", "lib_extra_dirs")) + self._parser.remove_option("platformio", "lib_extra_dirs") + self.warnings.append( + "`lib_extra_dirs` configuration option is deprecated in " + "section [platformio]! Please move it to global `env` section") + + renamed_options = {} + for option in ProjectOptions.values(): + if option.oldnames: + renamed_options.update( + {name: option.name + for name in option.oldnames}) + + for section in self._parser.sections(): + scope = section.split(":", 1)[0] + if scope not in ("platformio", "env"): + continue + for option in self._parser.options(section): + if option in renamed_options: + self.warnings.append( + "`%s` configuration option in section [%s] is " + "deprecated and will be removed in the next release! " + "Please use `%s` instead" % + (option, section, renamed_options[option])) + # rename on-the-fly + self._parser.set(section, renamed_options[option], + self._parser.get(section, option)) + self._parser.remove_option(section, option) + continue + + # unknown + unknown_conditions = [ + ("%s.%s" % (scope, option)) not in ProjectOptions, + scope != "env" or + not option.startswith(("custom_", "board_")) + ] # yapf: disable + if all(unknown_conditions): + self.warnings.append( + "Ignore unknown configuration option `%s` " + "in section [%s]" % (option, section)) + return True + def options(self, section=None, env=None): assert section or env if not section: @@ -239,72 +293,14 @@ class ProjectConfig(object): known = set(self.envs()) if not known: raise exception.ProjectEnvsNotAvailable() - unknown = set(list(envs or []) + self.default_envs()) - known if unknown: raise exception.UnknownEnvNames(", ".join(unknown), ", ".join(known)) - return self.validate_options(silent) - - def validate_options(self, silent=False): - warnings = [] - # legacy `lib_extra_dirs` in [platformio] - if (self._parser.has_section("platformio") - and self._parser.has_option("platformio", "lib_extra_dirs")): - if not self._parser.has_section("env"): - self._parser.add_section("env") - self._parser.set("env", "lib_extra_dirs", - self._parser.get("platformio", "lib_extra_dirs")) - self._parser.remove_option("platformio", "lib_extra_dirs") - warnings.append( - "`lib_extra_dirs` configuration option is deprecated in " - "section [platformio]! Please move it to global `env` section") - - warnings.extend(self._validate_unknown_options()) - if not silent: - for warning in warnings: + for warning in self.warnings: click.secho("Warning! %s" % warning, fg="yellow") - - return warnings - - def _validate_unknown_options(self): - warnings = [] - renamed_options = {} - for option in ProjectOptions.values(): - if option.oldnames: - renamed_options.update( - {name: option.name - for name in option.oldnames}) - - for section in self._parser.sections(): - if not section.startswith("env:") and section != "platformio": - continue - for option in self._parser.options(section): - # obsolete - if option in renamed_options: - warnings.append( - "`%s` configuration option in section [%s] is " - "deprecated and will be removed in the next release! " - "Please use `%s` instead" % - (option, section, renamed_options[option])) - # rename on-the-fly - self._parser.set(section, renamed_options[option], - self._parser.get(section, option)) - self._parser.remove_option(section, option) - continue - - # unknown - scope = section.split(":", 1)[0] - unknown_conditions = [ - ("%s.%s" % (scope, option)) not in ProjectOptions, - scope != "env" or - not option.startswith(("custom_", "board_")) - ] # yapf: disable - if all(unknown_conditions): - warnings.append("Ignore unknown configuration option `%s` " - "in section [%s]" % (option, section)) - return warnings + return True def to_json(self): result = {} diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index eeafb54e..1448cdb2 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -74,8 +74,10 @@ def test_real_config(tmpdir): with tmpdir.as_cwd(): config = ProjectConfig(tmpdir.join("platformio.ini").strpath) assert config + assert len(config.warnings) == 1 + assert "lib_install" in config.warnings[0] - len(config.validate(["extra_2", "base"], silent=True)) == 1 + config.validate(["extra_2", "base"], silent=True) with pytest.raises(UnknownEnvNames): config.validate(["non-existing-env"]) From 6257480d0d5bbc49c5f451cc06e2c54a6637791d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Jun 2019 19:44:45 +0300 Subject: [PATCH 214/333] Print platform package details, such as version, VSC source and commit // Resolve #2155 --- HISTORY.rst | 12 ++- platformio/builder/tools/piolib.py | 29 +++--- platformio/builder/tools/pioplatform.py | 131 ++++++++++++++++-------- platformio/exception.py | 8 +- platformio/managers/package.py | 7 +- 5 files changed, 115 insertions(+), 72 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 225b42bd..c63cd919 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -29,6 +29,14 @@ PlatformIO 4.0 - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - Switched to workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` +* **Build System** + + - Print platform package details, such as version, VSC source and commit (`issue #2155 `_) + +* **PIO Remote** + + - Added support for `shared_dir `__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a remote machine + * **Infrastructure** - Python 3 support (`issue #895 `_) @@ -42,10 +50,6 @@ PlatformIO 4.0 - Fixed an issue with hardcoded C standard version when generating project for CLion IDE (`issue #2527 `_) - Fixed an issue with Project Generator when include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) -* **PIO Remote** - - - Added support for `shared_dir `__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a remote machine - * **Miscellaneous** - Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 5b2c2e2b..c7629f11 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -22,7 +22,6 @@ import hashlib import os import re import sys -from glob import glob from os.path import (basename, commonprefix, dirname, expanduser, isdir, isfile, join, realpath, sep) @@ -35,7 +34,6 @@ from platformio import exception, util from platformio.builder.tools import platformio as piotool from platformio.compat import PY2, WINDOWS, get_file_contents, string_types from platformio.managers.lib import LibraryManager -from platformio.managers.package import PackageManager class LibBuilderFactory(object): @@ -148,13 +146,6 @@ class LibBuilderBase(object): def version(self): return self._manifest.get("version") - @property - def vcs_info(self): - items = glob(join(self.path, ".*", PackageManager.SRC_MANIFEST_NAME)) - if not items: - return None - return util.load_json(items[0]) - @property def dependencies(self): return LibraryManager.normalize_dependencies( @@ -990,7 +981,11 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches def ConfigureProjectLibBuilder(env): - def correct_found_libs(lib_builders): + def _get_vcs_info(lb): + path = LibraryManager.get_src_manifest_path(lb.path) + return util.load_json(path) if path else None + + def _correct_found_libs(lib_builders): # build full dependency graph found_lbs = [lb for lb in lib_builders if lb.dependent] for lb in lib_builders: @@ -1001,11 +996,11 @@ def ConfigureProjectLibBuilder(env): if deplb not in found_lbs: lb.depbuilders.remove(deplb) - def print_deps_tree(root, level=0): + def _print_deps_tree(root, level=0): margin = "| " * (level) for lb in root.depbuilders: title = "<%s>" % lb.name - vcs_info = lb.vcs_info + vcs_info = _get_vcs_info(lb) if lb.version: title += " %s" % lb.version if vcs_info and vcs_info.get("version"): @@ -1019,13 +1014,13 @@ def ConfigureProjectLibBuilder(env): sys.stdout.write(")") sys.stdout.write("\n") if lb.depbuilders: - print_deps_tree(lb, level + 1) + _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) - print("Library Dependency Finder -> http://bit.ly/configure-pio-ldf") - print("LDF MODES: FINDER(%s) COMPATIBILITY(%s)" % + print("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") + print("LDF Modes: Finder [%s] Compatibility [%s]" % (ldf_mode, project.lib_compat_mode)) lib_builders = env.GetLibBuilders() @@ -1035,11 +1030,11 @@ def ConfigureProjectLibBuilder(env): project.search_deps_recursive() if ldf_mode.startswith("chain") and project.depbuilders: - correct_found_libs(lib_builders) + _correct_found_libs(lib_builders) if project.depbuilders: print("Dependency Graph") - print_deps_tree(project) + _print_deps_tree(project) else: print("No dependencies") diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 6d0d1c59..56b27da4 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -17,6 +17,7 @@ from __future__ import absolute_import import sys from os.path import isdir, isfile, join +from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from platformio import exception, util @@ -118,58 +119,98 @@ def LoadPioPlatform(env): def PrintConfiguration(env): platform = env.PioPlatform() - platform_data = ["PLATFORM: %s >" % platform.title] - hardware_data = ["HARDWARE:"] - configuration_data = ["CONFIGURATION:"] - mcu = env.subst("$BOARD_MCU") - f_cpu = env.subst("$BOARD_F_CPU") - if mcu: - hardware_data.append(mcu.upper()) - if f_cpu: - f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()])) - hardware_data.append("%dMHz" % (f_cpu / 1000000)) + board_config = env.BoardConfig() if "BOARD" in env else None - debug_tools = None - if "BOARD" in env: - board_config = env.BoardConfig() - platform_data.append(board_config.get("name")) + def _get_configuration_data(): + return None if not board_config else [ + "CONFIGURATION:", + "https://docs.platformio.org/page/boards/%s/%s.html" % + (platform.name, board_config.id) + ] - debug_tools = board_config.get("debug", {}).get("tools") + def _get_plaform_data(): + data = ["PLATFORM: %s @ %s" % (platform.title, platform.version)] + src_manifest_path = platform.pm.get_src_manifest_path( + platform.get_dir()) + if src_manifest_path: + src_manifest = util.load_json(src_manifest_path) + if "version" in src_manifest: + data.append("#" + src_manifest['version']) + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + data.append("(%s)" % src_manifest['url']) + if board_config: + data.extend([">", board_config.get("name")]) + return data + + def _get_hardware_data(): + data = ["HARDWARE:"] + mcu = env.subst("$BOARD_MCU") + f_cpu = env.subst("$BOARD_F_CPU") + if mcu: + data.append(mcu.upper()) + if f_cpu: + f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()])) + data.append("%dMHz" % (f_cpu / 1000000)) + if not board_config: + return data ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") - hardware_data.append( - "%s RAM (%s Flash)" % - (util.format_filesize(ram), util.format_filesize(flash))) - configuration_data.append( - "https://docs.platformio.org/page/boards/%s/%s.html" % - (platform.name, board_config.id)) + data.append("%s RAM [%s Flash]" % + (util.format_filesize(ram), util.format_filesize(flash))) + return data - for data in (configuration_data, platform_data, hardware_data): - if len(data) > 1: + def _get_debug_data(): + debug_tools = board_config.get( + "debug", {}).get("tools") if board_config else None + if not debug_tools: + return None + data = [ + "DEBUG:", + "CURRENT(%s)" % board_config.get_debug_tool_name( + env.GetProjectOption("debug_tool")) + ] + onboard = [] + external = [] + for key, value in debug_tools.items(): + if value.get("onboard"): + onboard.append(key) + else: + external.append(key) + if onboard: + data.append("ON-BOARD(%s)" % ", ".join(sorted(onboard))) + if external: + data.append("EXTERNAL(%s)" % ", ".join(sorted(external))) + return data + + def _get_packages_data(): + data = [] + for name, options in platform.packages.items(): + if options.get("optional"): + continue + pkg_dir = platform.get_package_dir(name) + if not pkg_dir: + continue + manifest = platform.pm.load_manifest(pkg_dir) + original_version = util.get_original_version(manifest['version']) + info = "%s %s %s" % (manifest['name'], + "#" if "__src_url" in manifest else "@", + manifest['version']) + extra = [] + if original_version: + extra.append(original_version) + if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)): + extra.append(manifest['__src_url']) + if extra: + info += "(%s)" % ", ".join(extra) + data.append(info) + return ["PACKAGES:", ", ".join(data)] + + for data in (_get_configuration_data(), _get_plaform_data(), + _get_hardware_data(), _get_debug_data(), + _get_packages_data()): + if data and len(data) > 1: print(" ".join(data)) - # Debugging - if not debug_tools: - return - - data = [ - "CURRENT(%s)" % - board_config.get_debug_tool_name(env.GetProjectOption("debug_tool")) - ] - onboard = [] - external = [] - for key, value in debug_tools.items(): - if value.get("onboard"): - onboard.append(key) - else: - external.append(key) - if onboard: - data.append("ON-BOARD(%s)" % ", ".join(sorted(onboard))) - if external: - data.append("EXTERNAL(%s)" % ", ".join(sorted(external))) - - print("DEBUG: %s" % " ".join(data)) - def exists(_): return True diff --git a/platformio/exception.py b/platformio/exception.py index 09d7d2f9..823f68af 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -318,9 +318,11 @@ class CygwinEnvDetected(PlatformioException): class DebugSupportError(PlatformioException): - MESSAGE = ("Currently, PlatformIO does not support debugging for `{0}`.\n" - "Please contact support@pioplus.com or visit " - "< https://docs.platformio.org/page/plus/debugging.html >") + MESSAGE = ( + "Currently, PlatformIO does not support debugging for `{0}`.\n" + "Please request support at https://github.com/platformio/" + "platformio-core/issues \nor visit -> https://docs.platformio.org" + "/page/plus/debugging.html") class DebugInvalidOptions(PlatformioException): diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 905fb32f..3e9495e7 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -330,14 +330,15 @@ class PkgInstallerMixin(object): name += "_ID%d" % manifest['id'] return str(name) - def get_src_manifest_path(self, pkg_dir): + @classmethod + def get_src_manifest_path(cls, pkg_dir): if not isdir(pkg_dir): return None for item in os.listdir(pkg_dir): if not isdir(join(pkg_dir, item)): continue - if isfile(join(pkg_dir, item, self.SRC_MANIFEST_NAME)): - return join(pkg_dir, item, self.SRC_MANIFEST_NAME) + if isfile(join(pkg_dir, item, cls.SRC_MANIFEST_NAME)): + return join(pkg_dir, item, cls.SRC_MANIFEST_NAME) return None def get_manifest_path(self, pkg_dir): From d51cd9c277fdfcce08491c25281e58a7afef3bda Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Jun 2019 19:48:21 +0300 Subject: [PATCH 215/333] Bump version to 4.0.0a17 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 75e6e461..b9f72e6a 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a16") +VERSION = (4, 0, "0a17") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From c5177efd0b1f5a4bd70540c3ae8640202a92636c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Jun 2019 22:24:38 +0300 Subject: [PATCH 216/333] Minor fixes --- platformio/builder/tools/piolib.py | 4 ++-- platformio/builder/tools/pioplatform.py | 22 ++++++++++------------ platformio/managers/package.py | 9 ++++----- platformio/util.py | 11 +++++------ 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index c7629f11..29389e38 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -1020,11 +1020,11 @@ def ConfigureProjectLibBuilder(env): ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) print("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") - print("LDF Modes: Finder [%s] Compatibility [%s]" % + print("LDF Modes: Finder ~ %s, Compatibility ~ %s" % (ldf_mode, project.lib_compat_mode)) lib_builders = env.GetLibBuilders() - print("Collected %d compatible libraries" % len(lib_builders)) + print("Found %d compatible libraries" % len(lib_builders)) print("Scanning dependencies...") project.search_deps_recursive() diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 56b27da4..a2971bce 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -117,7 +117,7 @@ def LoadPioPlatform(env): env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript")) -def PrintConfiguration(env): +def PrintConfiguration(env): # pylint: disable=too-many-statements platform = env.PioPlatform() board_config = env.BoardConfig() if "BOARD" in env else None @@ -129,7 +129,7 @@ def PrintConfiguration(env): ] def _get_plaform_data(): - data = ["PLATFORM: %s @ %s" % (platform.title, platform.version)] + data = ["PLATFORM: %s %s" % (platform.title, platform.version)] src_manifest_path = platform.pm.get_src_manifest_path( platform.get_dir()) if src_manifest_path: @@ -150,12 +150,12 @@ def PrintConfiguration(env): data.append(mcu.upper()) if f_cpu: f_cpu = int("".join([c for c in str(f_cpu) if c.isdigit()])) - data.append("%dMHz" % (f_cpu / 1000000)) + data.append("%dMHz," % (f_cpu / 1000000)) if not board_config: return data ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") - data.append("%s RAM [%s Flash]" % + data.append("%s RAM, %s Flash" % (util.format_filesize(ram), util.format_filesize(flash))) return data @@ -165,8 +165,8 @@ def PrintConfiguration(env): if not debug_tools: return None data = [ - "DEBUG:", - "CURRENT(%s)" % board_config.get_debug_tool_name( + "DEBUG:", "Current", + "(%s)" % board_config.get_debug_tool_name( env.GetProjectOption("debug_tool")) ] onboard = [] @@ -177,9 +177,9 @@ def PrintConfiguration(env): else: external.append(key) if onboard: - data.append("ON-BOARD(%s)" % ", ".join(sorted(onboard))) + data.extend(["On-board", "(%s)" % ", ".join(sorted(onboard))]) if external: - data.append("EXTERNAL(%s)" % ", ".join(sorted(external))) + data.extend(["External", "(%s)" % ", ".join(sorted(external))]) return data def _get_packages_data(): @@ -192,16 +192,14 @@ def PrintConfiguration(env): continue manifest = platform.pm.load_manifest(pkg_dir) original_version = util.get_original_version(manifest['version']) - info = "%s %s %s" % (manifest['name'], - "#" if "__src_url" in manifest else "@", - manifest['version']) + info = "%s %s" % (manifest['name'], manifest['version']) extra = [] if original_version: extra.append(original_version) if "__src_url" in manifest and int(ARGUMENTS.get("PIOVERBOSE", 0)): extra.append(manifest['__src_url']) if extra: - info += "(%s)" % ", ".join(extra) + info += " (%s)" % ", ".join(extra) data.append(info) return ["PACKAGES:", ", ".join(data)] diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 3e9495e7..fc3f4105 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -741,11 +741,10 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): action="Install", label=manifest['name']) - if not silent: - click.secho( - "{name} @ {version} has been successfully installed!". - format(**manifest), - fg="green") + click.secho( + "{name} @ {version} has been successfully installed!".format( + **manifest), + fg="green") return pkg_dir diff --git a/platformio/util.py b/platformio/util.py index a7657526..e07ce391 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -507,13 +507,12 @@ def ensure_udev_rules(): def get_original_version(version): if version.count(".") != 2: return None - _, y = version.split(".")[:2] - if int(y) < 100: + _, raw = version.split(".")[:2] + if int(raw) <= 99: return None - if len(y) % 2 != 0: - y = "0" + y - parts = [str(int(y[i * 2:i * 2 + 2])) for i in range(int(len(y) / 2))] - return ".".join(parts) + if int(raw) <= 999: + return "%s.%s" % (raw[:-2], int(raw[-2:])) + return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) def rmtree_(path): From 23a2022f046bbd46151aa4b3938aafed294238ec Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 1 Jun 2019 22:43:44 +0300 Subject: [PATCH 217/333] Add support for PLATFORMIO_DEFAULT_ENVS system environment variable // Resolve #1967 --- HISTORY.rst | 1 + docs | 2 +- examples | 2 +- platformio/project/config.py | 2 +- platformio/project/options.py | 5 +++-- tests/test_projectconf.py | 4 ++-- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c63cd919..51a05c9c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ PlatformIO 4.0 - Added support for Unix shell-style wildcards for `monitor_port `__ option (`issue #2541 `_) - Added new `monitor_flags `__ option which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) + - Added support for `PLATFORMIO_DEFAULT_ENVS `__ system environment variable (`issue #1967 `_) - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) * **Library Management** diff --git a/docs b/docs index c030b967..84973be5 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c030b967a5fa6d2e234f536d99ae2fe0b4d1a88d +Subproject commit 84973be53b7465e37ab4b20ce245ffa12a294098 diff --git a/examples b/examples index 89963fed..b5ae9ea1 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 89963fed9ebd0e7123d58dad5f7fc5020769ca63 +Subproject commit b5ae9ea1bb754d9e06b195d65f24853ef58f376d diff --git a/platformio/project/config.py b/platformio/project/config.py index a7001cf3..ee7e71d1 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -284,7 +284,7 @@ class ProjectConfig(object): return [s[4:] for s in self._parser.sections() if s.startswith("env:")] def default_envs(self): - return self.get("platformio", "env_default", []) + return self.get("platformio", "default_envs", []) def validate(self, envs=None, silent=False): if not isfile(self.path): diff --git a/platformio/project/options.py b/platformio/project/options.py index 1c5ff5e9..66fb13fc 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -48,9 +48,10 @@ ProjectOptions = OrderedDict([ # [platformio] # ConfigPlatformioOption(name="description"), - ConfigPlatformioOption(name="env_default", + ConfigPlatformioOption(name="default_envs", + oldnames=["env_default"], multiple=True, - sysenvvar="PLATFORMIO_ENV_DEFAULT"), + sysenvvar="PLATFORMIO_DEFAULT_ENVS"), ConfigPlatformioOption(name="extra_configs", multiple=True), # Dirs diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 1448cdb2..38725f8a 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -74,8 +74,8 @@ def test_real_config(tmpdir): with tmpdir.as_cwd(): config = ProjectConfig(tmpdir.join("platformio.ini").strpath) assert config - assert len(config.warnings) == 1 - assert "lib_install" in config.warnings[0] + assert len(config.warnings) == 2 + assert "lib_install" in config.warnings[1] config.validate(["extra_2", "base"], silent=True) with pytest.raises(UnknownEnvNames): From d0b3c5ee86140d2012cdfd48b66feac866fe90b8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 2 Jun 2019 14:11:31 +0300 Subject: [PATCH 218/333] Switch between Build Configurations (release and debug) with a new project configuration option build_type // Resolve #2184 --- HISTORY.rst | 16 +++++++--------- docs | 2 +- platformio/builder/main.py | 1 - platformio/builder/tools/piomisc.py | 2 +- platformio/builder/tools/platformio.py | 5 +++-- platformio/commands/debug/helpers.py | 15 +++++++++++---- platformio/project/options.py | 2 ++ 7 files changed, 25 insertions(+), 18 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 51a05c9c..621b6e52 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,32 +11,30 @@ PlatformIO 4.0 - New project configuration parser with a strict options typing (`API `__) - Unified workspace storage (`workspace_dir `__ -> ``.pio``) for PlatformIO Build System, Library Manager, and other internal services (`issue #1778 `_) - - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` - Share common (global) options between project environments using `[env] `__ section (`issue #1643 `_) - Include external configuration files with `extra_configs `__ option (`issue #1590 `_) - - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - Custom project ``***_dir`` options declared in `platformio `__ section have higher priority than `Environment variables `__ - Added support for Unix shell-style wildcards for `monitor_port `__ option (`issue #2541 `_) - Added new `monitor_flags `__ option which allows passing extra flags and options to `platformio device monitor `__ command (`issue #2165 `_) - - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) - Added support for `PLATFORMIO_DEFAULT_ENVS `__ system environment variable (`issue #1967 `_) - - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) + - Added support for `shared_dir `__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a `PIO Remote `__ machine * **Library Management** + - Switched to workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` - Save libraries passed to `platformio lib install `__ command into the project dependency list (`lib_deps `__) with a new ``--save`` flag (`issue #1028 `_) - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - Use isolated library dependency storage per project build environment (`issue #1696 `_) - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - - Switched to workspace ``.pio/libdeps`` folder for project dependencies instead of ``.piolibdeps`` * **Build System** + - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` + - Switch between `Build Configurations `__ (``release`` and ``debug``) with a new project configuration option `build_type `__ - Print platform package details, such as version, VSC source and commit (`issue #2155 `_) - -* **PIO Remote** - - - Added support for `shared_dir `__ where you can place an extra files (extra scripts, LD scripts, etc.) which should be transferred to a remote machine + - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) + - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) + - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) * **Infrastructure** diff --git a/docs b/docs index 84973be5..ddf3e192 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 84973be53b7465e37ab4b20ce245ffa12a294098 +Subproject commit ddf3e1929ae0d23a5a373ee4b15edb04b480415c diff --git a/platformio/builder/main.py b/platformio/builder/main.py index cedea7ca..bf1b001e 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -142,7 +142,6 @@ env.AddPreAction( "Configuring upload protocol...")) AlwaysBuild(env.Alias("debug", DEFAULT_TARGETS)) -AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS)) AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) ############################################################################## diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index a7fef99a..442ffdd7 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -298,7 +298,7 @@ def ProcessDebug(env): if not env.subst("$PIODEBUGFLAGS"): env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"]) env.Append(BUILD_FLAGS=list(env['PIODEBUGFLAGS']) + - ["-D__PLATFORMIO_DEBUG__"]) + ["-D__PLATFORMIO_BUILD_DEBUG__"]) unflags = ["-Os"] for level in [0, 1, 2]: for flag in ("O", "g", "ggdb"): diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index f8f3656c..c584d654 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -69,7 +69,7 @@ def _build_project_deps(env): if is_test: projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER") - if not is_test or env.get("TEST_BUILD_PROJECT_SRC") == "true": + if not is_test or env.GetProjectOption("test_build_project_src", False): projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", env.get("SRC_FILTER")) @@ -97,7 +97,8 @@ def BuildProgram(env): if not Util.case_sensitive_suffixes(".s", ".S"): env.Replace(AS="$CC", ASCOM="$ASPPCOM") - if set(["__debug", "debug"]) & set(COMMAND_LINE_TARGETS): + if ("debug" in COMMAND_LINE_TARGETS + or env.GetProjectOption("build_type") == "debug"): env.ProcessDebug() # process extra flags from board diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 2d1f5205..6c215512 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -21,7 +21,7 @@ from hashlib import sha1 from io import BytesIO from os.path import isfile -from platformio import VERSION, exception, util +from platformio import exception, util from platformio.commands.platform import \ platform_install as cmd_platform_install from platformio.commands.run import cli as cmd_run @@ -49,7 +49,14 @@ def escape_path(path): def get_default_debug_env(config): default_envs = config.default_envs() - return default_envs[0] if default_envs else config.envs()[0] + all_envs = config.envs() + for env in default_envs: + if config.get("env:" + env, "build_type") == "debug": + return env + for env in all_envs: + if config.get("env:" + env, "build_type") == "debug": + return env + return default_envs[0] if default_envs else all_envs[0] def validate_debug_options(cmd_ctx, env_options): @@ -141,7 +148,7 @@ def predebug_project(ctx, project_dir, env_name, preload, verbose): ctx.invoke(cmd_run, project_dir=project_dir, environment=[env_name], - target=["__debug"] + (["upload"] if preload else []), + target=["debug"] + (["upload"] if preload else []), verbose=verbose) if preload: time.sleep(5) @@ -206,7 +213,7 @@ def has_debug_symbols(prog_path): b".debug_abbrev": False, b" -Og": False, b" -g": False, - b"__PLATFORMIO_DEBUG__": (3, 6) > VERSION[:2] + b"__PLATFORMIO_BUILD_DEBUG__": False } with open(prog_path, "rb") as fp: last_data = b"" diff --git a/platformio/project/options.py b/platformio/project/options.py index 66fb13fc..fe5e9368 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -111,6 +111,8 @@ ProjectOptions = OrderedDict([ buildenvvar="BOARD_FLASH_MODE"), # Build + ConfigEnvOption(name="build_type", + type=click.Choice(["release", "debug"])), ConfigEnvOption(name="build_flags", multiple=True, sysenvvar="PLATFORMIO_BUILD_FLAGS", From ac3236693fed462924e0be58d2e95a1241f044ba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jun 2019 01:14:02 +0300 Subject: [PATCH 219/333] Bump version to 4.0.0a18 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b9f72e6a..1808eab8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a17") +VERSION = (4, 0, "0a18") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From e269c91d26e5fe78973b9e232182793b84c13966 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jun 2019 13:30:35 +0300 Subject: [PATCH 220/333] Improve compatibility with hashlib Py2/Py3 --- platformio/app.py | 10 ++++------ platformio/builder/main.py | 12 ++++++------ platformio/builder/tools/piolib.py | 6 +++--- platformio/builder/tools/piowinhooks.py | 7 +++---- platformio/commands/debug/client.py | 5 ++--- platformio/compat.py | 16 ++++++++++++++++ platformio/managers/package.py | 10 ++++++---- platformio/managers/platform.py | 20 +++++++++++++------- platformio/proc.py | 6 +++--- platformio/project/helpers.py | 9 ++++----- platformio/telemetry.py | 12 ++++++++---- 11 files changed, 68 insertions(+), 45 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index c0923bf1..49f24625 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -25,7 +25,7 @@ from time import time import requests from platformio import exception, lockfile, util -from platformio.compat import PY2, WINDOWS +from platformio.compat import WINDOWS, hashlib_encode_data from platformio.proc import is_ci from platformio.project.helpers import (get_project_cache_dir, get_project_core_dir) @@ -174,10 +174,8 @@ class ContentCache(object): def key_from_args(*args): h = hashlib.md5() for arg in args: - if not arg: - continue - arg = str(arg) - h.update(arg if PY2 else arg.encode()) + if arg: + h.update(hashlib_encode_data(arg)) return h.hexdigest() def get(self, key): @@ -363,7 +361,7 @@ def get_cid(): pass if not uid: uid = uuid.getnode() - cid = uuid.UUID(bytes=hashlib.md5(str(uid).encode()).digest()) + cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) cid = str(cid) if WINDOWS or os.getuid() > 0: # yapf: disable pylint: disable=no-member set_state_item("cid", cid) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index bf1b001e..1fd76b24 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64 import json import sys from os import environ @@ -31,6 +30,7 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import util from platformio.compat import PY2, path_to_unicode +from platformio.managers.platform import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project import helpers as project_helpers @@ -91,11 +91,11 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)): env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) # Load variables from CLI -for key in list(clivars.keys()): - if key in env: - env[key] = base64.b64decode(env[key]) - if isinstance(env[key], bytes): - env[key] = env[key].decode() +env.Replace( + **{ + key: PlatformBase.decode_scons_arg(env[key]) + for key in list(clivars.keys()) if key in env + }) if env.GetOption('clean'): env.PioClean(env.subst("$BUILD_DIR")) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 29389e38..b9855411 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -32,7 +32,8 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, util from platformio.builder.tools import platformio as piotool -from platformio.compat import PY2, WINDOWS, get_file_contents, string_types +from platformio.compat import (WINDOWS, get_file_contents, hashlib_encode_data, + string_types) from platformio.managers.lib import LibraryManager @@ -181,8 +182,7 @@ class LibBuilderBase(object): @property def build_dir(self): - lib_hash = hashlib.sha1( - self.path if PY2 else self.path.encode()).hexdigest()[:3] + lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3] return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path)) @property diff --git a/platformio/builder/tools/piowinhooks.py b/platformio/builder/tools/piowinhooks.py index 1105091f..3679897d 100644 --- a/platformio/builder/tools/piowinhooks.py +++ b/platformio/builder/tools/piowinhooks.py @@ -18,7 +18,7 @@ from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join -from platformio.compat import PY2, WINDOWS +from platformio.compat import WINDOWS, hashlib_encode_data # Windows CLI has limit with command length to 8192 # Leave 2000 chars for flags and other options @@ -61,9 +61,8 @@ def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") if not isdir(build_dir): makedirs(build_dir) - tmp_file = join( - build_dir, - "longcmd-%s" % md5(data if PY2 else data.encode()).hexdigest()) + tmp_file = join(build_dir, + "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest()) if isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index e7945284..03daed25 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -29,7 +29,7 @@ from platformio import app, exception, util from platformio.commands.debug import helpers, initcfgs from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer -from platformio.compat import PY2 +from platformio.compat import hashlib_encode_data from platformio.project.helpers import get_project_cache_dir from platformio.telemetry import MeasurementProtocol @@ -61,8 +61,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes def spawn(self, gdb_path, prog_path): session_hash = gdb_path + prog_path - self._session_id = sha1( - session_hash if PY2 else session_hash.encode()).hexdigest() + self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest() self._kill_previous_session() patterns = { diff --git a/platformio/compat.py b/platformio/compat.py index 3c3afc52..975400fe 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -35,12 +35,21 @@ if PY2: return isinstance(x, (buffer, bytearray)) def path_to_unicode(path): + if isinstance(path, unicode): + return path return path.decode(get_filesystem_encoding()).encode("utf-8") def get_file_contents(path): with open(path) as f: return f.read() + def hashlib_encode_data(data): + if is_bytes(data): + return data + if not isinstance(data, string_types): + data = str(data) + return data + _magic_check = re.compile('([*?[])') _magic_check_bytes = re.compile(b'([*?[])') @@ -74,3 +83,10 @@ else: except UnicodeDecodeError: with open(path, encoding="latin-1") as f: return f.read() + + def hashlib_encode_data(data): + if is_bytes(data): + return data + if not isinstance(data, string_types): + data = str(data) + return data.encode() diff --git a/platformio/managers/package.py b/platformio/managers/package.py index fc3f4105..56a33f0b 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -26,7 +26,7 @@ import requests import semantic_version from platformio import __version__, app, exception, telemetry, util -from platformio.compat import path_to_unicode +from platformio.compat import hashlib_encode_data, path_to_unicode from platformio.downloader import FileDownloader from platformio.lockfile import LockFile from platformio.unpacker import FileUnpacker @@ -173,7 +173,7 @@ class PkgInstallerMixin(object): cache_key_data = app.ContentCache.key_from_args(url, "data") if self.FILE_CACHE_VALID: with app.ContentCache() as cc: - fname = cc.get(cache_key_fname) + fname = str(cc.get(cache_key_fname)) cache_path = cc.get_cache_path(cache_key_data) if fname and isfile(cache_path): dst_path = join(dest_dir, fname) @@ -591,7 +591,8 @@ class PkgInstallerMixin(object): target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5( - cur_manifest['__src_url'].encode()).hexdigest()) + hashlib_encode_data( + cur_manifest['__src_url'])).hexdigest()) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: @@ -601,7 +602,8 @@ class PkgInstallerMixin(object): target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5( - tmp_manifest['__src_url'].encode()).hexdigest()) + hashlib_encode_data( + tmp_manifest['__src_url'])).hexdigest()) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 7aa153f8..e7a29027 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -24,7 +24,7 @@ import click import semantic_version from platformio import __version__, app, exception, util -from platformio.compat import PY2 +from platformio.compat import get_filesystem_encoding, string_types from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, @@ -359,6 +359,17 @@ class PlatformRunMixin(object): LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) + @staticmethod + def encode_scons_arg(value): + if isinstance(value, string_types): + value = value.encode(get_filesystem_encoding()) + return base64.urlsafe_b64encode(value).decode() + + @staticmethod + def decode_scons_arg(value): + return base64.urlsafe_b64decode(value).decode( + get_filesystem_encoding()) + def run(self, variables, targets, silent, verbose): assert isinstance(variables, dict) assert isinstance(targets, list) @@ -402,12 +413,7 @@ class PlatformRunMixin(object): # encode and append variables for key, value in variables.items(): - if PY2: - cmd.append("%s=%s" % (key.upper(), base64.b64encode(value))) - else: - cmd.append( - "%s=%s" % - (key.upper(), base64.b64encode(value.encode()).decode())) + cmd.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) def _write_and_flush(stream, data): stream.write(data) diff --git a/platformio/proc.py b/platformio/proc.py index 9ffad954..7808fa42 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -19,7 +19,7 @@ from os.path import isdir, isfile, join, normpath from threading import Thread from platformio import exception -from platformio.compat import PY2, WINDOWS, string_types +from platformio.compat import WINDOWS, get_filesystem_encoding, string_types class AsyncPipeBase(object): @@ -123,8 +123,8 @@ def exec_command(*args, **kwargs): result[s[3:]] = kwargs[s].get_buffer() for k, v in result.items(): - if not PY2 and isinstance(result[k], bytes): - result[k] = result[k].decode() + if isinstance(result[k], bytes): + result[k] = result[k].decode(get_filesystem_encoding()) if v and isinstance(v, string_types): result[k] = result[k].strip() diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index bf63f8b0..71f73649 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -19,7 +19,7 @@ from os.path import (basename, dirname, expanduser, isdir, isfile, join, realpath, splitdrive) from platformio import __version__ -from platformio.compat import PY2, WINDOWS +from platformio.compat import WINDOWS, hashlib_encode_data from platformio.project.config import ProjectConfig @@ -54,9 +54,8 @@ def get_project_optional_dir(name, default=None): if "$PROJECT_HASH" in optional_dir: optional_dir = optional_dir.replace( "$PROJECT_HASH", "%s-%s" % - (basename(project_dir), - sha1(project_dir if PY2 else project_dir.encode()).hexdigest() - [:10])) + (basename(project_dir), sha1( + hashlib_encode_data(project_dir)).hexdigest()[:10])) if optional_dir.startswith("~"): optional_dir = expanduser(optional_dir) @@ -179,4 +178,4 @@ def calculate_project_hash(): # Fix issue with useless project rebuilding for case insensitive FS. # A case of disk drive can differ... chunks_to_str = chunks_to_str.lower() - return sha1(chunks_to_str if PY2 else chunks_to_str.encode()).hexdigest() + return sha1(hashlib_encode_data(chunks_to_str)).hexdigest() diff --git a/platformio/telemetry.py b/platformio/telemetry.py index f697a241..d35aac40 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -28,6 +28,7 @@ import requests from platformio import __version__, app, exception, util from platformio.commands import PlatformioCLI +from platformio.compat import string_types from platformio.proc import is_ci, is_container try: @@ -135,12 +136,15 @@ class MeasurementProtocol(TelemetryBase): return _arg return None - args = [ - str(arg).lower() for arg in PlatformioCLI.leftover_args - if not str(arg).startswith("-") - ] + args = [] + for arg in PlatformioCLI.leftover_args: + if not isinstance(arg, string_types): + arg = str(arg) + if not arg.startswith("-"): + args.append(arg.lower()) if not args: return + cmd_path = args[:1] if args[0] in ("platform", "platforms", "serialports", "device", "settings", "account"): From 37653d8446716c84b1faaca7562c0c48cad0536c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jun 2019 13:57:58 +0300 Subject: [PATCH 221/333] Better decoding SCons arguments --- platformio/downloader.py | 7 +------ platformio/managers/platform.py | 14 +++++++------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/platformio/downloader.py b/platformio/downloader.py index be5e8888..60f0e159 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -22,7 +22,6 @@ import click import requests from platformio import util -from platformio.compat import PY2, get_filesystem_encoding from platformio.exception import (FDSHASumMismatch, FDSizeMismatch, FDUnrecognizedStatusCode) from platformio.proc import exec_command @@ -46,15 +45,11 @@ class FileDownloader(object): if disposition and "filename=" in disposition: self._fname = disposition[disposition.index("filename=") + 9:].replace('"', "").replace("'", "") - if PY2: - self._fname = self._fname.encode("utf8") else: self._fname = [p for p in url.split("/") if p][-1] - + self._fname = str(self._fname) self._destination = self._fname if dest_dir: - if PY2: - dest_dir = dest_dir.decode(get_filesystem_encoding()) self.set_destination(join(dest_dir, self._fname)) def set_destination(self, destination): diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index e7a29027..2e4d2ebf 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -24,7 +24,8 @@ import click import semantic_version from platformio import __version__, app, exception, util -from platformio.compat import get_filesystem_encoding, string_types +from platformio.compat import (get_filesystem_encoding, hashlib_encode_data, + is_bytes) from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, @@ -361,14 +362,13 @@ class PlatformRunMixin(object): @staticmethod def encode_scons_arg(value): - if isinstance(value, string_types): - value = value.encode(get_filesystem_encoding()) - return base64.urlsafe_b64encode(value).decode() + data = base64.urlsafe_b64encode(hashlib_encode_data(value)) + return data.decode() if is_bytes(data) else data @staticmethod - def decode_scons_arg(value): - return base64.urlsafe_b64decode(value).decode( - get_filesystem_encoding()) + def decode_scons_arg(data): + value = base64.urlsafe_b64decode(data) + return value.decode() if is_bytes(value) else value def run(self, variables, targets, silent, verbose): assert isinstance(variables, dict) From 9eb18ca72dfc15f3fbfc2c4fe41a086af86777c4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jun 2019 14:29:22 +0300 Subject: [PATCH 222/333] PyLint fix --- platformio/managers/platform.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 2e4d2ebf..b2e467d4 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -24,8 +24,7 @@ import click import semantic_version from platformio import __version__, app, exception, util -from platformio.compat import (get_filesystem_encoding, hashlib_encode_data, - is_bytes) +from platformio.compat import hashlib_encode_data, is_bytes from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, From 80a1b95887c8ff5bfcb10505565b72b59bf775e1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jun 2019 14:48:05 +0300 Subject: [PATCH 223/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index ddf3e192..a2d06177 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ddf3e1929ae0d23a5a373ee4b15edb04b480415c +Subproject commit a2d061775a51e1181db8aac8bc05058e21da7e96 From 4416c12747c4b82c2dbd9a49d29e8d090241d25b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jun 2019 17:44:41 +0300 Subject: [PATCH 224/333] Fix numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars // Resolve #2100 --- HISTORY.rst | 1 + platformio/builder/main.py | 23 ++++++----------------- platformio/compat.py | 9 +++++++++ platformio/ide/projectgenerator.py | 12 ++++++------ platformio/proc.py | 7 +++++-- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 621b6e52..810f89c5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -42,6 +42,7 @@ PlatformIO 4.0 - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) + - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #2100 `_) * **Integration** diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 1fd76b24..70f71ad6 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -import sys from os import environ from os.path import join from time import time @@ -29,7 +27,7 @@ from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error from platformio import util -from platformio.compat import PY2, path_to_unicode +from platformio.compat import PY2, dump_json_to_unicode from platformio.managers.platform import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project import helpers as project_helpers @@ -151,17 +149,8 @@ if "envdump" in COMMAND_LINE_TARGETS: env.Exit(0) if "idedata" in COMMAND_LINE_TARGETS: - try: - Import("projenv") - print("\n%s\n" % path_to_unicode( - json.dumps( - env.DumpIDEData(projenv), # pylint: disable=undefined-variable - ensure_ascii=False))) - env.Exit(0) - except UnicodeDecodeError: - sys.stderr.write( - "\nUnicodeDecodeError: Non-ASCII characters found in build " - "environment\n" - "See explanation in FAQ > Troubleshooting > Building\n" - "https://docs.platformio.org/page/faq.html\n\n") - env.Exit(1) + Import("projenv") + print("\n%s\n" % dump_json_to_unicode( + env.DumpIDEData(projenv) # pylint: disable=undefined-variable + )) + env.Exit(0) diff --git a/platformio/compat.py b/platformio/compat.py index 975400fe..53ba736a 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -14,6 +14,7 @@ # pylint: disable=unused-import +import json import os import re import sys @@ -50,6 +51,11 @@ if PY2: data = str(data) return data + def dump_json_to_unicode(obj): + return json.dumps(obj, + encoding=get_filesystem_encoding(), + ensure_ascii=False).encode("utf8") + _magic_check = re.compile('([*?[])') _magic_check_bytes = re.compile(b'([*?[])') @@ -90,3 +96,6 @@ else: if not isinstance(data, string_types): data = str(data) return data.encode() + + def dump_json_to_unicode(obj): + return json.dumps(obj, ensure_ascii=False) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index e420802a..eb3ce6ec 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import codecs import json import os import re @@ -23,7 +24,7 @@ from click.testing import CliRunner from platformio import exception, util from platformio.commands.run import cli as cmd_run -from platformio.compat import PY2, WINDOWS, get_file_contents +from platformio.compat import WINDOWS, get_file_contents from platformio.proc import where_is_program from platformio.project.config import ProjectConfig from platformio.project.helpers import (get_project_lib_dir, @@ -35,7 +36,7 @@ class ProjectGenerator(object): def __init__(self, project_dir, ide, env_name): self.project_dir = project_dir - self.ide = ide + self.ide = str(ide) self.env_name = env_name self._tplvars = {} @@ -121,8 +122,7 @@ class ProjectGenerator(object): file_name = basename(tpl_path)[:-4] contents = self._render_tpl(tpl_path) - self._merge_contents(join(dst_dir, file_name), - contents.encode("utf8") if PY2 else contents) + self._merge_contents(join(dst_dir, file_name), contents) def _render_tpl(self, tpl_path): return bottle.template(get_file_contents(tpl_path), **self._tplvars) @@ -131,8 +131,8 @@ class ProjectGenerator(object): def _merge_contents(dst_path, contents): if basename(dst_path) == ".gitignore" and isfile(dst_path): return - with open(dst_path, "w") as f: - f.write(contents) + with codecs.open(dst_path, "w", encoding="utf8") as fp: + fp.write(contents) def _gather_tplvars(self): self._tplvars.update(self.get_project_env()) diff --git a/platformio/proc.py b/platformio/proc.py index 7808fa42..c90590e3 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -19,7 +19,7 @@ from os.path import isdir, isfile, join, normpath from threading import Thread from platformio import exception -from platformio.compat import WINDOWS, get_filesystem_encoding, string_types +from platformio.compat import WINDOWS, string_types class AsyncPipeBase(object): @@ -124,7 +124,10 @@ def exec_command(*args, **kwargs): for k, v in result.items(): if isinstance(result[k], bytes): - result[k] = result[k].decode(get_filesystem_encoding()) + try: + result[k] = result[k].decode(sys.getdefaultencoding()) + except UnicodeDecodeError: + result[k] = result[k].decode("latin-1") if v and isinstance(v, string_types): result[k] = result[k].strip() From d09964a8979c376032be68d97592d711188ff8e4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 3 Jun 2019 19:20:10 +0300 Subject: [PATCH 225/333] Use common IDE data loading for IDE and DEBUG --- platformio/commands/debug/command.py | 7 +- platformio/commands/debug/helpers.py | 61 +++++--------- platformio/commands/init.py | 2 +- platformio/commands/run/command.py | 8 +- platformio/commands/run/helpers.py | 6 +- platformio/commands/run/processor.py | 4 +- platformio/ide/projectgenerator.py | 115 +++++++++------------------ platformio/project/helpers.py | 24 +++++- 8 files changed, 97 insertions(+), 130 deletions(-) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 891363f1..70111e6f 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -24,7 +24,8 @@ from platformio import exception, util from platformio.commands.debug import helpers from platformio.managers.core import inject_contrib_pysite from platformio.project.config import ProjectConfig -from platformio.project.helpers import is_platformio_project +from platformio.project.helpers import (is_platformio_project, + load_project_ide_data) @click.command("debug", @@ -81,7 +82,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, return helpers.predebug_project(ctx, project_dir, env_name, False, verbose) - configuration = helpers.load_configuration(ctx, project_dir, env_name) + configuration = load_project_ide_data(project_dir, env_name) if not configuration: raise exception.DebugInvalidOptions( "Could not load debug configuration") @@ -116,8 +117,8 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, if rebuild_prog: if helpers.is_mi_mode(__unprocessed): - output = helpers.GDBBytesIO() click.echo('~"Preparing firmware for debugging...\\n"') + output = helpers.GDBBytesIO() with helpers.capture_std_streams(output): helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 6c215512..c9b36897 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import sys import time from contextlib import contextmanager @@ -39,6 +38,17 @@ class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods self.STDOUT.flush() +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr + + def is_mi_mode(args): return "--interpreter" in " ".join(args) @@ -59,6 +69,16 @@ def get_default_debug_env(config): return default_envs[0] if default_envs else all_envs[0] +def predebug_project(ctx, project_dir, env_name, preload, verbose): + ctx.invoke(cmd_run, + project_dir=project_dir, + environment=[env_name], + target=["debug"] + (["upload"] if preload else []), + verbose=verbose) + if preload: + time.sleep(5) + + def validate_debug_options(cmd_ctx, env_options): def _cleanup_cmds(items): @@ -144,45 +164,6 @@ def validate_debug_options(cmd_ctx, env_options): return result -def predebug_project(ctx, project_dir, env_name, preload, verbose): - ctx.invoke(cmd_run, - project_dir=project_dir, - environment=[env_name], - target=["debug"] + (["upload"] if preload else []), - verbose=verbose) - if preload: - time.sleep(5) - - -@contextmanager -def capture_std_streams(stdout, stderr=None): - _stdout = sys.stdout - _stderr = sys.stderr - sys.stdout = stdout - sys.stderr = stderr or stdout - yield - sys.stdout = _stdout - sys.stderr = _stderr - - -def load_configuration(ctx, project_dir, env_name): - output = BytesIO() - with capture_std_streams(output): - ctx.invoke(cmd_run, - project_dir=project_dir, - environment=[env_name], - target=["idedata"]) - result = output.getvalue().decode() - output.close() - if '"includes":' not in result: - return None - for line in result.split("\n"): - line = line.strip() - if line.startswith('{"') and "cxx_path" in line: - return json.loads(line[:line.rindex("}") + 1]) - return None - - def configure_esp32_load_cmds(debug_options, configuration): ignore_conds = [ debug_options['load_cmds'] != ["load"], diff --git a/platformio/commands/init.py b/platformio/commands/init.py index d3d8f52f..09eddacf 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -132,7 +132,7 @@ def cli( def get_best_envname(project_dir, boards=None): - config = ProjectConfig(join(project_dir, "platformio.ini")) + config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) config.validate() envname = None diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 23ca09d4..c738749e 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -20,8 +20,8 @@ import click from platformio import exception, util from platformio.commands.device import device_monitor as cmd_device_monitor -from platformio.commands.run.helpers import (_clean_build_dir, - _handle_legacy_libdeps, +from platformio.commands.run.helpers import (clean_build_dir, + handle_legacy_libdeps, print_summary) from platformio.commands.run.processor import EnvironmentProcessor from platformio.project.config import ProjectConfig @@ -64,7 +64,7 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, # clean obsolete build dir if not disable_auto_clean: try: - _clean_build_dir(get_project_build_dir()) + clean_build_dir(get_project_build_dir()) except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " @@ -76,7 +76,7 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, project_conf or join(project_dir, "platformio.ini")) config.validate(environment) - _handle_legacy_libdeps(project_dir, config) + handle_legacy_libdeps(project_dir, config) results = [] start_time = time() diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index d1e6b59f..ddb1fb9d 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -27,7 +27,7 @@ from platformio.project.helpers import (calculate_project_hash, get_project_libdeps_dir) -def _handle_legacy_libdeps(project_dir, config): +def handle_legacy_libdeps(project_dir, config): legacy_libdeps_dir = join(project_dir, ".piolibdeps") if (not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == get_project_libdeps_dir()): @@ -46,7 +46,7 @@ def _handle_legacy_libdeps(project_dir, config): fg="yellow") -def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): +def autoinstall_libdeps(ctx, envname, libraries, verbose=False): if not libraries: return libdeps_dir = join(get_project_libdeps_dir(), envname) @@ -62,7 +62,7 @@ def _autoinstall_libdeps(ctx, envname, libraries, verbose=False): click.secho(str(e), fg="yellow") -def _clean_build_dir(build_dir): +def clean_build_dir(build_dir): # remove legacy ".pioenvs" folder legacy_build_dir = join(get_project_dir(), ".pioenvs") if isdir(legacy_build_dir) and legacy_build_dir != build_dir: diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 66e65637..66c07296 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -19,7 +19,7 @@ import click from platformio import exception, telemetry from platformio.commands.platform import \ platform_install as cmd_platform_install -from platformio.commands.run.helpers import _autoinstall_libdeps, print_header +from platformio.commands.run.helpers import autoinstall_libdeps, print_header from platformio.commands.test.processor import (CTX_META_TEST_IS_RUNNING, CTX_META_TEST_RUNNING_NAME) from platformio.managers.platform import PlatformFactory @@ -104,7 +104,7 @@ class EnvironmentProcessor(object): if "monitor" in build_targets: build_targets.remove("monitor") if "nobuild" not in build_targets and "lib_deps" in self.options: - _autoinstall_libdeps( + autoinstall_libdeps( self.cmd_ctx, self.name, self.config.get("env:" + self.name, "lib_deps"), self.verbose) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index eb3ce6ec..a83f7e6e 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -13,23 +13,21 @@ # limitations under the License. import codecs -import json import os import re import sys from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath import bottle -from click.testing import CliRunner -from platformio import exception, util -from platformio.commands.run import cli as cmd_run +from platformio import util from platformio.compat import WINDOWS, get_file_contents from platformio.proc import where_is_program from platformio.project.config import ProjectConfig from platformio.project.helpers import (get_project_lib_dir, get_project_libdeps_dir, - get_project_src_dir) + get_project_src_dir, + load_project_ide_data) class ProjectGenerator(object): @@ -39,57 +37,45 @@ class ProjectGenerator(object): self.ide = str(ide) self.env_name = env_name - self._tplvars = {} - self._gather_tplvars() - @staticmethod def get_supported_ides(): tpls_dir = join(util.get_source_dir(), "ide", "tpls") return sorted( [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) - @util.memoized() - def get_project_env(self): - data = {} - config = ProjectConfig.get_instance( - join(self.project_dir, "platformio.ini")) - for env in config.envs(): - if self.env_name != env: - continue - data = config.items(env=env, as_dict=True) - data['env_name'] = self.env_name - return data + def _load_tplvars(self): + tpl_vars = {"env_name": self.env_name} + # default env configuration + tpl_vars.update( + ProjectConfig.get_instance(join( + self.project_dir, "platformio.ini")).items(env=self.env_name, + as_dict=True)) + # build data + tpl_vars.update( + load_project_ide_data(self.project_dir, self.env_name) or {}) - def get_project_build_data(self): - data = { - "defines": [], - "includes": [], - "cxx_path": None, - "prog_path": None - } - envdata = self.get_project_env() - if not envdata: - return data + with util.cd(self.project_dir): + tpl_vars.update({ + "project_name": basename(self.project_dir), + "src_files": self.get_src_files(), + "user_home_dir": abspath(expanduser("~")), + "project_dir": self.project_dir, + "project_src_dir": get_project_src_dir(), + "project_lib_dir": get_project_lib_dir(), + "project_libdeps_dir": join( + get_project_libdeps_dir(), self.env_name), + "systype": util.get_systype(), + "platformio_path": self._fix_os_path( + sys.argv[0] if isfile(sys.argv[0]) + else where_is_program("platformio")), + "env_pathsep": os.pathsep, + "env_path": self._fix_os_path(os.getenv("PATH")) + }) # yapf: disable + return tpl_vars - result = CliRunner().invoke(cmd_run, [ - "--project-dir", self.project_dir, "--environment", - envdata['env_name'], "--target", "idedata" - ]) - - if result.exit_code != 0 and not isinstance(result.exception, - exception.ReturnErrorCode): - raise result.exception - if '"includes":' not in result.output: - raise exception.PlatformioException(result.output) - - for line in result.output.split("\n"): - line = line.strip() - if line.startswith('{"') and line.endswith("}"): - data = json.loads(line) - return data - - def get_project_name(self): - return basename(self.project_dir) + @staticmethod + def _fix_os_path(path): + return (re.sub(r"[\\]+", '\\' * 4, path) if WINDOWS else path) def get_src_files(self): result = [] @@ -113,6 +99,7 @@ class ProjectGenerator(object): return tpls def generate(self): + tpl_vars = self._load_tplvars() for tpl_relpath, tpl_path in self.get_tpls(): dst_dir = self.project_dir if tpl_relpath: @@ -121,11 +108,12 @@ class ProjectGenerator(object): os.makedirs(dst_dir) file_name = basename(tpl_path)[:-4] - contents = self._render_tpl(tpl_path) + contents = self._render_tpl(tpl_path, tpl_vars) self._merge_contents(join(dst_dir, file_name), contents) - def _render_tpl(self, tpl_path): - return bottle.template(get_file_contents(tpl_path), **self._tplvars) + @staticmethod + def _render_tpl(tpl_path, tpl_vars): + return bottle.template(get_file_contents(tpl_path), **tpl_vars) @staticmethod def _merge_contents(dst_path, contents): @@ -133,28 +121,3 @@ class ProjectGenerator(object): return with codecs.open(dst_path, "w", encoding="utf8") as fp: fp.write(contents) - - def _gather_tplvars(self): - self._tplvars.update(self.get_project_env()) - self._tplvars.update(self.get_project_build_data()) - with util.cd(self.project_dir): - self._tplvars.update({ - "project_name": self.get_project_name(), - "src_files": self.get_src_files(), - "user_home_dir": abspath(expanduser("~")), - "project_dir": self.project_dir, - "project_src_dir": get_project_src_dir(), - "project_lib_dir": get_project_lib_dir(), - "project_libdeps_dir": join( - get_project_libdeps_dir(), self.env_name), - "systype": util.get_systype(), - "platformio_path": self._fix_os_path( - sys.argv[0] if isfile(sys.argv[0]) - else where_is_program("platformio")), - "env_pathsep": os.pathsep, - "env_path": self._fix_os_path(os.getenv("PATH")) - }) # yapf: disable - - @staticmethod - def _fix_os_path(path): - return (re.sub(r"[\\]+", '\\' * 4, path) if WINDOWS else path) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 71f73649..3c017666 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -12,13 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os from hashlib import sha1 from os import walk from os.path import (basename, dirname, expanduser, isdir, isfile, join, realpath, splitdrive) -from platformio import __version__ +from click.testing import CliRunner + +from platformio import __version__, exception from platformio.compat import WINDOWS, hashlib_encode_data from platformio.project.config import ProjectConfig @@ -179,3 +182,22 @@ def calculate_project_hash(): # A case of disk drive can differ... chunks_to_str = chunks_to_str.lower() return sha1(hashlib_encode_data(chunks_to_str)).hexdigest() + + +def load_project_ide_data(project_dir, env_name): + from platformio.commands.run import cli as cmd_run + result = CliRunner().invoke(cmd_run, [ + "--project-dir", project_dir, "--environment", env_name, "--target", + "idedata" + ]) + if result.exit_code != 0 and not isinstance(result.exception, + exception.ReturnErrorCode): + raise result.exception + if '"includes":' not in result.output: + raise exception.PlatformioException(result.output) + + for line in result.output.split("\n"): + line = line.strip() + if line.startswith('{"') and line.endswith("}"): + return json.loads(line) + return None From a504a13fa8a66c307f8573623ec893776ad9ad9a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 4 Jun 2019 00:50:45 +0300 Subject: [PATCH 226/333] Fix broken example with ConfigParser using // Resolve #2616 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index a2d06177..56a4c2ac 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a2d061775a51e1181db8aac8bc05058e21da7e96 +Subproject commit 56a4c2ac24d59218ea29ba03ee7f413ee3d541d8 From 5c8c10e7d380d53ae9856ae37a28bfabd10dc531 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 4 Jun 2019 13:55:11 +0300 Subject: [PATCH 227/333] Do not check that lib_extra_dirs exist // Resolve #2624 --- platformio/project/options.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/platformio/project/options.py b/platformio/project/options.py index fe5e9368..846f28dc 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -159,9 +159,7 @@ ProjectOptions = OrderedDict([ ConfigEnvOption(name="lib_ignore", multiple=True), ConfigEnvOption(name="lib_extra_dirs", multiple=True, - sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS", - type=click.Path( - exists=True, file_okay=False, dir_okay=True)), + sysenvvar="PLATFORMIO_LIB_EXTRA_DIRS"), ConfigEnvOption(name="lib_ldf_mode", type=click.Choice( ["off", "chain", "deep", "chain+", "deep+"])), From f873bd41f809545543bc2aac9349e67e6c07054e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Jun 2019 16:47:02 +0300 Subject: [PATCH 228/333] Better printing of relative path for removed objects --- platformio/builder/tools/piomisc.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 442ffdd7..65596d10 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -285,10 +285,13 @@ def PioClean(env, clean_dir): if not isdir(clean_dir): print("Build environment is clean") env.Exit(0) + clean_rel_path = relpath(clean_dir) for root, _, files in walk(clean_dir): - for file_ in files: - remove(join(root, file_)) - print("Removed %s" % relpath(join(root, file_))) + for f in files: + dst = join(root, f) + remove(dst) + print("Removed %s" % + (dst if clean_rel_path.startswith(".") else relpath(dst))) print("Done cleaning") util.rmtree_(clean_dir) env.Exit(0) From 84ce7db3e3cb25cef5084f55f5846b971aab1d79 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Jun 2019 17:53:02 +0300 Subject: [PATCH 229/333] Fixed an issue when library keeps reinstalling for non-latin path // Resolve #1252 --- HISTORY.rst | 1 + platformio/managers/package.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 810f89c5..c1122952 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -26,6 +26,7 @@ PlatformIO 4.0 - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - Use isolated library dependency storage per project build environment (`issue #1696 `_) - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields + - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 `_) * **Build System** diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 56a33f0b..617237e9 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -26,7 +26,7 @@ import requests import semantic_version from platformio import __version__, app, exception, telemetry, util -from platformio.compat import hashlib_encode_data, path_to_unicode +from platformio.compat import hashlib_encode_data from platformio.downloader import FileDownloader from platformio.lockfile import LockFile from platformio.unpacker import FileUnpacker @@ -394,7 +394,7 @@ class PkgInstallerMixin(object): if "version" not in manifest: manifest['version'] = "0.0.0" - manifest['__pkg_dir'] = path_to_unicode(pkg_dir) + manifest['__pkg_dir'] = pkg_dir self.cache_set(cache_key, manifest) return manifest @@ -451,7 +451,7 @@ class PkgInstallerMixin(object): def get_package_by_dir(self, pkg_dir): for manifest in self.get_installed(): - if manifest['__pkg_dir'] == path_to_unicode(abspath(pkg_dir)): + if manifest['__pkg_dir'] == abspath(pkg_dir): return manifest return None From 394d272324735e547fbcf2305876c0c6b1cb002e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Jun 2019 17:57:22 +0300 Subject: [PATCH 230/333] Fix numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars // Resolve #143, Resolve #1342, Resolve #1959, Resolve #2100 --- HISTORY.rst | 2 +- platformio/app.py | 16 ++-- platformio/commands/boards.py | 3 +- platformio/commands/device.py | 5 +- platformio/commands/home/rpc/handlers/app.py | 80 ++++++++----------- platformio/commands/home/rpc/handlers/misc.py | 12 ++- platformio/commands/home/rpc/handlers/os.py | 4 +- .../commands/home/rpc/handlers/piocore.py | 12 +-- .../commands/home/rpc/handlers/project.py | 15 ++-- platformio/commands/home/rpc/server.py | 9 ++- platformio/commands/lib.py | 18 ++--- platformio/commands/platform.py | 16 ++-- platformio/commands/run/processor.py | 2 +- platformio/managers/platform.py | 11 ++- 14 files changed, 103 insertions(+), 102 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c1122952..d896f7e3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -43,7 +43,7 @@ PlatformIO 4.0 - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) - - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #2100 `_) + - Fixed numerous issues related to "UnicodeDecodeError" and international locales, or when project path contains non-ASCII chars (`issue #143 `_, `issue #1342 `_, `issue #1959 `_, `issue #2100 `_) * **Integration** diff --git a/platformio/app.py b/platformio/app.py index 49f24625..c35c4baf 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -14,7 +14,6 @@ import codecs import hashlib -import json import os import uuid from copy import deepcopy @@ -25,7 +24,8 @@ from time import time import requests from platformio import exception, lockfile, util -from platformio.compat import WINDOWS, hashlib_encode_data +from platformio.compat import (WINDOWS, dump_json_to_unicode, + hashlib_encode_data) from platformio.proc import is_ci from platformio.project.helpers import (get_project_cache_dir, get_project_core_dir) @@ -102,16 +102,19 @@ class State(object): self._lock_state_file() if isfile(self.path): self._state = util.load_json(self.path) - except exception.PlatformioException: + assert isinstance(self._state, dict) + except (AssertionError, UnicodeDecodeError, + exception.PlatformioException): self._state = {} self._prev_state = deepcopy(self._state) return self._state def __exit__(self, type_, value, traceback): - if self._prev_state != self._state: + new_state = dump_json_to_unicode(self._state) + if self._prev_state != new_state: try: - with codecs.open(self.path, "w", encoding="utf8") as fp: - json.dump(self._state, fp) + with open(self.path, "w") as fp: + fp.write(new_state) except IOError: raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() @@ -167,6 +170,7 @@ class ContentCache(object): return True def get_cache_path(self, key): + key = str(key) assert len(key) > 3 return join(self.cache_dir, key[-2:], key) diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index b48be4df..6aff1681 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -17,6 +17,7 @@ import json import click from platformio import util +from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformManager @@ -82,4 +83,4 @@ def _print_boards_json(query, installed=False): if query.lower() not in search_data.lower(): continue result.append(board) - click.echo(json.dumps(result)) + click.echo(dump_json_to_unicode(result)) diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 7876fae7..d5ffe030 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import sys from fnmatch import fnmatch from os import getcwd @@ -22,6 +21,7 @@ import click from serial.tools import miniterm from platformio import exception, util +from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig @@ -50,7 +50,8 @@ def device_list( # pylint: disable=too-many-branches single_key = list(data)[0] if len(list(data)) == 1 else None if json_output: - return click.echo(json.dumps(data[single_key] if single_key else data)) + return click.echo( + dump_json_to_unicode(data[single_key] if single_key else data)) titles = { "serial": "Serial Ports", diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index e356fa06..0f8b8285 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -14,11 +14,9 @@ from __future__ import absolute_import -import json -from os.path import expanduser, isfile, join +from os.path import expanduser, join -from platformio import __version__, app, exception, util -from platformio.compat import path_to_unicode +from platformio import __version__, app, util from platformio.project.helpers import (get_project_core_dir, is_platformio_project) @@ -29,57 +27,45 @@ class AppRPC(object): @staticmethod def load_state(): - state = None - try: - if isfile(AppRPC.APPSTATE_PATH): - state = util.load_json(AppRPC.APPSTATE_PATH) - except exception.PlatformioException: - pass - if not isinstance(state, dict): - state = {} - storage = state.get("storage", {}) + with app.State(AppRPC.APPSTATE_PATH, lock=True) as state: + storage = state.get("storage", {}) - # base data - caller_id = app.get_session_var("caller_id") - storage['cid'] = app.get_cid() - storage['coreVersion'] = __version__ - storage['coreSystype'] = util.get_systype() - storage['coreCaller'] = (str(caller_id).lower() if caller_id else None) - storage['coreSettings'] = { - name: { - "description": data['description'], - "default_value": data['value'], - "value": app.get_setting(name) + # base data + caller_id = app.get_session_var("caller_id") + storage['cid'] = app.get_cid() + storage['coreVersion'] = __version__ + storage['coreSystype'] = util.get_systype() + storage['coreCaller'] = (str(caller_id).lower() + if caller_id else None) + storage['coreSettings'] = { + name: { + "description": data['description'], + "default_value": data['value'], + "value": app.get_setting(name) + } + for name, data in app.DEFAULT_SETTINGS.items() } - for name, data in app.DEFAULT_SETTINGS.items() - } - # encode to UTF-8 - for key in storage['coreSettings']: - if not key.endswith("dir"): - continue - storage['coreSettings'][key]['default_value'] = path_to_unicode( - storage['coreSettings'][key]['default_value']) - storage['coreSettings'][key]['value'] = path_to_unicode( - storage['coreSettings'][key]['value']) - storage['homeDir'] = path_to_unicode(expanduser("~")) - storage['projectsDir'] = storage['coreSettings']['projects_dir'][ - 'value'] + storage['homeDir'] = expanduser("~") + storage['projectsDir'] = storage['coreSettings']['projects_dir'][ + 'value'] - # skip non-existing recent projects - storage['recentProjects'] = [ - p for p in storage.get("recentProjects", []) - if is_platformio_project(p) - ] + # skip non-existing recent projects + storage['recentProjects'] = [ + p for p in storage.get("recentProjects", []) + if is_platformio_project(p) + ] - state['storage'] = storage - return state + state['storage'] = storage + return state @staticmethod def get_state(): return AppRPC.load_state() - def save_state(self, state): - with open(self.APPSTATE_PATH, "w") as fp: - json.dump(state, fp) + @staticmethod + def save_state(state): + with app.State(AppRPC.APPSTATE_PATH, lock=True) as s: + s.clear() + s.update(state) return True diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index 8de0e195..77df7bc8 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -27,7 +27,7 @@ from platformio.commands.home.rpc.handlers.os import OSRPC class MiscRPC(object): def load_latest_tweets(self, username): - cache_key = "piohome_latest_tweets_%s" % username + cache_key = "piohome_latest_tweets_" + str(username) cache_valid = "7d" with app.ContentCache() as cc: cache_data = cc.get(cache_key) @@ -60,13 +60,11 @@ class MiscRPC(object): "include_new_items_bar=true") % username if helpers.is_twitter_blocked(): api_url = self._get_proxed_uri(api_url) - html_or_json = yield OSRPC.fetch_content( + content = yield OSRPC.fetch_content( api_url, headers=self._get_twitter_headers(username)) - # issue with PIO Core < 3.5.3 and ContentCache - if not isinstance(html_or_json, dict): - html_or_json = json.loads(html_or_json) - assert "items_html" in html_or_json - soup = BeautifulSoup(html_or_json['items_html'], "html.parser") + content = json.loads(content) + assert "items_html" in content + soup = BeautifulSoup(content['items_html'], "html.parser") tweet_nodes = soup.find_all("div", attrs={ "class": "tweet", diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index c394d935..d26a0c00 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -25,7 +25,7 @@ from twisted.internet import defer # pylint: disable=import-error from platformio import app, util from platformio.commands.home import helpers -from platformio.compat import PY2, get_filesystem_encoding, path_to_unicode +from platformio.compat import PY2, get_filesystem_encoding class OSRPC(object): @@ -150,6 +150,6 @@ class OSRPC(object): items = [] for item in util.get_logical_devices(): if item['name']: - item['name'] = path_to_unicode(item['name']) + item['name'] = item['name'] items.append(item) return items diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index b651b498..198552d1 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -23,7 +23,7 @@ from twisted.internet import utils # pylint: disable=import-error from platformio import __version__ from platformio.commands.home import helpers -from platformio.compat import get_filesystem_encoding, string_types +from platformio.compat import string_types class PIOCoreRPC(object): @@ -33,8 +33,8 @@ class PIOCoreRPC(object): json_output = "--json-output" in args try: args = [ - arg.encode(get_filesystem_encoding()) if isinstance( - arg, string_types) else str(arg) for arg in args + str(arg) if not isinstance(arg, string_types) else arg + for arg in args ] except UnicodeError: raise jsonrpc.exceptions.JSONRPCDispatchException( @@ -51,18 +51,12 @@ class PIOCoreRPC(object): @staticmethod def _call_callback(result, json_output=False): - result = list(result) - assert len(result) == 3 - for i in (0, 1): - result[i] = result[i].decode(get_filesystem_encoding()).strip() out, err, code = result text = ("%s\n\n%s" % (out, err)).strip() if code != 0: raise Exception(text) - if not json_output: return text - try: return json.loads(out) except ValueError as e: diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 6e029aca..bb301ea9 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -25,7 +25,7 @@ import jsonrpc # pylint: disable=import-error from platformio import exception, util from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC -from platformio.compat import get_filesystem_encoding +from platformio.compat import PY2, get_filesystem_encoding from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager from platformio.project.config import ProjectConfig @@ -172,6 +172,10 @@ class ProjectRPC(object): return project_dir def import_arduino(self, board, use_arduino_libs, arduino_project_dir): + board = str(board) + if arduino_project_dir and PY2: + arduino_project_dir = arduino_project_dir.encode( + get_filesystem_encoding()) # don't import PIO Project if is_platformio_project(arduino_project_dir): return arduino_project_dir @@ -188,7 +192,7 @@ class ProjectRPC(object): message="Not an Arduino project: %s" % arduino_project_dir) state = AppRPC.load_state() - project_dir = join(state['storage']['projectsDir'].decode("utf-8"), + project_dir = join(state['storage']['projectsDir'], time.strftime("%y%m%d-%H%M%S-") + board) if not isdir(project_dir): os.makedirs(project_dir) @@ -213,8 +217,7 @@ class ProjectRPC(object): src_dir = get_project_src_dir() if isdir(src_dir): util.rmtree_(src_dir) - shutil.copytree( - arduino_project_dir.encode(get_filesystem_encoding()), src_dir) + shutil.copytree(arduino_project_dir, src_dir) return project_dir @staticmethod @@ -255,12 +258,14 @@ class ProjectRPC(object): @staticmethod def import_pio(project_dir): + if project_dir and PY2: + project_dir = project_dir.encode(get_filesystem_encoding()) if not project_dir or not is_platformio_project(project_dir): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4001, message="Not an PlatformIO project: %s" % project_dir) new_project_dir = join( - AppRPC.load_state()['storage']['projectsDir'].decode("utf-8"), + AppRPC.load_state()['storage']['projectsDir'], time.strftime("%y%m%d-%H%M%S-") + basename(project_dir)) shutil.copytree(project_dir, new_project_dir) diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index d62faa3d..f99b09b3 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -14,14 +14,14 @@ # pylint: disable=import-error -import json - import jsonrpc from autobahn.twisted.websocket import (WebSocketServerFactory, WebSocketServerProtocol) from jsonrpc.exceptions import JSONRPCDispatchException from twisted.internet import defer +from platformio.compat import PY2, dump_json_to_unicode, is_bytes + class JSONRPCServerProtocol(WebSocketServerProtocol): @@ -57,7 +57,10 @@ class JSONRPCServerProtocol(WebSocketServerProtocol): def sendJSONResponse(self, response): # print("< %s" % response) - self.sendMessage(json.dumps(response).encode("utf8")) + response = dump_json_to_unicode(response) + if not PY2 and not is_bytes(response): + response = response.encode("utf-8") + self.sendMessage(response) class JSONRPCServerFactory(WebSocketServerFactory): diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 23094f43..ad39bf53 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -14,7 +14,6 @@ # pylint: disable=too-many-branches, too-many-locals -import json import time from os.path import isdir, join @@ -22,6 +21,7 @@ import click import semantic_version from platformio import exception, util +from platformio.compat import dump_json_to_unicode from platformio.managers.lib import (LibraryManager, get_builtin_libs, is_builtin_lib) from platformio.proc import is_ci @@ -247,8 +247,8 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output): if json_output: return click.echo( - json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) == - 1 else json_result)) + dump_json_to_unicode(json_result[storage_dirs[0]] + if len(storage_dirs) == 1 else json_result)) return True @@ -274,8 +274,8 @@ def lib_list(ctx, json_output): if json_output: return click.echo( - json.dumps(json_result[storage_dirs[0]] if len(storage_dirs) == - 1 else json_result)) + dump_json_to_unicode(json_result[storage_dirs[0]] + if len(storage_dirs) == 1 else json_result)) return True @@ -309,7 +309,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): cache_valid="1d") if json_output: - click.echo(json.dumps(result)) + click.echo(dump_json_to_unicode(result)) return if result['total'] == 0: @@ -361,7 +361,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): def lib_builtin(storage, json_output): items = get_builtin_libs(storage) if json_output: - return click.echo(json.dumps(items)) + return click.echo(dump_json_to_unicode(items)) for storage_ in items: if not storage_['items']: @@ -390,7 +390,7 @@ def lib_show(library, json_output): interactive=not json_output) lib = util.get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") if json_output: - return click.echo(json.dumps(lib)) + return click.echo(dump_json_to_unicode(lib)) click.secho(lib['name'], fg="cyan") click.echo("=" * len(lib['name'])) @@ -478,7 +478,7 @@ def lib_stats(json_output): result = util.get_api_result("/lib/stats", cache_valid="1h") if json_output: - return click.echo(json.dumps(result)) + return click.echo(dump_json_to_unicode(result)) printitem_tpl = "{name:<33} {url}" printitemdate_tpl = "{name:<33} {date:23} {url}" diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 9f56f8d6..49a61812 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json from os.path import dirname, isdir import click from platformio import app, exception, util from platformio.commands.boards import print_boards +from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformFactory, PlatformManager @@ -156,7 +156,7 @@ def platform_search(query, json_output): for platform in _get_registry_platforms(): if query == "all": query = "" - search_data = json.dumps(platform) + search_data = dump_json_to_unicode(platform) if query and query.lower() not in search_data.lower(): continue platforms.append( @@ -165,7 +165,7 @@ def platform_search(query, json_output): expose_packages=False)) if json_output: - click.echo(json.dumps(platforms)) + click.echo(dump_json_to_unicode(platforms)) else: _print_platforms(platforms) @@ -178,7 +178,7 @@ def platform_frameworks(query, json_output): for framework in util.get_api_result("/frameworks", cache_valid="7d"): if query == "all": query = "" - search_data = json.dumps(framework) + search_data = dump_json_to_unicode(framework) if query and query.lower() not in search_data.lower(): continue framework['homepage'] = ("https://platformio.org/frameworks/" + @@ -191,7 +191,7 @@ def platform_frameworks(query, json_output): frameworks = sorted(frameworks, key=lambda manifest: manifest['name']) if json_output: - click.echo(json.dumps(frameworks)) + click.echo(dump_json_to_unicode(frameworks)) else: _print_platforms(frameworks) @@ -209,7 +209,7 @@ def platform_list(json_output): platforms = sorted(platforms, key=lambda manifest: manifest['name']) if json_output: - click.echo(json.dumps(platforms)) + click.echo(dump_json_to_unicode(platforms)) else: _print_platforms(platforms) @@ -222,7 +222,7 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches if not data: raise exception.UnknownPlatform(platform) if json_output: - return click.echo(json.dumps(data)) + return click.echo(dump_json_to_unicode(data)) click.echo("{name} ~ {title}".format(name=click.style(data['name'], fg="cyan"), @@ -361,7 +361,7 @@ def platform_update( # pylint: disable=too-many-locals if latest: data['versionLatest'] = latest result.append(data) - return click.echo(json.dumps(result)) + return click.echo(dump_json_to_unicode(result)) # cleanup cached board and platform lists app.clean_cache() diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 66c07296..2fbba4be 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -37,7 +37,7 @@ class EnvironmentProcessor(object): self.cmd_ctx = cmd_ctx self.name = name self.config = config - self.targets = targets + self.targets = [str(t) for t in targets] self.upload_port = upload_port self.silent = silent self.verbose = verbose diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index b2e467d4..d34b9880 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -24,7 +24,7 @@ import click import semantic_version from platformio import __version__, app, exception, util -from platformio.compat import hashlib_encode_data, is_bytes +from platformio.compat import PY2, hashlib_encode_data, is_bytes from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, @@ -693,6 +693,15 @@ class PlatformBoardConfig(object): value = self._manifest for k in path.split("."): value = value[k] + # pylint: disable=undefined-variable + if PY2 and isinstance(value, unicode): + # cast to plain string from unicode for PY2, resolves issue in + # dev/platform when BoardConfig.get() is used in pair with + # os.path.join(file_encoding, unicode_encoding) + try: + value = value.encode("utf-8") + except UnicodeEncodeError: + pass return value except KeyError: if default is not None: From 522f8148113ffa8fb40566c4a9135df3488a182e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Jun 2019 18:26:39 +0300 Subject: [PATCH 231/333] Show detailed info about a platform when is installed from local folder or VCS // Resolve #2081 --- platformio/commands/platform.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 49a61812..5b35c4df 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -42,7 +42,11 @@ def _print_platforms(platforms): if "packages" in platform: click.echo("Packages: %s" % ", ".join(platform['packages'])) if "version" in platform: - click.echo("Version: " + platform['version']) + if "__src_url" in platform: + click.echo("Version: #%s (%s)" % + (platform['version'], platform['__src_url'])) + else: + click.echo("Version: " + platform['version']) click.echo() From 898d79956d19d915ee31f9e48aa70d41c56b6cf3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 5 Jun 2019 18:28:23 +0300 Subject: [PATCH 232/333] Bump version to 4.0.0a19 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 1808eab8..b21aa74b 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a18") +VERSION = (4, 0, "0a19") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From bcff26d4d7e42d35ab34145ac48d2242d491110b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 6 Jun 2019 00:13:04 +0300 Subject: [PATCH 233/333] Refactor using "@util.memoized" --- platformio/commands/home/helpers.py | 6 ++--- platformio/commands/lib.py | 6 ++++- platformio/managers/lib.py | 12 +++++---- platformio/managers/package.py | 41 +++++++++++++---------------- platformio/managers/platform.py | 3 +-- platformio/util.py | 13 ++++++--- 6 files changed, 44 insertions(+), 37 deletions(-) diff --git a/platformio/commands/home/helpers.py b/platformio/commands/home/helpers.py index a8ff898f..46c1750c 100644 --- a/platformio/commands/home/helpers.py +++ b/platformio/commands/home/helpers.py @@ -43,18 +43,18 @@ class AsyncSession(requests.Session): return defer.ensureDeferred(*args, **kwargs) -@util.memoized(expire=5000) +@util.memoized(expire="60s") def requests_session(): return AsyncSession(n=5) -@util.memoized() +@util.memoized(expire="60s") def get_core_fullpath(): return where_is_program( "platformio" + (".exe" if "windows" in util.get_systype() else "")) -@util.memoized(expire=10000) +@util.memoized(expire="10s") def is_twitter_blocked(): ip = "104.244.42.1" timeout = 2 diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index ad39bf53..c66dc77b 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -147,6 +147,7 @@ def lib_install( # pylint: disable=too-many-arguments force=force) installed_manifests[library] = lm.load_manifest(pkg_dir) elif storage_dir in storage_libdeps: + builtin_lib_storages = None for library in storage_libdeps[storage_dir]: try: pkg_dir = lm.install(library, @@ -155,7 +156,10 @@ def lib_install( # pylint: disable=too-many-arguments force=force) installed_manifests[library] = lm.load_manifest(pkg_dir) except exception.LibNotFound as e: - if not silent or not is_builtin_lib(library): + if builtin_lib_storages is None: + builtin_lib_storages = get_builtin_libs() + if not silent or not is_builtin_lib( + builtin_lib_storages, library): click.secho("Warning! %s" % e, fg="yellow") if not save or not libraries: diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 1c0d04dd..e995d751 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -361,6 +361,7 @@ class LibraryManager(BasePkgManager): if not silent: click.secho("Installing dependencies", fg="yellow") + builtin_lib_storages = None for filters in self.normalize_dependencies(manifest['dependencies']): assert "name" in filters @@ -382,7 +383,10 @@ class LibraryManager(BasePkgManager): try: lib_id = self.search_lib_id(filters, silent, interactive) except exception.LibNotFound as e: - if not silent or is_builtin_lib(filters['name']): + if builtin_lib_storages is None: + builtin_lib_storages = get_builtin_libs() + if not silent or is_builtin_lib(builtin_lib_storages, + filters['name']): click.secho("Warning! %s" % e, fg="yellow") continue @@ -402,7 +406,6 @@ class LibraryManager(BasePkgManager): return pkg_dir -@util.memoized() def get_builtin_libs(storage_names=None): items = [] storage_names = storage_names or [] @@ -421,9 +424,8 @@ def get_builtin_libs(storage_names=None): return items -@util.memoized() -def is_builtin_lib(name): - for storage in get_builtin_libs(): +def is_builtin_lib(storages, name): + for storage in storages or []: if any(l.get("name") == name for l in storage['items']): return True return False diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 617237e9..6eb20188 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -37,8 +37,6 @@ from platformio.vcsclient import VCSClientFactory class PackageRepoIterator(object): - _MANIFEST_CACHE = {} - def __init__(self, package, repositories): assert isinstance(repositories, list) self.package = package @@ -50,27 +48,25 @@ class PackageRepoIterator(object): def __next__(self): return self.next() - def next(self): - manifest = {} - repo = next(self.repositories) - if isinstance(repo, dict): - manifest = repo - elif repo in PackageRepoIterator._MANIFEST_CACHE: - manifest = PackageRepoIterator._MANIFEST_CACHE[repo] - else: - r = None - try: - r = requests.get(repo, headers=util.get_request_defheaders()) - r.raise_for_status() - manifest = r.json() - except: # pylint: disable=bare-except - pass - finally: - if r: - r.close() - PackageRepoIterator._MANIFEST_CACHE[repo] = manifest + @staticmethod + @util.memoized(expire="60s") + def load_manifest(url): + r = None + try: + r = requests.get(url, headers=util.get_request_defheaders()) + r.raise_for_status() + return r.json() + except: # pylint: disable=bare-except + pass + finally: + if r: + r.close() + return None - if self.package in manifest: + def next(self): + repo = next(self.repositories) + manifest = repo if isinstance(repo, dict) else self.load_manifest(repo) + if manifest and self.package in manifest: return manifest[self.package] return next(self) @@ -795,6 +791,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): return True def update(self, package, requirements=None, only_check=False): + self.cache_reset() if isdir(package) and self.get_package_by_dir(package): pkg_dir = package else: diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index d34b9880..506275c5 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -173,7 +173,7 @@ class PlatformManager(BasePkgManager): self.cache_reset() return True - @util.memoized(expire=5000) + @util.memoized(expire="5s") def get_installed_boards(self): boards = [] for manifest in self.get_installed(): @@ -185,7 +185,6 @@ class PlatformManager(BasePkgManager): return boards @staticmethod - @util.memoized() def get_registered_boards(): return util.get_api_result("/boards", cache_valid="7d") diff --git a/platformio/util.py b/platformio/util.py index e07ce391..097cfa8e 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -50,7 +50,12 @@ class cd(object): class memoized(object): def __init__(self, expire=0): - self.expire = expire / 1000 # milliseconds + expire = str(expire) + if expire.isdigit(): + expire = "%ss" % (int(expire) / 1000) + tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} + assert expire.endswith(tuple(tdmap)) + self.expire = int(tdmap[expire[-1]] * int(expire[:-1])) self.cache = {} def __call__(self, func): @@ -68,7 +73,7 @@ class memoized(object): return wrapper def _reset(self): - self.cache = {} + self.cache.clear() class throttle(object): @@ -296,7 +301,7 @@ def get_request_defheaders(): return {"User-Agent": "PlatformIO/%s CI/%d %s" % data} -@memoized(expire=10000) +@memoized(expire="60s") def _api_request_session(): return requests.Session() @@ -394,7 +399,7 @@ PING_INTERNET_IPS = [ ] -@memoized(expire=5000) +@memoized(expire="5s") def _internet_on(): timeout = 2 socket.setdefaulttimeout(timeout) From ce66033190a71cb5f2d142c83143194ebc87f411 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 7 Jun 2019 00:18:34 +0300 Subject: [PATCH 234/333] Docs: update migration from 3.0 to 4.0 --- HISTORY.rst | 2 ++ docs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index d896f7e3..65a57d1c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,8 @@ Release Notes ============= +.. _release_notes_4_0: + PlatformIO 4.0 -------------- diff --git a/docs b/docs index 56a4c2ac..6affd345 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 56a4c2ac24d59218ea29ba03ee7f413ee3d541d8 +Subproject commit 6affd34531e3dda49729d9f6239241229f43267f From d205370e9b7b3bd2b8ec510a4107d527ae13ea2f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 7 Jun 2019 01:06:16 +0300 Subject: [PATCH 235/333] Docs: Minor changes to migration guide --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 6affd345..bded2604 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6affd34531e3dda49729d9f6239241229f43267f +Subproject commit bded2604c7c9d1f0fa91e7b34c275dfaf04231d2 From 686d615639f8d2dda21e90b9e8a4ad016fcec301 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 7 Jun 2019 13:14:14 +0300 Subject: [PATCH 236/333] Cast env_name to string // Resolve #2644 --- platformio/ide/projectgenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index a83f7e6e..debdeeab 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -35,7 +35,7 @@ class ProjectGenerator(object): def __init__(self, project_dir, ide, env_name): self.project_dir = project_dir self.ide = str(ide) - self.env_name = env_name + self.env_name = str(env_name) @staticmethod def get_supported_ides(): From d9010230a4d7b36e9c0de4729acc6255521eea8a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 7 Jun 2019 15:01:27 +0300 Subject: [PATCH 237/333] Make internal in-memory cache for package manager to be instance related --- platformio/managers/package.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 6eb20188..4bbdca9c 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -138,19 +138,16 @@ class PkgInstallerMixin(object): FILE_CACHE_VALID = None # for example, 1 week = "7d" FILE_CACHE_MAX_SIZE = 1024 * 1024 * 50 # 50 Mb - MEMORY_CACHE = {} + MEMORY_CACHE = {} # cache for package manifests and read dirs - @staticmethod - def cache_get(key, default=None): - return PkgInstallerMixin.MEMORY_CACHE.get(key, default) + def cache_get(self, key, default=None): + return self.MEMORY_CACHE.get(key, default) - @staticmethod - def cache_set(key, value): - PkgInstallerMixin.MEMORY_CACHE[key] = value + def cache_set(self, key, value): + self.MEMORY_CACHE[key] = value - @staticmethod - def cache_reset(): - PkgInstallerMixin.MEMORY_CACHE = {} + def cache_reset(self): + self.MEMORY_CACHE.clear() def read_dirs(self, src_dir): cache_key = "read_dirs-%s" % src_dir From d18b4f12d0bc363fabd1b0bf7c6b717b030687d2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 7 Jun 2019 15:01:52 +0300 Subject: [PATCH 238/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index bded2604..66e41937 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit bded2604c7c9d1f0fa91e7b34c275dfaf04231d2 +Subproject commit 66e419376752449b73bfb37aec9bb4cb47580bc3 From 5f1bd286c7e2e530b189ce30e5d49052239c4852 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 7 Jun 2019 15:12:24 +0300 Subject: [PATCH 239/333] Fix "ValueError: invalid literal for int() with base 10: '0.0'" // Resolve #2646 --- platformio/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/util.py b/platformio/util.py index 097cfa8e..24148617 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -52,7 +52,7 @@ class memoized(object): def __init__(self, expire=0): expire = str(expire) if expire.isdigit(): - expire = "%ss" % (int(expire) / 1000) + expire = "%ss" % int((int(expire) / 1000)) tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} assert expire.endswith(tuple(tdmap)) self.expire = int(tdmap[expire[-1]] * int(expire[:-1])) From 68a3b3f9e721e638d1378460a12c4f88c53b396d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 7 Jun 2019 17:22:02 +0300 Subject: [PATCH 240/333] Custom platform_packages per a build environment with an option to override default // Resolve #1367 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/tools/pioplatform.py | 8 ++----- platformio/managers/platform.py | 29 +++++++++++++++++++------ platformio/project/options.py | 3 ++- 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 65a57d1c..34c2cd80 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -34,6 +34,7 @@ PlatformIO 4.0 - Switched to workspace ``.pio/build`` folder for build artifacts instead of ``.pioenvs`` - Switch between `Build Configurations `__ (``release`` and ``debug``) with a new project configuration option `build_type `__ + - Custom `platform_packages `__ per a build environment with an option to override default (`issue #1367 `_) - Print platform package details, such as version, VSC source and commit (`issue #2155 `_) - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) diff --git a/docs b/docs index 66e41937..7ea74ad0 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 66e419376752449b73bfb37aec9bb4cb47580bc3 +Subproject commit 7ea74ad00610cdcb3155c5ccdcf6f85d667ffb81 diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index a2971bce..09d71287 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -29,22 +29,18 @@ from platformio.project.config import ProjectOptions @util.memoized() -def initPioPlatform(name): - return PlatformFactory.newPlatform(name) - - def PioPlatform(env): variables = env.GetProjectOptions(as_dict=True) if "framework" in variables: # support PIO Core 3.0 dev/platforms variables['pioframework'] = variables['framework'] - p = initPioPlatform(env['PLATFORM_MANIFEST']) + p = PlatformFactory.newPlatform(env['PLATFORM_MANIFEST']) p.configure_default_packages(variables, COMMAND_LINE_TARGETS) return p def BoardConfig(env, board=None): - p = initPioPlatform(env['PLATFORM_MANIFEST']) + p = env.PioPlatform() try: board = board or env.get("BOARD") assert board, "BoardConfig: Board is not defined" diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 506275c5..75b24417 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -490,16 +490,17 @@ class PlatformBase( # pylint: disable=too-many-public-methods _BOARDS_CACHE = {} def __init__(self, manifest_path): - self._BOARDS_CACHE = {} self.manifest_path = manifest_path + self.silent = False + self.verbose = False + + self._BOARDS_CACHE = {} self._manifest = util.load_json(manifest_path) + self._custom_packages = None self.pm = PackageManager(get_project_packages_dir(), self.package_repositories) - self.silent = False - self.verbose = False - # if self.engines and "platformio" in self.engines: # if self.PIO_VERSION not in semantic_version.Spec( # self.engines['platformio']): @@ -560,9 +561,20 @@ class PlatformBase( # pylint: disable=too-many-public-methods @property def packages(self): - if "packages" not in self._manifest: - self._manifest['packages'] = {} - return self._manifest['packages'] + packages = self._manifest.get("packages", {}) + for item in (self._custom_packages or []): + name = item + version = "*" + if "@" in item: + name, version = item.split("@", 2) + name = name.strip() + if name not in packages: + packages[name] = {} + packages[name].update({ + "version": version.strip(), + "optional": False + }) + return packages def get_dir(self): return dirname(self.manifest_path) @@ -626,6 +638,9 @@ class PlatformBase( # pylint: disable=too-many-public-methods return self.packages[name].get("type") def configure_default_packages(self, options, targets): + # override user custom packages + self._custom_packages = options.get("platform_packages") + # enable used frameworks for framework in options.get("framework", []): if not self.frameworks: diff --git a/platformio/project/options.py b/platformio/project/options.py index 846f28dc..1b49bf42 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -91,9 +91,9 @@ ProjectOptions = OrderedDict([ # Generic ConfigEnvOption(name="platform", buildenvvar="PIOPLATFORM"), + ConfigEnvOption(name="platform_packages", multiple=True), ConfigEnvOption( name="framework", multiple=True, buildenvvar="PIOFRAMEWORK"), - ConfigEnvOption(name="targets", multiple=True), # Board ConfigEnvOption(name="board", buildenvvar="BOARD"), @@ -129,6 +129,7 @@ ProjectOptions = OrderedDict([ multiple=True, sysenvvar="PLATFORMIO_SRC_FILTER", buildenvvar="SRC_FILTER"), + ConfigEnvOption(name="targets", multiple=True), # Upload ConfigEnvOption(name="upload_port", From 15b5a149954290ab2f6d6782176d842a14abb3fd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 7 Jun 2019 17:24:07 +0300 Subject: [PATCH 241/333] Bump version to 4.0.0a20 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b21aa74b..fc8e9085 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a19") +VERSION = (4, 0, "0a20") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 643d11806282bf288c6ed25e16ae087910b635bf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 7 Jun 2019 17:42:33 +0300 Subject: [PATCH 242/333] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7ea74ad0..abd830b4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7ea74ad00610cdcb3155c5ccdcf6f85d667ffb81 +Subproject commit abd830b430a4ad021e83699818f80de59d91e2fd From 80718ebb9536a8871b2f1f66aeaf1715686fc823 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 10 Jun 2019 15:25:59 +0300 Subject: [PATCH 243/333] Fix "UnicodeEncodeError: 'ascii' codec can't encode characters" // Resolve #2644 --- platformio/commands/run/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 2fbba4be..35150c25 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -35,7 +35,7 @@ class EnvironmentProcessor(object): self, cmd_ctx, name, config, targets, upload_port, silent, verbose): self.cmd_ctx = cmd_ctx - self.name = name + self.name = str(name) self.config = config self.targets = [str(t) for t in targets] self.upload_port = upload_port From 97acf23a6da6c0e8de60d9eb9efc37b86d167756 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 10 Jun 2019 16:58:34 +0300 Subject: [PATCH 244/333] Docs: Add info how to enable built-in PIO Core for VSCode --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index abd830b4..70b53d06 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit abd830b430a4ad021e83699818f80de59d91e2fd +Subproject commit 70b53d06509d1a82308430a685e59aac57f20016 From 9de7297d38da8f931235bf1a5abe887dbad460f6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 10 Jun 2019 19:44:18 +0300 Subject: [PATCH 245/333] Fix "UnicodeEncodeError: 'ascii' codec can't encode characters" // Resolve #2644 --- platformio/commands/run/processor.py | 2 +- platformio/compat.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 35150c25..2fbba4be 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -35,7 +35,7 @@ class EnvironmentProcessor(object): self, cmd_ctx, name, config, targets, upload_port, silent, verbose): self.cmd_ctx = cmd_ctx - self.name = str(name) + self.name = name self.config = config self.targets = [str(t) for t in targets] self.upload_port = upload_port diff --git a/platformio/compat.py b/platformio/compat.py index 53ba736a..4aad4ea2 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -47,7 +47,9 @@ if PY2: def hashlib_encode_data(data): if is_bytes(data): return data - if not isinstance(data, string_types): + if isinstance(data, unicode): + data = data.encode(get_filesystem_encoding()) + elif not isinstance(data, string_types): data = str(data) return data From b7e9bcb609b51510f98a70c8a770b4afdc57a0d4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 11 Jun 2019 20:30:06 +0300 Subject: [PATCH 246/333] Do not check udev rules for gdb --version --- platformio/commands/debug/command.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 70111e6f..86d7cb4d 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -53,16 +53,6 @@ from platformio.project.helpers import (is_platformio_project, @click.pass_context def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unprocessed): - try: - util.ensure_udev_rules() - except NameError: - pass - except exception.InvalidUdevRules as e: - for line in str(e).split("\n") + [""]: - click.echo( - ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") % - line) - if not is_platformio_project(project_dir) and os.getenv("CWD"): project_dir = os.getenv("CWD") @@ -94,6 +84,16 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, raise exception.PlatformioException("\n".join( [result['out'], result['err']])) + try: + util.ensure_udev_rules() + except NameError: + pass + except exception.InvalidUdevRules as e: + for line in str(e).split("\n") + [""]: + click.echo( + ('~"%s\\n"' if helpers.is_mi_mode(__unprocessed) else "%s") % + line) + debug_options['load_cmds'] = helpers.configure_esp32_load_cmds( debug_options, configuration) From d5c98e4f27d74225bcb1a99e7012dc86303f58db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Jun 2019 22:02:59 +0300 Subject: [PATCH 247/333] Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry // Resolve #1654 --- HISTORY.rst | 1 + platformio/__main__.py | 17 ++++++++++---- platformio/builder/main.py | 5 ++++ platformio/builder/tools/piolib.py | 34 ++++++++++++++++++++-------- platformio/commands/run/helpers.py | 21 +---------------- platformio/commands/run/processor.py | 6 +---- platformio/managers/package.py | 1 + platformio/managers/platform.py | 3 +++ 8 files changed, 49 insertions(+), 39 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 34c2cd80..4a6dcaa2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ PlatformIO 4.0 - Save libraries passed to `platformio lib install `__ command into the project dependency list (`lib_deps `__) with a new ``--save`` flag (`issue #1028 `_) - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - Use isolated library dependency storage per project build environment (`issue #1696 `_) + - Look for built-in library instead of PlatformIO Registry (`issue #1654 `_) - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 `_) diff --git a/platformio/__main__.py b/platformio/__main__.py index f8358867..8b1ab093 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -18,7 +18,7 @@ from traceback import format_exc import click -from platformio import __version__, exception, maintenance +from platformio import __version__, exception, maintenance, util from platformio.commands import PlatformioCLI from platformio.compat import CYGWIN @@ -42,6 +42,7 @@ def process_result(ctx, result, force, caller): # pylint: disable=W0613 maintenance.on_platformio_end(ctx, result) +@util.memoized() def configure(): if CYGWIN: raise exception.CygwinEnvDetected() @@ -76,10 +77,17 @@ def configure(): click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs) -def main(): +def main(argv=None): + exit_code = 0 + prev_sys_argv = sys.argv[:] + if argv: + assert isinstance(argv, list) + sys.argv = argv try: configure() cli(None, None, None) + except SystemExit: + pass except Exception as e: # pylint: disable=broad-except if not isinstance(e, exception.ReturnErrorCode): maintenance.on_platformio_exception(e) @@ -105,8 +113,9 @@ An unexpected error occurred. Further steps: ============================================================ """ click.secho(error_str, fg="red", err=True) - return int(str(e)) if str(e).isdigit() else 1 - return 0 + exit_code = int(str(e)) if str(e).isdigit() else 1 + sys.argv = prev_sys_argv + return exit_code def debug_gdb_main(): diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 70f71ad6..e1ac0bd0 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -16,6 +16,7 @@ from os import environ from os.path import join from time import time +import click from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import DEFAULT_TARGETS # pylint: disable=import-error @@ -95,6 +96,10 @@ env.Replace( for key in list(clivars.keys()) if key in env }) +if int(ARGUMENTS.get("ISATTY", 0)): + # pylint: disable=protected-access + click._compat.isatty = lambda stream: True + if env.GetOption('clean'): env.PioClean(env.subst("$BUILD_DIR")) env.Exit(0) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index b9855411..0abc856a 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -25,6 +25,7 @@ import sys from os.path import (basename, commonprefix, dirname, expanduser, isdir, isfile, join, realpath, sep) +import click import SCons.Scanner # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error @@ -862,15 +863,15 @@ class ProjectAsLibBuilder(LibBuilderBase): pass def process_dependencies(self): # pylint: disable=too-many-branches - uris = self.env.GetProjectOption("lib_deps", []) - if not uris: + lib_deps = self.env.GetProjectOption("lib_deps") + if not lib_deps: return storage_dirs = [] for lb in self.env.GetLibBuilders(): if dirname(lb.path) not in storage_dirs: storage_dirs.append(dirname(lb.path)) - for uri in uris: + for uri in lib_deps: found = False for storage_dir in storage_dirs: if found: @@ -888,12 +889,26 @@ class ProjectAsLibBuilder(LibBuilderBase): break if not found: + # look for built-in libraries by a name + # which don't have package manifest for lb in self.env.GetLibBuilders(): - if lb.name != uri: - continue - if lb not in self.depbuilders: - self.depend_recursive(lb) - break + if lb.name == uri: + if lb not in self.depbuilders: + self.depend_recursive(lb) + found = True + break + + if not found: + lm = LibraryManager( + self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) + try: + lm.install(uri) + # delete cached lib builders + if "__PIO_LIB_BUILDERS" in DefaultEnvironment(): + del DefaultEnvironment()['__PIO_LIB_BUILDERS'] + except (exception.LibNotFound, + exception.InternetIsOffline) as e: + click.secho("Warning! %s" % e, fg="yellow") def build(self): self._is_built = True # do not build Project now @@ -917,8 +932,7 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches key=lambda lb: 0 if lb.dependent else 1) items = [] - verbose = int(ARGUMENTS.get("PIOVERBOSE", - 0)) and not env.GetOption('clean') + verbose = int(ARGUMENTS.get("PIOVERBOSE", 0)) def _check_lib_builder(lb): compat_mode = lb.lib_compat_mode diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index ddb1fb9d..0c51012b 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -18,10 +18,7 @@ from time import time import click -from platformio import exception, util -from platformio.commands.lib import (CTX_META_STORAGE_DIRS_KEY, - CTX_META_STORAGE_LIBDEPS_KEY) -from platformio.commands.lib import lib_install as cmd_lib_install +from platformio import util from platformio.project.helpers import (calculate_project_hash, get_project_dir, get_project_libdeps_dir) @@ -46,22 +43,6 @@ def handle_legacy_libdeps(project_dir, config): fg="yellow") -def autoinstall_libdeps(ctx, envname, libraries, verbose=False): - if not libraries: - return - libdeps_dir = join(get_project_libdeps_dir(), envname) - ctx.meta.update({ - CTX_META_STORAGE_DIRS_KEY: [libdeps_dir], - CTX_META_STORAGE_LIBDEPS_KEY: { - libdeps_dir: libraries - } - }) - try: - ctx.invoke(cmd_lib_install, silent=not verbose) - except exception.InternetIsOffline as e: - click.secho(str(e), fg="yellow") - - def clean_build_dir(build_dir): # remove legacy ".pioenvs" folder legacy_build_dir = join(get_project_dir(), ".pioenvs") diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 2fbba4be..4f495298 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -19,7 +19,7 @@ import click from platformio import exception, telemetry from platformio.commands.platform import \ platform_install as cmd_platform_install -from platformio.commands.run.helpers import autoinstall_libdeps, print_header +from platformio.commands.run.helpers import print_header from platformio.commands.test.processor import (CTX_META_TEST_IS_RUNNING, CTX_META_TEST_RUNNING_NAME) from platformio.managers.platform import PlatformFactory @@ -103,10 +103,6 @@ class EnvironmentProcessor(object): # skip monitor target, we call it above if "monitor" in build_targets: build_targets.remove("monitor") - if "nobuild" not in build_targets and "lib_deps" in self.options: - autoinstall_libdeps( - self.cmd_ctx, self.name, - self.config.get("env:" + self.name, "lib_deps"), self.verbose) try: p = PlatformFactory.newPlatform(self.options['platform']) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 4bbdca9c..82043114 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -171,6 +171,7 @@ class PkgInstallerMixin(object): if fname and isfile(cache_path): dst_path = join(dest_dir, fname) shutil.copy(cache_path, dst_path) + click.echo("Using cache: %s" % cache_path) return dst_path with_progress = not app.is_disabled_progressbar() diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 75b24417..8cb07223 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -407,6 +407,9 @@ class PlatformRunMixin(object): join(util.get_source_dir(), "builder", "main.py") ] cmd.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + # pylint: disable=protected-access + cmd.append("ISATTY=%d" % + (1 if click._compat.isatty(sys.stdout) else 0)) cmd += targets # encode and append variables From b6688db8b731a34e936530833c911a441ce980ac Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Jun 2019 22:03:28 +0300 Subject: [PATCH 248/333] Bump version to 4.0.0a21 --- HISTORY.rst | 2 +- platformio/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4a6dcaa2..e5f4e304 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,7 +27,7 @@ PlatformIO 4.0 - Save libraries passed to `platformio lib install `__ command into the project dependency list (`lib_deps `__) with a new ``--save`` flag (`issue #1028 `_) - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - Use isolated library dependency storage per project build environment (`issue #1696 `_) - - Look for built-in library instead of PlatformIO Registry (`issue #1654 `_) + - Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry (`issue #1654 `_) - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 `_) diff --git a/platformio/__init__.py b/platformio/__init__.py index fc8e9085..63c11ee8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a20") +VERSION = (4, 0, "0a21") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 98513c9967a7bb6afeef9d5532438aa324116b61 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Jun 2019 23:47:22 +0300 Subject: [PATCH 249/333] Fix nested import --- platformio/managers/lib.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index e995d751..b6f630c8 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -22,7 +22,7 @@ from os.path import isdir, join import click -from platformio import app, commands, exception, util +from platformio import app, exception, util from platformio.compat import glob_escape, string_types from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager @@ -255,8 +255,9 @@ class LibraryManager(BasePkgManager): "by request %s:" % json.dumps(filters), fg="yellow", err=True) + from platformio.commands.lib import print_lib_item for item in result['items']: - commands.lib.print_lib_item(item) + print_lib_item(item) if not interactive: click.secho( From 4ae302762a18419253bbbd66141b450d7ddd8669 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Jun 2019 00:42:10 +0300 Subject: [PATCH 250/333] Revert back "Look firstly in built-in library storages" --- HISTORY.rst | 1 - platformio/__main__.py | 17 ++++---------- platformio/builder/main.py | 5 ---- platformio/builder/tools/piolib.py | 34 ++++++++-------------------- platformio/commands/run/helpers.py | 21 ++++++++++++++++- platformio/commands/run/processor.py | 6 ++++- platformio/managers/package.py | 1 - platformio/managers/platform.py | 3 --- 8 files changed, 39 insertions(+), 49 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e5f4e304..34c2cd80 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,7 +27,6 @@ PlatformIO 4.0 - Save libraries passed to `platformio lib install `__ command into the project dependency list (`lib_deps `__) with a new ``--save`` flag (`issue #1028 `_) - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - Use isolated library dependency storage per project build environment (`issue #1696 `_) - - Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry (`issue #1654 `_) - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 `_) diff --git a/platformio/__main__.py b/platformio/__main__.py index 8b1ab093..f8358867 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -18,7 +18,7 @@ from traceback import format_exc import click -from platformio import __version__, exception, maintenance, util +from platformio import __version__, exception, maintenance from platformio.commands import PlatformioCLI from platformio.compat import CYGWIN @@ -42,7 +42,6 @@ def process_result(ctx, result, force, caller): # pylint: disable=W0613 maintenance.on_platformio_end(ctx, result) -@util.memoized() def configure(): if CYGWIN: raise exception.CygwinEnvDetected() @@ -77,17 +76,10 @@ def configure(): click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs) -def main(argv=None): - exit_code = 0 - prev_sys_argv = sys.argv[:] - if argv: - assert isinstance(argv, list) - sys.argv = argv +def main(): try: configure() cli(None, None, None) - except SystemExit: - pass except Exception as e: # pylint: disable=broad-except if not isinstance(e, exception.ReturnErrorCode): maintenance.on_platformio_exception(e) @@ -113,9 +105,8 @@ An unexpected error occurred. Further steps: ============================================================ """ click.secho(error_str, fg="red", err=True) - exit_code = int(str(e)) if str(e).isdigit() else 1 - sys.argv = prev_sys_argv - return exit_code + return int(str(e)) if str(e).isdigit() else 1 + return 0 def debug_gdb_main(): diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e1ac0bd0..70f71ad6 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -16,7 +16,6 @@ from os import environ from os.path import join from time import time -import click from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import DEFAULT_TARGETS # pylint: disable=import-error @@ -96,10 +95,6 @@ env.Replace( for key in list(clivars.keys()) if key in env }) -if int(ARGUMENTS.get("ISATTY", 0)): - # pylint: disable=protected-access - click._compat.isatty = lambda stream: True - if env.GetOption('clean'): env.PioClean(env.subst("$BUILD_DIR")) env.Exit(0) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 0abc856a..b9855411 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -25,7 +25,6 @@ import sys from os.path import (basename, commonprefix, dirname, expanduser, isdir, isfile, join, realpath, sep) -import click import SCons.Scanner # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error @@ -863,15 +862,15 @@ class ProjectAsLibBuilder(LibBuilderBase): pass def process_dependencies(self): # pylint: disable=too-many-branches - lib_deps = self.env.GetProjectOption("lib_deps") - if not lib_deps: + uris = self.env.GetProjectOption("lib_deps", []) + if not uris: return storage_dirs = [] for lb in self.env.GetLibBuilders(): if dirname(lb.path) not in storage_dirs: storage_dirs.append(dirname(lb.path)) - for uri in lib_deps: + for uri in uris: found = False for storage_dir in storage_dirs: if found: @@ -889,26 +888,12 @@ class ProjectAsLibBuilder(LibBuilderBase): break if not found: - # look for built-in libraries by a name - # which don't have package manifest for lb in self.env.GetLibBuilders(): - if lb.name == uri: - if lb not in self.depbuilders: - self.depend_recursive(lb) - found = True - break - - if not found: - lm = LibraryManager( - self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) - try: - lm.install(uri) - # delete cached lib builders - if "__PIO_LIB_BUILDERS" in DefaultEnvironment(): - del DefaultEnvironment()['__PIO_LIB_BUILDERS'] - except (exception.LibNotFound, - exception.InternetIsOffline) as e: - click.secho("Warning! %s" % e, fg="yellow") + if lb.name != uri: + continue + if lb not in self.depbuilders: + self.depend_recursive(lb) + break def build(self): self._is_built = True # do not build Project now @@ -932,7 +917,8 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches key=lambda lb: 0 if lb.dependent else 1) items = [] - verbose = int(ARGUMENTS.get("PIOVERBOSE", 0)) + verbose = int(ARGUMENTS.get("PIOVERBOSE", + 0)) and not env.GetOption('clean') def _check_lib_builder(lb): compat_mode = lb.lib_compat_mode diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index 0c51012b..ddb1fb9d 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -18,7 +18,10 @@ from time import time import click -from platformio import util +from platformio import exception, util +from platformio.commands.lib import (CTX_META_STORAGE_DIRS_KEY, + CTX_META_STORAGE_LIBDEPS_KEY) +from platformio.commands.lib import lib_install as cmd_lib_install from platformio.project.helpers import (calculate_project_hash, get_project_dir, get_project_libdeps_dir) @@ -43,6 +46,22 @@ def handle_legacy_libdeps(project_dir, config): fg="yellow") +def autoinstall_libdeps(ctx, envname, libraries, verbose=False): + if not libraries: + return + libdeps_dir = join(get_project_libdeps_dir(), envname) + ctx.meta.update({ + CTX_META_STORAGE_DIRS_KEY: [libdeps_dir], + CTX_META_STORAGE_LIBDEPS_KEY: { + libdeps_dir: libraries + } + }) + try: + ctx.invoke(cmd_lib_install, silent=not verbose) + except exception.InternetIsOffline as e: + click.secho(str(e), fg="yellow") + + def clean_build_dir(build_dir): # remove legacy ".pioenvs" folder legacy_build_dir = join(get_project_dir(), ".pioenvs") diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 4f495298..2fbba4be 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -19,7 +19,7 @@ import click from platformio import exception, telemetry from platformio.commands.platform import \ platform_install as cmd_platform_install -from platformio.commands.run.helpers import print_header +from platformio.commands.run.helpers import autoinstall_libdeps, print_header from platformio.commands.test.processor import (CTX_META_TEST_IS_RUNNING, CTX_META_TEST_RUNNING_NAME) from platformio.managers.platform import PlatformFactory @@ -103,6 +103,10 @@ class EnvironmentProcessor(object): # skip monitor target, we call it above if "monitor" in build_targets: build_targets.remove("monitor") + if "nobuild" not in build_targets and "lib_deps" in self.options: + autoinstall_libdeps( + self.cmd_ctx, self.name, + self.config.get("env:" + self.name, "lib_deps"), self.verbose) try: p = PlatformFactory.newPlatform(self.options['platform']) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 82043114..4bbdca9c 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -171,7 +171,6 @@ class PkgInstallerMixin(object): if fname and isfile(cache_path): dst_path = join(dest_dir, fname) shutil.copy(cache_path, dst_path) - click.echo("Using cache: %s" % cache_path) return dst_path with_progress = not app.is_disabled_progressbar() diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8cb07223..75b24417 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -407,9 +407,6 @@ class PlatformRunMixin(object): join(util.get_source_dir(), "builder", "main.py") ] cmd.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) - # pylint: disable=protected-access - cmd.append("ISATTY=%d" % - (1 if click._compat.isatty(sys.stdout) else 0)) cmd += targets # encode and append variables From 5338a9caa39f843c4dd4ebdc819407c4ec09193d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Jun 2019 00:42:24 +0300 Subject: [PATCH 251/333] Bump version to 4.0.0a22 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 63c11ee8..5f315c40 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a21") +VERSION = (4, 0, "0a22") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 1ccc52696001146426c8d66f5a32dd424223c28e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Jun 2019 13:08:53 +0300 Subject: [PATCH 252/333] Revert "Revert back "Look firstly in built-in library storages"" This reverts commit 4ae302762a18419253bbbd66141b450d7ddd8669. --- HISTORY.rst | 1 + platformio/__main__.py | 17 ++++++++++---- platformio/builder/main.py | 5 ++++ platformio/builder/tools/piolib.py | 34 ++++++++++++++++++++-------- platformio/commands/run/helpers.py | 21 +---------------- platformio/commands/run/processor.py | 6 +---- platformio/managers/package.py | 1 + platformio/managers/platform.py | 3 +++ 8 files changed, 49 insertions(+), 39 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 34c2cd80..e5f4e304 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ PlatformIO 4.0 - Save libraries passed to `platformio lib install `__ command into the project dependency list (`lib_deps `__) with a new ``--save`` flag (`issue #1028 `_) - Install all project dependencies declared via `lib_deps `__ option using a simple `platformio lib install `__ command (`issue #2147 `_) - Use isolated library dependency storage per project build environment (`issue #1696 `_) + - Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry (`issue #1654 `_) - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 `_) diff --git a/platformio/__main__.py b/platformio/__main__.py index f8358867..8b1ab093 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -18,7 +18,7 @@ from traceback import format_exc import click -from platformio import __version__, exception, maintenance +from platformio import __version__, exception, maintenance, util from platformio.commands import PlatformioCLI from platformio.compat import CYGWIN @@ -42,6 +42,7 @@ def process_result(ctx, result, force, caller): # pylint: disable=W0613 maintenance.on_platformio_end(ctx, result) +@util.memoized() def configure(): if CYGWIN: raise exception.CygwinEnvDetected() @@ -76,10 +77,17 @@ def configure(): click.secho = lambda *args, **kwargs: _safe_echo(1, *args, **kwargs) -def main(): +def main(argv=None): + exit_code = 0 + prev_sys_argv = sys.argv[:] + if argv: + assert isinstance(argv, list) + sys.argv = argv try: configure() cli(None, None, None) + except SystemExit: + pass except Exception as e: # pylint: disable=broad-except if not isinstance(e, exception.ReturnErrorCode): maintenance.on_platformio_exception(e) @@ -105,8 +113,9 @@ An unexpected error occurred. Further steps: ============================================================ """ click.secho(error_str, fg="red", err=True) - return int(str(e)) if str(e).isdigit() else 1 - return 0 + exit_code = int(str(e)) if str(e).isdigit() else 1 + sys.argv = prev_sys_argv + return exit_code def debug_gdb_main(): diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 70f71ad6..e1ac0bd0 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -16,6 +16,7 @@ from os import environ from os.path import join from time import time +import click from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import DEFAULT_TARGETS # pylint: disable=import-error @@ -95,6 +96,10 @@ env.Replace( for key in list(clivars.keys()) if key in env }) +if int(ARGUMENTS.get("ISATTY", 0)): + # pylint: disable=protected-access + click._compat.isatty = lambda stream: True + if env.GetOption('clean'): env.PioClean(env.subst("$BUILD_DIR")) env.Exit(0) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index b9855411..0abc856a 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -25,6 +25,7 @@ import sys from os.path import (basename, commonprefix, dirname, expanduser, isdir, isfile, join, realpath, sep) +import click import SCons.Scanner # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error @@ -862,15 +863,15 @@ class ProjectAsLibBuilder(LibBuilderBase): pass def process_dependencies(self): # pylint: disable=too-many-branches - uris = self.env.GetProjectOption("lib_deps", []) - if not uris: + lib_deps = self.env.GetProjectOption("lib_deps") + if not lib_deps: return storage_dirs = [] for lb in self.env.GetLibBuilders(): if dirname(lb.path) not in storage_dirs: storage_dirs.append(dirname(lb.path)) - for uri in uris: + for uri in lib_deps: found = False for storage_dir in storage_dirs: if found: @@ -888,12 +889,26 @@ class ProjectAsLibBuilder(LibBuilderBase): break if not found: + # look for built-in libraries by a name + # which don't have package manifest for lb in self.env.GetLibBuilders(): - if lb.name != uri: - continue - if lb not in self.depbuilders: - self.depend_recursive(lb) - break + if lb.name == uri: + if lb not in self.depbuilders: + self.depend_recursive(lb) + found = True + break + + if not found: + lm = LibraryManager( + self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) + try: + lm.install(uri) + # delete cached lib builders + if "__PIO_LIB_BUILDERS" in DefaultEnvironment(): + del DefaultEnvironment()['__PIO_LIB_BUILDERS'] + except (exception.LibNotFound, + exception.InternetIsOffline) as e: + click.secho("Warning! %s" % e, fg="yellow") def build(self): self._is_built = True # do not build Project now @@ -917,8 +932,7 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches key=lambda lb: 0 if lb.dependent else 1) items = [] - verbose = int(ARGUMENTS.get("PIOVERBOSE", - 0)) and not env.GetOption('clean') + verbose = int(ARGUMENTS.get("PIOVERBOSE", 0)) def _check_lib_builder(lb): compat_mode = lb.lib_compat_mode diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index ddb1fb9d..0c51012b 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -18,10 +18,7 @@ from time import time import click -from platformio import exception, util -from platformio.commands.lib import (CTX_META_STORAGE_DIRS_KEY, - CTX_META_STORAGE_LIBDEPS_KEY) -from platformio.commands.lib import lib_install as cmd_lib_install +from platformio import util from platformio.project.helpers import (calculate_project_hash, get_project_dir, get_project_libdeps_dir) @@ -46,22 +43,6 @@ def handle_legacy_libdeps(project_dir, config): fg="yellow") -def autoinstall_libdeps(ctx, envname, libraries, verbose=False): - if not libraries: - return - libdeps_dir = join(get_project_libdeps_dir(), envname) - ctx.meta.update({ - CTX_META_STORAGE_DIRS_KEY: [libdeps_dir], - CTX_META_STORAGE_LIBDEPS_KEY: { - libdeps_dir: libraries - } - }) - try: - ctx.invoke(cmd_lib_install, silent=not verbose) - except exception.InternetIsOffline as e: - click.secho(str(e), fg="yellow") - - def clean_build_dir(build_dir): # remove legacy ".pioenvs" folder legacy_build_dir = join(get_project_dir(), ".pioenvs") diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 2fbba4be..4f495298 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -19,7 +19,7 @@ import click from platformio import exception, telemetry from platformio.commands.platform import \ platform_install as cmd_platform_install -from platformio.commands.run.helpers import autoinstall_libdeps, print_header +from platformio.commands.run.helpers import print_header from platformio.commands.test.processor import (CTX_META_TEST_IS_RUNNING, CTX_META_TEST_RUNNING_NAME) from platformio.managers.platform import PlatformFactory @@ -103,10 +103,6 @@ class EnvironmentProcessor(object): # skip monitor target, we call it above if "monitor" in build_targets: build_targets.remove("monitor") - if "nobuild" not in build_targets and "lib_deps" in self.options: - autoinstall_libdeps( - self.cmd_ctx, self.name, - self.config.get("env:" + self.name, "lib_deps"), self.verbose) try: p = PlatformFactory.newPlatform(self.options['platform']) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 4bbdca9c..82043114 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -171,6 +171,7 @@ class PkgInstallerMixin(object): if fname and isfile(cache_path): dst_path = join(dest_dir, fname) shutil.copy(cache_path, dst_path) + click.echo("Using cache: %s" % cache_path) return dst_path with_progress = not app.is_disabled_progressbar() diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 75b24417..8cb07223 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -407,6 +407,9 @@ class PlatformRunMixin(object): join(util.get_source_dir(), "builder", "main.py") ] cmd.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + # pylint: disable=protected-access + cmd.append("ISATTY=%d" % + (1 if click._compat.isatty(sys.stdout) else 0)) cmd += targets # encode and append variables From 461d71c2c7728511105569f11cd5d65e0b5d8830 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Jun 2019 18:22:36 +0300 Subject: [PATCH 253/333] Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry // Resolve #1654 --- platformio/builder/tools/piolib.py | 184 ++++++++++++++--------------- 1 file changed, 90 insertions(+), 94 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 0abc856a..850c8f79 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -41,7 +41,7 @@ from platformio.managers.lib import LibraryManager class LibBuilderFactory(object): @staticmethod - def new(env, path, verbose=False): + def new(env, path, verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): clsname = "UnknownLibBuilder" if isfile(join(path, "library.json")): clsname = "PlatformIOLibBuilder" @@ -138,6 +138,8 @@ class LibBuilderBase(object): if WINDOWS: p1 = p1.lower() p2 = p2.lower() + if p1 == p2: + return True return commonprefix((p1 + sep, p2)) == p1 + sep @property @@ -274,40 +276,17 @@ class LibBuilderBase(object): if not self.dependencies: return for item in self.dependencies: - skip = False - for key in ("platforms", "frameworks"): - env_key = "PIO" + key.upper()[:-1] - if env_key not in self.env: - continue - if (key in item and - not util.items_in_list(self.env[env_key], item[key])): - if self.verbose: - sys.stderr.write( - "Skip %s incompatible dependency %s\n" % - (key[:-1], item)) - skip = True - if skip: - continue - found = False for lb in self.env.GetLibBuilders(): if item['name'] != lb.name: continue - elif "frameworks" in item and \ - not lb.is_frameworks_compatible(item["frameworks"]): - continue - elif "platforms" in item and \ - not lb.is_platforms_compatible(item["platforms"]): - continue found = True self.depend_recursive(lb) break - if not found: - sys.stderr.write( - "Error: Could not find `%s` dependency for `%s` " - "library\n" % (item['name'], self.name)) - self.env.Exit(1) + if not found and self.verbose: + sys.stderr.write("Warning: Ignored `%s` dependency for `%s` " + "library\n" % (item['name'], self.name)) def get_search_files(self): items = [ @@ -384,6 +363,8 @@ class LibBuilderBase(object): return result def depend_recursive(self, lb, search_files=None): + if lb in self.depbuilders: + return def _already_depends(_lb): if self in _lb.depbuilders: @@ -858,57 +839,72 @@ class ProjectAsLibBuilder(LibBuilderBase): return (self.env.get("SRC_FILTER") or LibBuilderBase.src_filter.fget(self)) + @property + def dependencies(self): + return self.env.GetProjectOption("lib_deps", []) + def process_extra_options(self): # skip for project, options are already processed pass + def install_dependencies(self): + + def _is_builtin(uri): + for lb in self.env.GetLibBuilders(): + if lb.name == uri: + return True + return False + + lm = LibraryManager( + self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) + did_install = False + for item in self.dependencies: + # check if built-in library or already installed + if (_is_builtin(item) + or lm.get_package_dir(*lm.parse_pkg_uri(item))): + continue + try: + lm.install(item) + did_install = True + except (exception.LibNotFound, exception.InternetIsOffline) as e: + click.secho("Warning! %s" % e, fg="yellow") + + # reset cache + if did_install: + DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None) + def process_dependencies(self): # pylint: disable=too-many-branches - lib_deps = self.env.GetProjectOption("lib_deps") - if not lib_deps: - return storage_dirs = [] for lb in self.env.GetLibBuilders(): if dirname(lb.path) not in storage_dirs: storage_dirs.append(dirname(lb.path)) - for uri in lib_deps: + for uri in self.dependencies: found = False for storage_dir in storage_dirs: if found: break lm = LibraryManager(storage_dir) - pkg_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri)) - if not pkg_dir: + lib_dir = lm.get_package_dir(*lm.parse_pkg_uri(uri)) + if not lib_dir: continue for lb in self.env.GetLibBuilders(): - if lb.path != pkg_dir: + if lib_dir not in lb: continue - if lb not in self.depbuilders: - self.depend_recursive(lb) + self.depend_recursive(lb) found = True break + if found: + continue - if not found: - # look for built-in libraries by a name - # which don't have package manifest - for lb in self.env.GetLibBuilders(): - if lb.name == uri: - if lb not in self.depbuilders: - self.depend_recursive(lb) - found = True - break - - if not found: - lm = LibraryManager( - self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) - try: - lm.install(uri) - # delete cached lib builders - if "__PIO_LIB_BUILDERS" in DefaultEnvironment(): - del DefaultEnvironment()['__PIO_LIB_BUILDERS'] - except (exception.LibNotFound, - exception.InternetIsOffline) as e: - click.secho("Warning! %s" % e, fg="yellow") + # look for built-in libraries by a name + # which don't have package manifest + for lb in self.env.GetLibBuilders(): + if lb.name != uri: + continue + self.depend_recursive(lb) + found = True + break def build(self): self._is_built = True # do not build Project now @@ -925,61 +921,60 @@ def GetLibSourceDirs(env): ] -def GetLibBuilders(env): # pylint: disable=too-many-branches +def IsCompatibleLibBuilder(env, + lb, + verbose=int(ARGUMENTS.get("PIOVERBOSE", 0))): + compat_mode = lb.lib_compat_mode + if lb.name in env.GetProjectOption("lib_ignore", []): + if verbose: + sys.stderr.write("Ignored library %s\n" % lb.path) + return None + if compat_mode == "strict" and not lb.is_platforms_compatible( + env['PIOPLATFORM']): + if verbose: + sys.stderr.write("Platform incompatible library %s\n" % lb.path) + return False + if (compat_mode == "soft" and "PIOFRAMEWORK" in env + and not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", []))): + if verbose: + sys.stderr.write("Framework incompatible library %s\n" % lb.path) + return False + return True - if "__PIO_LIB_BUILDERS" in DefaultEnvironment(): + +def GetLibBuilders(env): # pylint: disable=too-many-branches + if DefaultEnvironment().get("__PIO_LIB_BUILDERS", None) is not None: return sorted(DefaultEnvironment()['__PIO_LIB_BUILDERS'], key=lambda lb: 0 if lb.dependent else 1) - items = [] + DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=[]) + verbose = int(ARGUMENTS.get("PIOVERBOSE", 0)) - - def _check_lib_builder(lb): - compat_mode = lb.lib_compat_mode - if lb.name in env.GetProjectOption("lib_ignore", []): - if verbose: - sys.stderr.write("Ignored library %s\n" % lb.path) - return None - if compat_mode == "strict" and not lb.is_platforms_compatible( - env['PIOPLATFORM']): - if verbose: - sys.stderr.write("Platform incompatible library %s\n" % - lb.path) - return False - if compat_mode == "soft" and "PIOFRAMEWORK" in env and \ - not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])): - if verbose: - sys.stderr.write("Framework incompatible library %s\n" % - lb.path) - return False - return True - found_incompat = False + for libs_dir in env.GetLibSourceDirs(): - libs_dir = env.subst(libs_dir) + libs_dir = realpath(env.subst(libs_dir)) if not isdir(libs_dir): continue for item in sorted(os.listdir(libs_dir)): - if item == "__cores__" or not isdir(join(libs_dir, item)): + lib_dir = join(libs_dir, item) + if item == "__cores__" or not isdir(lib_dir): continue try: - lb = LibBuilderFactory.new(env, - join(libs_dir, item), - verbose=verbose) + lb = LibBuilderFactory.new(env, lib_dir) except exception.InvalidJSONFile: if verbose: sys.stderr.write( - "Skip library with broken manifest: %s\n" % - join(libs_dir, item)) + "Skip library with broken manifest: %s\n" % lib_dir) continue - if _check_lib_builder(lb): - items.append(lb) + if env.IsCompatbileLibBuilder(lb): + DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb]) else: found_incompat = True for lb in env.get("EXTRA_LIB_BUILDERS", []): - if _check_lib_builder(lb): - items.append(lb) + if env.IsCompatbileLibBuilder(lb): + DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb]) else: found_incompat = True @@ -989,8 +984,7 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches "https://docs.platformio.org/page/librarymanager/ldf.html#" "ldf-compat-mode\n") - DefaultEnvironment()['__PIO_LIB_BUILDERS'] = items - return items + return DefaultEnvironment()['__PIO_LIB_BUILDERS'] def ConfigureProjectLibBuilder(env): @@ -1031,6 +1025,7 @@ def ConfigureProjectLibBuilder(env): _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") + project.install_dependencies() ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) print("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") @@ -1061,6 +1056,7 @@ def exists(_): def generate(env): env.AddMethod(GetLibSourceDirs) + env.AddMethod(IsCompatibleLibBuilder) env.AddMethod(GetLibBuilders) env.AddMethod(ConfigureProjectLibBuilder) return env From 46eab99888498397772bb0a7be16ce9d99ab28a9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Jun 2019 18:31:03 +0300 Subject: [PATCH 254/333] Typo fix --- platformio/builder/tools/piolib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 850c8f79..56987d5f 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -967,13 +967,13 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches sys.stderr.write( "Skip library with broken manifest: %s\n" % lib_dir) continue - if env.IsCompatbileLibBuilder(lb): + if env.IsCompatibleLibBuilder(lb): DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb]) else: found_incompat = True for lb in env.get("EXTRA_LIB_BUILDERS", []): - if env.IsCompatbileLibBuilder(lb): + if env.IsCompatibleLibBuilder(lb): DefaultEnvironment().Append(__PIO_LIB_BUILDERS=[lb]) else: found_incompat = True From 1339924c2e25ab0ffda61933354d29d9a8153865 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Jun 2019 19:54:40 +0300 Subject: [PATCH 255/333] Print installation progress for "lib_deps" after LDF banner --- platformio/builder/tools/piolib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 56987d5f..013038d9 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -1025,13 +1025,14 @@ def ConfigureProjectLibBuilder(env): _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") - project.install_dependencies() ldf_mode = LibBuilderBase.lib_ldf_mode.fget(project) print("LDF: Library Dependency Finder -> http://bit.ly/configure-pio-ldf") print("LDF Modes: Finder ~ %s, Compatibility ~ %s" % (ldf_mode, project.lib_compat_mode)) + project.install_dependencies() + lib_builders = env.GetLibBuilders() print("Found %d compatible libraries" % len(lib_builders)) From bd3b29c304cdd01482aea8d2d161a465dcfd6cec Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Jun 2019 20:24:55 +0300 Subject: [PATCH 256/333] Fix converting to real version --- platformio/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/util.py b/platformio/util.py index 24148617..b93b4d8a 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -515,7 +515,7 @@ def get_original_version(version): _, raw = version.split(".")[:2] if int(raw) <= 99: return None - if int(raw) <= 999: + if int(raw) <= 9999: return "%s.%s" % (raw[:-2], int(raw[-2:])) return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) From 8b8b6c3b9e2fbeff422cda8c5812f32d81b43a1f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Jun 2019 22:57:53 +0300 Subject: [PATCH 257/333] Bump version to 4.0.0a23 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 5f315c40..40c6874e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a22") +VERSION = (4, 0, "0a23") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From bd8ba738cf537a55a9c9619a52e9cf0f0a6c3f2b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Jun 2019 23:46:53 +0300 Subject: [PATCH 258/333] Improve docs how to enable PIO Core dev/version in PIO IDE --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 70b53d06..668c8b0d 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 70b53d06509d1a82308430a685e59aac57f20016 +Subproject commit 668c8b0d89fa0248e8384545538a14261531c980 From fbdfe31f17c3a806542087e87b942521ad3fdc22 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 15 Jun 2019 18:53:13 +0300 Subject: [PATCH 259/333] Shared cache directory for the build derived files // Resolve #2674 --- HISTORY.rst | 1 + docs | 2 +- platformio/builder/main.py | 10 ++++++++-- platformio/builder/tools/piomisc.py | 2 ++ platformio/project/options.py | 2 ++ 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e5f4e304..fece4e97 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -39,6 +39,7 @@ PlatformIO 4.0 - Print platform package details, such as version, VSC source and commit (`issue #2155 `_) - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) + - Configure a shared folder for the derived files (objects, firmwares, ELFs) from a build system using `build_cache_dir `__ option (`issue #2674 `_) - Fixed an issue when ``-U`` in ``build_flags`` does not remove macro previously defined via ``-D`` flag (`issue #2508 `_) * **Infrastructure** diff --git a/docs b/docs index 668c8b0d..6ca558b2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 668c8b0d89fa0248e8384545538a14261531c980 +Subproject commit 6ca558b253058d4176be8dc894fdc439bb844c32 diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e1ac0bd0..8fa012d0 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import environ -from os.path import join +from os import environ, makedirs +from os.path import isdir, join from time import time import click @@ -67,6 +67,7 @@ DEFAULT_ENV_OPTIONS = dict( PROJECTTEST_DIR=project_helpers.get_project_test_dir(), PROJECTDATA_DIR=project_helpers.get_project_data_dir(), PROJECTBUILD_DIR=project_helpers.get_project_build_dir(), + BUILDCACHE_DIR=project_helpers.get_project_optional_dir("build_cache_dir"), BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"), BUILDSRC_DIR=join("$BUILD_DIR", "src"), BUILDTEST_DIR=join("$BUILD_DIR", "test"), @@ -96,6 +97,11 @@ env.Replace( for key in list(clivars.keys()) if key in env }) +if env.subst("$BUILDCACHE_DIR"): + if not isdir(env.subst("$BUILDCACHE_DIR")): + makedirs(env.subst("$BUILDCACHE_DIR")) + env.CacheDir("$BUILDCACHE_DIR") + if int(ARGUMENTS.get("ISATTY", 0)): # pylint: disable=protected-access click._compat.isatty = lambda stream: True diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 65596d10..1e01d59e 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -210,6 +210,8 @@ def _delete_file(path): @util.memoized() def _get_compiler_type(env): + if env.subst("$CC").endswith("-gcc"): + return "gcc" try: sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) diff --git a/platformio/project/options.py b/platformio/project/options.py index 1b49bf42..bc2f5c3d 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -66,6 +66,8 @@ ProjectOptions = OrderedDict([ sysenvvar="PLATFORMIO_PACKAGES_DIR"), ConfigPlatformioOption(name="cache_dir", sysenvvar="PLATFORMIO_CACHE_DIR"), + ConfigPlatformioOption(name="build_cache_dir", + sysenvvar="PLATFORMIO_BUILD_CACHE_DIR"), ConfigPlatformioOption(name="workspace_dir", sysenvvar="PLATFORMIO_WORKSPACE_DIR"), ConfigPlatformioOption(name="build_dir", From 38408c1e1f2006480951d6252e926a66aac71863 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 15 Jun 2019 21:09:52 +0300 Subject: [PATCH 260/333] Use only 1 example per framework for CI --- tests/test_examples.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 8e655c14..3cb365d6 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -14,8 +14,8 @@ import random from glob import glob -from os import getenv, listdir, walk -from os.path import dirname, getsize, isdir, isfile, join, normpath +from os import listdir, walk +from os.path import basename, dirname, getsize, isdir, isfile, join, normpath import pytest @@ -50,20 +50,21 @@ def pytest_generate_tests(metafunc): project_dirs = [] for examples_dir in examples_dirs: - platform_examples = [] + candidates = {} for root, _, files in walk(examples_dir): if "platformio.ini" not in files or ".skiptest" in files: continue - platform_examples.append(root) + group = basename(root) + if "-" in group: + group = group.split("-", 1)[0] + if group not in candidates: + candidates[group] = [] + candidates[group].append(root) - random.shuffle(platform_examples) - - if getenv("APPVEYOR"): - # use only 1 example for AppVeyor CI - project_dirs.append(platform_examples[0]) - else: - # test random 3 examples - project_dirs.extend(platform_examples[:3]) + project_dirs.extend([ + random.choice(examples) for examples in candidates.values() + if examples + ]) metafunc.parametrize("pioproject_dir", sorted(project_dirs)) @@ -75,7 +76,8 @@ def test_run(pioproject_dir): if isdir(build_dir): util.rmtree_(build_dir) - env_names = ProjectConfig(join(pioproject_dir, "platformio.ini")).envs() + env_names = ProjectConfig(join(pioproject_dir, + "platformio.ini")).envs() result = util.exec_command( ["platformio", "run", "-e", random.choice(env_names)]) From 5fabadd05968d7125d5726c0fb403012dee1ba4d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 15 Jun 2019 21:12:40 +0300 Subject: [PATCH 261/333] Use build cache for CI examples --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d9c8e3a..e4b65227 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,11 @@ matrix: - os: linux sudo: required python: 2.7 - env: TOX_ENV=py27 + env: TOX_ENV=py27 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d) - os: linux sudo: required python: 3.6 - env: TOX_ENV=py36 + env: TOX_ENV=py36 PLATFORMIO_BUILD_CACHE_DIR=$(mktemp -d) - os: osx language: generic env: TOX_ENV=skipexamples From fa761f9616edee122f1d4ab43d5d4675ec4b2f55 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 15 Jun 2019 22:06:55 +0300 Subject: [PATCH 262/333] Use build cache for AppVeyor CI examples --- .appveyor.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index ddee3c59..c9703e56 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -6,7 +6,10 @@ platform: environment: matrix: - TOXENV: "py27" + PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P2_{build} + - TOXENV: "py36" + PLATFORMIO_BUILD_CACHE_DIR: C:/Temp/PIO_Build_Cache_P3_{build} install: - cmd: git submodule update --init --recursive From 40c80465466520f11e7b7d593302499c5f0c2a86 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 16 Jun 2019 19:36:41 +1000 Subject: [PATCH 263/333] Fix Teensy serial lockout (#2676) Fixes `[Errno 16] Device or resource busy: '/dev/ttyACM0` issue on Teensy 3.2 and 3.5 (at minimum) due to ModemManager probing the devices and making them unavailable. Updated rules sourced from current https://www.pjrc.com/teensy/49-teensy.rules configuration. --- scripts/99-platformio-udev.rules | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index f3249682..51a4c9eb 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -63,10 +63,10 @@ SUBSYSTEMS=="usb", ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE="0666 SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="05dc", MODE:="0666" # Teensy boards -ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", ENV{ID_MM_DEVICE_IGNORE}="1" -ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", ENV{MTP_NO_PROBE}="1" -SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", MODE:="0666" -KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789]?", MODE:="0666" +ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789A]?", ENV{MTP_NO_PROBE}="1" +SUBSYSTEMS=="usb", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789ABCD]?", MODE:="0666" +KERNEL=="ttyACM*", ATTRS{idVendor}=="16c0", ATTRS{idProduct}=="04[789B]?", MODE:="0666" #TI Stellaris Launchpad SUBSYSTEMS=="usb", ATTRS{idVendor}=="1cbe", ATTRS{idProduct}=="00fd", MODE="0666" From 51115c125447a85e7ecec63828a6cedd30fb5346 Mon Sep 17 00:00:00 2001 From: Peter Date: Sun, 16 Jun 2019 19:37:32 +1000 Subject: [PATCH 264/333] Ignore vscode-cpptools cache (#2623) One of the recent updates to the vscode-cpptools extension made it start caching in .vscode/ipch, and this cache can be quite big for even small projects. Since they're cache files, they should be ignored by default. This may be a short-lived suggestion, as there is already some mention that the IntelliSense cache folder may be moved to workspace storage, rather than 'per project' storage... although the 'when' is anyone's guess. See here for more: https://github.com/microsoft/vscode-cpptools/issues/3347 --- platformio/ide/tpls/vscode/.gitignore.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/ide/tpls/vscode/.gitignore.tpl b/platformio/ide/tpls/vscode/.gitignore.tpl index 18453b74..89cc49cb 100644 --- a/platformio/ide/tpls/vscode/.gitignore.tpl +++ b/platformio/ide/tpls/vscode/.gitignore.tpl @@ -2,3 +2,4 @@ .vscode/.browse.c_cpp.db* .vscode/c_cpp_properties.json .vscode/launch.json +.vscode/ipch From 04dc6230e7f4a8799e90ca7f8e0e9bcc58019eb3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 19 Jun 2019 00:29:21 +0300 Subject: [PATCH 265/333] Open source PIO Unified Debugger and Unit Testing --- HISTORY.rst | 7 +++++++ docs | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index fece4e97..9e931722 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,13 @@ PlatformIO 4.0 4.0.0 (2019-??-??) ~~~~~~~~~~~~~~~~~~ +`Migration Guide from 3.0 to 4.0 `__. + +* `PlatformIO Plus Goes Open Source... `__ + + - Open source `PIO Unified Debugger `__ + - Open source `PIO Unit Testing `__ + * **Project Configuration** - New project configuration parser with a strict options typing (`API `__) diff --git a/docs b/docs index 6ca558b2..a67924d7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6ca558b253058d4176be8dc894fdc439bb844c32 +Subproject commit a67924d7a34678ffe6d01b5a7db273f6357253e3 From 5ab564a6d0a3ded4e08678d1b81715a17c2cd382 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 20 Jun 2019 17:41:06 +0300 Subject: [PATCH 266/333] Add doc page about ULP programming --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index a67924d7..2e05c6e7 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a67924d7a34678ffe6d01b5a7db273f6357253e3 +Subproject commit 2e05c6e788fa4585ddaf6299e8d98e74881e9d67 From 3ebeb1bab23dde6d30ce0f81524e5052c22dd9db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 20 Jun 2019 19:10:21 +0300 Subject: [PATCH 267/333] Create FUNDING.yml --- .github/FUNDING.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..3b7200ff --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms + +github: [ivankravets, valeros] +custom: https://platformio.org/donate From 634afdcf8a52f5b928fa9aa42263322d23b13888 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 20 Jun 2019 19:11:50 +0300 Subject: [PATCH 268/333] Update FUNDING.yml --- .github/FUNDING.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 3b7200ff..6f70f7e9 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,4 +1 @@ -# These are supported funding model platforms - -github: [ivankravets, valeros] custom: https://platformio.org/donate From 6d968a70933909c16ca58e61b22328b636f3ac83 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 21 Jun 2019 15:11:51 +0300 Subject: [PATCH 269/333] Docs: Add info about using custom targets with mbed framework --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 2e05c6e7..b4390901 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2e05c6e788fa4585ddaf6299e8d98e74881e9d67 +Subproject commit b439090186777b3dcea0c9434cc8c4d2d05e41df From 4b7916c2af281ed1f487fb0deda96f16ef79597e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 21 Jun 2019 17:50:18 +0300 Subject: [PATCH 270/333] Bump version to 4.0.0b1 --- HISTORY.rst | 2 +- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9e931722..b60e44ff 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,7 +11,7 @@ PlatformIO 4.0 `Migration Guide from 3.0 to 4.0 `__. -* `PlatformIO Plus Goes Open Source... `__ +* `PlatformIO Plus Goes Open Source `__ - Open source `PIO Unified Debugger `__ - Open source `PIO Unit Testing `__ diff --git a/platformio/__init__.py b/platformio/__init__.py index 40c6874e..9d89adae 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0a23") +VERSION = (4, 0, "0b1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 6efc19f9..2a144399 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -30,7 +30,7 @@ CORE_PACKAGES = { "contrib-piohome": "^2.1.0", "contrib-pysite": "~2.%d%d.190418" % (sys.version_info[0], sys.version_info[1]), - "tool-pioplus": "^2.4.0", + "tool-pioplus": "^2.5.2", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.7" if PY2 else "~3.30005.0" } From c0a9ae5c70741e622762c4ff02cf6f8590602a60 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Jun 2019 21:21:34 +0300 Subject: [PATCH 271/333] Better handling of library.properties "architectures" meta data --- platformio/builder/tools/piolib.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 013038d9..9d8f5ecc 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -507,22 +507,23 @@ class ArduinoLibBuilder(LibBuilderBase): def is_platforms_compatible(self, platforms): platforms_map = { - "avr": "atmelavr", - "sam": "atmelsam", - "samd": "atmelsam", - "esp8266": "espressif8266", - "esp32": "espressif32", - "arc32": "intel_arc32", - "stm32": "ststm32" + "avr": ["atmelavr"], + "sam": ["atmelsam"], + "samd": ["atmelsam"], + "esp8266": ["espressif8266"], + "esp32": ["espressif32"], + "arc32": ["intel_arc32"], + "stm32": ["ststm32"], + "nrf5": ["nordicnrf51", "nordicnrf52"] } items = [] for arch in self._manifest.get("architectures", "").split(","): - arch = arch.strip() + arch = arch.strip().lower() if arch == "*": items = "*" break if arch in platforms_map: - items.append(platforms_map[arch]) + items.extend(platforms_map[arch]) if not items: return LibBuilderBase.is_platforms_compatible(self, platforms) return util.items_in_list(platforms, items) From 17fa5f77d503b86faefc5caf8d47aa9626a11dd7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Jun 2019 21:23:19 +0300 Subject: [PATCH 272/333] Fix LDF recursive behaviour --- platformio/builder/tools/piolib.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 9d8f5ecc..5593e328 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -281,7 +281,8 @@ class LibBuilderBase(object): if item['name'] != lb.name: continue found = True - self.depend_recursive(lb) + if lb not in self.depbuilders: + self.depend_recursive(lb) break if not found and self.verbose: @@ -363,8 +364,6 @@ class LibBuilderBase(object): return result def depend_recursive(self, lb, search_files=None): - if lb in self.depbuilders: - return def _already_depends(_lb): if self in _lb.depbuilders: @@ -892,7 +891,8 @@ class ProjectAsLibBuilder(LibBuilderBase): for lb in self.env.GetLibBuilders(): if lib_dir not in lb: continue - self.depend_recursive(lb) + if lb not in self.depbuilders: + self.depend_recursive(lb) found = True break if found: @@ -903,7 +903,8 @@ class ProjectAsLibBuilder(LibBuilderBase): for lb in self.env.GetLibBuilders(): if lb.name != uri: continue - self.depend_recursive(lb) + if lb not in self.depbuilders: + self.depend_recursive(lb) found = True break From 2c3d8ce695bc15e9a4c33701ab9ae4b2179935a5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 23 Jun 2019 21:23:44 +0300 Subject: [PATCH 273/333] Bump version to 4.0.0b2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 9d89adae..dac8e821 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0b1") +VERSION = (4, 0, "0b2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 6daf387c905b40927fbc1e7fdc754912f84b698d Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 26 Jun 2019 06:50:06 +1000 Subject: [PATCH 274/333] Missing parentheses and depreciated syntax (#2700) Fixes ``` File "get-platformio.py", line 93 print r['out'] ^ SyntaxError: Missing parentheses in call to 'print'. Did you mean print(r['out'])? ``` and ``` File "get-platformio.py", line 146 except Exception, e: ^ SyntaxError: invalid syntax ``` --- scripts/get-platformio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/get-platformio.py b/scripts/get-platformio.py index 90ad5fea..ec201d62 100644 --- a/scripts/get-platformio.py +++ b/scripts/get-platformio.py @@ -90,7 +90,7 @@ def exec_python_cmd(args): def install_pip(): r = exec_python_cmd(["-m", "pip", "--version"]) if r['returncode'] == 0: - print r['out'] + print(r['out']) return try: from urllib2 import urlopen @@ -143,7 +143,7 @@ def main(): try: s[1]() print("[SUCCESS]") - except Exception, e: + except Exception as e: is_error = True print(str(e)) print("[FAILURE]") From 89b951e7d5bf3f0f417b151a6d416debd539720d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Jun 2019 00:29:49 +0300 Subject: [PATCH 275/333] Switch to news from registry --- .isort.cfg | 2 +- docs | 2 +- platformio/commands/home/rpc/handlers/misc.py | 149 +----------------- platformio/commands/home/rpc/handlers/os.py | 8 +- 4 files changed, 9 insertions(+), 152 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 5f6d6207..c147908f 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=79 -known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,bs4,jsonrpc +known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial,twisted,autobahn,jsonrpc diff --git a/docs b/docs index b4390901..f660192c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b439090186777b3dcea0c9434cc8c4d2d05e41df +Subproject commit f660192cb1dfe1de0b27e7348e35d17f73faca08 diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index 77df7bc8..2422da78 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -13,14 +13,11 @@ # limitations under the License. import json -import re import time -from bs4 import BeautifulSoup # pylint: disable=import-error from twisted.internet import defer, reactor # pylint: disable=import-error from platformio import app -from platformio.commands.home import helpers from platformio.commands.home.rpc.handlers.os import OSRPC @@ -42,9 +39,12 @@ class MiscRPC(object): result = self._preload_latest_tweets(username, cache_key, cache_valid) return result + @staticmethod @defer.inlineCallbacks - def _preload_latest_tweets(self, username, cache_key, cache_valid): - result = yield self._fetch_tweets(username) + def _preload_latest_tweets(username, cache_key, cache_valid): + result = yield OSRPC.fetch_content( + "https://api.platformio.org/tweets/" + username) + result = json.loads(result) with app.ContentCache() as cc: cc.set(cache_key, json.dumps({ @@ -52,142 +52,3 @@ class MiscRPC(object): "result": result }), cache_valid) defer.returnValue(result) - - @defer.inlineCallbacks - def _fetch_tweets(self, username): - api_url = ("https://twitter.com/i/profiles/show/%s/timeline/tweets?" - "include_available_features=1&include_entities=1&" - "include_new_items_bar=true") % username - if helpers.is_twitter_blocked(): - api_url = self._get_proxed_uri(api_url) - content = yield OSRPC.fetch_content( - api_url, headers=self._get_twitter_headers(username)) - content = json.loads(content) - assert "items_html" in content - soup = BeautifulSoup(content['items_html'], "html.parser") - tweet_nodes = soup.find_all("div", - attrs={ - "class": "tweet", - "data-tweet-id": True - }) - result = yield defer.DeferredList( - [self._parse_tweet_node(node, username) for node in tweet_nodes], - consumeErrors=True) - defer.returnValue([r[1] for r in result if r[0]]) - - @defer.inlineCallbacks - def _parse_tweet_node(self, tweet, username): - # remove non-visible items - for node in tweet.find_all(class_=["invisible", "u-hidden"]): - node.decompose() - twitter_url = "https://twitter.com" - time_node = tweet.find("span", attrs={"data-time": True}) - text_node = tweet.find(class_="tweet-text") - quote_text_node = tweet.find(class_="QuoteTweet-text") - if quote_text_node and not text_node.get_text().strip(): - text_node = quote_text_node - photos = [ - node.get("data-image-url") for node in (tweet.find_all(class_=[ - "AdaptiveMedia-photoContainer", "QuoteMedia-photoContainer" - ]) or []) - ] - urls = [ - node.get("data-expanded-url") - for node in (quote_text_node or text_node).find_all( - class_="twitter-timeline-link", - attrs={"data-expanded-url": True} - ) - ] # yapf: disable - - # fetch data from iframe card - if (not photos or not urls) and tweet.get("data-card2-type"): - iframe_node = tweet.find("div", - attrs={"data-full-card-iframe-url": True}) - if iframe_node: - iframe_card = yield self._fetch_iframe_card( - twitter_url + iframe_node.get("data-full-card-iframe-url"), - username) - if not photos and iframe_card['photo']: - photos.append(iframe_card['photo']) - if not urls and iframe_card['url']: - urls.append(iframe_card['url']) - if iframe_card['text_node']: - text_node = iframe_card['text_node'] - - if not photos: - photos.append(tweet.find("img", class_="avatar").get("src")) - - def _fetch_text(text_node): - text = text_node.decode_contents(formatter="html").strip() - text = re.sub(r'href="/', 'href="%s/' % twitter_url, text) - if "

" not in text and "", text) - return text - - defer.returnValue({ - "tweetId": - tweet.get("data-tweet-id"), - "tweetUrl": - twitter_url + tweet.get("data-permalink-path"), - "author": - tweet.get("data-name"), - "time": - int(time_node.get("data-time")), - "timeFormatted": - time_node.string, - "text": - _fetch_text(text_node), - "entries": { - "urls": - urls, - "photos": [ - self._get_proxed_uri(uri) - if helpers.is_twitter_blocked() else uri for uri in photos - ] - }, - "isPinned": - "user-pinned" in tweet.get("class") - }) - - @defer.inlineCallbacks - def _fetch_iframe_card(self, url, username): - if helpers.is_twitter_blocked(): - url = self._get_proxed_uri(url) - html = yield OSRPC.fetch_content( - url, headers=self._get_twitter_headers(username), cache_valid="7d") - soup = BeautifulSoup(html, "html.parser") - photo_node = soup.find("img", attrs={"data-src": True}) - url_node = soup.find("a", class_="TwitterCard-container") - text_node = soup.find("div", class_="SummaryCard-content") - if text_node: - text_node.find("span", - class_="SummaryCard-destination").decompose() - defer.returnValue({ - "photo": - photo_node.get("data-src") if photo_node else None, - "text_node": - text_node, - "url": - url_node.get("href") if url_node else None - }) - - @staticmethod - def _get_proxed_uri(uri): - index = uri.index("://") - return "https://dl.platformio.org/__prx__/" + uri[index + 3:] - - @staticmethod - def _get_twitter_headers(username): - return { - "Accept": - "application/json, text/javascript, */*; q=0.01", - "Referer": - "https://twitter.com/%s" % username, - "User-Agent": - ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit" - "/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8"), - "X-Twitter-Active-User": - "yes", - "X-Requested-With": - "XMLHttpRequest" - } diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index d26a0c00..afb2e29f 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -33,7 +33,6 @@ class OSRPC(object): @staticmethod @defer.inlineCallbacks def fetch_content(uri, data=None, headers=None, cache_valid=None): - timeout = 2 if not headers: headers = { "User-Agent": @@ -54,12 +53,9 @@ class OSRPC(object): session = helpers.requests_session() if data: - r = yield session.post(uri, - data=data, - headers=headers, - timeout=timeout) + r = yield session.post(uri, data=data, headers=headers) else: - r = yield session.get(uri, headers=headers, timeout=timeout) + r = yield session.get(uri, headers=headers) r.raise_for_status() result = r.text From 5cb2a970fa4d6c4baa2226a716d59e94c24341b6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Jun 2019 00:52:22 +0300 Subject: [PATCH 276/333] Suppress IOError --- platformio/managers/platform.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8cb07223..d0ab898a 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -417,8 +417,11 @@ class PlatformRunMixin(object): cmd.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) def _write_and_flush(stream, data): - stream.write(data) - stream.flush() + try: + stream.write(data) + stream.flush() + except IOError: + pass copy_pythonpath_to_osenv() result = exec_command( From 54921c5dbdc68893fe45649d07d067818c36889b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 26 Jun 2019 01:02:08 +0300 Subject: [PATCH 277/333] Bump version to 4.0.0b3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index dac8e821..b02e522f 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0b2") +VERSION = (4, 0, "0b3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From d07a1d265ee37e1adfd29e9df1f58dc534fa3c32 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Jun 2019 13:38:46 +0300 Subject: [PATCH 278/333] Log stdout of GDB client --- platformio/commands/debug/client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 03daed25..a7ed9b2b 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -170,11 +170,12 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes stdio.StandardIO(p) def onStdInData(self, data): - self._last_server_activity = time.time() if LOG_FILE: with open(LOG_FILE, "ab") as fp: fp.write(data) + self._last_server_activity = time.time() + if b"-exec-run" in data: if self._target_is_run: token, _ = data.split(b"-", 1) @@ -198,6 +199,10 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes reactor.stop() def outReceived(self, data): + if LOG_FILE: + with open(LOG_FILE, "ab") as fp: + fp.write(data) + self._last_server_activity = time.time() super(GDBClient, self).outReceived(data) self._handle_error(data) From e61caa37a8c8fecf976b2388de5aaa0d981530c4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Jun 2019 14:04:16 +0300 Subject: [PATCH 279/333] Fix issue when CMakeListsUser.txt is not included // Resolve #2712 --- platformio/ide/tpls/clion/CMakeLists.txt.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 526aeafe..4680f6b4 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -10,7 +10,7 @@ project({{project_name}}) include(CMakeListsPrivate.txt) -if(EXISTS CMakeListsUser.txt) +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/CMakeListsUser.txt) include(CMakeListsUser.txt) endif() From 1d80914070462ae34d951397960167332d647a1c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Jun 2019 14:56:21 +0300 Subject: [PATCH 280/333] Remove upgrade hooks from Core 2.0 to Core 3.0 --- platformio/maintenance.py | 49 +-------------------------------------- 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index b1e2d7b9..5dfff12f 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -import os from os import getenv -from os.path import isdir, join +from os.path import join from time import time import click @@ -25,17 +23,12 @@ from platformio import __version__, app, exception, telemetry, util from platformio.commands import PlatformioCLI from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib import lib_update as cmd_lib_update -from platformio.commands.platform import \ - platform_install as cmd_platform_install -from platformio.commands.platform import \ - platform_uninstall as cmd_platform_uninstall from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager from platformio.managers.platform import PlatformFactory, PlatformManager from platformio.proc import is_ci, is_container -from platformio.project.helpers import get_project_core_dir def on_platformio_start(ctx, force, caller): @@ -93,9 +86,6 @@ class Upgrader(object): util.pepver_to_semver(to_version)) self._upgraders = [ - (semantic_version.Version("3.0.0-a.1"), self._upgrade_to_3_0_0), - (semantic_version.Version("3.0.0-b.11"), - self._upgrade_to_3_0_0b11), (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms) ] @@ -111,43 +101,6 @@ class Upgrader(object): return all(result) - @staticmethod - def _upgrade_to_3_0_0(ctx): - # convert custom board configuration - boards_dir = join(get_project_core_dir(), "boards") - if isdir(boards_dir): - for item in os.listdir(boards_dir): - if not item.endswith(".json"): - continue - data = util.load_json(join(boards_dir, item)) - if set(["name", "url", "vendor"]) <= set(data): - continue - os.remove(join(boards_dir, item)) - for key, value in data.items(): - with open(join(boards_dir, "%s.json" % key), "w") as f: - json.dump(value, f, sort_keys=True, indent=2) - - # re-install PlatformIO 2.0 development platforms - installed_platforms = app.get_state_item("installed_platforms", []) - if installed_platforms: - if "espressif" in installed_platforms: - installed_platforms[installed_platforms.index( - "espressif")] = "espressif8266" - ctx.invoke(cmd_platform_install, platforms=installed_platforms) - - return True - - @staticmethod - def _upgrade_to_3_0_0b11(ctx): - current_platforms = [ - m['name'] for m in PlatformManager().get_installed() - ] - if "espressif" not in current_platforms: - return True - ctx.invoke(cmd_platform_install, platforms=["espressif8266"]) - ctx.invoke(cmd_platform_uninstall, platforms=["espressif"]) - return True - @staticmethod def _update_dev_platforms(ctx): ctx.invoke(cmd_platform_update) From e27b40390d98ae381bc15cb747bad51f84738756 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Jun 2019 14:57:05 +0300 Subject: [PATCH 281/333] Look for project dir in sys env variables (hooks for Eclipse, CLion) --- platformio/commands/debug/command.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 86d7cb4d..7892cfee 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -53,8 +53,12 @@ from platformio.project.helpers import (is_platformio_project, @click.pass_context def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unprocessed): - if not is_platformio_project(project_dir) and os.getenv("CWD"): - project_dir = os.getenv("CWD") + # use env variables from Eclipse or CLion + for sysenv in ("CWD", "PWD", "PLATFORMIO_PROJECT_DIR"): + if is_platformio_project(project_dir): + break + if os.getenv(sysenv): + project_dir = os.getenv(sysenv) with util.cd(project_dir): config = ProjectConfig.get_instance( From c89793eab9ef4e82228819949df355a465c120a7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Jun 2019 15:06:12 +0300 Subject: [PATCH 282/333] Use piped openOCD for RISC-V --- platformio/commands/debug/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index 8719b1f0..33800bd5 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -59,9 +59,9 @@ class DebugServer(BaseProcess): self._debug_port = ":3333" openocd_pipe_allowed = all([ - not self.debug_options['port'], "openocd" in server_executable, - self.env_options['platform'] != "riscv" - ]) + not self.debug_options['port'], + "openocd" in server_executable + ]) # yapf: disable if openocd_pipe_allowed: args = [] if server['cwd']: From 96d337388b22aea0e8f1a1a728e017b05c470649 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Jun 2019 15:07:13 +0300 Subject: [PATCH 283/333] Debug: Initial configuration for QEMU --- platformio/commands/debug/client.py | 2 ++ platformio/commands/debug/initcfgs.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index a7ed9b2b..9b208a48 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -119,6 +119,8 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes cfg = initcfgs.GDB_STUTIL_INIT_CONFIG elif "mspdebug" in server_exe: cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG + elif "qemu" in server_exe: + cfg = initcfgs.GDB_QEMU_INIT_CONFIG elif self.debug_options['require_debug_port']: cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG else: diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index 658418ae..a9d71d32 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -107,3 +107,18 @@ monitor erase $LOAD_CMDS pio_reset_halt_target """ + +GDB_QEMU_INIT_CONFIG = """ +define pio_reset_halt_target + monitor system_reset +end + +define pio_reset_target + pio_reset_halt_target +end + +target extended-remote $DEBUG_PORT +$INIT_BREAK +$LOAD_CMDS +pio_reset_halt_target +""" From a75823227d0bd301573be32980c2a891616f0a0f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 27 Jun 2019 15:13:16 +0300 Subject: [PATCH 284/333] Use :1234 as default GDB port for QEMU --- platformio/commands/debug/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index 33800bd5..6d81585e 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -100,6 +100,8 @@ class DebugServer(BaseProcess): self._debug_port = ":2000" elif "jlink" in server_executable.lower(): self._debug_port = ":2331" + elif "qemu" in server_executable.lower(): + self._debug_port = ":1234" return self._transport From 4d23ad03c302570187f8231cb6198f1f695e4a87 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Jun 2019 13:03:49 +0300 Subject: [PATCH 285/333] Remove upgrade hooks from Core 2.0 to Core 3.0 --- tests/test_maintenance.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 4a69227e..07a50c66 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -22,39 +22,6 @@ from platformio.commands import upgrade as cmd_upgrade from platformio.managers.platform import PlatformManager -def test_after_upgrade_2_to_3(clirunner, validate_cliresult, - isolated_pio_home): - app.set_state_item("last_version", "2.11.2") - app.set_state_item("installed_platforms", ["native"]) - - # generate PlatformIO 2.0 boards - boards = isolated_pio_home.mkdir("boards") - board_ids = set() - for prefix in ("foo", "bar"): - data = {} - for i in range(3): - board_id = "board_%s_%d" % (prefix, i) - board_ids.add(board_id) - data[board_id] = { - "name": "Board %s #%d" % (prefix, i), - "url": "", - "vendor": "" - } - boards.join(prefix + ".json").write(json.dumps(data)) - - result = clirunner.invoke(cli_pio, ["settings", "get"]) - validate_cliresult(result) - assert "upgraded to " in result.output - - # check PlatformIO 3.0 boards - assert board_ids == set([p.basename[:-5] for p in boards.listdir()]) - - result = clirunner.invoke(cli_pio, - ["boards", "--installed", "--json-output"]) - validate_cliresult(result) - assert board_ids == set([b['id'] for b in json.loads(result.output)]) - - def test_after_upgrade_silence(clirunner, validate_cliresult): app.set_state_item("last_version", "2.11.2") result = clirunner.invoke(cli_pio, ["boards", "--json-output"]) From 936b04e07524f86695b514bbc765bff6f33d1872 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Jun 2019 13:04:34 +0300 Subject: [PATCH 286/333] Remove upgrade hooks from Core 2.0 to Core 3.0 --- platformio/maintenance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 5dfff12f..56712c25 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -85,9 +85,8 @@ class Upgrader(object): self.to_version = semantic_version.Version.coerce( util.pepver_to_semver(to_version)) - self._upgraders = [ - (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms) - ] + self._upgraders = [(semantic_version.Version("3.5.0-a.2"), + self._update_dev_platforms)] def run(self, ctx): if self.from_version > self.to_version: From 1bff3c6615b6cc64fe25daa4dfef13e2d69479e3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Jun 2019 13:21:11 +0300 Subject: [PATCH 287/333] Use isolated Core folder for test_maintenance --- tests/conftest.py | 4 ++-- tests/test_maintenance.py | 22 ++++++++-------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d53d92e8..c61166bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,10 +40,10 @@ def clirunner(): @pytest.fixture(scope="module") def isolated_pio_home(request, tmpdir_factory): home_dir = tmpdir_factory.mktemp(".platformio") - os.environ['PLATFORMIO_HOME_DIR'] = str(home_dir) + os.environ['PLATFORMIO_CORE_DIR'] = str(home_dir) def fin(): - del os.environ['PLATFORMIO_HOME_DIR'] + del os.environ['PLATFORMIO_CORE_DIR'] request.addfinalizer(fin) return home_dir diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 07a50c66..d7b4dea0 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -22,15 +22,7 @@ from platformio.commands import upgrade as cmd_upgrade from platformio.managers.platform import PlatformManager -def test_after_upgrade_silence(clirunner, validate_cliresult): - app.set_state_item("last_version", "2.11.2") - result = clirunner.invoke(cli_pio, ["boards", "--json-output"]) - validate_cliresult(result) - boards = json.loads(result.output) - assert any([b['id'] == "uno" for b in boards]) - - -def test_check_pio_upgrade(clirunner, validate_cliresult): +def test_check_pio_upgrade(clirunner, isolated_pio_home, validate_cliresult): def _patch_pio_version(version): maintenance.__version__ = version @@ -60,7 +52,7 @@ def test_check_pio_upgrade(clirunner, validate_cliresult): _patch_pio_version(origin_version) -def test_check_lib_updates(clirunner, validate_cliresult): +def test_check_lib_updates(clirunner, isolated_pio_home, validate_cliresult): # install obsolete library result = clirunner.invoke(cli_pio, ["lib", "-g", "install", "ArduinoJson@<5.7"]) @@ -77,7 +69,8 @@ def test_check_lib_updates(clirunner, validate_cliresult): result.output) -def test_check_and_update_libraries(clirunner, validate_cliresult): +def test_check_and_update_libraries(clirunner, isolated_pio_home, + validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_libraries", "Yes"]) @@ -108,8 +101,8 @@ def test_check_and_update_libraries(clirunner, validate_cliresult): assert prev_data[0]['version'] != json.loads(result.output)[0]['version'] -def test_check_platform_updates(clirunner, validate_cliresult, - isolated_pio_home): +def test_check_platform_updates(clirunner, isolated_pio_home, + validate_cliresult): # install obsolete platform result = clirunner.invoke(cli_pio, ["platform", "install", "native"]) validate_cliresult(result) @@ -131,7 +124,8 @@ def test_check_platform_updates(clirunner, validate_cliresult, assert "There are the new updates for platforms (native)" in result.output -def test_check_and_update_platforms(clirunner, validate_cliresult): +def test_check_and_update_platforms(clirunner, isolated_pio_home, + validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_platforms", "Yes"]) From 4123aa4c234ed99f9b4550f94d9d0dc6036b426c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Jun 2019 13:27:05 +0300 Subject: [PATCH 288/333] Look in for "lib_deps" in all declared library storages // Resolve #2708 --- platformio/builder/tools/piolib.py | 49 +++++++++++++++++------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 5593e328..f9ab05f0 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -22,8 +22,8 @@ import hashlib import os import re import sys -from os.path import (basename, commonprefix, dirname, expanduser, isdir, - isfile, join, realpath, sep) +from os.path import (basename, commonprefix, expanduser, isdir, isfile, join, + realpath, sep) import click import SCons.Scanner # pylint: disable=import-error @@ -855,16 +855,27 @@ class ProjectAsLibBuilder(LibBuilderBase): return True return False + not_found_uri = [] + for uri in self.dependencies: + # check if built-in library + if _is_builtin(uri): + continue + + found = False + for storage_dir in self.env.GetLibSourceDirs(): + lm = LibraryManager(storage_dir) + if lm.get_package_dir(*lm.parse_pkg_uri(uri)): + found = True + break + if not found: + not_found_uri.append(uri) + + did_install = False lm = LibraryManager( self.env.subst(join("$PROJECTLIBDEPS_DIR", "$PIOENV"))) - did_install = False - for item in self.dependencies: - # check if built-in library or already installed - if (_is_builtin(item) - or lm.get_package_dir(*lm.parse_pkg_uri(item))): - continue + for uri in not_found_uri: try: - lm.install(item) + lm.install(uri) did_install = True except (exception.LibNotFound, exception.InternetIsOffline) as e: click.secho("Warning! %s" % e, fg="yellow") @@ -874,14 +885,9 @@ class ProjectAsLibBuilder(LibBuilderBase): DefaultEnvironment().Replace(__PIO_LIB_BUILDERS=None) def process_dependencies(self): # pylint: disable=too-many-branches - storage_dirs = [] - for lb in self.env.GetLibBuilders(): - if dirname(lb.path) not in storage_dirs: - storage_dirs.append(dirname(lb.path)) - for uri in self.dependencies: found = False - for storage_dir in storage_dirs: + for storage_dir in self.env.GetLibSourceDirs(): if found: break lm = LibraryManager(storage_dir) @@ -919,7 +925,8 @@ def GetLibSourceDirs(env): items = env.GetProjectOption("lib_extra_dirs", []) items.extend(env['LIBSOURCE_DIRS']) return [ - expanduser(item) if item.startswith("~") else item for item in items + env.subst(expanduser(item) if item.startswith("~") else item) + for item in items ] @@ -954,12 +961,12 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches verbose = int(ARGUMENTS.get("PIOVERBOSE", 0)) found_incompat = False - for libs_dir in env.GetLibSourceDirs(): - libs_dir = realpath(env.subst(libs_dir)) - if not isdir(libs_dir): + for storage_dir in env.GetLibSourceDirs(): + storage_dir = realpath(storage_dir) + if not isdir(storage_dir): continue - for item in sorted(os.listdir(libs_dir)): - lib_dir = join(libs_dir, item) + for item in sorted(os.listdir(storage_dir)): + lib_dir = join(storage_dir, item) if item == "__cores__" or not isdir(lib_dir): continue try: From 0035f56e15362e87da6266f97418604e22601d7b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Jun 2019 19:07:44 +0300 Subject: [PATCH 289/333] Sync docs and examples --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index f660192c..8681d7e1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f660192cb1dfe1de0b27e7348e35d17f73faca08 +Subproject commit 8681d7e151393d653cb54275be3e9757823ebbde diff --git a/examples b/examples index b5ae9ea1..76278680 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit b5ae9ea1bb754d9e06b195d65f24853ef58f376d +Subproject commit 762786807e88b68784492f908738346277b74be8 From f30bd18bdc41b29d7447ff4814ed3e626b5ea1d5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Jun 2019 19:07:59 +0300 Subject: [PATCH 290/333] Skip async output when empty byte is given --- platformio/commands/debug/helpers.py | 7 +++++-- platformio/proc.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index c9b36897..ddc9db97 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -33,8 +33,11 @@ class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods STDOUT = sys.stdout def write(self, text): - for line in text.strip().split("\n"): - self.STDOUT.write('~"%s\\n"\n' % line) + if "\n" in text: + for line in text.strip().split("\n"): + self.STDOUT.write('~"%s\\n"\n' % line) + else: + self.STDOUT.write('~"%s"' % text) self.STDOUT.flush() diff --git a/platformio/proc.py b/platformio/proc.py index c90590e3..066813ed 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -66,7 +66,7 @@ class BuildAsyncPipe(AsyncPipeBase): for byte in iter(lambda: self._pipe_reader.read(1), ""): self._buffer += byte - if line and line[-3:] == (line[-1] * 3): + if line and byte.strip() and line[-3:] == (byte * 3): print_immediately = True if print_immediately: From 8d459d86d33992129726ef177ed24fe8a00e9b75 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Jun 2019 19:09:47 +0300 Subject: [PATCH 291/333] Bump version to 4.0.0rc1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b02e522f..4b354750 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0b3") +VERSION = (4, 0, "0rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f8dd90c9a9a7f8f8a287efe6ac4b3cb2269e28c9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 28 Jun 2019 23:34:08 +0300 Subject: [PATCH 292/333] Docs: Fix invalid references to Sipeed boards --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 8681d7e1..cb42b34c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8681d7e151393d653cb54275be3e9757823ebbde +Subproject commit cb42b34c54e6036985bef8ee5bc6d66cd43cac51 From 6739d5a570bf9f4d3ccc1ec7a69be3a81d6629db Mon Sep 17 00:00:00 2001 From: Axel W Date: Sat, 29 Jun 2019 11:48:24 +0200 Subject: [PATCH 293/333] In the current version of this template all build targets result (#2714) in pio -c -f eclipse debug run target The commit fixes this to be pio -c -f eclipse run target See also discussion in forum: https://community.platformio.org/t/pio-4-0-0b3-potential-bug-in-cprojet-tpl/8390 --- platformio/ide/tpls/eclipse/.cproject.tpl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/platformio/ide/tpls/eclipse/.cproject.tpl b/platformio/ide/tpls/eclipse/.cproject.tpl index 29b76ebc..01460e13 100644 --- a/platformio/ide/tpls/eclipse/.cproject.tpl +++ b/platformio/ide/tpls/eclipse/.cproject.tpl @@ -221,7 +221,7 @@ -f -c eclipse run -t program true - true + false false @@ -229,7 +229,7 @@ -f -c eclipse run -t uploadfs true - true + false false @@ -237,7 +237,7 @@ -f -c eclipse run true - true + false false @@ -245,7 +245,7 @@ -f -c eclipse run -t upload true - true + false false @@ -253,7 +253,7 @@ -f -c eclipse run -t clean true - true + false false @@ -261,7 +261,7 @@ -f -c eclipse test true - true + false false @@ -269,7 +269,7 @@ -f -c eclipse update true - true + false false @@ -277,7 +277,7 @@ -f -c eclipse init --ide eclipse true - true + false false From 500b3e08fed358f7937eadcee82dd91df6bf61d0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 29 Jun 2019 13:29:10 +0300 Subject: [PATCH 294/333] Docs: Fix default OTA port for ESP32 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index cb42b34c..f59f1e26 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit cb42b34c54e6036985bef8ee5bc6d66cd43cac51 +Subproject commit f59f1e262d6bc370c32e6ed85319392c89de9af4 From dfca7f0b6877e9f3fe06920e9276339a778c8fd6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Jul 2019 15:55:42 +0300 Subject: [PATCH 295/333] Speedup PIO Home via internal calling of PIO Core CLI --- platformio/__main__.py | 3 +- platformio/commands/debug/command.py | 2 +- platformio/commands/debug/helpers.py | 12 -------- platformio/commands/home/rpc/handlers/os.py | 9 +++--- .../commands/home/rpc/handlers/piocore.py | 28 +++++++++++-------- platformio/commands/platform.py | 2 +- platformio/compat.py | 4 +++ platformio/util.py | 12 ++++++++ 8 files changed, 41 insertions(+), 31 deletions(-) diff --git a/platformio/__main__.py b/platformio/__main__.py index 8b1ab093..d4664935 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -119,8 +119,7 @@ An unexpected error occurred. Further steps: def debug_gdb_main(): - sys.argv = [sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:] - return main() + return main([sys.argv[0], "debug", "--interface", "gdb"] + sys.argv[1:]) if __name__ == "__main__": diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 7892cfee..8599a962 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -123,7 +123,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, if helpers.is_mi_mode(__unprocessed): click.echo('~"Preparing firmware for debugging...\\n"') output = helpers.GDBBytesIO() - with helpers.capture_std_streams(output): + with util.capture_std_streams(output): helpers.predebug_project(ctx, project_dir, env_name, preload, verbose) output.close() diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index ddc9db97..daaa8d93 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -14,7 +14,6 @@ import sys import time -from contextlib import contextmanager from fnmatch import fnmatch from hashlib import sha1 from io import BytesIO @@ -41,17 +40,6 @@ class GDBBytesIO(BytesIO): # pylint: disable=too-few-public-methods self.STDOUT.flush() -@contextmanager -def capture_std_streams(stdout, stderr=None): - _stdout = sys.stdout - _stderr = sys.stderr - sys.stdout = stdout - sys.stderr = stderr or stdout - yield - sys.stdout = _stdout - sys.stderr = _stderr - - def is_mi_mode(args): return "--interpreter" in " ".join(args) diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index afb2e29f..c84f486e 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -14,6 +14,7 @@ from __future__ import absolute_import +import codecs import glob import os import shutil @@ -67,10 +68,10 @@ class OSRPC(object): def request_content(self, uri, data=None, headers=None, cache_valid=None): if uri.startswith('http'): return self.fetch_content(uri, data, headers, cache_valid) - if isfile(uri): - with open(uri) as fp: - return fp.read() - return None + if not isfile(uri): + return None + with codecs.open(uri, encoding="utf-8") as fp: + return fp.read() @staticmethod def open_url(url): diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 198552d1..19d7d370 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -17,12 +17,12 @@ from __future__ import absolute_import import json import os import re +from io import BytesIO import jsonrpc # pylint: disable=import-error -from twisted.internet import utils # pylint: disable=import-error +from twisted.internet import threads # pylint: disable=import-error -from platformio import __version__ -from platformio.commands.home import helpers +from platformio import __main__, __version__, util from platformio.compat import string_types @@ -30,7 +30,6 @@ class PIOCoreRPC(object): @staticmethod def call(args, options=None): - json_output = "--json-output" in args try: args = [ str(arg) if not isinstance(arg, string_types) else arg @@ -39,13 +38,20 @@ class PIOCoreRPC(object): except UnicodeError: raise jsonrpc.exceptions.JSONRPCDispatchException( code=4002, message="PIO Core: non-ASCII chars in arguments") - d = utils.getProcessOutputAndValue( - helpers.get_core_fullpath(), - args, - path=(options or {}).get("cwd"), - env={k: v - for k, v in os.environ.items() if "%" not in k}) - d.addCallback(PIOCoreRPC._call_callback, json_output) + + def _call_cli(): + outbuff = BytesIO() + errbuff = BytesIO() + with util.capture_std_streams(outbuff, errbuff): + with util.cd((options or {}).get("cwd") or os.getcwd()): + exit_code = __main__.main(["-c"] + args) + result = (outbuff.getvalue(), errbuff.getvalue(), exit_code) + outbuff.close() + errbuff.close() + return result + + d = threads.deferToThread(_call_cli) + d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) d.addErrback(PIOCoreRPC._call_errback) return d diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 5b35c4df..26f86666 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -182,7 +182,7 @@ def platform_frameworks(query, json_output): for framework in util.get_api_result("/frameworks", cache_valid="7d"): if query == "all": query = "" - search_data = dump_json_to_unicode(framework) + search_data = framework if query and query.lower() not in search_data.lower(): continue framework['homepage'] = ("https://platformio.org/frameworks/" + diff --git a/platformio/compat.py b/platformio/compat.py index 4aad4ea2..686518a8 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -54,6 +54,8 @@ if PY2: return data def dump_json_to_unicode(obj): + if isinstance(obj, unicode): + return obj return json.dumps(obj, encoding=get_filesystem_encoding(), ensure_ascii=False).encode("utf8") @@ -100,4 +102,6 @@ else: return data.encode() def dump_json_to_unicode(obj): + if isinstance(obj, string_types): + return obj return json.dumps(obj, ensure_ascii=False) diff --git a/platformio/util.py b/platformio/util.py index b93b4d8a..377160b1 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -20,6 +20,7 @@ import socket import stat import sys import time +from contextlib import contextmanager from functools import wraps from glob import glob from os.path import abspath, basename, dirname, isfile, join @@ -107,6 +108,17 @@ def singleton(cls): return get_instance +@contextmanager +def capture_std_streams(stdout, stderr=None): + _stdout = sys.stdout + _stderr = sys.stderr + sys.stdout = stdout + sys.stderr = stderr or stdout + yield + sys.stdout = _stdout + sys.stderr = _stderr + + def load_json(file_path): try: with open(file_path, "r") as f: From 8525fd6ae8a3077293e9f8d322c05655208b861b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Jul 2019 18:33:05 +0300 Subject: [PATCH 296/333] Docs: Sync Atmel SAMD boards --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index f59f1e26..2de2e4a1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f59f1e262d6bc370c32e6ed85319392c89de9af4 +Subproject commit 2de2e4a1069dd320252474b8040324b0b90ab016 From bf77d70d8295e2c00ac46adc6ee38c91ad708a1d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Jul 2019 20:39:52 +0300 Subject: [PATCH 297/333] Thread safe internal PIO Core calling for PIO Home --- .../commands/home/rpc/handlers/piocore.py | 64 +++++++++++++------ platformio/commands/home/rpc/server.py | 7 +- platformio/commands/platform.py | 2 +- 3 files changed, 50 insertions(+), 23 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 19d7d370..4370b5a9 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -16,7 +16,8 @@ from __future__ import absolute_import import json import os -import re +import sys +import thread from io import BytesIO import jsonrpc # pylint: disable=import-error @@ -26,8 +27,46 @@ from platformio import __main__, __version__, util from platformio.compat import string_types +class ThreadSafeStdBuffer(object): + + def __init__(self, parent_stream, parent_thread_id): + self.parent_stream = parent_stream + self.parent_thread_id = parent_thread_id + self._buffer = {} + + def write(self, value): + thread_id = thread.get_ident() + if thread_id == self.parent_thread_id: + return self.parent_stream.write(value) + if thread_id not in self._buffer: + self._buffer[thread_id] = BytesIO() + return self._buffer[thread_id].write(value) + + def flush(self): + return (self.parent_stream.flush() + if thread.get_ident() == self.parent_thread_id else None) + + def getvalue_and_close(self, thread_id=None): + thread_id = thread_id or thread.get_ident() + if thread_id not in self._buffer: + return "" + result = self._buffer.get(thread_id).getvalue() + self._buffer.get(thread_id).close() + del self._buffer[thread_id] + return result + + class PIOCoreRPC(object): + def __init__(self): + cur_thread_id = thread.get_ident() + PIOCoreRPC.thread_stdout = ThreadSafeStdBuffer(sys.stdout, + cur_thread_id) + PIOCoreRPC.thread_stderr = ThreadSafeStdBuffer(sys.stderr, + cur_thread_id) + sys.stdout = PIOCoreRPC.thread_stdout + sys.stderr = PIOCoreRPC.thread_stderr + @staticmethod def call(args, options=None): try: @@ -40,15 +79,10 @@ class PIOCoreRPC(object): code=4002, message="PIO Core: non-ASCII chars in arguments") def _call_cli(): - outbuff = BytesIO() - errbuff = BytesIO() - with util.capture_std_streams(outbuff, errbuff): - with util.cd((options or {}).get("cwd") or os.getcwd()): - exit_code = __main__.main(["-c"] + args) - result = (outbuff.getvalue(), errbuff.getvalue(), exit_code) - outbuff.close() - errbuff.close() - return result + with util.cd((options or {}).get("cwd") or os.getcwd()): + exit_code = __main__.main(["-c"] + args) + return (PIOCoreRPC.thread_stdout.getvalue_and_close(), + PIOCoreRPC.thread_stderr.getvalue_and_close(), exit_code) d = threads.deferToThread(_call_cli) d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) @@ -61,15 +95,7 @@ class PIOCoreRPC(object): text = ("%s\n\n%s" % (out, err)).strip() if code != 0: raise Exception(text) - if not json_output: - return text - try: - return json.loads(out) - except ValueError as e: - if "sh: " in out: - return json.loads( - re.sub(r"^sh: [^\n]+$", "", out, flags=re.M).strip()) - raise e + return json.loads(out) if json_output else text @staticmethod def _call_errback(failure): diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index f99b09b3..b77e1b94 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -14,6 +14,7 @@ # pylint: disable=import-error +import click import jsonrpc from autobahn.twisted.websocket import (WebSocketServerFactory, WebSocketServerProtocol) @@ -26,7 +27,7 @@ from platformio.compat import PY2, dump_json_to_unicode, is_bytes class JSONRPCServerProtocol(WebSocketServerProtocol): def onMessage(self, payload, isBinary): # pylint: disable=unused-argument - # print("> %s" % payload) + # click.echo("> %s" % payload) response = jsonrpc.JSONRPCResponseManager.handle( payload, self.factory.dispatcher).data # if error @@ -52,11 +53,11 @@ class JSONRPCServerProtocol(WebSocketServerProtocol): message=failure.getErrorMessage()) del response["result"] response['error'] = e.error._data # pylint: disable=protected-access - print(response['error']) + click.secho(str(response['error']), fg="red", err=True) self.sendJSONResponse(response) def sendJSONResponse(self, response): - # print("< %s" % response) + # click.echo("< %s" % response) response = dump_json_to_unicode(response) if not PY2 and not is_bytes(response): response = response.encode("utf-8") diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 26f86666..5b35c4df 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -182,7 +182,7 @@ def platform_frameworks(query, json_output): for framework in util.get_api_result("/frameworks", cache_valid="7d"): if query == "all": query = "" - search_data = framework + search_data = dump_json_to_unicode(framework) if query and query.lower() not in search_data.lower(): continue framework['homepage'] = ("https://platformio.org/frameworks/" + From 6d9de80f121d80ff008221ad9577095551763cc1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 1 Jul 2019 20:42:23 +0300 Subject: [PATCH 298/333] Better comparison for app state changes --- platformio/app.py | 19 ++++++++++--------- platformio/compat.py | 5 +++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index c35c4baf..12dcbb41 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -14,9 +14,9 @@ import codecs import hashlib +import json import os import uuid -from copy import deepcopy from os import environ, getenv, listdir, remove from os.path import abspath, dirname, expanduser, isdir, isfile, join from time import time @@ -94,27 +94,28 @@ class State(object): if not self.path: self.path = join(get_project_core_dir(), "appstate.json") self._state = {} - self._prev_state = {} + self._prev_state_raw = "" self._lockfile = None def __enter__(self): try: self._lock_state_file() if isfile(self.path): - self._state = util.load_json(self.path) + with open(self.path) as fp: + self._prev_state_raw = fp.read().strip() + self._state = json.loads(self._prev_state_raw) assert isinstance(self._state, dict) - except (AssertionError, UnicodeDecodeError, - exception.PlatformioException): + except (AssertionError, ValueError, UnicodeDecodeError): self._state = {} - self._prev_state = deepcopy(self._state) + self._prev_state_raw = "" return self._state def __exit__(self, type_, value, traceback): - new_state = dump_json_to_unicode(self._state) - if self._prev_state != new_state: + new_state_raw = dump_json_to_unicode(self._state) + if self._prev_state_raw != new_state_raw: try: with open(self.path, "w") as fp: - fp.write(new_state) + fp.write(new_state_raw) except IOError: raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() diff --git a/platformio/compat.py b/platformio/compat.py index 686518a8..8b082b4b 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -58,7 +58,8 @@ if PY2: return obj return json.dumps(obj, encoding=get_filesystem_encoding(), - ensure_ascii=False).encode("utf8") + ensure_ascii=False, + sort_keys=True).encode("utf8") _magic_check = re.compile('([*?[])') _magic_check_bytes = re.compile(b'([*?[])') @@ -104,4 +105,4 @@ else: def dump_json_to_unicode(obj): if isinstance(obj, string_types): return obj - return json.dumps(obj, ensure_ascii=False) + return json.dumps(obj, ensure_ascii=False, sort_keys=True) From d2c86ab71c4bb1ecae9a710e092358cc1e467ee0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 00:41:47 +0300 Subject: [PATCH 299/333] Refactor state to a proxied dictionary --- platformio/app.py | 51 +++++++++++++------ platformio/commands/home/rpc/handlers/app.py | 4 +- .../commands/home/rpc/handlers/piocore.py | 17 ++++--- platformio/commands/home/rpc/server.py | 3 +- 4 files changed, 50 insertions(+), 25 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 12dcbb41..a6fd6e45 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -14,7 +14,6 @@ import codecs import hashlib -import json import os import uuid from os import environ, getenv, listdir, remove @@ -93,29 +92,26 @@ class State(object): self.lock = lock if not self.path: self.path = join(get_project_core_dir(), "appstate.json") - self._state = {} - self._prev_state_raw = "" + self._storage = {} self._lockfile = None + self._modified = False def __enter__(self): try: self._lock_state_file() if isfile(self.path): - with open(self.path) as fp: - self._prev_state_raw = fp.read().strip() - self._state = json.loads(self._prev_state_raw) - assert isinstance(self._state, dict) - except (AssertionError, ValueError, UnicodeDecodeError): - self._state = {} - self._prev_state_raw = "" - return self._state + self._storage = util.load_json(self.path) + assert isinstance(self._storage, dict) + except (AssertionError, ValueError, UnicodeDecodeError, + exception.InvalidJSONFile): + self._storage = {} + return self def __exit__(self, type_, value, traceback): - new_state_raw = dump_json_to_unicode(self._state) - if self._prev_state_raw != new_state_raw: + if self._modified: try: with open(self.path, "w") as fp: - fp.write(new_state_raw) + fp.write(dump_json_to_unicode(self._storage)) except IOError: raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() @@ -133,8 +129,31 @@ class State(object): if hasattr(self, "_lockfile") and self._lockfile: self._lockfile.release() - def __del__(self): - self._unlock_state_file() + # Dictionary Proxy + + def as_dict(self): + return self._storage + + def get(self, key, default=True): + return self._storage.get(key, default) + + def update(self, *args, **kwargs): + self._modified = True + return self._storage.update(*args, **kwargs) + + def __getitem__(self, key): + return self._storage[key] + + def __setitem__(self, key, value): + self._modified = True + self._storage[key] = value + + def __delitem__(self, key): + self._modified = True + del self._storage[key] + + def __contains__(self, item): + return item in self._storage class ContentCache(object): diff --git a/platformio/commands/home/rpc/handlers/app.py b/platformio/commands/home/rpc/handlers/app.py index 0f8b8285..1666dc17 100644 --- a/platformio/commands/home/rpc/handlers/app.py +++ b/platformio/commands/home/rpc/handlers/app.py @@ -57,7 +57,7 @@ class AppRPC(object): ] state['storage'] = storage - return state + return state.as_dict() @staticmethod def get_state(): @@ -66,6 +66,6 @@ class AppRPC(object): @staticmethod def save_state(state): with app.State(AppRPC.APPSTATE_PATH, lock=True) as s: - s.clear() + # s.clear() s.update(state) return True diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 4370b5a9..62dfdd3e 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -17,7 +17,6 @@ from __future__ import absolute_import import json import os import sys -import thread from io import BytesIO import jsonrpc # pylint: disable=import-error @@ -26,6 +25,11 @@ from twisted.internet import threads # pylint: disable=import-error from platformio import __main__, __version__, util from platformio.compat import string_types +try: + from thread import get_ident as thread_get_ident +except ImportError: + from threading import get_ident as thread_get_ident + class ThreadSafeStdBuffer(object): @@ -35,19 +39,20 @@ class ThreadSafeStdBuffer(object): self._buffer = {} def write(self, value): - thread_id = thread.get_ident() + thread_id = thread_get_ident() if thread_id == self.parent_thread_id: - return self.parent_stream.write(value) + return self.parent_stream.write( + value if isinstance(value, string_types) else value.decode()) if thread_id not in self._buffer: self._buffer[thread_id] = BytesIO() return self._buffer[thread_id].write(value) def flush(self): return (self.parent_stream.flush() - if thread.get_ident() == self.parent_thread_id else None) + if thread_get_ident() == self.parent_thread_id else None) def getvalue_and_close(self, thread_id=None): - thread_id = thread_id or thread.get_ident() + thread_id = thread_id or thread_get_ident() if thread_id not in self._buffer: return "" result = self._buffer.get(thread_id).getvalue() @@ -59,7 +64,7 @@ class ThreadSafeStdBuffer(object): class PIOCoreRPC(object): def __init__(self): - cur_thread_id = thread.get_ident() + cur_thread_id = thread_get_ident() PIOCoreRPC.thread_stdout = ThreadSafeStdBuffer(sys.stdout, cur_thread_id) PIOCoreRPC.thread_stderr = ThreadSafeStdBuffer(sys.stderr, diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index b77e1b94..36aa1dff 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -53,11 +53,12 @@ class JSONRPCServerProtocol(WebSocketServerProtocol): message=failure.getErrorMessage()) del response["result"] response['error'] = e.error._data # pylint: disable=protected-access - click.secho(str(response['error']), fg="red", err=True) self.sendJSONResponse(response) def sendJSONResponse(self, response): # click.echo("< %s" % response) + if "error" in response: + click.secho("Error: %s" % response['error'], fg="red", err=True) response = dump_json_to_unicode(response) if not PY2 and not is_bytes(response): response = response.encode("utf-8") From bf9552bd562bc15fe5912631e6da378b2088f7db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 00:45:35 +0300 Subject: [PATCH 300/333] Free lock when state is deleted --- platformio/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/platformio/app.py b/platformio/app.py index a6fd6e45..d1283dec 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -129,6 +129,9 @@ class State(object): if hasattr(self, "_lockfile") and self._lockfile: self._lockfile.release() + def __del__(self): + self._unlock_state_file() + # Dictionary Proxy def as_dict(self): From 69de40c409627740e1c27850652e9979690a4ee6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 12:21:06 +0300 Subject: [PATCH 301/333] Fix an issue when empty multiple values for project option were not casted to list // Resolve #2734 --- platformio/project/config.py | 2 +- tests/test_projectconf.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/platformio/project/config.py b/platformio/project/config.py index ee7e71d1..dc455809 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -243,7 +243,7 @@ class ProjectConfig(object): if not option_meta: return value or default - if value and option_meta.multiple: + if option_meta.multiple: value = self.parse_multi_values(value) if option_meta.sysenvvar: diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 38725f8a..882c2dd6 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -42,6 +42,7 @@ lib_ignore = LibIgnoreCustom [env:base] build_flags = ${custom.debug_flags} ${custom.extra_flags} +targets = """ EXTRA_ENVS_CONFIG = """ @@ -102,7 +103,7 @@ def test_real_config(tmpdir): # options assert config.options(env="base") == [ - "build_flags", "monitor_speed", "lib_deps", "lib_ignore" + "build_flags", "targets", "monitor_speed", "lib_deps", "lib_ignore" ] # has_option @@ -126,6 +127,7 @@ def test_real_config(tmpdir): assert config.get("env:extra_2", "upload_port") == "/dev/extra_2/port" # getraw + assert config.getraw("env:base", "targets") == "" assert config.getraw("env:extra_1", "lib_deps") == "574" assert config.getraw("env:extra_1", "build_flags") == "-lc -lm -D DEBUG=1" @@ -148,6 +150,16 @@ def test_real_config(tmpdir): ("extra_flags", "-L /usr/local/lib"), ("lib_ignore", "LibIgnoreCustom") ] # yapf: disable + print(config.items(env="base")) + assert config.items(env="base") == [ + ("build_flags", [ + "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), + ("targets", []), + ("monitor_speed", "115200"), + ("lib_deps", ["Lib1", "Lib2"]), + ("lib_ignore", ["LibIgnoreCustom"]), + ("upload_port", "/dev/sysenv/port") + ] # yapf: disable assert config.items(env="extra_1") == [ ("build_flags", ["-lc -lm -D DEBUG=1", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), ("lib_deps", ["574"]), From 83f64cebbd9921eea99bf3f385c8c5fc3ff35b81 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 12:23:35 +0300 Subject: [PATCH 302/333] Remove test code --- tests/test_projectconf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 882c2dd6..1ad8b2cd 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -150,7 +150,6 @@ def test_real_config(tmpdir): ("extra_flags", "-L /usr/local/lib"), ("lib_ignore", "LibIgnoreCustom") ] # yapf: disable - print(config.items(env="base")) assert config.items(env="base") == [ ("build_flags", [ "-D DEBUG=1 -L /usr/local/lib", "-DSYSENVDEPS1 -DSYSENVDEPS2"]), From 148b7dccfdd51ed589d649cac0219b639279fc42 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 15:47:03 +0300 Subject: [PATCH 303/333] Refactor ThreadSafeStdBuffer --- .../commands/home/rpc/handlers/piocore.py | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 62dfdd3e..d4cc170f 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -33,42 +33,42 @@ except ImportError: class ThreadSafeStdBuffer(object): - def __init__(self, parent_stream, parent_thread_id): - self.parent_stream = parent_stream - self.parent_thread_id = parent_thread_id - self._buffer = {} + def __init__(self, parent_stream): + self._buffers = {thread_get_ident(): parent_stream} + + def __getattr__(self, name): + thread_id = thread_get_ident() + if thread_id not in self._buffers: + raise AttributeError(name) + return getattr(self._buffers[thread_id], name) def write(self, value): thread_id = thread_get_ident() - if thread_id == self.parent_thread_id: - return self.parent_stream.write( - value if isinstance(value, string_types) else value.decode()) - if thread_id not in self._buffer: - self._buffer[thread_id] = BytesIO() - return self._buffer[thread_id].write(value) + if thread_id not in self._buffers: + self._buffers[thread_id] = BytesIO() + try: + return self._buffers[thread_id].write(value) + except TypeError: + return self._buffers[thread_id].write(value.encode()) - def flush(self): - return (self.parent_stream.flush() - if thread_get_ident() == self.parent_thread_id else None) - - def getvalue_and_close(self, thread_id=None): - thread_id = thread_id or thread_get_ident() - if thread_id not in self._buffer: - return "" - result = self._buffer.get(thread_id).getvalue() - self._buffer.get(thread_id).close() - del self._buffer[thread_id] + def get_value_and_close(self): + thread_id = thread_get_ident() + result = "" + try: + result = self.getvalue() + self.close() + if thread_id in self._buffers: + del self._buffers[thread_id] + except AttributeError: + pass return result class PIOCoreRPC(object): def __init__(self): - cur_thread_id = thread_get_ident() - PIOCoreRPC.thread_stdout = ThreadSafeStdBuffer(sys.stdout, - cur_thread_id) - PIOCoreRPC.thread_stderr = ThreadSafeStdBuffer(sys.stderr, - cur_thread_id) + PIOCoreRPC.thread_stdout = ThreadSafeStdBuffer(sys.stdout) + PIOCoreRPC.thread_stderr = ThreadSafeStdBuffer(sys.stderr) sys.stdout = PIOCoreRPC.thread_stdout sys.stderr = PIOCoreRPC.thread_stderr @@ -86,8 +86,8 @@ class PIOCoreRPC(object): def _call_cli(): with util.cd((options or {}).get("cwd") or os.getcwd()): exit_code = __main__.main(["-c"] + args) - return (PIOCoreRPC.thread_stdout.getvalue_and_close(), - PIOCoreRPC.thread_stderr.getvalue_and_close(), exit_code) + return (PIOCoreRPC.thread_stdout.get_value_and_close(), + PIOCoreRPC.thread_stderr.get_value_and_close(), exit_code) d = threads.deferToThread(_call_cli) d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) From c79d5f0cf1fc3b24af3812f279612f93ff58bc5a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 15:52:12 +0300 Subject: [PATCH 304/333] Fix an issue saving modified State --- platformio/app.py | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index d1283dec..2217c671 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -94,7 +94,7 @@ class State(object): self.path = join(get_project_core_dir(), "appstate.json") self._storage = {} self._lockfile = None - self._modified = False + self.modified = False def __enter__(self): try: @@ -108,7 +108,7 @@ class State(object): return self def __exit__(self, type_, value, traceback): - if self._modified: + if self.modified: try: with open(self.path, "w") as fp: fp.write(dump_json_to_unicode(self._storage)) @@ -141,18 +141,18 @@ class State(object): return self._storage.get(key, default) def update(self, *args, **kwargs): - self._modified = True + self.modified = True return self._storage.update(*args, **kwargs) def __getitem__(self, key): return self._storage[key] def __setitem__(self, key, value): - self._modified = True + self.modified = True self._storage[key] = value def __delitem__(self, key): - self._modified = True + self.modified = True del self._storage[key] def __contains__(self, item): @@ -315,19 +315,20 @@ def sanitize_setting(name, value): def get_state_item(name, default=None): - with State() as data: - return data.get(name, default) + with State() as state: + return state.get(name, default) def set_state_item(name, value): - with State(lock=True) as data: - data[name] = value + with State(lock=True) as state: + state[name] = value + state.modified = True def delete_state_item(name): - with State(lock=True) as data: - if name in data: - del data[name] + with State(lock=True) as state: + if name in state: + del state[name] def get_setting(name): @@ -335,24 +336,25 @@ def get_setting(name): if _env_name in environ: return sanitize_setting(name, getenv(_env_name)) - with State() as data: - if "settings" in data and name in data['settings']: - return data['settings'][name] + with State() as state: + if "settings" in state and name in state['settings']: + return state['settings'][name] return DEFAULT_SETTINGS[name]['value'] def set_setting(name, value): - with State(lock=True) as data: - if "settings" not in data: - data['settings'] = {} - data['settings'][name] = sanitize_setting(name, value) + with State(lock=True) as state: + if "settings" not in state: + state['settings'] = {} + state['settings'][name] = sanitize_setting(name, value) + state.modified = True def reset_settings(): - with State(lock=True) as data: - if "settings" in data: - del data['settings'] + with State(lock=True) as state: + if "settings" in state: + del state['settings'] def get_session_var(name, default=None): From 29887244563363cac9685e677bbe2bd8356cfac2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 20:44:33 +0300 Subject: [PATCH 305/333] Setup MultiThreadingStdStreams from child threads --- .../commands/home/rpc/handlers/piocore.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index d4cc170f..9df38461 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -31,7 +31,7 @@ except ImportError: from threading import get_ident as thread_get_ident -class ThreadSafeStdBuffer(object): +class MultiThreadingStdStream(object): def __init__(self, parent_stream): self._buffers = {thread_get_ident(): parent_stream} @@ -66,9 +66,12 @@ class ThreadSafeStdBuffer(object): class PIOCoreRPC(object): - def __init__(self): - PIOCoreRPC.thread_stdout = ThreadSafeStdBuffer(sys.stdout) - PIOCoreRPC.thread_stderr = ThreadSafeStdBuffer(sys.stderr) + @staticmethod + def setup_multithreading_std_streams(): + if isinstance(sys.stdout, MultiThreadingStdStream): + return + PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout) + PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr) sys.stdout = PIOCoreRPC.thread_stdout sys.stderr = PIOCoreRPC.thread_stderr @@ -83,13 +86,17 @@ class PIOCoreRPC(object): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4002, message="PIO Core: non-ASCII chars in arguments") - def _call_cli(): - with util.cd((options or {}).get("cwd") or os.getcwd()): + PIOCoreRPC.setup_multithreading_std_streams() + cwd = (options or {}).get("cwd") or os.getcwd() + + def _call_inline(): + with util.cd(cwd): exit_code = __main__.main(["-c"] + args) return (PIOCoreRPC.thread_stdout.get_value_and_close(), PIOCoreRPC.thread_stderr.get_value_and_close(), exit_code) - d = threads.deferToThread(_call_cli) + d = threads.deferToThread(_call_inline) + d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) d.addErrback(PIOCoreRPC._call_errback) return d From 4b24d6e3e47ab04b724c7891fc14a8485c4d1b55 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 20:48:43 +0300 Subject: [PATCH 306/333] Minor changes --- platformio/commands/home/rpc/handlers/piocore.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 9df38461..b481afca 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -77,6 +77,7 @@ class PIOCoreRPC(object): @staticmethod def call(args, options=None): + PIOCoreRPC.setup_multithreading_std_streams() try: args = [ str(arg) if not isinstance(arg, string_types) else arg @@ -86,11 +87,8 @@ class PIOCoreRPC(object): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4002, message="PIO Core: non-ASCII chars in arguments") - PIOCoreRPC.setup_multithreading_std_streams() - cwd = (options or {}).get("cwd") or os.getcwd() - def _call_inline(): - with util.cd(cwd): + with util.cd((options or {}).get("cwd") or os.getcwd()): exit_code = __main__.main(["-c"] + args) return (PIOCoreRPC.thread_stdout.get_value_and_close(), PIOCoreRPC.thread_stderr.get_value_and_close(), exit_code) From 900a4d463fca9d0096af881c40302e5a5bc613b8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 20:49:06 +0300 Subject: [PATCH 307/333] Bump version to 4.0.0rc2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 4b354750..8b5979e1 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0rc1") +VERSION = (4, 0, "0rc2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 18d93dfcc90c141caa2fcc19048de49e16cfeb57 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 21:04:43 +0300 Subject: [PATCH 308/333] Use StringIO for PY3 --- platformio/commands/home/rpc/handlers/piocore.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index b481afca..376f1606 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -17,13 +17,13 @@ from __future__ import absolute_import import json import os import sys -from io import BytesIO +from io import BytesIO, StringIO import jsonrpc # pylint: disable=import-error from twisted.internet import threads # pylint: disable=import-error from platformio import __main__, __version__, util -from platformio.compat import string_types +from platformio.compat import PY2, is_bytes, string_types try: from thread import get_ident as thread_get_ident @@ -45,11 +45,9 @@ class MultiThreadingStdStream(object): def write(self, value): thread_id = thread_get_ident() if thread_id not in self._buffers: - self._buffers[thread_id] = BytesIO() - try: - return self._buffers[thread_id].write(value) - except TypeError: - return self._buffers[thread_id].write(value.encode()) + self._buffers[thread_id] = BytesIO() if PY2 else StringIO() + return self._buffers[thread_id].write( + value.decode() if is_bytes(value) else value) def get_value_and_close(self): thread_id = thread_get_ident() From 6c47c7506e4733af17858a076a9a32deb1ba6bb6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 22:26:01 +0300 Subject: [PATCH 309/333] Do not remove thread buffers, reset them instead --- platformio/commands/home/rpc/handlers/piocore.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 376f1606..d22a6e80 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -49,14 +49,12 @@ class MultiThreadingStdStream(object): return self._buffers[thread_id].write( value.decode() if is_bytes(value) else value) - def get_value_and_close(self): - thread_id = thread_get_ident() + def get_value_and_reset(self): result = "" try: result = self.getvalue() - self.close() - if thread_id in self._buffers: - del self._buffers[thread_id] + self.truncate(0) + self.seek(0) except AttributeError: pass return result @@ -88,11 +86,10 @@ class PIOCoreRPC(object): def _call_inline(): with util.cd((options or {}).get("cwd") or os.getcwd()): exit_code = __main__.main(["-c"] + args) - return (PIOCoreRPC.thread_stdout.get_value_and_close(), - PIOCoreRPC.thread_stderr.get_value_and_close(), exit_code) + return (PIOCoreRPC.thread_stdout.get_value_and_reset(), + PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code) d = threads.deferToThread(_call_inline) - d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) d.addErrback(PIOCoreRPC._call_errback) return d From fd9ca0cd15e86fc44bd564db65333e667fb44585 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 2 Jul 2019 22:35:35 +0300 Subject: [PATCH 310/333] Ensure buffer is created for thread stream --- platformio/commands/home/rpc/handlers/piocore.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index d22a6e80..ef290ea3 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -38,14 +38,16 @@ class MultiThreadingStdStream(object): def __getattr__(self, name): thread_id = thread_get_ident() - if thread_id not in self._buffers: - raise AttributeError(name) + self._ensure_thread_buffer(thread_id) return getattr(self._buffers[thread_id], name) + def _ensure_thread_buffer(self, thread_id): + if thread_id not in self._buffers: + self._buffers[thread_id] = BytesIO() if PY2 else StringIO() + def write(self, value): thread_id = thread_get_ident() - if thread_id not in self._buffers: - self._buffers[thread_id] = BytesIO() if PY2 else StringIO() + self._ensure_thread_buffer(thread_id) return self._buffers[thread_id].write( value.decode() if is_bytes(value) else value) From b68e5db46b7a1eb8294180149f0bb9efe009d4ae Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jul 2019 12:37:00 +0300 Subject: [PATCH 311/333] Fix broken example when using custom uploader // Resolve #2735 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 2de2e4a1..a9c7a292 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 2de2e4a1069dd320252474b8040324b0b90ab016 +Subproject commit a9c7a292d85dd7d2d11f3c2fd0e8009ebc789368 From dc07ea56d2ffc567d8c1ff366e71780fb10b159b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jul 2019 14:01:14 +0300 Subject: [PATCH 312/333] Bump version to 4.0.0rc3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 8b5979e1..b8fd88a3 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0rc2") +VERSION = (4, 0, "0rc3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 7c6fabaee29205fbca7e301cdb89137219ffca9b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jul 2019 15:10:37 +0300 Subject: [PATCH 313/333] Fix an issue with unhandled warnings from PIO Core when calling it internally // Resolved #2727 --- .../commands/home/rpc/handlers/piocore.py | 18 +++++++++++++++++- platformio/commands/lib.py | 5 ++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index ef290ea3..07809171 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -19,6 +19,7 @@ import os import sys from io import BytesIO, StringIO +import click import jsonrpc # pylint: disable=import-error from twisted.internet import threads # pylint: disable=import-error @@ -102,7 +103,22 @@ class PIOCoreRPC(object): text = ("%s\n\n%s" % (out, err)).strip() if code != 0: raise Exception(text) - return json.loads(out) if json_output else text + if not json_output: + return text + try: + return json.loads(out) + except ValueError as e: + click.secho("%s => `%s`" % (e, out), fg="red", err=True) + # if PIO Core prints unhandled warnings + for line in out.split("\n"): + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except ValueError: + pass + raise e @staticmethod def _call_errback(failure): diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index c66dc77b..cdc3d2cc 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -21,6 +21,7 @@ import click import semantic_version from platformio import exception, util +from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import (LibraryManager, get_builtin_libs, is_builtin_lib) @@ -89,6 +90,7 @@ def cli(ctx, **options): get_project_global_lib_dir(), ctx.invoked_subcommand) + in_silence = PlatformioCLI.in_silence() ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY] = options['environment'] ctx.meta[CTX_META_INPUT_DIRS_KEY] = storage_dirs ctx.meta[CTX_META_STORAGE_DIRS_KEY] = [] @@ -101,7 +103,7 @@ def cli(ctx, **options): libdeps_dir = get_project_libdeps_dir() config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini")) - config.validate(options['environment']) + config.validate(options['environment'], silent=in_silence) for env in config.envs(): if options['environment'] and env not in options['environment']: continue @@ -261,6 +263,7 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output): @click.option("--json-output", is_flag=True) @click.pass_context def lib_list(ctx, json_output): + print("hello") storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] json_result = {} for storage_dir in storage_dirs: From 26dda104dd970c7828d8057d88e3fa8f54268bcf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jul 2019 15:22:04 +0300 Subject: [PATCH 314/333] Remove debug code --- platformio/commands/lib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index cdc3d2cc..1d75c961 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -263,7 +263,6 @@ def lib_update(ctx, libraries, only_check, dry_run, json_output): @click.option("--json-output", is_flag=True) @click.pass_context def lib_list(ctx, json_output): - print("hello") storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY] json_result = {} for storage_dir in storage_dirs: From 0a907627beb46a7403818bd14d38cd4a4aee3e8c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jul 2019 15:22:22 +0300 Subject: [PATCH 315/333] Bump version to 4.0.0rc4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index b8fd88a3..f0ff548e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0rc3") +VERSION = (4, 0, "0rc4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From c3ac10fe29f421ad93da94936b596b27d5fd78c5 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 3 Jul 2019 22:58:27 +0300 Subject: [PATCH 316/333] Update docs and examples for SiFive dev/platform --- docs | 2 +- examples | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index a9c7a292..49a86db3 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a9c7a292d85dd7d2d11f3c2fd0e8009ebc789368 +Subproject commit 49a86db3aa4489d12633a212bd291b041c1e8347 diff --git a/examples b/examples index 76278680..70f28968 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 762786807e88b68784492f908738346277b74be8 +Subproject commit 70f28968f2fa3f76374d236156581ddc4e2e8670 From affa54e5fc8aefdac7b820db5df491724be4cff9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jul 2019 16:46:54 +0300 Subject: [PATCH 317/333] Fix an issue when `lib_compat_mode = strict` does not ignore libraries incompatible with a project framework --- HISTORY.rst | 5 +++-- platformio/builder/tools/piolib.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b60e44ff..79e86f67 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,8 +13,8 @@ PlatformIO 4.0 * `PlatformIO Plus Goes Open Source `__ - - Open source `PIO Unified Debugger `__ - - Open source `PIO Unit Testing `__ + - Open source `PIO Unified Debugger `__ + - Open source `PIO Unit Testing `__ * **Project Configuration** @@ -37,6 +37,7 @@ PlatformIO 4.0 - Look firstly in built-in library storages for a missing dependency instead of PlatformIO Registry (`issue #1654 `_) - Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields - Fixed an issue when library keeps reinstalling for non-latin path (`issue #1252 `_) + - Fixed an issue when `lib_compat_mode = strict `__ does not ignore libraries incompatible with a project framework * **Build System** diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index f9ab05f0..66f9db57 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -943,7 +943,7 @@ def IsCompatibleLibBuilder(env, if verbose: sys.stderr.write("Platform incompatible library %s\n" % lb.path) return False - if (compat_mode == "soft" and "PIOFRAMEWORK" in env + if (compat_mode in ("soft", "strict") and "PIOFRAMEWORK" in env and not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", []))): if verbose: sys.stderr.write("Framework incompatible library %s\n" % lb.path) From d9b6842b6a7bca98c4a804c009fd905ff740047e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jul 2019 16:49:29 +0300 Subject: [PATCH 318/333] Update history with refactored PIO Home back-end --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 79e86f67..a4d8907d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -53,6 +53,7 @@ PlatformIO 4.0 * **Infrastructure** - Python 3 support (`issue #895 `_) + - Significantly speedup back-end for PIO Home. It works super fast now! - Added support for the latest Python "Click" package (CLI) (`issue #349 `_) - Added options to override default locations used by PlatformIO Core (`core_dir `__, `globallib_dir `__, `platforms_dir `__, `packages_dir `__, `cache_dir `__) (`issue #1615 `_) - Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools (`issue #856 `_) From caf51590029cd45628c21249e40a99cdaa1fdedd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jul 2019 17:47:26 +0300 Subject: [PATCH 319/333] Fixed an issue when Ctrl+C(SIGINT) terminates debugging session instead of halting // Resolve #2733 --- platformio/commands/debug/client.py | 3 +++ platformio/commands/debug/command.py | 2 ++ platformio/commands/debug/process.py | 7 +++++++ platformio/commands/debug/server.py | 12 +++++++++++- 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index 9b208a48..f6d403bb 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -15,6 +15,7 @@ import json import os import re +import signal import time from hashlib import sha1 from os.path import abspath, basename, dirname, isdir, join, splitext @@ -188,6 +189,8 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes if b"-exec-continue" in data: self._target_is_run = True if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"): + # Allow terminating via SIGINT/CTRL+C + signal.signal(signal.SIGINT, signal.default_int_handler) self.transport.write(b"pio_reset_target\n") self.transport.write(data) diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 8599a962..14c4f665 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -16,6 +16,7 @@ # pylint: disable=too-many-locals, too-many-branches import os +import signal from os.path import isfile, join import click @@ -146,6 +147,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, client = GDBClient(project_dir, __unprocessed, debug_options, env_options) client.spawn(configuration['gdb_path'], configuration['prog_path']) + signal.signal(signal.SIGINT, lambda *args, **kwargs: None) reactor.run() return True diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index a9064492..98c7cc1a 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import signal + import click from twisted.internet import protocol # pylint: disable=import-error @@ -71,3 +73,8 @@ class BaseProcess(protocol.ProcessProtocol, object): with open(LOG_FILE, "ab") as fp: fp.write(data) click.echo(data, nl=False, err=True) + + @staticmethod + def processEnded(_): + # Allow terminating via SIGINT/CTRL+C + signal.signal(signal.SIGINT, signal.default_int_handler) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index 6d81585e..83bba340 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -15,6 +15,7 @@ import os from os.path import isdir, isfile, join +from twisted.internet import error # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error from platformio import exception, util @@ -31,6 +32,7 @@ class DebugServer(BaseProcess): self._debug_port = None self._transport = None + self._process_ended = False def spawn(self, patterns): # pylint: disable=too-many-branches systype = util.get_systype() @@ -108,6 +110,14 @@ class DebugServer(BaseProcess): def get_debug_port(self): return self._debug_port + def processEnded(self, reason): + self._process_ended = True + super(DebugServer, self).processEnded(reason) + def terminate(self): - if self._transport: + if self._process_ended or not self._transport: + return + try: self._transport.signalProcess("KILL") + except (OSError, error.ProcessExitedAlready): + pass From 73dacd9418868f1317837522c8b3e64de6755a4b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jul 2019 19:02:13 +0300 Subject: [PATCH 320/333] Add "make profile" hotkey --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bc8ba602..c82b7c54 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,3 @@ - lint: pylint --rcfile=./.pylintrc ./platformio @@ -23,4 +22,9 @@ clean: clean-docs rm -rf .cache rm -rf build rm -rf htmlcov - rm -f .coverage \ No newline at end of file + rm -f .coverage + +profile: + # Usage $ > make PIOARGS="boards" profile + python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS} + snakeviz .tox/.tmp/cprofile.prof From 412a1f78cd72c36cd7ed5da0636b608d0c49e15a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jul 2019 20:07:44 +0300 Subject: [PATCH 321/333] Fixed an issue when generating invalid "Eclipse CDT Cross GCC Built-in Compiler Settings" if a custom PLATFORMIO_CORE_DIR is used // Resolve #806 --- HISTORY.rst | 3 ++- .../.settings/language.settings.xml.tpl | 20 +++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a4d8907d..df69c43c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -63,7 +63,8 @@ PlatformIO 4.0 - Support custom CMake configuration for CLion IDE using ``CMakeListsUser.txt`` file - Fixed an issue with hardcoded C standard version when generating project for CLion IDE (`issue #2527 `_) - - Fixed an issue with Project Generator when include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) + - Fixed an issue with Project Generator when an include path search order is inconsistent to what passed to the compiler (`issue #2509 `_) + - Fixed an issue when generating invalid "Eclipse CDT Cross GCC Built-in Compiler Settings" if a custom `PLATFORMIO_CORE_DIR `__ is used (`issue #806 `_) * **Miscellaneous** diff --git a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl index a119c8b2..eefe1a52 100644 --- a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl +++ b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl @@ -3,6 +3,14 @@ % cxx_stds = STD_RE.findall(cxx_flags) % cxx_std = cxx_stds[-1] if cxx_stds else "" % +% if cxx_path.startswith(user_home_dir): +% if "windows" in systype: +% cxx_path = "${USERPROFILE}" + cxx_path.replace(user_home_dir, "") +% else: +% cxx_path = "${HOME}" + cxx_path.replace(user_home_dir, "") +% end +% end +% @@ -10,11 +18,7 @@ - % if "windows" in systype: - - % else: - - % end + @@ -25,11 +29,7 @@ - % if "windows" in systype: - - % else: - - % end + From 2a4b25705ce7a215e812c3b4de0a1f5827f71740 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jul 2019 22:53:36 +0300 Subject: [PATCH 322/333] More pre-configured target for Eclipse IDE --- platformio/ide/tpls/eclipse/.cproject.tpl | 60 ++++++++++++++++++++--- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/platformio/ide/tpls/eclipse/.cproject.tpl b/platformio/ide/tpls/eclipse/.cproject.tpl index 01460e13..8a237f8d 100644 --- a/platformio/ide/tpls/eclipse/.cproject.tpl +++ b/platformio/ide/tpls/eclipse/.cproject.tpl @@ -219,7 +219,7 @@ platformio -f -c eclipse - run -t program + run --target program true false false @@ -227,7 +227,7 @@ platformio -f -c eclipse - run -t uploadfs + run --target uploadfs true false false @@ -240,10 +240,26 @@ false false + + platformio + -f -c eclipse + run --verbose + true + false + false + platformio -f -c eclipse - run -t upload + run --target upload + true + false + false + + + platformio + -f -c eclipse + run --target upload --verbose true false false @@ -251,7 +267,7 @@ platformio -f -c eclipse - run -t clean + run --target clean true false false @@ -264,10 +280,10 @@ false false - + platformio -f -c eclipse - update + remote run --target upload true false false @@ -280,6 +296,38 @@ false false + + platformio + -f -c eclipse + device list + true + false + false + + + platformio + -f -c eclipse + lib update + true + false + false + + + platformio + -f -c eclipse + update + true + false + false + + + platformio + -f -c eclipse + upgrade + true + false + false + From c69e80249d0e875d33176944412b807db4621b24 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 4 Jul 2019 22:56:54 +0300 Subject: [PATCH 323/333] Append system PATH to overridden in CDT project // Resolve #810 --- .../ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl b/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl index e87cf90b..13b8c382 100644 --- a/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl +++ b/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl @@ -1,11 +1,11 @@ eclipse.preferences.version=1 environment/project/0.910961921/PATH/delimiter={{env_pathsep.replace(":", "\\:")}} environment/project/0.910961921/PATH/operation=replace -environment/project/0.910961921/PATH/value={{env_path.replace(":", "\\:")}} +environment/project/0.910961921/PATH/value={{env_path.replace(":", "\\:")}}${PathDelimiter}${PATH} environment/project/0.910961921/append=true environment/project/0.910961921/appendContributed=true environment/project/0.910961921.1363900502/PATH/delimiter={{env_pathsep.replace(":", "\\:")}} environment/project/0.910961921.1363900502/PATH/operation=replace -environment/project/0.910961921.1363900502/PATH/value={{env_path.replace(":", "\\:")}} +environment/project/0.910961921.1363900502/PATH/value={{env_path.replace(":", "\\:")}}${PathDelimiter}${PATH} environment/project/0.910961921.1363900502/append=true environment/project/0.910961921.1363900502/appendContributed=true \ No newline at end of file From ffcf6b873a5f16f843c991394a82ebfbbd4f3c3c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 8 Jul 2019 17:21:28 +0300 Subject: [PATCH 324/333] Use native Windows API for getting My Documents folder path --- platformio/app.py | 15 ++++++++++++++- platformio/commands/settings.py | 6 ++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/platformio/app.py b/platformio/app.py index 2217c671..c2b95967 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -30,6 +30,19 @@ from platformio.project.helpers import (get_project_cache_dir, get_project_core_dir) +def get_default_projects_dir(): + docs_dir = join(expanduser("~"), "Documents") + try: + assert WINDOWS + import ctypes.wintypes + buf = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) + ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf) + docs_dir = buf.value + except: # pylint: disable=bare-except + pass + return join(docs_dir, "PlatformIO", "Projects") + + def projects_dir_validate(projects_dir): assert isdir(projects_dir) return abspath(projects_dir) @@ -77,7 +90,7 @@ DEFAULT_SETTINGS = { }, "projects_dir": { "description": "Default location for PlatformIO projects (PIO Home)", - "value": join(expanduser("~"), "Documents", "PlatformIO", "Projects"), + "value": get_default_projects_dir(), "validator": projects_dir_validate }, } diff --git a/platformio/commands/settings.py b/platformio/commands/settings.py index d1d5ad59..5fa20993 100644 --- a/platformio/commands/settings.py +++ b/platformio/commands/settings.py @@ -15,6 +15,7 @@ import click from platformio import app +from platformio.compat import string_types @click.group(short_help="Manage PlatformIO settings") @@ -26,7 +27,7 @@ def cli(): @click.argument("name", required=False) def settings_get(name): - list_tpl = "{name:<40} {value:<35} {description}" + list_tpl = u"{name:<40} {value:<35} {description}" terminal_width, _ = click.get_terminal_size() click.echo( @@ -41,7 +42,8 @@ def settings_get(name): continue _value = app.get_setting(_name) - _value_str = str(_value) + _value_str = (str(_value) + if not isinstance(_value, string_types) else _value) if isinstance(_value, bool): _value_str = "Yes" if _value else "No" _value_str = click.style(_value_str, fg="green") From 0a638b7ea5944d7fb1f52d5e187d6ac8a1af427d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Jul 2019 00:14:11 +0300 Subject: [PATCH 325/333] Fix issues with Unicode arguments when calling inline PIO Core --- .../commands/home/rpc/handlers/piocore.py | 16 ++-- .../commands/home/rpc/handlers/project.py | 86 +++++++++---------- 2 files changed, 49 insertions(+), 53 deletions(-) diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 07809171..22ffb373 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -24,7 +24,8 @@ import jsonrpc # pylint: disable=import-error from twisted.internet import threads # pylint: disable=import-error from platformio import __main__, __version__, util -from platformio.compat import PY2, is_bytes, string_types +from platformio.compat import (PY2, get_filesystem_encoding, is_bytes, + string_types) try: from thread import get_ident as thread_get_ident @@ -77,14 +78,11 @@ class PIOCoreRPC(object): @staticmethod def call(args, options=None): PIOCoreRPC.setup_multithreading_std_streams() - try: - args = [ - str(arg) if not isinstance(arg, string_types) else arg - for arg in args - ] - except UnicodeError: - raise jsonrpc.exceptions.JSONRPCDispatchException( - code=4002, message="PIO Core: non-ASCII chars in arguments") + for i, arg in enumerate(args): + if isinstance(arg, string_types): + args[i] = arg.encode(get_filesystem_encoding()) if PY2 else arg + else: + args[i] = str(arg) def _call_inline(): with util.cd((options or {}).get("cwd") or os.getcwd()): diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index bb301ea9..4ca5af63 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -113,18 +113,54 @@ class ProjectRPC(object): def get_projects(self, project_dirs=None): return self._get_projects(project_dirs) + @staticmethod + def get_project_examples(): + result = [] + for manifest in PlatformManager().get_installed(): + examples_dir = join(manifest['__pkg_dir'], "examples") + if not isdir(examples_dir): + continue + items = [] + for project_dir, _, __ in os.walk(examples_dir): + project_description = None + try: + config = ProjectConfig(join(project_dir, "platformio.ini")) + config.validate(silent=True) + project_description = config.get("platformio", + "description") + except exception.PlatformIOProjectException: + continue + + path_tokens = project_dir.split(sep) + items.append({ + "name": + "/".join(path_tokens[path_tokens.index("examples") + 1:]), + "path": + project_dir, + "description": + project_description + }) + result.append({ + "platform": { + "title": manifest['title'], + "version": manifest['version'] + }, + "items": sorted(items, key=lambda item: item['name']) + }) + return sorted(result, key=lambda data: data['platform']['title']) + def init(self, board, framework, project_dir): assert project_dir state = AppRPC.load_state() if not isdir(project_dir): os.makedirs(project_dir) - args = ["init", "--project-dir", project_dir, "--board", board] + args = ["init", "--board", board] if framework: args.extend(["--project-option", "framework = %s" % framework]) if (state['storage']['coreCaller'] and state['storage']['coreCaller'] in ProjectGenerator.get_supported_ides()): args.extend(["--ide", state['storage']['coreCaller']]) - d = PIOCoreRPC.call(args) + d = PIOCoreRPC.call(args, options={"cwd": project_dir}) d.addCallback(self._generate_project_main, project_dir, framework) return d @@ -196,7 +232,7 @@ class ProjectRPC(object): time.strftime("%y%m%d-%H%M%S-") + board) if not isdir(project_dir): os.makedirs(project_dir) - args = ["init", "--project-dir", project_dir, "--board", board] + args = ["init", "--board", board] args.extend(["--project-option", "framework = arduino"]) if use_arduino_libs: args.extend([ @@ -206,7 +242,7 @@ class ProjectRPC(object): if (state['storage']['coreCaller'] and state['storage']['coreCaller'] in ProjectGenerator.get_supported_ides()): args.extend(["--ide", state['storage']['coreCaller']]) - d = PIOCoreRPC.call(args) + d = PIOCoreRPC.call(args, options={"cwd": project_dir}) d.addCallback(self._finalize_arduino_import, project_dir, arduino_project_dir) return d @@ -220,46 +256,8 @@ class ProjectRPC(object): shutil.copytree(arduino_project_dir, src_dir) return project_dir - @staticmethod - def get_project_examples(): - result = [] - for manifest in PlatformManager().get_installed(): - examples_dir = join(manifest['__pkg_dir'], "examples") - if not isdir(examples_dir): - continue - items = [] - for project_dir, _, __ in os.walk(examples_dir): - project_description = None - try: - config = ProjectConfig(join(project_dir, "platformio.ini")) - config.validate(silent=True) - project_description = config.get("platformio", - "description") - except exception.PlatformIOProjectException: - continue - - path_tokens = project_dir.split(sep) - items.append({ - "name": - "/".join(path_tokens[path_tokens.index("examples") + 1:]), - "path": - project_dir, - "description": - project_description - }) - result.append({ - "platform": { - "title": manifest['title'], - "version": manifest['version'] - }, - "items": sorted(items, key=lambda item: item['name']) - }) - return sorted(result, key=lambda data: data['platform']['title']) - @staticmethod def import_pio(project_dir): - if project_dir and PY2: - project_dir = project_dir.encode(get_filesystem_encoding()) if not project_dir or not is_platformio_project(project_dir): raise jsonrpc.exceptions.JSONRPCDispatchException( code=4001, @@ -270,10 +268,10 @@ class ProjectRPC(object): shutil.copytree(project_dir, new_project_dir) state = AppRPC.load_state() - args = ["init", "--project-dir", new_project_dir] + args = ["init"] if (state['storage']['coreCaller'] and state['storage']['coreCaller'] in ProjectGenerator.get_supported_ides()): args.extend(["--ide", state['storage']['coreCaller']]) - d = PIOCoreRPC.call(args) + d = PIOCoreRPC.call(args, options={"cwd": new_project_dir}) d.addCallback(lambda _: new_project_dir) return d From a300168658f58784468e230d962f03e1ff325702 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 9 Jul 2019 00:14:46 +0300 Subject: [PATCH 326/333] Bump version to 4.0.0rc5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index f0ff548e..afeba5d9 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0rc4") +VERSION = (4, 0, "0rc5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From d003dffc5a820410c1444f1ea77befd88500a87c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jul 2019 00:20:23 +0300 Subject: [PATCH 327/333] Fixed an issue with broken internal call for PIO Account // Resolve #2748 --- docs | 2 +- platformio/commands/home/rpc/handlers/piocore.py | 16 ++++++++++++++-- platformio/managers/core.py | 5 +++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs b/docs index 49a86db3..8a81324c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 49a86db3aa4489d12633a212bd291b041c1e8347 +Subproject commit 8a81324c3628900daf667b0fb763cefcfdc6eb67 diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 22ffb373..a88f495d 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -22,8 +22,10 @@ from io import BytesIO, StringIO import click import jsonrpc # pylint: disable=import-error from twisted.internet import threads # pylint: disable=import-error +from twisted.internet import utils # pylint: disable=import-error from platformio import __main__, __version__, util +from platformio.commands.home import helpers from platformio.compat import (PY2, get_filesystem_encoding, is_bytes, string_types) @@ -78,6 +80,7 @@ class PIOCoreRPC(object): @staticmethod def call(args, options=None): PIOCoreRPC.setup_multithreading_std_streams() + cwd = (options or {}).get("cwd") or os.getcwd() for i, arg in enumerate(args): if isinstance(arg, string_types): args[i] = arg.encode(get_filesystem_encoding()) if PY2 else arg @@ -85,12 +88,21 @@ class PIOCoreRPC(object): args[i] = str(arg) def _call_inline(): - with util.cd((options or {}).get("cwd") or os.getcwd()): + with util.cd(cwd): exit_code = __main__.main(["-c"] + args) return (PIOCoreRPC.thread_stdout.get_value_and_reset(), PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code) - d = threads.deferToThread(_call_inline) + if args and args[0] in ("account", "remote"): + d = utils.getProcessOutputAndValue( + helpers.get_core_fullpath(), + args, + path=cwd, + env={k: v + for k, v in os.environ.items() if "%" not in k}) + else: + d = threads.deferToThread(_call_inline) + d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args) d.addErrback(PIOCoreRPC._call_errback) return d diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 2a144399..f93145fb 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -139,8 +139,9 @@ def pioplus_call(args, **kwargs): os.environ['PYTHONEXEPATH'] = pythonexe_path os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite") os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "") - os.environ['PATH'] = (os.pathsep).join( - [dirname(pythonexe_path), os.environ['PATH']]) + if dirname(pythonexe_path) not in os.environ['PATH'].split(os.pathsep): + os.environ['PATH'] = (os.pathsep).join( + [dirname(pythonexe_path), os.environ['PATH']]) copy_pythonpath_to_osenv() code = subprocess.call([pioplus_path] + args, **kwargs) From fee748d38400db63de403d8279ac44b05a0834da Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jul 2019 00:20:43 +0300 Subject: [PATCH 328/333] Bump version to 4.0.0rc6 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index afeba5d9..6e4aee59 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0rc5") +VERSION = (4, 0, "0rc6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f81b1089c1871dbcf8f4cd893a5843d1f8e1f6db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jul 2019 13:00:51 +0300 Subject: [PATCH 329/333] Control a number of parallel build jobs with a new `-j, --jobs` option --- HISTORY.rst | 1 + docs | 2 +- platformio/commands/run/command.py | 17 ++++++++++-- platformio/commands/run/processor.py | 8 +++--- platformio/managers/platform.py | 39 ++++++++++++---------------- 5 files changed, 38 insertions(+), 29 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index df69c43c..07738598 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -45,6 +45,7 @@ PlatformIO 4.0 - Switch between `Build Configurations `__ (``release`` and ``debug``) with a new project configuration option `build_type `__ - Custom `platform_packages `__ per a build environment with an option to override default (`issue #1367 `_) - Print platform package details, such as version, VSC source and commit (`issue #2155 `_) + - Control a number of parallel build jobs with a new `-j, --jobs `__ option - Override default `"platformio.ini" (Project Configuration File) `__ with a custom using ``-c, --project-conf`` option for `platformio run `__, `platformio debug `__, or `platformio test `__ commands (`issue #1913 `_) - Override default development platform upload command with a custom `upload_command `__ (`issue #2599 `_) - Configure a shared folder for the derived files (objects, firmwares, ELFs) from a build system using `build_cache_dir `__ option (`issue #2674 `_) diff --git a/docs b/docs index 8a81324c..36cb5748 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8a81324c3628900daf667b0fb763cefcfdc6eb67 +Subproject commit 36cb5748e7adbbe332b1744810208e2bc9e58a84 diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index c738749e..84d19857 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from multiprocessing import cpu_count from os import getcwd from os.path import isfile, join from time import time @@ -30,6 +31,11 @@ from platformio.project.helpers import (find_project_dir_above, # pylint: disable=too-many-arguments,too-many-locals,too-many-branches +try: + DEFAULT_JOB_NUMS = cpu_count() +except NotImplementedError: + DEFAULT_JOB_NUMS = 1 + @click.command("run", short_help="Process project environments") @click.option("-e", "--environment", multiple=True) @@ -50,11 +56,18 @@ from platformio.project.helpers import (find_project_dir_above, dir_okay=False, readable=True, resolve_path=True)) +@click.option("-j", + "--jobs", + type=int, + default=DEFAULT_JOB_NUMS, + help=("Allow N jobs at once. " + "Default is a number of CPUs in a system (N=%d)" % + DEFAULT_JOB_NUMS)) @click.option("-s", "--silent", is_flag=True) @click.option("-v", "--verbose", is_flag=True) @click.option("--disable-auto-clean", is_flag=True) @click.pass_context -def cli(ctx, environment, target, upload_port, project_dir, project_conf, +def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, silent, verbose, disable_auto_clean): # find project directory on upper level if isfile(project_dir): @@ -95,7 +108,7 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, click.echo() ep = EnvironmentProcessor(ctx, envname, config, target, - upload_port, silent, verbose) + upload_port, silent, verbose, jobs) result = (envname, ep.process()) results.append(result) diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 4f495298..9ff1e056 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -32,8 +32,8 @@ class EnvironmentProcessor(object): DEFAULT_PRINT_OPTIONS = ("platform", "framework", "board") def __init__( # pylint: disable=too-many-arguments - self, cmd_ctx, name, config, targets, upload_port, silent, - verbose): + self, cmd_ctx, name, config, targets, upload_port, silent, verbose, + jobs): self.cmd_ctx = cmd_ctx self.name = name self.config = config @@ -41,6 +41,7 @@ class EnvironmentProcessor(object): self.upload_port = upload_port self.silent = silent self.verbose = verbose + self.jobs = jobs self.options = config.items(env=name, as_dict=True) def process(self): @@ -112,4 +113,5 @@ class EnvironmentProcessor(object): skip_default_package=True) p = PlatformFactory.newPlatform(self.options['platform']) - return p.run(build_vars, build_targets, self.silent, self.verbose) + return p.run(build_vars, build_targets, self.silent, self.verbose, + self.jobs) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index d0ab898a..1a45e433 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -17,7 +17,6 @@ import os import re import sys from imp import load_source -from multiprocessing import cpu_count from os.path import basename, dirname, isdir, isfile, join import click @@ -368,7 +367,8 @@ class PlatformRunMixin(object): value = base64.urlsafe_b64decode(data) return value.decode() if is_bytes(value) else value - def run(self, variables, targets, silent, verbose): + def run( # pylint: disable=too-many-arguments + self, variables, targets, silent, verbose, jobs): assert isinstance(variables, dict) assert isinstance(targets, list) @@ -393,28 +393,28 @@ class PlatformRunMixin(object): if not isfile(variables['build_script']): raise exception.BuildScriptNotFound(variables['build_script']) - result = self._run_scons(variables, targets) + result = self._run_scons(variables, targets, jobs) assert "returncode" in result return result - def _run_scons(self, variables, targets): - cmd = [ + def _run_scons(self, variables, targets, jobs): + args = [ get_pythonexe_path(), - join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q", - "-j %d" % self.get_job_nums(), "--warn=no-no-parallel-support", - "-f", - join(util.get_source_dir(), "builder", "main.py") - ] - cmd.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + join(get_core_package_dir("tool-scons"), "script", "scons"), + "-Q", "--warn=no-no-parallel-support", + "--jobs", str(jobs), + "--sconstruct", join(util.get_source_dir(), "builder", "main.py") + ] # yapf: disable + args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) # pylint: disable=protected-access - cmd.append("ISATTY=%d" % - (1 if click._compat.isatty(sys.stdout) else 0)) - cmd += targets + args.append("ISATTY=%d" % + (1 if click._compat.isatty(sys.stdout) else 0)) + args += targets # encode and append variables for key, value in variables.items(): - cmd.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) + args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) def _write_and_flush(stream, data): try: @@ -425,7 +425,7 @@ class PlatformRunMixin(object): copy_pythonpath_to_osenv() result = exec_command( - cmd, + args, stdout=BuildAsyncPipe( line_callback=self._on_stdout_line, data_callback=lambda data: _write_and_flush(sys.stdout, data)), @@ -481,13 +481,6 @@ class PlatformRunMixin(object): dots="*" * (56 + len(filename))) click.echo(banner, err=True) - @staticmethod - def get_job_nums(): - try: - return cpu_count() - except NotImplementedError: - return 1 - class PlatformBase( # pylint: disable=too-many-public-methods PlatformPackagesMixin, PlatformRunMixin): From 24d2b94e791721f76b5ae0234c42e2fe77fe7a2c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jul 2019 13:17:42 +0300 Subject: [PATCH 330/333] Drop Python 2.7 limitation from a docs // Resolve #1991 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 36cb5748..4bdf3094 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 36cb5748e7adbbe332b1744810208e2bc9e58a84 +Subproject commit 4bdf309414cc1c5191d9c3fb7a9100a7da8ba07f From e9276696324882522683e547b3f9208e7e6d4fb8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jul 2019 13:49:59 +0300 Subject: [PATCH 331/333] Document how to override board configuration --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 4bdf3094..dc041afd 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4bdf309414cc1c5191d9c3fb7a9100a7da8ba07f +Subproject commit dc041afd947bdb44609e62af04ee7853c39388ac From 6fcc76e1996326d370799dd0c9bd2b5a58df026d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jul 2019 16:07:13 +0300 Subject: [PATCH 332/333] Docs for VSCode: Proxy Server Support --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index dc041afd..e4c3ba24 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit dc041afd947bdb44609e62af04ee7853c39388ac +Subproject commit e4c3ba241e17e419a9b8b62706fe08e9a8987590 From fe210aadcf6386bafec66dec338d63f22ccc5bda Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 10 Jul 2019 16:22:37 +0300 Subject: [PATCH 333/333] Bump version to 4.0.0 --- HISTORY.rst | 6 +++--- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 07738598..efa87c51 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,15 +6,15 @@ Release Notes PlatformIO 4.0 -------------- -4.0.0 (2019-??-??) +4.0.0 (2019-07-10) ~~~~~~~~~~~~~~~~~~ `Migration Guide from 3.0 to 4.0 `__. * `PlatformIO Plus Goes Open Source `__ - - Open source `PIO Unified Debugger `__ - - Open source `PIO Unit Testing `__ + - Built-in `PIO Unified Debugger `__ + - Built-in `PIO Unit Testing `__ * **Project Configuration** diff --git a/docs b/docs index e4c3ba24..ae7deefa 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e4c3ba241e17e419a9b8b62706fe08e9a8987590 +Subproject commit ae7deefa584f8c0fbb98c36b45649cc86bdf46b7 diff --git a/platformio/__init__.py b/platformio/__init__.py index 6e4aee59..9fb531a5 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 0, "0rc6") +VERSION = (4, 0, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"