From ecaa9d90b30623cf9ba7f7ed88168b042ae7c478 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Jun 2018 17:02:49 +0300 Subject: [PATCH 01/86] Continue work on PIO Core 3.5.4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6de67c99..da19c94d 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 5, 3) +VERSION = (3, 5, "4a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 20c1ce40d387f7f39b99e277eea9a431396a677a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 1 Jun 2018 18:07:47 +0300 Subject: [PATCH 02/86] Check maximum allowed firmware size for uploading with "pio run --target checkprogsize" // Resolve #1412 --- HISTORY.rst | 6 ++++++ docs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4bcb68f6..bfb85fe7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,12 @@ Release Notes PlatformIO 3.0 -------------- +3.5.4 (2018-??-??) +~~~~~~~~~~~~~~~~~~ + +* Check maximum allowed firmware size for programming/uploading using `platformio run --target checkprogsize `__ + (`issue #1412 `_) + 3.5.3 (2018-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 3ad76be8..91eae264 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 3ad76be8f73ab1b3766bafa7ffca4284051aca4c +Subproject commit 91eae264171d7c7662ba39be5954d9b76f267373 From 2fb8128791bc268ad06fc7a7113a768d411d1de0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 2 Jun 2018 16:14:58 +0300 Subject: [PATCH 03/86] YAPF --- platformio/builder/tools/piolib.py | 11 +++++------ platformio/builder/tools/piomisc.py | 18 +++++++++--------- platformio/commands/lib.py | 4 ++-- platformio/commands/platform.py | 4 ++-- platformio/commands/run.py | 16 ++++++++-------- platformio/commands/settings.py | 4 ++-- platformio/downloader.py | 5 ++--- platformio/maintenance.py | 18 +++++++++--------- platformio/managers/lib.py | 18 +++++++++--------- platformio/managers/package.py | 26 +++++++++++++------------- platformio/managers/platform.py | 10 +++++----- 11 files changed, 66 insertions(+), 68 deletions(-) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 9048ea1c..3570e232 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -279,9 +279,8 @@ class LibBuilderBase(object): if (key in item and not util.items_in_list(self.env[env_key], item[key])): if self.verbose: - sys.stderr.write( - "Skip %s incompatible dependency %s\n" % (key[:-1], - item)) + sys.stderr.write("Skip %s incompatible dependency %s\n" + % (key[:-1], item)) skip = True if skip: continue @@ -394,9 +393,9 @@ class LibBuilderBase(object): if self != lb: if _already_depends(lb): if self.verbose: - sys.stderr.write("Warning! Circular dependencies detected " - "between `%s` and `%s`\n" % (self.path, - lb.path)) + sys.stderr.write( + "Warning! Circular dependencies detected " + "between `%s` and `%s`\n" % (self.path, lb.path)) self._circular_deps.append(lb) elif lb not in self._depbuilders: self._depbuilders.append(lb) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index b01ea1a2..87331f5d 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -30,7 +30,8 @@ from platformio.managers.core import get_core_package_dir class InoToCPPConverter(object): - PROTOTYPE_RE = re.compile(r"""^( + PROTOTYPE_RE = re.compile( + r"""^( (?:template\<.*\>\s*)? # template ([a-z_\d]+\*?\s+){1,2} # return type ([a-z_\d]+\s*) # name of prototype @@ -89,8 +90,8 @@ class InoToCPPConverter(object): self.env.Execute( self.env.VerboseAction( '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( - out_file, - tmp_path), "Converting " + basename(out_file[:-4]))) + out_file, tmp_path), + "Converting " + basename(out_file[:-4]))) atexit.register(_delete_file, tmp_path) return isfile(out_file) @@ -163,18 +164,17 @@ class InoToCPPConverter(object): prototype_names = set([m.group(3).strip() for m in prototypes]) split_pos = prototypes[0].start() - match_ptrs = re.search(self.PROTOPTRS_TPLRE % - ("|".join(prototype_names)), - contents[:split_pos], re.M) + match_ptrs = re.search( + self.PROTOPTRS_TPLRE % ("|".join(prototype_names)), + contents[:split_pos], re.M) if match_ptrs: split_pos = contents.rfind("\n", 0, match_ptrs.start()) + 1 result = [] result.append(contents[:split_pos].strip()) result.append("%s;" % ";\n".join([m.group(1) for m in prototypes])) - result.append('#line %d "%s"' % - (self._get_total_lines(contents[:split_pos]), - self._main_ino.replace("\\", "/"))) + result.append('#line %d "%s"' % (self._get_total_lines( + contents[:split_pos]), self._main_ino.replace("\\", "/"))) result.append(contents[split_pos:].strip()) return "\n".join(result) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 81400684..826cd344 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -439,8 +439,8 @@ def lib_stats(json_output): printitem_tpl.format( name=click.style(name, fg="cyan"), url=click.style( - "https://platformio.org/lib/search?query=" + - quote("keyword:%s" % name), + "https://platformio.org/lib/search?query=" + quote( + "keyword:%s" % name), fg="blue"))) for key in ("updated", "added"): diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 8ec1c259..394803a1 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -273,8 +273,8 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches if item['type']: click.echo("Type: %s" % item['type']) click.echo("Requirements: %s" % item['requirements']) - click.echo("Installed: %s" % - ("Yes" if item.get("version") else "No (optional)")) + click.echo("Installed: %s" % ("Yes" if item.get("version") else + "No (optional)")) if "version" in item: click.echo("Version: %s" % item['version']) if "originalVersion" in item: diff --git a/platformio/commands/run.py b/platformio/commands/run.py index f2d4636c..a09ee01e 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -296,10 +296,10 @@ class EnvironmentProcessor(object): if d.strip() ], self.verbose) if "lib_deps" in self.options: - _autoinstall_libdeps(self.cmd_ctx, - util.parse_conf_multi_values( - self.options['lib_deps']), - self.verbose) + _autoinstall_libdeps( + self.cmd_ctx, + util.parse_conf_multi_values(self.options['lib_deps']), + self.verbose) try: p = PlatformFactory.newPlatform(self.options['platform']) @@ -385,10 +385,10 @@ def print_summary(results, start_time): err=status is False) print_header( - "[%s] Took %.2f seconds" % - ((click.style("SUCCESS", fg="green", bold=True) - if successed else click.style("ERROR", fg="red", bold=True)), - time() - start_time), + "[%s] Took %.2f seconds" % ( + (click.style("SUCCESS", fg="green", bold=True) + if successed else click.style("ERROR", fg="red", bold=True)), + time() - start_time), is_error=not successed) diff --git a/platformio/commands/settings.py b/platformio/commands/settings.py index f163ff6d..a29d3997 100644 --- a/platformio/commands/settings.py +++ b/platformio/commands/settings.py @@ -32,8 +32,8 @@ def settings_get(name): click.echo( list_tpl.format( name=click.style("Name", fg="cyan"), - value=(click.style("Value", fg="green") + - click.style(" [Default]", fg="yellow")), + value=(click.style("Value", fg="green") + click.style( + " [Default]", fg="yellow")), description="Description")) click.echo("-" * terminal_width) diff --git a/platformio/downloader.py b/platformio/downloader.py index 34f597b2..62c19385 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -43,9 +43,8 @@ class FileDownloader(object): disposition = self._request.headers.get("content-disposition") if disposition and "filename=" in disposition: - self._fname = disposition[ - disposition.index("filename=") + 9:].replace('"', "").replace( - "'", "") + self._fname = disposition[disposition.index("filename=") + + 9:].replace('"', "").replace("'", "") self._fname = self._fname.encode("utf8") else: self._fname = [p for p in url.split("/") if p][-1] diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 35f0e173..7b1a5f58 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -208,8 +208,8 @@ def after_upgrade(ctx): # PlatformIO banner click.echo("*" * terminal_width) - click.echo("If you like %s, please:" % - (click.style("PlatformIO", fg="cyan"))) + click.echo( + "If you like %s, please:" % (click.style("PlatformIO", fg="cyan"))) click.echo("- %s us on Twitter to stay up-to-date " "on the latest project news > %s" % (click.style("follow", fg="cyan"), @@ -224,9 +224,9 @@ def after_upgrade(ctx): (click.style("try", fg="cyan"), click.style("https://platformio.org/platformio-ide", fg="cyan"))) if not util.is_ci(): - click.echo("- %s us with PlatformIO Plus > %s" % - (click.style("support", fg="cyan"), - click.style("https://pioplus.com", fg="cyan"))) + click.echo("- %s us with PlatformIO Plus > %s" % (click.style( + "support", fg="cyan"), click.style( + "https://pioplus.com", fg="cyan"))) click.echo("*" * terminal_width) click.echo("") @@ -296,8 +296,8 @@ def check_internal_updates(ctx, what): if manifest['name'] in outdated_items: continue conds = [ - pm.outdated(manifest['__pkg_dir']), - what == "platforms" and PlatformFactory.newPlatform( + pm.outdated(manifest['__pkg_dir']), what == "platforms" + and PlatformFactory.newPlatform( manifest['__pkg_dir']).are_outdated_packages() ] if any(conds): @@ -318,8 +318,8 @@ def check_internal_updates(ctx, what): if not app.get_setting("auto_update_" + what): click.secho("Please update them via ", fg="yellow", nl=False) click.secho( - "`platformio %s update`" % - ("lib --global" if what == "libraries" else "platform"), + "`platformio %s update`" % ("lib --global" if what == "libraries" + else "platform"), fg="cyan", nl=False) click.secho(" command.\n", fg="yellow") diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index 5613b668..ba81870a 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -208,9 +208,9 @@ class LibraryManager(BasePkgManager): cache_valid="30d") assert dl_data - return self._install_from_url(name, dl_data['url'].replace( - "http://", "https://") if app.get_setting("enable_ssl") else - dl_data['url'], requirements) + return self._install_from_url( + name, dl_data['url'].replace("http://", "https://") + if app.get_setting("enable_ssl") else dl_data['url'], requirements) def search_lib_id( # pylint: disable=too-many-branches self, @@ -237,9 +237,9 @@ class LibraryManager(BasePkgManager): if not isinstance(values, list): values = [v.strip() for v in values.split(",") if v] for value in values: - query.append('%s:"%s"' % (key[:-1] - if key.endswith("s") else key, - value)) + query.append( + '%s:"%s"' % (key[:-1] + if key.endswith("s") else key, value)) lib_info = None result = util.get_api_result( @@ -290,9 +290,9 @@ class LibraryManager(BasePkgManager): def _get_lib_id_from_installed(self, filters): if filters['name'].startswith("id="): return int(filters['name'][3:]) - package_dir = self.get_package_dir(filters['name'], - filters.get("requirements", - filters.get("version"))) + package_dir = self.get_package_dir( + filters['name'], filters.get("requirements", + filters.get("version"))) if not package_dir: return None manifest = self.load_manifest(package_dir) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 3996bc06..58331980 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -500,8 +500,8 @@ class PkgInstallerMixin(object): cur_manifest['version']) if "__src_url" in cur_manifest: target_dirname = "%s@src-%s" % ( - pkg_dirname, - hashlib.md5(cur_manifest['__src_url']).hexdigest()) + pkg_dirname, hashlib.md5( + cur_manifest['__src_url']).hexdigest()) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: @@ -509,8 +509,8 @@ class PkgInstallerMixin(object): tmp_manifest['version']) if "__src_url" in tmp_manifest: target_dirname = "%s@src-%s" % ( - pkg_dirname, - hashlib.md5(tmp_manifest['__src_url']).hexdigest()) + pkg_dirname, hashlib.md5( + tmp_manifest['__src_url']).hexdigest()) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package @@ -722,13 +722,13 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): pkg_dir = self.get_package_dir(name, requirements, url) if not pkg_dir: - raise exception.UnknownPackage("%s @ %s" % (package, - requirements or "*")) + raise exception.UnknownPackage( + "%s @ %s" % (package, requirements or "*")) manifest = self.load_manifest(pkg_dir) click.echo( - "Uninstalling %s @ %s: \t" % - (click.style(manifest['name'], fg="cyan"), manifest['version']), + "Uninstalling %s @ %s: \t" % (click.style( + manifest['name'], fg="cyan"), manifest['version']), nl=False) if islink(pkg_dir): @@ -740,9 +740,9 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): # unfix package with the same name pkg_dir = self.get_package_dir(manifest['name']) if pkg_dir and "@" in pkg_dir: - shutil.move(pkg_dir, - join(self.package_dir, - self.get_install_dirname(manifest))) + shutil.move( + pkg_dir, + join(self.package_dir, self.get_install_dirname(manifest))) self.cache_reset() click.echo("[%s]" % click.style("OK", fg="green")) @@ -761,8 +761,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): pkg_dir = self.get_package_dir(*self.parse_pkg_uri(package)) if not pkg_dir: - raise exception.UnknownPackage("%s @ %s" % (package, - requirements or "*")) + raise exception.UnknownPackage( + "%s @ %s" % (package, requirements or "*")) manifest = self.load_manifest(pkg_dir) name = manifest['name'] diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 03010fdc..5a947c4c 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -39,9 +39,9 @@ class PlatformManager(BasePkgManager): "{0}://dl.platformio.org/platforms/manifest.json".format( "https" if app.get_setting("enable_ssl") else "http") ] - BasePkgManager.__init__(self, package_dir - or join(util.get_home_dir(), "platforms"), - repositories) + BasePkgManager.__init__( + self, package_dir or join(util.get_home_dir(), "platforms"), + repositories) @property def manifest_names(self): @@ -331,8 +331,8 @@ class PlatformPackagesMixin(object): def get_package_dir(self, name): version = self.packages[name].get("version", "") if ":" in version: - return self.pm.get_package_dir(*self.pm.parse_pkg_uri( - "%s=%s" % (name, version))) + return self.pm.get_package_dir( + *self.pm.parse_pkg_uri("%s=%s" % (name, version))) return self.pm.get_package_dir(name, version) def get_package_version(self, name): From bfc94d36e36bc9e2164734cf3894241db1a4831e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Jun 2018 14:09:48 +0300 Subject: [PATCH 04/86] Introduce "Program Memory Usage" --- HISTORY.rst | 11 ++- docs | 2 +- platformio/builder/main.py | 31 +++++-- platformio/builder/tools/pioupload.py | 108 +++++++++++++++++++------ platformio/builder/tools/platformio.py | 11 +-- 5 files changed, 121 insertions(+), 42 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bfb85fe7..5a2a955f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,11 +4,16 @@ Release Notes PlatformIO 3.0 -------------- -3.5.4 (2018-??-??) +3.6.0 (2018-??-??) ~~~~~~~~~~~~~~~~~~ -* Check maximum allowed firmware size for programming/uploading using `platformio run --target checkprogsize `__ - (`issue #1412 `_) +* `Program Memory Usage `_ + + - Print human-readable memory usage information after a build and before uploading + - Print detailed memory usage information with "sections" and "addresses" + in `verbose mode `__ + - Check maximum allowed "program" and "data" sizes before uploading/programming + (`issue #1412 `_) 3.5.3 (2018-06-01) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 91eae264..22c4d010 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 91eae264171d7c7662ba39be5954d9b76f267373 +Subproject commit 22c4d0109e6f2386153523e5de29e1a77c7ffce6 diff --git a/platformio/builder/main.py b/platformio/builder/main.py index caafda5b..150d848f 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -20,7 +20,7 @@ from os.path import expanduser, join from time import time from SCons.Script import (ARGUMENTS, COMMAND_LINE_TARGETS, DEFAULT_TARGETS, - Action, AllowSubstExceptions, AlwaysBuild, + AllowSubstExceptions, AlwaysBuild, Default, DefaultEnvironment, Variables) from platformio import util @@ -164,15 +164,34 @@ for item in env.GetExtraScripts("pre"): env.SConscript("$BUILD_SCRIPT") -AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS + ["size"])) -AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS + ["size"])) - if "UPLOAD_FLAGS" in env: env.Prepend(UPLOADERFLAGS=["$UPLOAD_FLAGS"]) for item in env.GetExtraScripts("post"): env.SConscript(item, exports="env") +############################################################################## + +# Checking program size +if env.get("SIZETOOL") and "nobuild" not in COMMAND_LINE_TARGETS: + env.Depends(["upload", "program"], "checkprogsize") + # Replace platform's "size" target with our + _new_targets = [t for t in DEFAULT_TARGETS if str(t) != "size"] + Default(None) + Default(_new_targets) + Default("checkprogsize") + +# Print configured protocols +env.AddPreAction( + ["upload", "program"], + env.VerboseAction(lambda source, target, env: env.PrintUploadInfo(), + "Configuring upload protocol...")) + +AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS)) +AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS)) + +############################################################################## + if "envdump" in COMMAND_LINE_TARGETS: print env.Dump() env.Exit(0) @@ -189,7 +208,3 @@ if "idedata" in COMMAND_LINE_TARGETS: "See explanation in FAQ > Troubleshooting > Building\n" "http://docs.platformio.org/page/faq.html\n\n") env.Exit(1) - -env.AddPreAction(["upload", "program"], - Action(lambda source, target, env: env.PrintUploadInfo(), - "Configuring upload protocol...")) diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 1ba523e4..88390509 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -14,6 +14,7 @@ from __future__ import absolute_import +import re import sys from fnmatch import fnmatch from os import environ @@ -21,11 +22,13 @@ from os.path import isfile, join from shutil import copyfile from time import sleep -from SCons.Node.Alias import Alias +from SCons.Script import ARGUMENTS from serial import Serial, SerialException from platformio import util +# pylint: disable=unused-argument + def FlushSerialBuffer(env, port): s = Serial(env.subst(port)) @@ -45,7 +48,7 @@ def TouchSerialPort(env, port, baudrate): s = Serial(port=port, baudrate=baudrate) s.setDTR(False) s.close() - except: # pylint: disable=W0702 + except: # pylint: disable=bare-except pass sleep(0.4) # DO NOT REMOVE THAT (required by SAM-BA based boards) @@ -88,7 +91,7 @@ def WaitForNewSerialPort(env, before): return new_port -def AutodetectUploadPort(*args, **kwargs): # pylint: disable=unused-argument +def AutodetectUploadPort(*args, **kwargs): env = args[0] def _get_pattern(): @@ -173,7 +176,7 @@ def AutodetectUploadPort(*args, **kwargs): # pylint: disable=unused-argument env.Exit(1) -def UploadToDisk(_, target, source, env): # pylint: disable=W0613,W0621 +def UploadToDisk(_, target, source, env): assert "UPLOAD_PORT" in env progname = env.subst("$PROGNAME") for ext in ("bin", "hex"): @@ -186,32 +189,87 @@ def UploadToDisk(_, target, source, env): # pylint: disable=W0613,W0621 "(Some boards may require manual hard reset)" -def CheckUploadSize(_, target, source, env): # pylint: disable=W0613,W0621 - if "BOARD" not in env: - return - max_size = int(env.BoardConfig().get("upload.maximum_size", 0)) - if max_size == 0 or "SIZETOOL" not in env: - return - - sysenv = environ.copy() - sysenv['PATH'] = str(env['ENV']['PATH']) - cmd = [ - env.subst("$SIZETOOL"), "-B", - str(source[0] if isinstance(target[0], Alias) else target[0]) +def CheckUploadSize(_, target, source, env): + check_conditions = [ + env.get("BOARD"), + env.get("SIZETOOL") or env.get("SIZECHECKCMD") ] - result = util.exec_command(cmd, env=sysenv) - if result['returncode'] != 0: + if not all(check_conditions): + return + program_max_size = int(env.BoardConfig().get("upload.maximum_size", 0)) + data_max_size = int(env.BoardConfig().get("upload.maximum_ram_size", 0)) + if program_max_size == 0: return - print result['out'].strip() - line = result['out'].strip().splitlines()[1] - values = [v.strip() for v in line.split("\t")] - used_size = int(values[0]) + int(values[1]) + def _configure_defaults(): + env.Replace( + SIZECHECKCMD="$SIZETOOL -B -d $SOURCES", + SIZEPROGREGEXP=r"^(\d+)\s+(\d+)\s+\d+\s", + SIZEDATAREGEXP=r"^\d+\s+(\d+)\s+(\d+)\s+\d+") - if used_size > max_size: + def _get_size_output(): + cmd = env.get("SIZECHECKCMD") + if not cmd: + return None + if not isinstance(cmd, list): + cmd = cmd.split() + cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg] + sysenv = environ.copy() + sysenv['PATH'] = str(env['ENV']['PATH']) + result = util.exec_command(env.subst(cmd), env=sysenv) + if result['returncode'] != 0: + return None + return result['out'].strip() + + def _calculate_size(output, pattern): + if not output or not pattern: + return -1 + size = 0 + regexp = re.compile(pattern) + for line in output.split("\n"): + line = line.strip() + if not line: + continue + match = regexp.search(line) + if not match: + continue + size += sum(int(value) for value in match.groups()) + return size + + def _format_availale_bytes(value, total): + percent_raw = float(value) / float(total) + blocks_per_progress = 10 + used_blocks = int(round(blocks_per_progress * percent_raw)) + if used_blocks > blocks_per_progress: + used_blocks = blocks_per_progress + return "[{:{}}] {: 6.1%} (used {:d} bytes from {:d} bytes)".format( + "=" * used_blocks, blocks_per_progress, percent_raw, value, total) + + if not env.get("SIZECHECKCMD") and not env.get("SIZEPROGREGEXP"): + _configure_defaults() + output = _get_size_output() + program_size = _calculate_size(output, env.get("SIZEPROGREGEXP")) + data_size = _calculate_size(output, env.get("SIZEDATAREGEXP")) + + print "Memory Usage -> http://bit.ly/pio-memory-usage" + if data_max_size and data_size > -1: + print "DATA: %s" % _format_availale_bytes(data_size, data_max_size) + if program_size > -1: + print "PROGRAM: %s" % _format_availale_bytes(program_size, + program_max_size) + if int(ARGUMENTS.get("PIOVERBOSE", 0)): + print output + + # raise error + if data_max_size and data_size > data_max_size: + sys.stderr.write( + "Error: The data size (%d bytes) is greater " + "than maximum allowed (%s bytes)\n" % (data_size, data_max_size)) + env.Exit(1) + if program_size > program_max_size: sys.stderr.write("Error: The program size (%d bytes) is greater " - "than maximum allowed (%s bytes)\n" % (used_size, - max_size)) + "than maximum allowed (%s bytes)\n" % + (program_size, program_max_size)) env.Exit(1) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 9c6f6d0d..491d7418 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -107,12 +107,13 @@ def BuildProgram(env): program = env.Program( join("$BUILD_DIR", env.subst("$PROGNAME")), env['PIOBUILDFILES']) + env.Replace(PIOMAINPROG=program) - checksize_action = env.VerboseAction(env.CheckUploadSize, - "Checking program size") - AlwaysBuild(env.Alias("checkprogsize", program, checksize_action)) - if set(["upload", "program"]) & set(COMMAND_LINE_TARGETS): - env.AddPostAction(program, checksize_action) + AlwaysBuild( + env.Alias( + "checkprogsize", program, + env.VerboseAction(env.CheckUploadSize, + "Checking size $PIOMAINPROG"))) return program From 4528ca036538ee54e6923804a9cc5ea0d2f6b7cd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 4 Jun 2018 16:38:53 +0300 Subject: [PATCH 05/86] Bump version to 3.6.0a1 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index da19c94d..461ae7b0 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 5, "4a1") +VERSION = (3, 6, "0a1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 2ff88837ec9549dcb14c4202bd39367bd2ff776e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 8 Jun 2018 18:37:05 +0300 Subject: [PATCH 06/86] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 22c4d010..4941b955 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 22c4d0109e6f2386153523e5de29e1a77c7ffce6 +Subproject commit 4941b95550efc76fb3ed6e88aa5a66e7e2d80a44 From 2ba7c47603c62cc0bf883f0e780e9265f98e9bc3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 10 Jun 2018 01:56:48 +0300 Subject: [PATCH 07/86] Temporary disable raising error when program data size excesses declared in board manfiest --- platformio/builder/tools/pioupload.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 88390509..8dbceaf7 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -261,11 +261,11 @@ def CheckUploadSize(_, target, source, env): print output # raise error - if data_max_size and data_size > data_max_size: - sys.stderr.write( - "Error: The data size (%d bytes) is greater " - "than maximum allowed (%s bytes)\n" % (data_size, data_max_size)) - env.Exit(1) + # if data_max_size and data_size > data_max_size: + # sys.stderr.write( + # "Error: The data size (%d bytes) is greater " + # "than maximum allowed (%s bytes)\n" % (data_size, data_max_size)) + # env.Exit(1) if program_size > program_max_size: sys.stderr.write("Error: The program size (%d bytes) is greater " "than maximum allowed (%s bytes)\n" % From 0f9c213796cde959e3aad56d0c76af113e1dad5a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 11 Jun 2018 22:55:28 +0300 Subject: [PATCH 08/86] Bump version to 3.6.0a2 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 461ae7b0..7168f826 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a1") +VERSION = (3, 6, "0a2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 22b5e4e5c01b93a12dfb266a53e465f7b1856248 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 15 Jun 2018 17:53:04 +0300 Subject: [PATCH 09/86] Remove examples with cart range for SemVer (issue with dependent Python package) --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 6d29169d..61dd68d5 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 6d29169d6e29c48561b58fa1505cc70d259e00e6 +Subproject commit 61dd68d585bfe3f15b4ebbdf1a91fae8f2f13d2d From a3cda59d702db6ba61019baf6cba7e9db271d184 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 16 Jun 2018 18:41:46 +0300 Subject: [PATCH 10/86] Refactor docs menu --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 61dd68d5..db967e2b 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 61dd68d585bfe3f15b4ebbdf1a91fae8f2f13d2d +Subproject commit db967e2b10cb52a338ff88b29d771c8a8704d5f2 From 4d48c365f5b73f9b0f7d72879d1cacf0864a5bed Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 16 Jun 2018 21:11:31 +0300 Subject: [PATCH 11/86] Autogenerate examples for platforms and frameworks --- docs | 2 +- examples | 2 +- scripts/docspregen.py | 104 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 88 insertions(+), 20 deletions(-) diff --git a/docs b/docs index db967e2b..0953dbb6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit db967e2b10cb52a338ff88b29d771c8a8704d5f2 +Subproject commit 0953dbb686a707dccd13f4c106796a619acda4ec diff --git a/examples b/examples index 41f3396c..ebff9ae7 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 41f3396c5883d54e6cc5603d380400ca46df8876 +Subproject commit ebff9ae73fa582c5edb7b47f628891b637b29d94 diff --git a/scripts/docspregen.py b/scripts/docspregen.py index cd56034c..444e7d04 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -396,10 +396,8 @@ def update_platform_docs(): rst_path = join(platforms_dir, "%s.rst" % name) with open(rst_path, "w") as f: f.write( - generate_platform(name, - isfile( - join(platforms_dir, - "%s_extra.rst" % name)))) + generate_platform( + name, isfile(join(platforms_dir, "%s_extra.rst" % name)))) def generate_framework(type_, data, has_extra=False): @@ -518,10 +516,9 @@ def update_framework_docs(): rst_path = join(frameworks_dir, "%s.rst" % name) with open(rst_path, "w") as f: f.write( - generate_framework(name, framework, - isfile( - join(frameworks_dir, - "%s_extra.rst" % name)))) + generate_framework( + name, framework, + isfile(join(frameworks_dir, "%s_extra.rst" % name)))) def update_create_platform_doc(): @@ -691,19 +688,78 @@ Boards "\n".join(lines)) -def update_examples_readme(): - examples_dir = join(util.get_source_dir(), "..", "examples") +def update_project_examples(): + platform_readme_tpl = """ +# {title}: development platform for [PlatformIO](https://platformio.org) - # Platforms +{description} + +* [Home](https://platformio.org/platforms/{name}) (home page in PlatformIO Registry) +* [Documentation](http://docs.platformio.org/page/platforms/{name}.html) (advanced usage, packages, boards, frameworks, etc.) + +# Examples + +{examples} +""" + framework_readme_tpl = """ +# {title}: framework for [PlatformIO](https://platformio.org) + +{description} + +* [Home](https://platformio.org/frameworks/{name}) (home page in PlatformIO Registry) +* [Documentation](http://docs.platformio.org/page/frameworks/{name}.html) + +# Examples + +{examples} +""" + + project_examples_dir = join(util.get_source_dir(), "..", "examples") + framework_examples_md_lines = {} embedded = [] desktop = [] + for manifest in PLATFORM_MANIFESTS: p = PlatformFactory.newPlatform(manifest['name']) - url = campaign_url( - "http://docs.platformio.org/en/latest/platforms/%s.html#examples" % - p.name, - source="github", - medium="examples") + github_platform_url = ( + "https://github.com/platformio/platform-%s" % p.name) + + # Platform README + platform_examples_dir = join(p.get_dir(), "examples") + examples_md_lines = [] + if isdir(platform_examples_dir): + for item in os.listdir(platform_examples_dir): + if not isdir(join(platform_examples_dir, item)): + continue + url = ("%s/tree/develop/examples/%s" % (github_platform_url, + item)) + examples_md_lines.append("* [%s](%s)" % (item, url)) + + readme_dir = join(project_examples_dir, "platforms", p.name) + if not isdir(readme_dir): + os.makedirs(readme_dir) + with open(join(readme_dir, "README.md"), "w") as fp: + fp.write( + platform_readme_tpl.format( + name=p.name, + title=p.title, + description=p.description, + examples="\n".join(examples_md_lines))) + + # Framework README + for framework in API_FRAMEWORKS: + if not is_compat_platform_and_framework(p.name, framework['name']): + continue + if framework['name'] not in framework_examples_md_lines: + framework_examples_md_lines[framework['name']] = [] + lines = [] + lines.append("- [%s](%s)" % (p.title, github_platform_url)) + lines.extend(" %s" % l for l in examples_md_lines) + lines.append("") + framework_examples_md_lines[framework['name']].extend(lines) + + # Root README + url = "https://github.com/platformio/platform-%s/tree/develop/examples" % p.name line = "* [%s](%s)" % (p.title, url) if p.is_embedded(): embedded.append(line) @@ -713,6 +769,18 @@ def update_examples_readme(): # Frameworks frameworks = [] for framework in API_FRAMEWORKS: + readme_dir = join(project_examples_dir, "frameworks", + framework['name']) + if not isdir(readme_dir): + os.makedirs(readme_dir) + with open(join(readme_dir, "README.md"), "w") as fp: + fp.write( + framework_readme_tpl.format( + name=framework['name'], + title=framework['title'], + description=framework['description'], + examples="\n".join( + framework_examples_md_lines[framework['name']]))) url = campaign_url( "http://docs.platformio.org/en/latest/frameworks/%s.html#examples" % framework['name'], @@ -720,7 +788,7 @@ def update_examples_readme(): medium="examples") frameworks.append("* [%s](%s)" % (framework['title'], url)) - with open(join(examples_dir, "README.md"), "w") as fp: + with open(join(project_examples_dir, "README.md"), "w") as fp: fp.write("""# PlatformIO Project Examples - [Development platforms](#development-platforms): @@ -750,7 +818,7 @@ def main(): update_framework_docs() update_embedded_boards() update_debugging() - update_examples_readme() + update_project_examples() if __name__ == "__main__": From 3c6f57ac5ca7b2f68a43af56c455862e50214778 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jun 2018 15:48:14 +0300 Subject: [PATCH 12/86] Custom tasks for VScode --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 0953dbb6..704add87 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0953dbb686a707dccd13f4c106796a619acda4ec +Subproject commit 704add877dc5119e4f264c8306aa7d541331a3f2 From ee69c13b2dd8f81d6a25cff12a20f64e04389f99 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jun 2018 16:31:03 +0300 Subject: [PATCH 13/86] Handle ConfigParser erros --- platformio/exception.py | 5 +++++ platformio/util.py | 19 +++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/platformio/exception.py b/platformio/exception.py index 8b3df0f2..ec0737de 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -192,6 +192,11 @@ class InvalidLibConfURL(PlatformioException): MESSAGE = "Invalid library config URL '{0}'" +class InvalidProjectConf(PlatformioException): + + MESSAGE = "Invalid `platformio.ini`, project configuration file: '{0}'" + + class BuildScriptNotFound(PlatformioException): MESSAGE = "Invalid path '{0}' to build script" diff --git a/platformio/util.py b/platformio/util.py index e37660e2..98c3cbc1 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -36,23 +36,27 @@ from platformio import __apiurl__, __version__, exception # pylint: disable=wrong-import-order, too-many-ancestors try: - from configparser import ConfigParser + import configparser as ConfigParser except ImportError: - from ConfigParser import ConfigParser + import ConfigParser as ConfigParser -class ProjectConfig(ConfigParser): +class ProjectConfig(ConfigParser.ConfigParser): VARTPL_RE = re.compile(r"\$\{([^\.\}]+)\.([^\}]+)\}") def items(self, section, **_): # pylint: disable=arguments-differ items = [] - for option in ConfigParser.options(self, section): + for option in ConfigParser.ConfigParser.options(self, section): items.append((option, self.get(section, option))) return items def get(self, section, option, **kwargs): - value = ConfigParser.get(self, section, option, **kwargs) + try: + value = ConfigParser.ConfigParser.get(self, section, option, + **kwargs) + except ConfigParser.Error as e: + raise exception.InvalidProjectConf(str(e)) if "${" not in value or "}" not in value: return value return self.VARTPL_RE.sub(self._re_sub_handler, value) @@ -331,7 +335,10 @@ def load_project_config(path=None): raise exception.NotPlatformIOProject( dirname(path) if path.endswith("platformio.ini") else path) cp = ProjectConfig() - cp.read(path) + try: + cp.read(path) + except ConfigParser.Error as e: + raise exception.InvalidProjectConf(str(e)) return cp From 7cac351d2574f038c157f6ec91615593128ddd96 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 20 Jun 2018 16:33:13 +0300 Subject: [PATCH 14/86] Bump version to 3.6.0a3 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 7168f826..03e078f8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a2") +VERSION = (3, 6, "0a3") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 08759700b6b389756c654355cf6eaf1c24fc66e0 Mon Sep 17 00:00:00 2001 From: Vladimir Dronnikov Date: Thu, 21 Jun 2018 13:45:43 +0300 Subject: [PATCH 15/86] Add udev rule for Maple Leaf board (#1699) --- scripts/99-platformio-udev.rules | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 6daaf922..ecc1c368 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -53,6 +53,9 @@ KERNEL=="ttyACM*", ATTRS{idVendor}=="16d0", ATTRS{idProduct}=="0753", MODE:="066 # STM32 discovery boards, with onboard st/linkv2 SUBSYSTEMS=="usb", ATTRS{idVendor}=="0483", ATTRS{idProduct}=="374?", MODE:="0666" +# Maple with DFU +SUBSYSTEMS=="usb", ATTRS{idVendor}=="1eaf", ATTRS{idProduct}=="000[34]", MODE:="0666" + # USBtiny SUBSYSTEMS=="usb", ATTRS{idProduct}=="0c9f", ATTRS{idVendor}=="1781", MODE="0666" From 89d403879ebeb1440728b5ade9bb5edfe284b4f7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 23 Jun 2018 19:32:58 +0300 Subject: [PATCH 16/86] Docs: Reorder TOP menu --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 704add87..29e05c1a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 704add877dc5119e4f264c8306aa7d541331a3f2 +Subproject commit 29e05c1a3c63847fb33f452d6dc3f7b6145a2011 From ab8497e7ce9735851c56f14cb168791ea9a6dc67 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Jun 2018 15:47:01 +0300 Subject: [PATCH 17/86] [VIDEO OVERVIEW] ThingForward - Intro to PIO Unified Debugger using ARM mbed OS and PlatformIO IDE for VSCode --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 29e05c1a..b5f518b1 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 29e05c1a3c63847fb33f452d6dc3f7b6145a2011 +Subproject commit b5f518b16804ca93de04d2f59f766b44d3001bab From 9cff2d3206ec6afcabf6d66d2d6d13272eb98e37 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 25 Jun 2018 15:52:57 +0300 Subject: [PATCH 18/86] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b5f518b1..83273171 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b5f518b16804ca93de04d2f59f766b44d3001bab +Subproject commit 832731712f6496fb136262c4f5e8386324442e3d From 0ba9b341cd1de7cb535c296a060f90068cdd48c1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 26 Jun 2018 01:01:37 +0300 Subject: [PATCH 19/86] Add new Olimex debug tools for ESP32: ARM-USB-OCD and ARM-USB-TINY --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 83273171..b5fcd517 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 832731712f6496fb136262c4f5e8386324442e3d +Subproject commit b5fcd517cccbcca2bdfa6d2b3994ff8ba1a6ad49 From 7564e00fc44235d36b10b967a4559527562915c3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 29 Jun 2018 17:01:13 +0300 Subject: [PATCH 20/86] Introduce Premium Support --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index b5fcd517..74ce3693 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit b5fcd517cccbcca2bdfa6d2b3994ff8ba1a6ad49 +Subproject commit 74ce36934de3930995950c93715f85877ccec56d From 9bd033e2887e2ddcde8619fe2bd4105e7841ecdc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 29 Jun 2018 19:29:32 +0300 Subject: [PATCH 21/86] Update ESP32 boards flash size // Resolve #30 --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 74ce3693..7b0030fc 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 74ce36934de3930995950c93715f85877ccec56d +Subproject commit 7b0030fc0471f86c7e6e25e789bd44c9aac8eef5 From 3e0feeabb41f9e1403e198a1b6e2744da6737099 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 30 Jun 2018 19:48:30 +0300 Subject: [PATCH 22/86] Bump version to 3.6.0a4 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 03e078f8..c518caa6 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a3") +VERSION = (3, 6, "0a4") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 3956dae01e91d96c732010d59b68a36227e9fe3e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 2 Jul 2018 18:02:00 +0300 Subject: [PATCH 23/86] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index ae2968e5..bcc2f38e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ae2968e52ac2bc86ec7e7fe67073ceb37c47f577 +Subproject commit bcc2f38e1dc573f7ed446c521cb3cfd1127d1ae5 From a58535d95c445023ebe5945484d0566d4eb6ceb6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 3 Jul 2018 15:15:44 +0300 Subject: [PATCH 24/86] Bump version to 3.6.0a5 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index c518caa6..5df10f16 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a4") +VERSION = (3, 6, "0a5") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 8f88939aa03be1aff25d16e8d72bc9004e78e985 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 4 Jul 2018 00:06:44 +0300 Subject: [PATCH 25/86] Shorten a name for VSCode debug launch configurations --- docs | 2 +- examples | 2 +- platformio/ide/tpls/vscode/.vscode/launch.json.tpl | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs b/docs index 0b8ac5fb..a23b54ff 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0b8ac5fbf71416aaa9420f0cf10027efb0d3fe9e +Subproject commit a23b54ff4e52bec425969c3a338c5c8de8b1193d diff --git a/examples b/examples index 41f3396c..ebff9ae7 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 41f3396c5883d54e6cc5603d380400ca46df8876 +Subproject commit ebff9ae73fa582c5edb7b47f628891b637b29d94 diff --git a/platformio/ide/tpls/vscode/.vscode/launch.json.tpl b/platformio/ide/tpls/vscode/.vscode/launch.json.tpl index 541e2485..8dbc503b 100644 --- a/platformio/ide/tpls/vscode/.vscode/launch.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/launch.json.tpl @@ -17,7 +17,7 @@ { "type": "platformio-debug", "request": "launch", - "name": "PlatformIO Debugger", + "name": "PIO Debug", "executable": "{{ _escape_path(prog_path) }}", "toolchainBinDir": "{{ _escape_path(dirname(gdb_path)) }}", % if svd_path: @@ -29,7 +29,7 @@ { "type": "platformio-debug", "request": "launch", - "name": "PlatformIO Debugger (Skip Pre-Debug)", + "name": "PIO Debug (Skip Pre-Debug)", "executable": "{{ _escape_path(prog_path) }}", "toolchainBinDir": "{{ _escape_path(dirname(gdb_path)) }}", % if svd_path: From 5c48233259a82f819040b44186108038e8ad2427 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 5 Jul 2018 17:49:27 +0300 Subject: [PATCH 26/86] Docs: Install PIO Core into Virtual Environment --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index a23b54ff..5f66a709 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a23b54ff4e52bec425969c3a338c5c8de8b1193d +Subproject commit 5f66a70960f7ef83747f3bd2573c65bad83325a4 From d8329a68684a0b1f2798f8df60ceefa9ffe782b9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 5 Jul 2018 18:53:47 +0300 Subject: [PATCH 27/86] =?UTF-8?q?ThingForward:=20First=20steps=20with=20Pl?= =?UTF-8?q?atformIO=E2=80=99s=20Unified=20Debugger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 5f66a709..1f6299c4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5f66a70960f7ef83747f3bd2573c65bad83325a4 +Subproject commit 1f6299c446c0b365fb30cb1f7503a70a8d5c5efe From fd88a249b4bb906d90096e2d57619be172f8f06f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 6 Jul 2018 15:23:23 +0300 Subject: [PATCH 28/86] Update core dependencies --- docs | 2 +- platformio/managers/core.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs b/docs index 1f6299c4..9d8e84d5 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1f6299c446c0b365fb30cb1f7503a70a8d5c5efe +Subproject commit 9d8e84d527aa83f19ce402e0c55964a1655535eb diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 9bead905..0dee74fc 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,9 +21,9 @@ from platformio import __version__, exception, util from platformio.managers.package import PackageManager CORE_PACKAGES = { - "contrib-piohome": ">=0.9.5,<2", - "contrib-pysite": ">=0.2.0,<2", - "tool-pioplus": ">=1.3.1,<2", + "contrib-piohome": ">=0.9.6,<2", + "contrib-pysite": ">=0.3.0,<2", + "tool-pioplus": ">=1.3.6,<2", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.4" } From 1c8ac970734f0c93eaaf953f8b27c343ee2ce766 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 6 Jul 2018 15:25:26 +0300 Subject: [PATCH 29/86] Update core dependencies --- docs | 2 +- platformio/managers/core.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs b/docs index a23b54ff..9d8e84d5 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a23b54ff4e52bec425969c3a338c5c8de8b1193d +Subproject commit 9d8e84d527aa83f19ce402e0c55964a1655535eb diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 9bead905..0dee74fc 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,9 +21,9 @@ from platformio import __version__, exception, util from platformio.managers.package import PackageManager CORE_PACKAGES = { - "contrib-piohome": ">=0.9.5,<2", - "contrib-pysite": ">=0.2.0,<2", - "tool-pioplus": ">=1.3.1,<2", + "contrib-piohome": ">=0.9.6,<2", + "contrib-pysite": ">=0.3.0,<2", + "tool-pioplus": ">=1.3.6,<2", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.4" } From c1f4b729eac0a3c098460f3a9a22b38074176516 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 6 Jul 2018 15:25:51 +0300 Subject: [PATCH 30/86] Export extra flash images for IDE --- platformio/builder/tools/pioide.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index 89b49e3d..ec1134b2 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -152,6 +152,10 @@ def DumpIDEData(env): util.where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), "prog_path": env.subst("$PROG_PATH"), + "flash_extra_images": [{ + "offset": item[0], + "path": env.subst(item[1]) + } for item in env.get("FLASH_EXTRA_IMAGES", [])], "svd_path": _get_svd_path(env), "compiler_type": From 7c5c5b5f70ebfad13670050f11a8de41f91a22a7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 6 Jul 2018 15:26:45 +0300 Subject: [PATCH 31/86] Bump version to 3.6.0a6 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 5df10f16..10a5e6d8 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a5") +VERSION = (3, 6, "0a6") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From ca37190da400b7f1f0b8b4c8e585211908b5873f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 7 Jul 2018 01:35:52 +0300 Subject: [PATCH 32/86] Update docs for PIO Unified Debugger; add wiring connections and debug probes pictures --- docs | 2 +- examples | 2 +- scripts/docspregen.py | 44 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/docs b/docs index 9d8e84d5..4600f5e4 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9d8e84d527aa83f19ce402e0c55964a1655535eb +Subproject commit 4600f5e495e655eb5171ec2eeb132b4cbfd6fd75 diff --git a/examples b/examples index ebff9ae7..cab374e8 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit ebff9ae73fa582c5edb7b47f628891b637b29d94 +Subproject commit cab374e8c972eeb38ac8d5ad93bdcf3b9d8965cd diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 444e7d04..90eb5872 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -141,22 +141,22 @@ You can switch between debugging :ref:`debugging_tools` using """) if onboard_debug: lines.append(""" -On-Board tools -~~~~~~~~~~~~~~ +On-Board Debug Tools +~~~~~~~~~~~~~~~~~~~~~ -Boards listed below have on-board debugging tools and **ARE READY** for debugging! -You do not need to use/buy external debugger. +Boards listed below have on-board debug tool and **ARE READY** for debugging! +You do not need to use/buy external debug tool. """) lines.extend( generate_boards( onboard_debug, extend_debug=True, skip_columns=skip_columns)) if external_debug: lines.append(""" -External tools -~~~~~~~~~~~~~~ +External Debug Tools +~~~~~~~~~~~~~~~~~~~~~ -Boards listed below are compatible with :ref:`piodebug` but depend on external -debugging tools. See "Debug" column for compatible debugging tools. +Boards listed below are compatible with :ref:`piodebug` but **DEPEND ON** +external debug tool. See "Debug" column for compatible debug tools. """) lines.extend( generate_boards( @@ -614,12 +614,20 @@ popular embedded boards and IDE. def update_debugging(): + tools_to_platforms = {} vendors = {} platforms = [] frameworks = [] for data in BOARDS: if not data['debug']: continue + + for tool in data['debug']['tools']: + tool = str(tool) + if tool not in tools_to_platforms: + tools_to_platforms[tool] = [] + tools_to_platforms[tool].append(data['platform']) + platforms.append(data['platform']) frameworks.extend(data['frameworks']) vendor = data['vendor'] @@ -628,6 +636,24 @@ def update_debugging(): else: vendors[vendor] = [data] + def _update_tool_compat_platforms(content): + begin_tpl = ".. begin_compatible_platforms_" + end_tpl = ".. end_compatible_platforms_" + for tool, platforms in tools_to_platforms.items(): + begin = begin_tpl + tool + end = end_tpl + tool + begin_index = content.index(begin) + end_index = content.index(end) + chunk = ["\n\n**Compatible development platforms:**\n"] + chunk.extend([ + "* :ref:`platform_%s`" % str(p) + for p in sorted(set(platforms)) + ]) + chunk.extend(["\n"]) + content = content[:begin_index + len(begin)] + "\n".join( + chunk) + content[end_index:] + return content + lines = [] # Platforms lines.append(""".. _debugging_platforms: @@ -681,7 +707,7 @@ Boards with open( join(util.get_source_dir(), "..", "docs", "plus", "debugging.rst"), "r+") as fp: - content = fp.read() + content = _update_tool_compat_platforms(fp.read()) fp.seek(0) fp.truncate() fp.write(content[:content.index(".. _debugging_platforms:")] + From dae50a32c0ecc6a1734c7ee0f3cc21941d60401f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 7 Jul 2018 20:34:34 +0300 Subject: [PATCH 33/86] Docs: Info about connection Vbus [CN3-1] to Vcc [CN3-3] of FT2232H Mini-Module --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 4600f5e4..f21fe976 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4600f5e495e655eb5171ec2eeb132b4cbfd6fd75 +Subproject commit f21fe97694ea9c1c26dd1b5f114c2570c1c977a6 From 9af715e87292c56c9996bb6c90aa11e49b0b30f2 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 8 Jul 2018 18:58:04 +0300 Subject: [PATCH 34/86] Improve docs for Advanced Scripting section --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index f21fe976..7b3de738 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f21fe97694ea9c1c26dd1b5f114c2570c1c977a6 +Subproject commit 7b3de738eb3b71c49070d0fe0e715522b2976436 From f7ac71d48ef4d163d103391fe5a0b4bedbea4e02 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 10 Jul 2018 00:59:01 +0300 Subject: [PATCH 35/86] Docs: ``projenv`` is available only for POST-type scripts --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 7b3de738..394a9811 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 7b3de738eb3b71c49070d0fe0e715522b2976436 +Subproject commit 394a9811cef4a48bdbe493b2aa8105f1749b7cc7 From 23f0ffdfeb13964fda51b1349df75a04d03ef3d0 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 10 Jul 2018 15:54:46 +0300 Subject: [PATCH 36/86] Append __PLATFORMIO_DEBUG__ macro in debug session --- platformio/builder/tools/piomisc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index c15812c3..ec565b51 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -288,6 +288,7 @@ def PioClean(env, clean_dir): def ProcessDebug(env): if not env.subst("$PIODEBUGFLAGS"): env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"]) + env.Append(PIODEBUGFLAGS=["-D__PLATFORMIO_DEBUG__"]) env.Append( BUILD_FLAGS=env.get("PIODEBUGFLAGS", []), BUILD_UNFLAGS=["-Os", "-O0", "-O1", "-O2", "-O3"]) From 7d6192b0697d7309ccb17aa9dd9dce1484c1775e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 12 Jul 2018 01:22:18 +0300 Subject: [PATCH 37/86] Update ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 9c025651..4a0f97a2 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,22 +1,37 @@ What kind of issue is this? -- [ ] Question. This issue tracker is not the place for questions. If you want to ask how to do - something, or to understand why something isn't working the way you expect it to, use - our Community Forums https://community.platformio.org +- [ ] **Question** -- [ ] PlatformIO IDE. All issues related to PlatformIO IDE should be reported to appropriate repository - https://github.com/platformio/platformio-atom-ide/issues + This issue tracker is not the place for questions. If you want to ask how to do something, + or to understand why something isn't working the way you expect it to, + use: + + * Community Forums https://community.platformio.org + * Premium Support https://platformio.org/support -- [ ] Development Platform or Board. All issues related to Development Platforms or Embedded Boards - should be reported to appropriate repository. - See full list with repositories and search for "platform-xxx" repository related to your hardware - https://github.com/platformio?query=platform- +- [ ] **PlatformIO IDE** -- [ ] Feature Request. Start by telling us what problem you’re trying to solve. Often a solution + All issues related to PlatformIO IDE should be reported to appropriate repository: + + * Atom https://github.com/platformio/platformio-atom-ide/issues + * VSCode https://github.com/platformio/platformio-vscode-ide/issues + +- [ ] **Development Platform or Board** + + All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms + should be reported to appropriate repository related to your hardware + https://github.com/topics/platformio-platform + +- [ ] **Feature Request** + + Start by telling us what problem you’re trying to solve. Often a solution already exists! Don’t send pull requests to implement new features without first getting our support. Sometimes we leave features out on purpose to keep the project small. -- [ ] PlatformIO Core. If you’ve found a bug, please provide an information below. +- [ ] **PlatformIO Core** + + If you’ve found a bug, please provide an information below. + *You can erase any parts of this template not applicable to your Issue.* From eced1c4c2a69617f5b4e31553d901338575d89cd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 12 Jul 2018 01:23:24 +0300 Subject: [PATCH 38/86] Update ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 4a0f97a2..8ec79846 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -6,8 +6,8 @@ What kind of issue is this? or to understand why something isn't working the way you expect it to, use: - * Community Forums https://community.platformio.org - * Premium Support https://platformio.org/support + * Community Forums [https://community.platformio.org](https://community.platformio.org) + * Premium Support [https://platformio.org/support](https://platformio.org/support) - [ ] **PlatformIO IDE** From 0fdfb273c6c40353078787098734bee304bf0e51 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 12 Jul 2018 01:23:59 +0300 Subject: [PATCH 39/86] Update ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 8ec79846..15fe3bcb 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,35 +1,30 @@ What kind of issue is this? -- [ ] **Question** - +- [ ] **Question**. This issue tracker is not the place for questions. If you want to ask how to do something, or to understand why something isn't working the way you expect it to, use: - * Community Forums [https://community.platformio.org](https://community.platformio.org) - * Premium Support [https://platformio.org/support](https://platformio.org/support) - -- [ ] **PlatformIO IDE** + * Community Forums https://community.platformio.org + * Premium Support https://platformio.org/support +- [ ] **PlatformIO IDE**. All issues related to PlatformIO IDE should be reported to appropriate repository: * Atom https://github.com/platformio/platformio-atom-ide/issues * VSCode https://github.com/platformio/platformio-vscode-ide/issues -- [ ] **Development Platform or Board** - +- [ ] **Development Platform or Board**. All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms should be reported to appropriate repository related to your hardware https://github.com/topics/platformio-platform -- [ ] **Feature Request** - +- [ ] **Feature Request**. Start by telling us what problem you’re trying to solve. Often a solution already exists! Don’t send pull requests to implement new features without first getting our support. Sometimes we leave features out on purpose to keep the project small. -- [ ] **PlatformIO Core** - +- [ ] **PlatformIO Core**. If you’ve found a bug, please provide an information below. From 28a76eb389e49131e55a0fff9e5b305dad28fe0d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 12 Jul 2018 01:26:02 +0300 Subject: [PATCH 40/86] Update ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 15fe3bcb..9a951da4 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -3,16 +3,12 @@ What kind of issue is this? - [ ] **Question**. This issue tracker is not the place for questions. If you want to ask how to do something, or to understand why something isn't working the way you expect it to, - use: - - * Community Forums https://community.platformio.org - * Premium Support https://platformio.org/support + use [Community Forums](https://community.platformio.org) or [Premium Support](https://platformio.org/support) - [ ] **PlatformIO IDE**. All issues related to PlatformIO IDE should be reported to appropriate repository: - - * Atom https://github.com/platformio/platformio-atom-ide/issues - * VSCode https://github.com/platformio/platformio-vscode-ide/issues + [PlatformIO IDE for Atom](https://github.com/platformio/platformio-atom-ide/issues) or + [PlatformIO IDE for VSCode](https://github.com/platformio/platformio-vscode-ide/issues) - [ ] **Development Platform or Board**. All issues (building, uploading, adding new boards, etc.) related to PlatformIO development platforms From cb7717eaf6157f64505b1088dba8df3145b0d3d8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 12 Jul 2018 20:52:25 +0300 Subject: [PATCH 41/86] Update docs for creating a custom dev/platform --- docs | 2 +- examples | 2 +- scripts/docspregen.py | 35 ----------------------------------- 3 files changed, 2 insertions(+), 37 deletions(-) diff --git a/docs b/docs index 394a9811..14ee4036 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 394a9811cef4a48bdbe493b2aa8105f1749b7cc7 +Subproject commit 14ee40368f8c3683985e04c6cb685befaf9b206d diff --git a/examples b/examples index cab374e8..8e11abaa 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit cab374e8c972eeb38ac8d5ad93bdcf3b9d8965cd +Subproject commit 8e11abaafcfac758252f1df820d694a873e09e36 diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 90eb5872..17c1b6ba 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -521,40 +521,6 @@ def update_framework_docs(): isfile(join(frameworks_dir, "%s_extra.rst" % name)))) -def update_create_platform_doc(): - lines = [] - lines.append(""".. _platform_creating_packages: - -Packages --------- - -*PlatformIO* has pre-built packages for the most popular operation systems: -*Mac OS*, *Linux (+ARM)* and *Windows*. - -.. list-table:: - :header-rows: 1 - - * - Name - - Description""") - for name, items in sorted(API_PACKAGES.iteritems()): - lines.append(""" - * - `{name} <{url}>`__ - - {description}""".format( - name=name, - url=API_PACKAGES[name]['url'], - description=API_PACKAGES[name]['description'])) - - with open( - join(util.get_source_dir(), "..", "docs", "platforms", - "creating_platform.rst"), "r+") as fp: - content = fp.read() - fp.seek(0) - fp.truncate() - fp.write(content[:content.index(".. _platform_creating_packages:")] + - "\n".join(lines) + "\n\n" + content[content.index( - ".. _platform_creating_manifest_file:"):]) - - def update_embedded_boards(): lines = [] @@ -839,7 +805,6 @@ def update_project_examples(): def main(): - update_create_platform_doc() update_platform_docs() update_framework_docs() update_embedded_boards() From 6b6c60e82cd3a2330c86c3a4d6633614e551091b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 13 Jul 2018 00:04:16 +0300 Subject: [PATCH 42/86] Fix "Cannot uninstall 'pyparsing'" for macOS CI build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 07e019a2..81b13df5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: install: - git submodule update --init --recursive - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl -fsSL https://bootstrap.pypa.io/get-pip.py | sudo python; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install tox; else pip install -U tox; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox<3.1.2"; else pip install -U tox; fi # ChipKIT issue: install 32-bit support for GCC PIC32 - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi From 735cfbf850380f02500715d42f51220811921185 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 13 Jul 2018 00:25:50 +0300 Subject: [PATCH 43/86] Fix "Cannot uninstall 'pyparsing'" for macOS CI build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 81b13df5..523931ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: install: - git submodule update --init --recursive - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl -fsSL https://bootstrap.pypa.io/get-pip.py | sudo python; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox<3.1.2"; else pip install -U tox; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo pip install "tox==3.0.0"; else pip install -U tox; fi # ChipKIT issue: install 32-bit support for GCC PIC32 - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libc6-i386; fi From 44a926b30a1b17b31793b05844aa64c8223708db Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 13 Jul 2018 01:54:37 +0300 Subject: [PATCH 44/86] Check package structure after unpacking and raise error when antivirus tool blocks PlatformIO package manager // Resolve #1462 --- HISTORY.rst | 4 ++++ docs | 2 +- platformio/exception.py | 7 +++++++ platformio/unpacker.py | 13 +++++++++---- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7bc19c63..e567092f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,10 @@ PlatformIO 3.0 - Check maximum allowed "program" and "data" sizes before uploading/programming (`issue #1412 `_) +* Check package structure after unpacking and raise error when antivirus tool + blocks PlatformIO package manager + (`issue #1462 `_) + 3.5.4 (2018-07-03) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 14ee4036..e3330b7e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 14ee40368f8c3683985e04c6cb685befaf9b206d +Subproject commit e3330b7e33e48c11547299183fd0a599c77b01b6 diff --git a/platformio/exception.py b/platformio/exception.py index 01c1be7f..c7d3cd2f 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -102,6 +102,13 @@ class PackageInstallError(PlatformioException): "Please try this solution -> http://bit.ly/faq-package-manager") +class ExtractArchiveItemError(PlatformioException): + + MESSAGE = ( + "Could not extract `{0}` to `{1}`. Try to disable antivirus " + "tool or check this solution -> http://bit.ly/faq-package-manager") + + class FDUnrecognizedStatusCode(PlatformioException): MESSAGE = "Got an unrecognized status code '{0}' when downloaded {1}" diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 38165c40..9c432e21 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -13,15 +13,14 @@ # limitations under the License. from os import chmod -from os.path import join +from os.path import exists, join from tarfile import open as tarfile_open from time import mktime from zipfile import ZipFile import click -from platformio import util -from platformio.exception import UnsupportedArchiveType +from platformio import exception, util class ArchiveBase(object): @@ -51,6 +50,10 @@ class TARArchive(ArchiveBase): def get_items(self): return self._afo.getmembers() + def after_extract(self, item, dest_dir): + if not exists(join(dest_dir, item.name)): + raise exception.ExtractArchiveItemError(item.name, dest_dir) + class ZIPArchive(ArchiveBase): @@ -73,6 +76,8 @@ class ZIPArchive(ArchiveBase): return self._afo.infolist() def after_extract(self, item, dest_dir): + if not exists(join(dest_dir, item.filename)): + raise exception.ExtractArchiveItemError(item.filename, dest_dir) self.preserve_permissions(item, dest_dir) self.preserve_mtime(item, dest_dir) @@ -89,7 +94,7 @@ class FileUnpacker(object): elif self.archpath.lower().endswith(".zip"): self._unpacker = ZIPArchive(self.archpath) if not self._unpacker: - raise UnsupportedArchiveType(self.archpath) + raise exception.UnsupportedArchiveType(self.archpath) return self def __exit__(self, *args): From 1d7d518ec5dc3d052a390029ebeb6cabe981f0fb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 13 Jul 2018 12:15:07 +0300 Subject: [PATCH 45/86] Temporary disable checking of extracted item from archive --- platformio/unpacker.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 9c432e21..10640d00 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -50,9 +50,9 @@ class TARArchive(ArchiveBase): def get_items(self): return self._afo.getmembers() - def after_extract(self, item, dest_dir): - if not exists(join(dest_dir, item.name)): - raise exception.ExtractArchiveItemError(item.name, dest_dir) + # def after_extract(self, item, dest_dir): + # if not exists(join(dest_dir, item.name)): + # raise exception.ExtractArchiveItemError(item.name, dest_dir) class ZIPArchive(ArchiveBase): @@ -76,8 +76,8 @@ class ZIPArchive(ArchiveBase): return self._afo.infolist() def after_extract(self, item, dest_dir): - if not exists(join(dest_dir, item.filename)): - raise exception.ExtractArchiveItemError(item.filename, dest_dir) + # if not exists(join(dest_dir, item.filename)): + # raise exception.ExtractArchiveItemError(item.filename, dest_dir) self.preserve_permissions(item, dest_dir) self.preserve_mtime(item, dest_dir) From 695a850979e9af28c41ae5b59582f3d3e275ccdc Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 13 Jul 2018 12:16:44 +0300 Subject: [PATCH 46/86] Bump version to 3.6.0a7 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 10a5e6d8..82f47b12 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a6") +VERSION = (3, 6, "0a7") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From f2c4ba1895a900c6a29f2584fcf546c7183ba6b7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 13 Jul 2018 13:10:24 +0300 Subject: [PATCH 47/86] Check item after unpacking only if not symbolic link --- platformio/unpacker.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 10640d00..9a4f7952 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -13,7 +13,7 @@ # limitations under the License. from os import chmod -from os.path import exists, join +from os.path import exists, islink, join from tarfile import open as tarfile_open from time import mktime from zipfile import ZipFile @@ -50,9 +50,10 @@ class TARArchive(ArchiveBase): def get_items(self): return self._afo.getmembers() - # def after_extract(self, item, dest_dir): - # if not exists(join(dest_dir, item.name)): - # raise exception.ExtractArchiveItemError(item.name, dest_dir) + def after_extract(self, item, dest_dir): + item_path = join(dest_dir, item.name) + if not islink(item_path) and not exists(item_path): + raise exception.ExtractArchiveItemError(item.name, dest_dir) class ZIPArchive(ArchiveBase): @@ -76,8 +77,9 @@ class ZIPArchive(ArchiveBase): return self._afo.infolist() def after_extract(self, item, dest_dir): - # if not exists(join(dest_dir, item.filename)): - # raise exception.ExtractArchiveItemError(item.filename, dest_dir) + item_path = join(dest_dir, item.filename) + if not islink(item_path) and not exists(item_path): + raise exception.ExtractArchiveItemError(item.filename, dest_dir) self.preserve_permissions(item, dest_dir) self.preserve_mtime(item, dest_dir) From e1578dabac936a1595ff127c07b113804e3d7212 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 14 Jul 2018 22:10:56 +0300 Subject: [PATCH 48/86] Lock interprocess requests to PlatformIO Package Manager for install/uninstall operations // Resolve #1462 --- .isort.cfg | 2 +- HISTORY.rst | 3 + platformio/app.py | 31 +++------ platformio/exception.py | 4 ++ platformio/lockfile.py | 108 +++++++++++++++++++++++++++++ platformio/managers/package.py | 120 ++++++++++++++++++--------------- setup.py | 1 - 7 files changed, 192 insertions(+), 77 deletions(-) create mode 100644 platformio/lockfile.py diff --git a/.isort.cfg b/.isort.cfg index d094cc19..9b58f629 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=79 -known_third_party=bottle,click,lockfile,python-dateutil,pytest,requests,SCons,semantic_version,serial +known_third_party=bottle,click,pytest,requests,SCons,semantic_version,serial diff --git a/HISTORY.rst b/HISTORY.rst index e567092f..df3add2a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,9 @@ PlatformIO 3.0 * Check package structure after unpacking and raise error when antivirus tool blocks PlatformIO package manager (`issue #1462 `_) +* Lock interprocess requests to PlatformIO Package Manager for + install/uninstall operations + (`issue #1594 `_) 3.5.4 (2018-07-03) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/app.py b/platformio/app.py index 06ca7cfb..6ace78ed 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -19,13 +19,12 @@ import os import uuid from copy import deepcopy from os import environ, getenv, listdir, remove -from os.path import abspath, dirname, expanduser, getmtime, isdir, isfile, join +from os.path import abspath, dirname, expanduser, isdir, isfile, join from time import time import requests -from lockfile import LockFailed, LockFile -from platformio import __version__, exception, util +from platformio import exception, lockfile, util def projects_dir_validate(projects_dir): @@ -108,10 +107,7 @@ class State(object): if self._prev_state != self._state: try: with codecs.open(self.path, "w", encoding="utf8") as fp: - if "dev" in __version__: - json.dump(self._state, fp, indent=4) - else: - json.dump(self._state, fp) + json.dump(self._state, fp) except IOError: raise exception.HomeDirPermissionsError(util.get_home_dir()) self._unlock_state_file() @@ -119,21 +115,19 @@ class State(object): def _lock_state_file(self): if not self.lock: return - self._lockfile = LockFile(self.path) - - if self._lockfile.is_locked() and \ - (time() - getmtime(self._lockfile.lock_file)) > 10: - self._lockfile.break_lock() - + self._lockfile = lockfile.LockFile(self.path) try: self._lockfile.acquire() - except LockFailed: + except IOError: raise exception.HomeDirPermissionsError(dirname(self.path)) def _unlock_state_file(self): if self._lockfile: self._lockfile.release() + def __del__(self): + self._unlock_state_file() + class ContentCache(object): @@ -155,15 +149,10 @@ class ContentCache(object): def _lock_dbindex(self): if not self.cache_dir: os.makedirs(self.cache_dir) - self._lockfile = LockFile(self.cache_dir) - if self._lockfile.is_locked() and \ - isfile(self._lockfile.lock_file) and \ - (time() - getmtime(self._lockfile.lock_file)) > 10: - self._lockfile.break_lock() - + self._lockfile = lockfile.LockFile(self.cache_dir) try: self._lockfile.acquire() - except LockFailed: + except: # pylint: disable=bare-except return False return True diff --git a/platformio/exception.py b/platformio/exception.py index c7d3cd2f..8a1b43fc 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -28,6 +28,10 @@ class ReturnErrorCode(PlatformioException): MESSAGE = "{0}" +class LockFileTimeoutError(PlatformioException): + pass + + class MinitermException(PlatformioException): pass diff --git a/platformio/lockfile.py b/platformio/lockfile.py new file mode 100644 index 00000000..f00c16e5 --- /dev/null +++ b/platformio/lockfile.py @@ -0,0 +1,108 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import remove +from os.path import abspath, exists, getmtime +from time import sleep, time + +from platformio import exception + +LOCKFILE_TIMEOUT = 3600 # in seconds, 1 hour +LOCKFILE_DELAY = 0.2 + +LOCKFILE_INTERFACE_FCNTL = 1 +LOCKFILE_INTERFACE_MSVCRT = 2 + +try: + import fcntl + LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_FCNTL +except ImportError: + try: + import msvcrt + LOCKFILE_CURRENT_INTERFACE = LOCKFILE_INTERFACE_MSVCRT + except ImportError: + LOCKFILE_CURRENT_INTERFACE = None + + +class LockFileExists(Exception): + pass + + +class LockFile(object): + + def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY): + self.timeout = timeout + self.delay = delay + self._lock_path = abspath(path) + ".lock" + self._fp = None + + def _lock(self): + if not LOCKFILE_CURRENT_INTERFACE and exists(self._lock_path): + # remove stale lock + if time() - getmtime(self._lock_path) > 10: + try: + remove(self._lock_path) + except: # pylint: disable=bare-except + pass + else: + raise LockFileExists + + self._fp = open(self._lock_path, "w") + try: + if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL: + fcntl.flock(self._fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT: + msvcrt.locking(self._fp.fileno(), msvcrt.LK_NBLCK, 1) + except IOError: + self._fp = None + raise LockFileExists + return True + + def _unlock(self): + if not self._fp: + return + if LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_FCNTL: + fcntl.flock(self._fp.fileno(), fcntl.LOCK_UN) + elif LOCKFILE_CURRENT_INTERFACE == LOCKFILE_INTERFACE_MSVCRT: + msvcrt.locking(self._fp.fileno(), msvcrt.LK_UNLCK, 1) + self._fp.close() + self._fp = None + + def acquire(self): + elapsed = 0 + while elapsed < self.timeout: + try: + return self._lock() + except LockFileExists: + sleep(self.delay) + elapsed += self.delay + + raise exception.LockFileTimeoutError() + + def release(self): + self._unlock() + if exists(self._lock_path): + try: + remove(self._lock_path) + except: # pylint: disable=bare-except + pass + + def __enter__(self): + self.acquire() + + def __exit__(self, type_, value, traceback): + self.release() + + def __del__(self): + self.release() diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 3e5bd4a0..686c718b 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -27,6 +27,7 @@ import semantic_version from platformio import __version__, app, exception, telemetry, util from platformio.downloader import FileDownloader +from platformio.lockfile import LockFile from platformio.unpacker import FileUnpacker from platformio.vcsclient import VCSClientFactory @@ -707,71 +708,82 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): fg="yellow") return package_dir - if url: - pkg_dir = self._install_from_url( - name, url, requirements, track=True) - else: - pkg_dir = self._install_from_piorepo(name, requirements) + pkg_dir = None + # interprocess lock + with LockFile(self.package_dir): + self.cache_reset() - if not pkg_dir or not self.manifest_exists(pkg_dir): - raise exception.PackageInstallError(name, requirements or "*", - util.get_systype()) + if url: + pkg_dir = self._install_from_url( + name, url, requirements, track=True) + else: + pkg_dir = self._install_from_piorepo(name, requirements) - manifest = self.load_manifest(pkg_dir) - assert manifest + if not pkg_dir or not self.manifest_exists(pkg_dir): + raise exception.PackageInstallError(name, requirements or "*", + util.get_systype()) - if not after_update: - telemetry.on_event( - category=self.__class__.__name__, - action="Install", - label=manifest['name']) + manifest = self.load_manifest(pkg_dir) + assert manifest - if not silent: - click.secho( - "{name} @ {version} has been successfully installed!".format( - **manifest), - fg="green") + if not after_update: + telemetry.on_event( + category=self.__class__.__name__, + action="Install", + label=manifest['name']) + + if not silent: + click.secho( + "{name} @ {version} has been successfully installed!". + format(**manifest), + fg="green") return pkg_dir def uninstall(self, package, requirements=None, after_update=False): - if isdir(package) and self.get_package_by_dir(package): - pkg_dir = package - else: - name, requirements, url = self.parse_pkg_uri(package, requirements) - pkg_dir = self.get_package_dir(name, requirements, url) - - if not pkg_dir: - raise exception.UnknownPackage( - "%s @ %s" % (package, requirements or "*")) - - manifest = self.load_manifest(pkg_dir) - click.echo( - "Uninstalling %s @ %s: \t" % (click.style( - manifest['name'], fg="cyan"), manifest['version']), - nl=False) - - if islink(pkg_dir): - os.unlink(pkg_dir) - else: - util.rmtree_(pkg_dir) - self.cache_reset() - - # unfix package with the same name - pkg_dir = self.get_package_dir(manifest['name']) - if pkg_dir and "@" in pkg_dir: - shutil.move( - pkg_dir, - join(self.package_dir, self.get_install_dirname(manifest))) + # interprocess lock + with LockFile(self.package_dir): self.cache_reset() - click.echo("[%s]" % click.style("OK", fg="green")) + if isdir(package) and self.get_package_by_dir(package): + pkg_dir = package + else: + name, requirements, url = self.parse_pkg_uri( + package, requirements) + pkg_dir = self.get_package_dir(name, requirements, url) + + if not pkg_dir: + raise exception.UnknownPackage( + "%s @ %s" % (package, requirements or "*")) + + manifest = self.load_manifest(pkg_dir) + click.echo( + "Uninstalling %s @ %s: \t" % (click.style( + manifest['name'], fg="cyan"), manifest['version']), + nl=False) + + if islink(pkg_dir): + os.unlink(pkg_dir) + else: + util.rmtree_(pkg_dir) + self.cache_reset() + + # unfix package with the same name + pkg_dir = self.get_package_dir(manifest['name']) + if pkg_dir and "@" in pkg_dir: + shutil.move( + pkg_dir, + join(self.package_dir, self.get_install_dirname(manifest))) + self.cache_reset() + + click.echo("[%s]" % click.style("OK", fg="green")) + + if not after_update: + telemetry.on_event( + category=self.__class__.__name__, + action="Uninstall", + label=manifest['name']) - if not after_update: - telemetry.on_event( - category=self.__class__.__name__, - action="Uninstall", - label=manifest['name']) return True def update(self, package, requirements=None, only_check=False): diff --git a/setup.py b/setup.py index b4b8b1aa..4159e183 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,6 @@ install_requires = [ "bottle<0.13", "click>=5,<6", "colorama", - "lockfile>=0.9.1,<0.13", "pyserial>=3,<4,!=3.3", "requests>=2.4.0,<3", "semantic_version>=2.5.0,<3" From 72d260c2955d74693fa4aa3303083cfac8ec956c Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 15 Jul 2018 00:43:12 +0300 Subject: [PATCH 49/86] Fix file locking of package installer // Issue #1594 --- platformio/managers/package.py | 56 +++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 686c718b..33f40f6e 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -680,39 +680,39 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): silent=False, after_update=False, force=False): - name, requirements, url = self.parse_pkg_uri(name, requirements) - package_dir = self.get_package_dir(name, requirements, url) - - # avoid circle dependencies - if not self.INSTALL_HISTORY: - self.INSTALL_HISTORY = [] - history_key = "%s-%s-%s" % (name, requirements or "", url or "") - if history_key in self.INSTALL_HISTORY: - return package_dir - self.INSTALL_HISTORY.append(history_key) - - if package_dir and force: - self.uninstall(package_dir) - package_dir = None - - if not package_dir or not silent: - msg = "Installing " + click.style(name, fg="cyan") - if requirements: - msg += " @ " + requirements - self.print_message(msg) - if package_dir: - if not silent: - click.secho( - "{name} @ {version} is already installed".format( - **self.load_manifest(package_dir)), - fg="yellow") - return package_dir - pkg_dir = None # interprocess lock with LockFile(self.package_dir): self.cache_reset() + name, requirements, url = self.parse_pkg_uri(name, requirements) + package_dir = self.get_package_dir(name, requirements, url) + + # avoid circle dependencies + if not self.INSTALL_HISTORY: + self.INSTALL_HISTORY = [] + history_key = "%s-%s-%s" % (name, requirements or "", url or "") + if history_key in self.INSTALL_HISTORY: + return package_dir + self.INSTALL_HISTORY.append(history_key) + + if package_dir and force: + self.uninstall(package_dir) + package_dir = None + + if not package_dir or not silent: + msg = "Installing " + click.style(name, fg="cyan") + if requirements: + msg += " @ " + requirements + self.print_message(msg) + if package_dir: + if not silent: + click.secho( + "{name} @ {version} is already installed".format( + **self.load_manifest(package_dir)), + fg="yellow") + return package_dir + if url: pkg_dir = self._install_from_url( name, url, requirements, track=True) From a14f2d291e7886f0b33aadad0a4d891c9177fe56 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sun, 15 Jul 2018 01:06:59 +0300 Subject: [PATCH 50/86] Improve checking of package structure after unpacking // Issue #1462 --- platformio/unpacker.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 9a4f7952..d747ee5e 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -31,6 +31,9 @@ class ArchiveBase(object): def get_items(self): raise NotImplementedError() + def get_item_filename(self, item): + raise NotImplementedError() + def extract_item(self, item, dest_dir): self._afo.extract(item, dest_dir) self.after_extract(item, dest_dir) @@ -50,10 +53,8 @@ class TARArchive(ArchiveBase): def get_items(self): return self._afo.getmembers() - def after_extract(self, item, dest_dir): - item_path = join(dest_dir, item.name) - if not islink(item_path) and not exists(item_path): - raise exception.ExtractArchiveItemError(item.name, dest_dir) + def get_item_filename(self, item): + return item.name class ZIPArchive(ArchiveBase): @@ -76,10 +77,10 @@ class ZIPArchive(ArchiveBase): def get_items(self): return self._afo.infolist() + def get_item_filename(self, item): + return item.filename + def after_extract(self, item, dest_dir): - item_path = join(dest_dir, item.filename) - if not islink(item_path) and not exists(item_path): - raise exception.ExtractArchiveItemError(item.filename, dest_dir) self.preserve_permissions(item, dest_dir) self.preserve_mtime(item, dest_dir) @@ -114,4 +115,12 @@ class FileUnpacker(object): with click.progressbar(items, label="Unpacking") as pb: for item in pb: self._unpacker.extract_item(item, dest_dir) + + # check on disk + for item in self._unpacker.get_items(): + filename = self._unpacker.get_item_filename(item) + item_path = join(dest_dir, filename) + if not islink(item_path) and not exists(item_path): + raise exception.ExtractArchiveItemError(filename, dest_dir) + return True From 4a3b616b0f4a01d453cf53e878a1df5d6acd8765 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 16 Jul 2018 17:45:52 +0300 Subject: [PATCH 51/86] Sync docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index e3330b7e..0a962842 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e3330b7e33e48c11547299183fd0a599c77b01b6 +Subproject commit 0a962842d653c3df31b773f96f33eb2ae2d67d83 From da89f57046270fe89c8fb89014a523c20ae2091f Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Jul 2018 16:24:02 +0300 Subject: [PATCH 52/86] PIO Home 1.0.0; PIO Plus 1.4.0 --- docs | 2 +- platformio/managers/core.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs b/docs index 0a962842..4c1af280 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 0a962842d653c3df31b773f96f33eb2ae2d67d83 +Subproject commit 4c1af2805ecc42961edcb8a3adda5fde2f4104d8 diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 0dee74fc..a4ff9be0 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,9 +21,9 @@ from platformio import __version__, exception, util from platformio.managers.package import PackageManager CORE_PACKAGES = { - "contrib-piohome": ">=0.9.6,<2", - "contrib-pysite": ">=0.3.0,<2", - "tool-pioplus": ">=1.3.6,<2", + "contrib-piohome": "^1.0.0", + "contrib-pysite": ">=0.3.2,<2", + "tool-pioplus": "^1.4.0", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.4" } From 734cb5c7aad1ed085701ec23fbab993c34767a39 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Jul 2018 16:24:38 +0300 Subject: [PATCH 53/86] Bump version to 3.6.0a8 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 82f47b12..f3132dbe 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a7") +VERSION = (3, 6, "0a8") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 4a14cc686c5167b487b31532d096ca72c869b6df Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Jul 2018 20:57:41 +0300 Subject: [PATCH 54/86] Add udev rules for J-Link devices --- scripts/99-platformio-udev.rules | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index ecc1c368..15c09217 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -208,3 +208,52 @@ ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="660", GROUP="plugdev", # CMSIS-DAP compatible adapters ATTRS{product}=="*CMSIS-DAP*", MODE="660", GROUP="plugdev", TAG+="uaccess" + +#SEGGER J-LIK +ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="666" From 2ba41cddc45666ac3dd7c619ea19158898680061 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Jul 2018 20:58:20 +0300 Subject: [PATCH 55/86] Disable exec command for Sublime Text debugger --- platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl b/platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl index bf27d97c..6f24d6a1 100644 --- a/platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl +++ b/platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl @@ -97,7 +97,7 @@ "settings": { "sublimegdb_workingdir": "{{project_dir}}", - "sublimegdb_exec_cmd": "-exec-continue", + "sublimegdb_exec_cmd": "", "sublimegdb_commandline": "{{platformio_path}} -f -c sublimetext debug --interface=gdb --interpreter=mi -x .pioinit" } From bc380714bdda6684154e03db2e829e054dabc143 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Jul 2018 20:58:20 +0300 Subject: [PATCH 56/86] Disable exec command for Sublime Text debugger --- platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl b/platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl index bf27d97c..6f24d6a1 100644 --- a/platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl +++ b/platformio/ide/tpls/sublimetext/platformio.sublime-project.tpl @@ -97,7 +97,7 @@ "settings": { "sublimegdb_workingdir": "{{project_dir}}", - "sublimegdb_exec_cmd": "-exec-continue", + "sublimegdb_exec_cmd": "", "sublimegdb_commandline": "{{platformio_path}} -f -c sublimetext debug --interface=gdb --interpreter=mi -x .pioinit" } From 3f2f79ade4e7cad2c8754333a64a7cdbdc67db07 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 17 Jul 2018 20:57:41 +0300 Subject: [PATCH 57/86] Add udev rules for J-Link devices --- scripts/99-platformio-udev.rules | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 6daaf922..ad6ed90c 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -205,3 +205,52 @@ ATTRS{idVendor}=="c251", ATTRS{idProduct}=="2710", MODE="660", GROUP="plugdev", # CMSIS-DAP compatible adapters ATTRS{product}=="*CMSIS-DAP*", MODE="660", GROUP="plugdev", TAG+="uaccess" + +#SEGGER J-LIK +ATTR{idProduct}=="1001", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1002", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1003", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1004", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1005", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1006", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1007", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1008", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1009", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100a", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100b", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100c", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100d", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100e", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="100f", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1010", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1011", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1012", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1013", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1014", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1015", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1016", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1017", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1018", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1019", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101a", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101b", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101c", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101d", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101e", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="101f", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1020", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1021", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1022", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1023", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1024", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1025", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1026", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1027", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1028", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="1029", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102a", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102b", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102c", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102d", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102e", ATTR{idVendor}=="1366", MODE="666" +ATTR{idProduct}=="102f", ATTR{idVendor}=="1366", MODE="666" From ff59dcefe0caca65ec770abb55f871b8b0c93aec Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 10 Jul 2018 15:54:46 +0300 Subject: [PATCH 58/86] Append __PLATFORMIO_DEBUG__ macro in debug session --- platformio/builder/tools/piomisc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index c15812c3..ec565b51 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -288,6 +288,7 @@ def PioClean(env, clean_dir): def ProcessDebug(env): if not env.subst("$PIODEBUGFLAGS"): env.Replace(PIODEBUGFLAGS=["-Og", "-g3", "-ggdb3"]) + env.Append(PIODEBUGFLAGS=["-D__PLATFORMIO_DEBUG__"]) env.Append( BUILD_FLAGS=env.get("PIODEBUGFLAGS", []), BUILD_UNFLAGS=["-Os", "-O0", "-O1", "-O2", "-O3"]) From ff3ce2d69e837e1fa40c1ae2ccc224f23c1fb143 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Jul 2018 15:27:00 +0300 Subject: [PATCH 59/86] Fix PIO Unit Testing issue when ``UNIT_TEST`` macro was not set in a build environment --- HISTORY.rst | 7 +++++++ docs | 2 +- examples | 2 +- platformio/builder/tools/platformio.py | 19 ++++++++++--------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ffa4b489..d5f4f19a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,13 @@ Release Notes PlatformIO 3.0 -------------- +3.5.5 (2018-??-??) +~~~~~~~~~~~~~~~~~~ + +* Fixed `PIO Unit Testing `__ + issue with a shared code between main and test programs when ``UNIT_TEST`` + macro was not set in a build environment + 3.5.4 (2018-07-03) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index 9d8e84d5..a77ecd12 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 9d8e84d527aa83f19ce402e0c55964a1655535eb +Subproject commit a77ecd12952ff66b799357d505e9a09c24a4a5ec diff --git a/examples b/examples index ebff9ae7..8e11abaa 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit ebff9ae73fa582c5edb7b47f628891b637b29d94 +Subproject commit 8e11abaafcfac758252f1df820d694a873e09e36 diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 1c72bf3d..ab4cd9ec 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -54,21 +54,20 @@ def _build_project_deps(env): if project_lib_builder.env.get(key) }) - if "__test" in COMMAND_LINE_TARGETS: - env.ProcessTest() - projenv = env.Clone() - projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR", - "$PIOTEST_SRC_FILTER") - else: - projenv = env.Clone() - projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", - env.get("SRC_FILTER")) + projenv = env.Clone() # CPPPATH from dependencies projenv.PrependUnique(CPPPATH=project_lib_builder.env.get("CPPPATH")) # extra build flags from `platformio.ini` projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS")) + if "__test" in COMMAND_LINE_TARGETS: + projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR", + "$PIOTEST_SRC_FILTER") + else: + projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", + env.get("SRC_FILTER")) + if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS: sys.stderr.write( "Error: Nothing to build. Please put your source code files " @@ -95,6 +94,8 @@ def BuildProgram(env): if "__debug" in COMMAND_LINE_TARGETS: env.ProcessDebug() + if "__test" in COMMAND_LINE_TARGETS: + env.ProcessTest() # process extra flags from board if "BOARD" in env and "build.extra_flags" in env.BoardConfig(): From e4462d7546737f1f0613a77d46f8e3f786a0b2ab Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Jul 2018 16:19:10 +0300 Subject: [PATCH 60/86] Add Infineon XMC dev/platform // Resolve #471 --- docs | 2 +- examples | 2 +- scripts/docspregen.py | 38 +++++++++++++++++--------------------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/docs b/docs index a77ecd12..eb2b343a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a77ecd12952ff66b799357d505e9a09c24a4a5ec +Subproject commit eb2b343ac608d9515d01f70ccfaefb0a3b85facc diff --git a/examples b/examples index 8e11abaa..965ab1cb 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 8e11abaafcfac758252f1df820d694a873e09e36 +Subproject commit 965ab1cb2852d06b652b49944830d2f81d563db1 diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 17c1b6ba..9da8ac18 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -240,6 +240,8 @@ def generate_platform(name, has_extra=False): limitations under the License. """) p = PlatformFactory.newPlatform(name) + assert p.repository_url.endswith(".git") + github_url = p.repository_url[:-4] lines.append(".. _platform_%s:" % p.name) lines.append("") @@ -272,17 +274,13 @@ Examples -------- Examples are listed from `%s development platform repository <%s>`_: -""" % (p.title, - campaign_url( - "https://github.com/platformio/platform-%s/tree/develop/examples" % - p.name))) +""" % (p.title, campaign_url("%s/tree/master/examples" % github_url))) examples_dir = join(p.get_dir(), "examples") if isdir(examples_dir): for eitem in os.listdir(examples_dir): if not isdir(join(examples_dir, eitem)): continue - url = ("https://github.com/platformio/platform-%s" - "/tree/develop/examples/%s" % (p.name, eitem)) + url = "%s/tree/master/examples/%s" % (github_url, eitem) lines.append("* `%s <%s>`_" % (eitem, campaign_url(url))) # @@ -300,7 +298,7 @@ Examples are listed from `%s development platform repository <%s>`_: Stable and upstream versions ---------------------------- -You can switch between `stable releases `__ +You can switch between `stable releases <{github_url}/releases>`__ of {title} development platform and the latest upstream version using :ref:`projectconf_env_platform` option in :ref:`projectconf` as described below. @@ -325,9 +323,9 @@ Upstream .. code-block:: ini [env:upstream_develop] - platform = https://github.com/platformio/platform-{name}.git + platform = {github_url}.git board = ... -""".format(name=p.name, title=p.title)) +""".format(name=p.name, title=p.title, github_url=github_url)) # # Packages @@ -460,11 +458,12 @@ Examples -------- """) for manifest in compatible_platforms: - lines.append("* `%s for %s <%s>`_" % ( - data['title'], manifest['title'], - campaign_url( - "https://github.com/platformio/platform-%s/tree/develop/examples" - % manifest['name']))) + p = PlatformFactory.newPlatform(manifest['name']) + lines.append( + "* `%s for %s <%s>`_" % + (data['title'], manifest['title'], + campaign_url( + "%s/tree/master/examples" % p.repository_url[:-4]))) # Platforms lines.append(""" @@ -713,8 +712,7 @@ def update_project_examples(): for manifest in PLATFORM_MANIFESTS: p = PlatformFactory.newPlatform(manifest['name']) - github_platform_url = ( - "https://github.com/platformio/platform-%s" % p.name) + github_url = p.repository_url[:-4] # Platform README platform_examples_dir = join(p.get_dir(), "examples") @@ -723,8 +721,7 @@ def update_project_examples(): for item in os.listdir(platform_examples_dir): if not isdir(join(platform_examples_dir, item)): continue - url = ("%s/tree/develop/examples/%s" % (github_platform_url, - item)) + url = "%s/tree/master/examples/%s" % (github_url, item) examples_md_lines.append("* [%s](%s)" % (item, url)) readme_dir = join(project_examples_dir, "platforms", p.name) @@ -745,14 +742,13 @@ def update_project_examples(): if framework['name'] not in framework_examples_md_lines: framework_examples_md_lines[framework['name']] = [] lines = [] - lines.append("- [%s](%s)" % (p.title, github_platform_url)) + lines.append("- [%s](%s)" % (p.title, github_url)) lines.extend(" %s" % l for l in examples_md_lines) lines.append("") framework_examples_md_lines[framework['name']].extend(lines) # Root README - url = "https://github.com/platformio/platform-%s/tree/develop/examples" % p.name - line = "* [%s](%s)" % (p.title, url) + line = "* [%s](%s)" % (p.title, "%s/tree/master/examples" % github_url) if p.is_embedded(): embedded.append(line) else: From fd8b6039100dc46e7281b138502f36b174df22b3 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 19 Jul 2018 19:00:09 +0300 Subject: [PATCH 61/86] Update README --- README.rst | 33 +++++++++++++++++++++++---------- docs | 2 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index bfb2ebf9..b3daff30 100644 --- a/README.rst +++ b/README.rst @@ -19,12 +19,12 @@ PlatformIO .. image:: https://img.shields.io/PlatformIO/Community.png :alt: Community Forums :target: https://community.platformio.org?utm_source=github&utm_medium=core -.. image:: https://img.shields.io/PlatformIO/Plus.png?color=orange - :alt: PlatformIO Plus: Professional solutions for an awesome open source PlatformIO ecosystem +.. image:: https://img.shields.io/PIO/Plus.png?color=orange + :alt: PIO Plus: Professional solutions for an awesome open source PlatformIO ecosystem :target: https://platformio.org/pricing?utm_source=github&utm_medium=core -**Quick Links:** `Home Page `_ | -`PlatformIO Plus `_ | +**Quick Links:** `Web `_ | +`PIO Plus `_ | `PlatformIO IDE `_ | `Project Examples `_ | `Docs `_ | @@ -32,6 +32,7 @@ PlatformIO `Contact Us `_ **Social:** `Twitter `_ | +`LinkedIn `_ | `Facebook `_ | `Hackaday `_ | `Bintray `_ | @@ -49,16 +50,25 @@ Get Started * `What is PlatformIO? `_ -Products --------- +Open Source +----------- * `PlatformIO IDE `_ * `PlatformIO Core (CLI) `_ -* `PIO Remote™ `_ -* `PIO Unified Debugger `_ +* `Library Management `_ +* `Project Examples `_ +* `Desktop IDEs Integration `_ +* `Continuous Integration `_ +* `Advanced Scripting API `_ + +PIO Plus +-------- + +* `PIO Remote™ `_ +* `PIO Unified Debugger `_ * `PIO Unit Testing `_ -* `PIO Delivery™ `_ -* `Cloud Builder `_ +* `Cloud IDEs Integration `_ +* `Integration Services `_ Registry -------- @@ -84,6 +94,7 @@ Development Platforms * `Espressif 32 `_ * `Espressif 8266 `_ * `Freescale Kinetis `_ +* * `Infineon XMC `_ * `Intel ARC32 `_ * `Lattice iCE40 `_ * `Maxim 32 `_ @@ -91,6 +102,8 @@ Development Platforms * `Nordic nRF51 `_ * `Nordic nRF52 `_ * `NXP LPC `_ +* `RISC-V `_ +* `Samsung ARTIK `_ * `Silicon Labs EFM32 `_ * `ST STM32 `_ * `Teensy `_ diff --git a/docs b/docs index eb2b343a..0c58805c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit eb2b343ac608d9515d01f70ccfaefb0a3b85facc +Subproject commit 0c58805cb5f37dc4c33af971490caea548f9b961 From ab24ca4ff69e636c9820a10c145288b22945a61a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Jul 2018 00:33:24 +0300 Subject: [PATCH 62/86] Cleanup --- README.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/README.rst b/README.rst index b3daff30..4be7de65 100644 --- a/README.rst +++ b/README.rst @@ -78,14 +78,6 @@ Registry * `Frameworks `_ * `Embedded Boards `_ -Solutions ---------- - -* `Library Manager `_ -* `Cloud IDEs Integration `_ -* `Standalone IDEs Integration `_ -* `Continuous Integration `_ - Development Platforms --------------------- From 903b41b3363c6babb0760a6e124b38b00a61d124 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Jul 2018 02:04:40 +0300 Subject: [PATCH 63/86] Add "test_build_project_src" option for PIO Unit Testing --- HISTORY.rst | 8 +++-- docs | 2 +- examples | 2 +- platformio/builder/main.py | 6 +++- platformio/builder/tools/platformio.py | 5 +-- platformio/commands/run.py | 49 +++++++++++++------------- 6 files changed, 40 insertions(+), 32 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d5f4f19a..34bddd26 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,9 +7,11 @@ PlatformIO 3.0 3.5.5 (2018-??-??) ~~~~~~~~~~~~~~~~~~ -* Fixed `PIO Unit Testing `__ - issue with a shared code between main and test programs when ``UNIT_TEST`` - macro was not set in a build environment +* `PIO Unit Testing `__: + + - Documented `Project Shared Code `__ + - Force building of project source code using `test_build_project_src `__ option + - Fixed missed ``UNIT_TEST`` macro for unit test components/libraries 3.5.4 (2018-07-03) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index a77ecd12..e2505d2f 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit a77ecd12952ff66b799357d505e9a09c24a4a5ec +Subproject commit e2505d2fa9af757f9984417edae98c271e1c52ee diff --git a/examples b/examples index 8e11abaa..965ab1cb 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 8e11abaafcfac758252f1df820d694a873e09e36 +Subproject commit 965ab1cb2852d06b652b49944830d2f81d563db1 diff --git a/platformio/builder/main.py b/platformio/builder/main.py index caafda5b..79732f1c 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -68,6 +68,9 @@ commonvars.AddVariables( ("UPLOAD_FLAGS",), ("UPLOAD_RESETMETHOD",), + # test options + ("TEST_BUILD_PROJECT_SRC",), + # debug options ("DEBUG_TOOL",), ("DEBUG_SVD_PATH",), @@ -76,7 +79,8 @@ commonvars.AddVariables( MULTILINE_VARS = [ "EXTRA_SCRIPTS", "PIOFRAMEWORK", "BUILD_FLAGS", "SRC_BUILD_FLAGS", - "BUILD_UNFLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE", "LIB_EXTRA_DIRS" + "BUILD_UNFLAGS", "UPLOAD_FLAGS", "SRC_FILTER", "LIB_DEPS", "LIB_IGNORE", + "LIB_EXTRA_DIRS" ] DEFAULT_ENV_OPTIONS = dict( diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index ab4cd9ec..d880d386 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -61,10 +61,11 @@ def _build_project_deps(env): # extra build flags from `platformio.ini` projenv.ProcessFlags(env.get("SRC_BUILD_FLAGS")) - if "__test" in COMMAND_LINE_TARGETS: + is_test = "__test" in COMMAND_LINE_TARGETS + if is_test: projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER") - else: + if not is_test or env['TEST_BUILD_PROJECT_SRC'] == "true": projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", env.get("SRC_FILTER")) diff --git a/platformio/commands/run.py b/platformio/commands/run.py index c61dbc06..4bb74482 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -125,32 +125,33 @@ class EnvironmentProcessor(object): DEFAULT_DUMP_OPTIONS = ("platform", "framework", "board") - KNOWN_PLATFORMIO_OPTIONS = ("description", "env_default", "home_dir", - "lib_dir", "libdeps_dir", "include_dir", - "src_dir", "build_dir", "data_dir", "test_dir", - "boards_dir", "lib_extra_dirs") + KNOWN_PLATFORMIO_OPTIONS = [ + "description", "env_default", "home_dir", "lib_dir", "libdeps_dir", + "include_dir", "src_dir", "build_dir", "data_dir", "test_dir", + "boards_dir", "lib_extra_dirs" + ] - KNOWN_ENV_OPTIONS = ("platform", "framework", "board", "build_flags", - "src_build_flags", "build_unflags", "src_filter", - "extra_scripts", "targets", "upload_port", - "upload_protocol", "upload_speed", "upload_flags", - "upload_resetmethod", "lib_deps", "lib_ignore", - "lib_extra_dirs", "lib_ldf_mode", "lib_compat_mode", - "lib_archive", "piotest", "test_transport", - "test_filter", "test_ignore", "test_port", - "test_speed", "debug_tool", "debug_port", - "debug_init_cmds", "debug_extra_cmds", "debug_server", - "debug_init_break", "debug_load_cmd", - "debug_load_mode", "debug_svd_path", "monitor_port", - "monitor_speed", "monitor_rts", "monitor_dtr") + KNOWN_ENV_OPTIONS = [ + "platform", "framework", "board", "build_flags", "src_build_flags", + "build_unflags", "src_filter", "extra_scripts", "targets", + "upload_port", "upload_protocol", "upload_speed", "upload_flags", + "upload_resetmethod", "lib_deps", "lib_ignore", "lib_extra_dirs", + "lib_ldf_mode", "lib_compat_mode", "lib_archive", "piotest", + "test_transport", "test_filter", "test_ignore", "test_port", + "test_speed", "test_build_project_src", "debug_tool", "debug_port", + "debug_init_cmds", "debug_extra_cmds", "debug_server", + "debug_init_break", "debug_load_cmd", "debug_load_mode", + "debug_svd_path", "monitor_port", "monitor_speed", "monitor_rts", + "monitor_dtr" + ] - IGNORE_BUILD_OPTIONS = ("test_transport", "test_filter", "test_ignore", - "test_port", "test_speed", "debug_port", - "debug_init_cmds", "debug_extra_cmds", - "debug_server", "debug_init_break", - "debug_load_cmd", "debug_load_mode", - "monitor_port", "monitor_speed", "monitor_rts", - "monitor_dtr") + IGNORE_BUILD_OPTIONS = [ + "test_transport", "test_filter", "test_ignore", "test_port", + "test_speed", "debug_port", "debug_init_cmds", "debug_extra_cmds", + "debug_server", "debug_init_break", "debug_load_cmd", + "debug_load_mode", "monitor_port", "monitor_speed", "monitor_rts", + "monitor_dtr" + ] REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"} From f130b5bfb6a5c5471e5e3000f4524754243162c1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Jul 2018 02:07:07 +0300 Subject: [PATCH 64/86] Disable requirements status temporary --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 4be7de65..bf87ba3e 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,6 @@ PlatformIO .. image:: https://ci.appveyor.com/api/projects/status/unnpw0n3c5k14btn/branch/develop?svg=true :target: https://ci.appveyor.com/project/ivankravets/platformio-core :alt: AppVeyor.CI Build Status -.. image:: https://requires.io/github/platformio/platformio-core/requirements.svg?branch=develop - :target: https://requires.io/github/platformio/platformio-core/requirements/?branch=develop - :alt: Requirements Status .. image:: https://img.shields.io/pypi/v/platformio.svg :target: https://pypi.python.org/pypi/platformio/ :alt: Latest Version From 7c2c0ba1aa6a97fc31f3b8236539e9e5328fc100 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Jul 2018 02:09:40 +0300 Subject: [PATCH 65/86] Bump version to 3.6.0a9 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index f3132dbe..6ddc35e0 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a8") +VERSION = (3, 6, "0a9") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index a4ff9be0..1218f5e8 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,7 +21,7 @@ from platformio import __version__, exception, util from platformio.managers.package import PackageManager CORE_PACKAGES = { - "contrib-piohome": "^1.0.0", + "contrib-piohome": "^1.0.1", "contrib-pysite": ">=0.3.2,<2", "tool-pioplus": "^1.4.0", "tool-unity": "~1.20403.0", From 6e8ce5620678011916accf3b69dab0238acd8caf Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Jul 2018 02:10:50 +0300 Subject: [PATCH 66/86] Disable requirements status temporary --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index bf87ba3e..cbbf0cc1 100644 --- a/README.rst +++ b/README.rst @@ -61,7 +61,7 @@ Open Source PIO Plus -------- -* `PIO Remote™ `_ +* `PIO Remote `_ * `PIO Unified Debugger `_ * `PIO Unit Testing `_ * `Cloud IDEs Integration `_ @@ -83,7 +83,7 @@ Development Platforms * `Espressif 32 `_ * `Espressif 8266 `_ * `Freescale Kinetis `_ -* * `Infineon XMC `_ +* `Infineon XMC `_ * `Intel ARC32 `_ * `Lattice iCE40 `_ * `Maxim 32 `_ From a206b2e4fd420318be5cceea310776d781c043a7 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Jul 2018 13:35:06 +0300 Subject: [PATCH 67/86] Typo fix --- platformio/builder/tools/platformio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index d880d386..12baebe0 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -65,7 +65,7 @@ def _build_project_deps(env): if is_test: projenv.BuildSources("$BUILDTEST_DIR", "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER") - if not is_test or env['TEST_BUILD_PROJECT_SRC'] == "true": + if not is_test or env.get("TEST_BUILD_PROJECT_SRC") == "true": projenv.BuildSources("$BUILDSRC_DIR", "$PROJECTSRC_DIR", env.get("SRC_FILTER")) From 72bfa4a1e2f44c737d895d8d28b1309546032b28 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 20 Jul 2018 13:35:52 +0300 Subject: [PATCH 68/86] Bump version to 3.6.0a10 --- platformio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 6ddc35e0..1458749c 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a9") +VERSION = (3, 6, "0a10") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" From 42540d42075d59bcb57f1b8e9ba949d2dc526d65 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 23 Jul 2018 22:49:25 +0300 Subject: [PATCH 69/86] Docs: Fix broken example for "Dynamic build flags" --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index e2505d2f..201c711e 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e2505d2fa9af757f9984417edae98c271e1c52ee +Subproject commit 201c711ed3f643eb5baf9e9703d65cb078a0986d From 9b15ec417b962d2821ecd535f584105ff23479fd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 25 Jul 2018 15:58:47 +0300 Subject: [PATCH 70/86] Document solution for "ImportError: cannot import name _remove_dead_weakref" --- docs | 2 +- platformio/commands/upgrade.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index 201c711e..acf6c86c 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 201c711ed3f643eb5baf9e9703d65cb078a0986d +Subproject commit acf6c86c84bcf77f006c5c7d98ad3e02593d6b22 diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 3c54764b..019643b3 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -150,7 +150,7 @@ def get_develop_latest_version(): def get_pypi_latest_version(): r = requests.get( - "https://pypi.python.org/pypi/platformio/json", + "https://pypi.org/pypi/platformio/json", headers=util.get_request_defheaders()) r.raise_for_status() return r.json()['info']['version'] From 5aaa9cf20551cdda79a1ca15e3b54ae9d4c5099a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 25 Jul 2018 22:25:47 +0300 Subject: [PATCH 71/86] Fix "Home: Internal Store Exception" // Resolve #1756 --- docs | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs b/docs index acf6c86c..20aa8266 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit acf6c86c84bcf77f006c5c7d98ad3e02593d6b22 +Subproject commit 20aa8266cb71d9107644eddd3b82105fe18df709 diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 1218f5e8..7adf504b 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,7 +21,7 @@ from platformio import __version__, exception, util from platformio.managers.package import PackageManager CORE_PACKAGES = { - "contrib-piohome": "^1.0.1", + "contrib-piohome": "^1.0.2", "contrib-pysite": ">=0.3.2,<2", "tool-pioplus": "^1.4.0", "tool-unity": "~1.20403.0", From 2004c9b0798167313c2e674a462aa554e1263706 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 26 Jul 2018 20:46:21 +0300 Subject: [PATCH 72/86] Fix "test_lib" test --- tests/commands/test_lib.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index e4db2e7e..4c9705fd 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -166,14 +166,12 @@ def test_global_lib_list(clirunner, validate_cliresult): assert sorted(items1) == sorted(items2) versions1 = [ - "{name}@{version}".format(**item) - for item in json.loads(result.output) + "{name}@{version}".format(**item) for item in json.loads(result.output) ] versions2 = [ 'ArduinoJson@5.8.2', 'ArduinoJson@5.10.1', 'AsyncMqttClient@0.8.2', - 'AsyncTCP@1.0.1', 'ESPAsyncTCP@1.1.3', 'NeoPixelBus@2.2.4', - 'PJON@07fe9aa', 'PJON@1fb26fd', 'PubSubClient@bef5814', - 'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0' + 'AsyncTCP@1.0.1', 'NeoPixelBus@2.2.4', 'PJON@07fe9aa', 'PJON@1fb26fd', + 'PubSubClient@bef5814', 'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0' ] assert set(versions1) >= set(versions2) From 89fc77d87a8a974644a2edcc623811c67c7a98f4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 26 Jul 2018 20:46:43 +0300 Subject: [PATCH 73/86] Docs: Add Community link in TOP menu --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 20aa8266..5d938eee 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 20aa8266cb71d9107644eddd3b82105fe18df709 +Subproject commit 5d938eee907df5b9dda02f9820a44c1105746634 From fedf3162f1edb37f0c0699e854d15089a1b03876 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 26 Jul 2018 20:46:21 +0300 Subject: [PATCH 74/86] Fix "test_lib" test --- tests/commands/test_lib.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index e4db2e7e..4c9705fd 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -166,14 +166,12 @@ def test_global_lib_list(clirunner, validate_cliresult): assert sorted(items1) == sorted(items2) versions1 = [ - "{name}@{version}".format(**item) - for item in json.loads(result.output) + "{name}@{version}".format(**item) for item in json.loads(result.output) ] versions2 = [ 'ArduinoJson@5.8.2', 'ArduinoJson@5.10.1', 'AsyncMqttClient@0.8.2', - 'AsyncTCP@1.0.1', 'ESPAsyncTCP@1.1.3', 'NeoPixelBus@2.2.4', - 'PJON@07fe9aa', 'PJON@1fb26fd', 'PubSubClient@bef5814', - 'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0' + 'AsyncTCP@1.0.1', 'NeoPixelBus@2.2.4', 'PJON@07fe9aa', 'PJON@1fb26fd', + 'PubSubClient@bef5814', 'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0' ] assert set(versions1) >= set(versions2) From 9191ea97feb80e66698842c723854027a7151cb9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Jul 2018 01:27:05 +0300 Subject: [PATCH 75/86] Fix an issue when "srcFilter" field in "library.json" breaks a library build // Resolve #1735 --- HISTORY.rst | 4 ++++ platformio/builder/tools/piolib.py | 9 --------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 34bddd26..4fd6b098 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,10 @@ PlatformIO 3.0 - Force building of project source code using `test_build_project_src `__ option - Fixed missed ``UNIT_TEST`` macro for unit test components/libraries +* Fixed an issue when ``srcFilter`` field in `library.json `__ + breaks a library build + (`issue #1735 `_) + 3.5.4 (2018-07-03) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 787b3bdf..04d595b0 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -586,15 +586,6 @@ class PlatformIOLibBuilder(LibBuilderBase): def _is_arduino_manifest(self): return isfile(join(self.path, "library.properties")) - @property - def src_dir(self): - if all([ - "srcFilter" in self._manifest.get("build", {}) - or self.env['SRC_FILTER'], not self._is_arduino_manifest() - ]): - return self.path - return LibBuilderBase.src_dir.fget(self) - @property def src_filter(self): if "srcFilter" in self._manifest.get("build", {}): From 98aa47c885839e5f79c570df17cf6afb12b2d408 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Jul 2018 15:38:46 +0300 Subject: [PATCH 76/86] Drop support for codecov.io --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 523931ca..589289f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,10 +29,6 @@ install: script: - tox -e $TOX_ENV -after_success: - - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then tox -e coverage; fi - - if [[ "$TOX_ENV" == "py27" ]] && [[ "$TRAVIS_OS_NAME" == "linux" ]]; then bash <(curl -s https://codecov.io/bash); fi - notifications: email: false From d9e908fceb16f0e908af67dcf3ad4164fb12352e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 27 Jul 2018 15:45:51 +0300 Subject: [PATCH 77/86] Test 3 random examples from each dev/platform --- tests/test_examples.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 20e9cb63..1e873dc6 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -42,10 +42,15 @@ def pytest_generate_tests(metafunc): project_dirs = [] for examples_dir in examples_dirs: + platform_examples = [] for root, _, files in walk(examples_dir): if "platformio.ini" not in files or ".skiptest" in files: continue - project_dirs.append(root) + platform_examples.append(root) + + # test random 3 examples + random.shuffle(platform_examples) + project_dirs.extend(platform_examples[:3]) project_dirs.sort() metafunc.parametrize("pioproject_dir", project_dirs) From 2f2cfc2d84986965bce256939ba19bb5e10d87eb Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 28 Jul 2018 12:49:22 +0300 Subject: [PATCH 78/86] Fix broken unit testing when mbed framework is used --- platformio/builder/tools/platformio.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 12baebe0..a3677e26 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -95,8 +95,6 @@ def BuildProgram(env): if "__debug" in COMMAND_LINE_TARGETS: env.ProcessDebug() - if "__test" in COMMAND_LINE_TARGETS: - env.ProcessTest() # process extra flags from board if "BOARD" in env and "build.extra_flags" in env.BoardConfig(): @@ -114,6 +112,9 @@ def BuildProgram(env): # remove specified flags env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) + if "__test" in COMMAND_LINE_TARGETS: + env.ProcessTest() + # build project with dependencies _build_project_deps(env) From 9a5f9843b962b107a60f3725b6cb13b6f73c455e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Sat, 28 Jul 2018 12:51:52 +0300 Subject: [PATCH 79/86] Bump version to 3.6.0a11 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index 1458749c..d72d9f15 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a10") +VERSION = (3, 6, "0a11") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 7adf504b..7faf4efd 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -23,7 +23,7 @@ from platformio.managers.package import PackageManager CORE_PACKAGES = { "contrib-piohome": "^1.0.2", "contrib-pysite": ">=0.3.2,<2", - "tool-pioplus": "^1.4.0", + "tool-pioplus": "^1.4.2", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.4" } From ae964fa7292439da8899cac54478591791b8ccd9 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Jul 2018 14:26:24 +0300 Subject: [PATCH 80/86] Fixed an issue with PIO Remote when upload process depends on the source code of a project framework --- HISTORY.rst | 2 ++ platformio/builder/tools/platformio.py | 3 ++- platformio/managers/platform.py | 13 +++++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 4fd6b098..5d42a5d5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,8 @@ PlatformIO 3.0 - Force building of project source code using `test_build_project_src `__ option - Fixed missed ``UNIT_TEST`` macro for unit test components/libraries +* Fixed an issue with `PIO Remote `__ + when upload process depends on the source code of a project framework * Fixed an issue when ``srcFilter`` field in `library.json `__ breaks a library build (`issue #1735 `_) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index a3677e26..b8e17fcd 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -306,7 +306,8 @@ def BuildFrameworks(env, frameworks): if f in ("arduino", "energia"): # Arduino IDE appends .o the end of filename Builder.match_splitext = scons_patched_match_splitext - env.ConvertInoToCpp() + if "nobuild" not in COMMAND_LINE_TARGETS: + env.ConvertInoToCpp() if f in board_frameworks: SConscript(env.GetFrameworkScript(f), exports="env") diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 5a947c4c..210dd5f5 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -604,12 +604,13 @@ class PlatformBase( # pylint: disable=too-many-public-methods # enable upload tools for upload targets if any(["upload" in t for t in targets] + ["program" in targets]): - for _name, _opts in self.packages.iteritems(): - if _opts.get("type") == "uploader": - self.packages[_name]['optional'] = False - elif "nobuild" in targets: - # skip all packages, allow only upload tools - self.packages[_name]['optional'] = True + for name, opts in self.packages.iteritems(): + if opts.get("type") == "uploader": + self.packages[name]['optional'] = False + # skip all packages in "nobuild" mode + # allow only upload tools and frameworks + elif "nobuild" in targets and opts.get("type") != "framework": + self.packages[name]['optional'] = True def get_lib_storages(self): storages = [] From 617f51b9eac0c68d26bc067847fc6ca824915cb1 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Jul 2018 18:13:04 +0300 Subject: [PATCH 81/86] Improve docs for PIO Unified Debugger --- docs | 2 +- examples | 2 +- scripts/docspregen.py | 65 +++++++++++++++++++++++++++++-------------- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/docs b/docs index 5d938eee..19b51562 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 5d938eee907df5b9dda02f9820a44c1105746634 +Subproject commit 19b51562c921c70cf1e89469a303de955dc9bedd diff --git a/examples b/examples index 965ab1cb..40bdd9e1 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 965ab1cb2852d06b652b49944830d2f81d563db1 +Subproject commit 40bdd9e1b4829d37923438d76378d9e3eb7584ca diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 9da8ac18..ce940ff6 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -118,7 +118,7 @@ def generate_boards(boards, extend_debug=False, skip_columns=None): return lines -def generate_debug_boards(boards, skip_columns=None): +def generate_debug_contents(boards, skip_board_columns=None, extra_rst=None): lines = [] onboard_debug = [ b for b in boards if b['debug'] and any( @@ -127,40 +127,61 @@ def generate_debug_boards(boards, skip_columns=None): external_debug = [ b for b in boards if b['debug'] and b not in onboard_debug ] - if onboard_debug or external_debug: - lines.append(""" + if not onboard_debug and not external_debug: + return lines + + lines.append(""" Debugging --------- :ref:`piodebug` - "1-click" solution for debugging with a zero configuration. +.. contents:: + :local: +""") + if extra_rst: + lines.append(".. include:: %s" % extra_rst) + + lines.append(""" +Debug Tools +~~~~~~~~~~~ + Supported debugging tools are listed in "Debug" column. For more detailed information, please scroll table by horizontal. You can switch between debugging :ref:`debugging_tools` using :ref:`projectconf_debug_tool` options. + +.. warning:: + You will need to install debug tool drivers depending on your system. + Please click on compatible debug tool below for the further instructions. """) + if onboard_debug: lines.append(""" On-Board Debug Tools -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ Boards listed below have on-board debug tool and **ARE READY** for debugging! You do not need to use/buy external debug tool. """) lines.extend( generate_boards( - onboard_debug, extend_debug=True, skip_columns=skip_columns)) + onboard_debug, + extend_debug=True, + skip_columns=skip_board_columns)) if external_debug: lines.append(""" External Debug Tools -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ Boards listed below are compatible with :ref:`piodebug` but **DEPEND ON** external debug tool. See "Debug" column for compatible debug tools. """) lines.extend( generate_boards( - external_debug, extend_debug=True, skip_columns=skip_columns)) + external_debug, + extend_debug=True, + skip_columns=skip_board_columns)) return lines @@ -218,7 +239,7 @@ Packages return "\n".join(lines) -def generate_platform(name, has_extra=False): +def generate_platform(name, rst_dir): print "Processing platform: %s" % name compatible_boards = [ @@ -263,7 +284,7 @@ For more detailed information please visit `vendor site <%s>`_.""" % # # Extra # - if has_extra: + if isfile(join(rst_dir, "%s_extra.rst" % name)): lines.append(".. include:: %s_extra.rst" % p.name) # @@ -288,8 +309,11 @@ Examples are listed from `%s development platform repository <%s>`_: # if compatible_boards: lines.extend( - generate_debug_boards( - compatible_boards, skip_columns=["Platform"])) + generate_debug_contents( + compatible_boards, + skip_board_columns=["Platform"], + extra_rst="%s_debug.rst" % name + if isfile(join(rst_dir, "%s_debug.rst" % name)) else None)) # # Development version of dev/platform @@ -393,12 +417,10 @@ def update_platform_docs(): dirname(realpath(__file__)), "..", "docs", "platforms") rst_path = join(platforms_dir, "%s.rst" % name) with open(rst_path, "w") as f: - f.write( - generate_platform( - name, isfile(join(platforms_dir, "%s_extra.rst" % name)))) + f.write(generate_platform(name, platforms_dir)) -def generate_framework(type_, data, has_extra=False): +def generate_framework(type_, data, rst_dir=None): print "Processing framework: %s" % type_ compatible_platforms = [ @@ -442,14 +464,18 @@ For more detailed information please visit `vendor site <%s>`_. :depth: 1""") # Extra - if has_extra: + if isfile(join(rst_dir, "%s_extra.rst" % type_)): lines.append(".. include:: %s_extra.rst" % type_) # # Debugging # if compatible_boards: - lines.extend(generate_debug_boards(compatible_boards)) + lines.extend( + generate_debug_contents( + compatible_boards, + extra_rst="%s_debug.rst" % type_ + if isfile(join(rst_dir, "%s_debug.rst" % type_)) else None)) if compatible_platforms: # examples @@ -514,10 +540,7 @@ def update_framework_docs(): dirname(realpath(__file__)), "..", "docs", "frameworks") rst_path = join(frameworks_dir, "%s.rst" % name) with open(rst_path, "w") as f: - f.write( - generate_framework( - name, framework, - isfile(join(frameworks_dir, "%s_extra.rst" % name)))) + f.write(generate_framework(name, framework, frameworks_dir)) def update_embedded_boards(): From ab1d1f248ca60b277a633ae91f12ba69ea87bc7d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 30 Jul 2018 19:39:31 +0300 Subject: [PATCH 82/86] Refactor docs for PIO Unified debug tools --- docs | 2 +- scripts/docspregen.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs b/docs index 19b51562..4e02d8a8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 19b51562c921c70cf1e89469a303de955dc9bedd +Subproject commit 4e02d8a81e15dc14e7a561318adc6056e91da5c3 diff --git a/scripts/docspregen.py b/scripts/docspregen.py index ce940ff6..4fd8c2b3 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -632,9 +632,9 @@ def update_debugging(): end = end_tpl + tool begin_index = content.index(begin) end_index = content.index(end) - chunk = ["\n\n**Compatible development platforms:**\n"] + chunk = ["\n\n:Compatible Platforms:\n"] chunk.extend([ - "* :ref:`platform_%s`" % str(p) + " * :ref:`platform_%s`" % str(p) for p in sorted(set(platforms)) ]) chunk.extend(["\n"]) From 8f19dd50fe245e7eb4f98469b2ef665f22e497f4 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 2 Aug 2018 17:57:57 +0300 Subject: [PATCH 83/86] Docs: Fix "build_flags" examples with a macro where special chars are used --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 4e02d8a8..05a4f485 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4e02d8a81e15dc14e7a561318adc6056e91da5c3 +Subproject commit 05a4f485243dba712a2d7a56fd551cbbd0bf79e2 From df923bf17e4ca058cef6ce919dbd5392e2fa67dd Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Aug 2018 21:43:40 +0300 Subject: [PATCH 84/86] Add package "lib" folder to LD_PATH --- platformio/builder/tools/pioplatform.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 340ebd37..0fe4aeda 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -69,15 +69,22 @@ def LoadPioPlatform(env, variables): # Ensure real platform name env['PIOPLATFORM'] = p.name - # Add toolchains and uploaders to $PATH + # Add toolchains and uploaders to $PATH and $*_LIBRARY_PATH + systype = util.get_systype() for name in installed_packages: type_ = p.get_package_type(name) if type_ not in ("toolchain", "uploader", "debugger"): continue - path = p.get_package_dir(name) - if isdir(join(path, "bin")): - path = join(path, "bin") - env.PrependENVPath("PATH", path) + pkg_dir = p.get_package_dir(name) + env.PrependENVPath( + "PATH", + join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir) + if ("windows" not in systype and isdir(join(pkg_dir, "lib")) + and type_ != "toolchain"): + env.PrependENVPath( + "DYLD_LIBRARY_PATH" + if "darwin" in systype else "LD_LIBRARY_PATH", + join(pkg_dir, "lib")) # Platform specific LD Scripts if isdir(join(p.get_dir(), "ldscripts")): From 129146e82e9c7807092aa369ee68c9fec57fecee Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 3 Aug 2018 21:44:41 +0300 Subject: [PATCH 85/86] Bump version to 3.6.0rc1 --- platformio/__init__.py | 2 +- platformio/managers/core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/__init__.py b/platformio/__init__.py index d72d9f15..c7f83f1e 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0a11") +VERSION = (3, 6, "0rc1") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 7faf4efd..ef185cf9 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -23,7 +23,7 @@ from platformio.managers.package import PackageManager CORE_PACKAGES = { "contrib-piohome": "^1.0.2", "contrib-pysite": ">=0.3.2,<2", - "tool-pioplus": "^1.4.2", + "tool-pioplus": "^1.4.5", "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.4" } From 6d5dc60b471ca5a295cbcead01df84b18e619cba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 6 Aug 2018 18:43:58 +0300 Subject: [PATCH 86/86] Bump version to 3.6.0 (issues #1594 #1412 #1462 #1735) --- HISTORY.rst | 18 +++++++----------- docs | 2 +- platformio/__init__.py | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6e2cde6d..39ce1ab7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,7 +4,7 @@ Release Notes PlatformIO 3.0 -------------- -3.6.0 (2018-??-??) +3.6.0 (2018-08-06) ~~~~~~~~~~~~~~~~~~ * `Program Memory Usage `_ @@ -15,22 +15,18 @@ PlatformIO 3.0 - Check maximum allowed "program" and "data" sizes before uploading/programming (`issue #1412 `_) -* Check package structure after unpacking and raise error when antivirus tool - blocks PlatformIO package manager - (`issue #1462 `_) -* Lock interprocess requests to PlatformIO Package Manager for - install/uninstall operations - (`issue #1594 `_) - -3.5.5 (2018-??-??) -~~~~~~~~~~~~~~~~~~ - * `PIO Unit Testing `__: - Documented `Project Shared Code `__ - Force building of project source code using `test_build_project_src `__ option - Fixed missed ``UNIT_TEST`` macro for unit test components/libraries +* Check package structure after unpacking and raise error when antivirus tool + blocks PlatformIO package manager + (`issue #1462 `_) +* Lock interprocess requests to PlatformIO Package Manager for + install/uninstall operations + (`issue #1594 `_) * Fixed an issue with `PIO Remote `__ when upload process depends on the source code of a project framework * Fixed an issue when ``srcFilter`` field in `library.json `__ diff --git a/docs b/docs index 05a4f485..21c1cf52 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 05a4f485243dba712a2d7a56fd551cbbd0bf79e2 +Subproject commit 21c1cf522ce5e1cf7c04914ba7dbaf33c8cf463d diff --git a/platformio/__init__.py b/platformio/__init__.py index c7f83f1e..f86af5f0 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 6, "0rc1") +VERSION = (3, 6, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio"