diff --git a/.gitignore b/.gitignore index 0a54e681..07c7b427 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ build coverage.xml .coverage htmlcov +.pytest_cache diff --git a/HISTORY.rst b/HISTORY.rst index b672ab6c..4bcb68f6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,38 @@ Release Notes PlatformIO 3.0 -------------- +3.5.3 (2018-06-01) +~~~~~~~~~~~~~~~~~~ + +* `PlatformIO Home `__ - + interact with PlatformIO ecosystem using modern and cross-platform GUI: + + - "Recent News" block on "Welcome" page + - Direct import of development platform's example + +* Simplify configuration for `PIO Unit Testing `__: separate main program from a test build process, drop + requirement for ``#ifdef UNIT_TEST`` guard +* Override any option from board manifest in `Project Configuration File "platformio.ini" `__ + (`issue #1612 `_) +* Configure a custom path to SVD file using `debug_svd_path `__ + option +* Custom project `description `_ + which will be used by `PlatformIO Home `_ +* Updated Unity tool to 2.4.3 +* Improved support for Black Magic Probe in "uploader" mode +* Renamed "monitor_baud" option to "monitor_speed" +* Fixed issue when a custom `lib_dir `__ + was not handled correctly + (`issue #1473 `_) +* Fixed issue with useless project rebuilding for case insensitive file + systems (Windows) +* Fixed issue with ``build_unflags`` option when a macro contains value + (e.g., ``-DNAME=VALUE``) +* Fixed issue which did not allow to override runtime build environment using + extra POST script +* Fixed "RuntimeError: maximum recursion depth exceeded" for library manager + (`issue #1528 `_) + 3.5.2 (2018-03-13) ~~~~~~~~~~~~~~~~~~ diff --git a/docs b/docs index e9e78d04..3ad76be8 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit e9e78d043e4a1ba699f3e8e2b12bfeeeb18b1bd7 +Subproject commit 3ad76be8f73ab1b3766bafa7ffca4284051aca4c diff --git a/examples b/examples index db8b4f3c..41f3396c 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit db8b4f3c77cf9694986eb55280038c37dd43548a +Subproject commit 41f3396c5883d54e6cc5603d380400ca46df8876 diff --git a/platformio/__init__.py b/platformio/__init__.py index 1a98a778..6de67c99 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 5, 2) +VERSION = (3, 5, 3) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/app.py b/platformio/app.py index 8ca1ba4c..06ca7cfb 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import codecs import hashlib import json import os @@ -106,7 +107,7 @@ class State(object): def __exit__(self, type_, value, traceback): if self._prev_state != self._state: try: - with open(self.path, "w") as fp: + with codecs.open(self.path, "w", encoding="utf8") as fp: if "dev" in __version__: json.dump(self._state, fp, indent=4) else: @@ -187,11 +188,8 @@ class ContentCache(object): cache_path = self.get_cache_path(key) if not isfile(cache_path): return None - with open(cache_path, "rb") as fp: - data = fp.read() - if data and data[0] in ("{", "["): - return json.loads(data) - return data + with codecs.open(cache_path, "rb", encoding="utf8") as fp: + return fp.read() def set(self, key, data, valid): if not get_setting("enable_cache"): @@ -212,13 +210,17 @@ class ContentCache(object): if not isdir(dirname(cache_path)): os.makedirs(dirname(cache_path)) - with open(cache_path, "wb") as fp: - if isinstance(data, (dict, list)): - json.dump(data, fp) - else: - fp.write(str(data)) - with open(self._db_path, "a") as fp: - fp.write("%s=%s\n" % (str(expire_time), cache_path)) + try: + with codecs.open(cache_path, "wb", encoding="utf8") as fp: + fp.write(data) + with open(self._db_path, "a") as fp: + fp.write("%s=%s\n" % (str(expire_time), cache_path)) + except UnicodeError: + if isfile(cache_path): + try: + remove(cache_path) + except OSError: + pass return self._unlock_dbindex() diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 04a19eba..caafda5b 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -54,10 +54,12 @@ commonvars.AddVariables( # board options ("BOARD",), + # deprecated options, use board_{object.path} instead ("BOARD_MCU",), ("BOARD_F_CPU",), ("BOARD_F_FLASH",), ("BOARD_FLASH_MODE",), + # end of deprecated options # upload options ("UPLOAD_PORT",), @@ -68,7 +70,7 @@ commonvars.AddVariables( # debug options ("DEBUG_TOOL",), - + ("DEBUG_SVD_PATH",), ) # yapf: disable @@ -99,6 +101,7 @@ DEFAULT_ENV_OPTIONS = dict( BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"), BUILDSRC_DIR=join("$BUILD_DIR", "src"), BUILDTEST_DIR=join("$BUILD_DIR", "test"), + LIBPATH=["$BUILD_DIR"], LIBSOURCE_DIRS=[ util.get_projectlib_dir(), util.get_projectlibdeps_dir(), @@ -156,7 +159,7 @@ env.LoadPioPlatform(commonvars) env.SConscriptChdir(0) env.SConsignFile(join("$PROJECTBUILD_DIR", ".sconsign.dblite")) -for item in env.GetPreExtraScripts(): +for item in env.GetExtraScripts("pre"): env.SConscript(item, exports="env") env.SConscript("$BUILD_SCRIPT") @@ -165,9 +168,9 @@ AlwaysBuild(env.Alias("__debug", DEFAULT_TARGETS + ["size"])) AlwaysBuild(env.Alias("__test", DEFAULT_TARGETS + ["size"])) if "UPLOAD_FLAGS" in env: - env.Append(UPLOADERFLAGS=["$UPLOAD_FLAGS"]) + env.Prepend(UPLOADERFLAGS=["$UPLOAD_FLAGS"]) -for item in env.GetPostExtraScripts(): +for item in env.GetExtraScripts("post"): env.SConscript(item, exports="env") if "envdump" in COMMAND_LINE_TARGETS: diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index e3f98ca4..89b49e3d 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -16,7 +16,7 @@ from __future__ import absolute_import from glob import glob from os import environ -from os.path import join +from os.path import abspath, isfile, join from SCons.Defaults import processDefines @@ -53,11 +53,11 @@ def _dump_includes(env): if unity_dir: includes.append(unity_dir) - # remove dupicates + # remove duplicates result = [] for item in includes: if item not in result: - result.append(item) + result.append(abspath(item)) return result @@ -101,12 +101,34 @@ def _dump_defines(env): .replace("ATMEGA", "ATmega").replace("ATTINY", "ATtiny"))) # built-in GCC marcos - if env.GetCompilerType() == "gcc": - defines.extend(_get_gcc_defines(env)) + # if env.GetCompilerType() == "gcc": + # defines.extend(_get_gcc_defines(env)) return defines +def _get_svd_path(env): + svd_path = env.subst("$DEBUG_SVD_PATH") + if svd_path: + return abspath(svd_path) + + if "BOARD" not in env: + return None + try: + svd_path = env.BoardConfig().get("debug.svd_path") + assert svd_path + except (AssertionError, KeyError): + return None + # custom path to SVD file + if isfile(svd_path): + return svd_path + # default file from ./platform/misc/svd folder + p = env.PioPlatform() + if isfile(join(p.get_dir(), "misc", "svd", svd_path)): + return abspath(join(p.get_dir(), "misc", "svd", svd_path)) + return None + + def DumpIDEData(env): LINTCCOM = "$CFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS" LINTCXXCOM = "$CXXFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS" @@ -130,6 +152,8 @@ def DumpIDEData(env): util.where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")), "prog_path": env.subst("$PROG_PATH"), + "svd_path": + _get_svd_path(env), "compiler_type": env.GetCompilerType() } diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index b865771b..9048ea1c 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -27,7 +27,7 @@ from os.path import (basename, commonprefix, dirname, isdir, isfile, join, import SCons.Scanner from SCons.Script import ARGUMENTS, COMMAND_LINE_TARGETS, DefaultEnvironment -from platformio import util +from platformio import exception, util from platformio.builder.tools import platformio as piotool from platformio.managers.lib import LibraryManager from platformio.managers.package import PackageManager @@ -86,8 +86,8 @@ class LibBuilderBase(object): LDF_MODES = ["off", "chain", "deep", "chain+", "deep+"] LDF_MODE_DEFAULT = "chain" - COMPAT_MODES = ["off", "light", "strict"] - COMPAT_MODE_DEFAULT = "light" + COMPAT_MODES = ["off", "soft", "strict"] + COMPAT_MODE_DEFAULT = "soft" CLASSIC_SCANNER = SCons.Scanner.C.CScanner() CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner() @@ -758,7 +758,7 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches sys.stderr.write( "Platform incompatible library %s\n" % lb.path) return False - if compat_mode == "light" and "PIOFRAMEWORK" in env and \ + if compat_mode == "soft" and "PIOFRAMEWORK" in env and \ not lb.is_frameworks_compatible(env.get("PIOFRAMEWORK", [])): if verbose: sys.stderr.write( @@ -777,7 +777,7 @@ def GetLibBuilders(env): # pylint: disable=too-many-branches try: lb = LibBuilderFactory.new( env, join(libs_dir, item), verbose=verbose) - except ValueError: + except exception.InvalidJSONFile: if verbose: sys.stderr.write("Skip library with broken manifest: %s\n" % join(libs_dir, item)) diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 7c08237f..b01ea1a2 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -18,7 +18,7 @@ import atexit import re import sys from os import environ, remove, walk -from os.path import basename, isdir, isfile, join, relpath, sep +from os.path import basename, isdir, isfile, join, realpath, relpath, sep from tempfile import mkstemp from SCons.Action import Action @@ -199,7 +199,7 @@ def _delete_file(path): pass -@util.memoized +@util.memoized() def _get_compiler_type(env): try: sysenv = environ.copy() @@ -295,25 +295,21 @@ def ProcessTest(env): src_filter.append("+<%s%s>" % (env['PIOTEST'], sep)) env.Replace(PIOTEST_SRC_FILTER=src_filter) - return env.CollectBuildFiles( - "$BUILDTEST_DIR", - "$PROJECTTEST_DIR", - "$PIOTEST_SRC_FILTER", - duplicate=False) + return env.CollectBuildFiles("$BUILDTEST_DIR", "$PROJECTTEST_DIR", + "$PIOTEST_SRC_FILTER") -def GetPreExtraScripts(env): - return [ - item[4:] for item in env.get("EXTRA_SCRIPTS", []) - if item.startswith("pre:") - ] - - -def GetPostExtraScripts(env): - return [ - item[5:] if item.startswith("post:") else item - for item in env.get("EXTRA_SCRIPTS", []) if not item.startswith("pre:") - ] +def GetExtraScripts(env, scope): + items = [] + for item in env.get("EXTRA_SCRIPTS", []): + if scope == "post" and ":" not in item: + items.append(item) + elif item.startswith("%s:" % scope): + items.append(item[len(scope) + 1:]) + if not items: + return items + with util.cd(env.subst("$PROJECT_DIR")): + return [realpath(item) for item in items] def exists(_): @@ -328,6 +324,5 @@ def generate(env): env.AddMethod(PioClean) env.AddMethod(ProcessDebug) env.AddMethod(ProcessTest) - env.AddMethod(GetPreExtraScripts) - env.AddMethod(GetPostExtraScripts) + env.AddMethod(GetExtraScripts) return env diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 6a77bc74..340ebd37 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -14,6 +14,7 @@ from __future__ import absolute_import +import base64 import sys from os.path import isdir, isfile, join @@ -22,8 +23,10 @@ from SCons.Script import COMMAND_LINE_TARGETS from platformio import exception, util from platformio.managers.platform import PlatformFactory +# pylint: disable=too-many-branches -@util.memoized + +@util.memoized() def initPioPlatform(name): return PlatformFactory.newPlatform(name) @@ -69,7 +72,7 @@ def LoadPioPlatform(env, variables): # Add toolchains and uploaders to $PATH for name in installed_packages: type_ = p.get_package_type(name) - if type_ not in ("toolchain", "uploader"): + if type_ not in ("toolchain", "uploader", "debugger"): continue path = p.get_package_dir(name) if isdir(join(path, "bin")): @@ -81,24 +84,37 @@ def LoadPioPlatform(env, variables): env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")]) if "BOARD" not in env: + # handle _MCU and _F_CPU variables for AVR native + for key, value in variables.UnknownVariables().items(): + if not key.startswith("BOARD_"): + continue + env.Replace( + **{key.upper().replace("BUILD.", ""): base64.b64decode(value)}) return + # update board manifest with a custom data board_config = env.BoardConfig() - for k in variables.keys(): - if k in env or \ - not any([k.startswith("BOARD_"), k.startswith("UPLOAD_")]): + for key, value in variables.UnknownVariables().items(): + if not key.startswith("BOARD_"): continue - _opt, _val = k.lower().split("_", 1) + board_config.update(key.lower()[6:], base64.b64decode(value)) + + # update default environment variables + for key in variables.keys(): + if key in env or \ + not any([key.startswith("BOARD_"), key.startswith("UPLOAD_")]): + continue + _opt, _val = key.lower().split("_", 1) if _opt == "board": _opt = "build" if _val in board_config.get(_opt): - env.Replace(**{k: board_config.get("%s.%s" % (_opt, _val))}) + env.Replace(**{key: board_config.get("%s.%s" % (_opt, _val))}) if "build.ldscript" in board_config: env.Replace(LDSCRIPT_PATH=board_config.get("build.ldscript")) -def PrintConfiguration(env): # pylint: disable=too-many-branches +def PrintConfiguration(env): platform_data = ["PLATFORM: %s >" % env.PioPlatform().title] system_data = ["SYSTEM:"] mcu = env.subst("$BOARD_MCU") diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index 7bc036fb..1ba523e4 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -130,10 +130,12 @@ def AutodetectUploadPort(*args, **kwargs): # pylint: disable=unused-argument if not _is_match_pattern(item['port']): continue port = item['port'] - if upload_protocol.startswith("blackmagic") \ - and "GDB" in item['description']: - return ("\\\\.\\%s" % port if "windows" in util.get_systype() - and port.startswith("COM") and len(port) > 4 else port) + if upload_protocol.startswith("blackmagic"): + if "windows" in util.get_systype() and \ + port.startswith("COM") and len(port) > 4: + port = "\\\\.\\%s" % port + if "GDB" in item['description']: + return port for hwid in board_hwids: hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "") if hwid_str in item['hwid']: @@ -220,7 +222,7 @@ def PrintUploadInfo(env): available.extend(env.BoardConfig().get("upload", {}).get( "protocols", [])) if available: - print "AVAILABLE: %s" % ", ".join(sorted(available)) + print "AVAILABLE: %s" % ", ".join(sorted(set(available))) if configured: print "CURRENT: upload_protocol = %s" % configured diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index da82de85..9c6f6d0d 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -20,7 +20,7 @@ from glob import glob from os import sep, walk from os.path import basename, dirname, isdir, join, realpath -from SCons import Action, Builder, Util +from SCons import Builder, Util from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild, DefaultEnvironment, SConscript) @@ -30,12 +30,11 @@ SRC_HEADER_EXT = ["h", "hpp"] SRC_C_EXT = ["c", "cc", "cpp"] SRC_BUILD_EXT = SRC_C_EXT + ["S", "spp", "SPP", "sx", "s", "asm", "ASM"] SRC_FILTER_DEFAULT = ["+<*>", "-<.git%s>" % sep, "-" % sep] +SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>") def scons_patched_match_splitext(path, suffixes=None): - """ - Patch SCons Builder, append $OBJSUFFIX to the end of each target - """ + """Patch SCons Builder, append $OBJSUFFIX to the end of each target""" tokens = Util.splitext(path) if suffixes and tokens[1] and tokens[1] in suffixes: return (path, tokens[1]) @@ -63,8 +62,6 @@ def BuildProgram(env): # process extra flags from board if "BOARD" in env and "build.extra_flags" in env.BoardConfig(): env.ProcessFlags(env.BoardConfig().get("build.extra_flags")) - # remove base flags - env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) # apply user flags env.ProcessFlags(env.get("BUILD_FLAGS")) @@ -74,6 +71,9 @@ def BuildProgram(env): # restore PIO macros if it was deleted by framework _append_pio_macros() + # remove specified flags + env.ProcessUnFlags(env.get("BUILD_UNFLAGS")) + # build dependent libs; place them before built-in libs env.Prepend(LIBS=env.BuildProjectLibraries()) @@ -90,16 +90,14 @@ def BuildProgram(env): # Handle SRC_BUILD_FLAGS env.ProcessFlags(env.get("SRC_BUILD_FLAGS")) - env.Append( - LIBPATH=["$BUILD_DIR"], - PIOBUILDFILES=env.CollectBuildFiles( - "$BUILDSRC_DIR", - "$PROJECTSRC_DIR", - src_filter=env.get("SRC_FILTER"), - duplicate=False)) - if "__test" in COMMAND_LINE_TARGETS: env.Append(PIOBUILDFILES=env.ProcessTest()) + else: + env.Append( + PIOBUILDFILES=env.CollectBuildFiles( + "$BUILDSRC_DIR", + "$PROJECTSRC_DIR", + src_filter=env.get("SRC_FILTER"))) if not env['PIOBUILDFILES'] and not COMMAND_LINE_TARGETS: sys.stderr.write( @@ -110,8 +108,8 @@ def BuildProgram(env): program = env.Program( join("$BUILD_DIR", env.subst("$PROGNAME")), env['PIOBUILDFILES']) - checksize_action = Action.Action(env.CheckUploadSize, - "Checking program size") + checksize_action = env.VerboseAction(env.CheckUploadSize, + "Checking program size") AlwaysBuild(env.Alias("checkprogsize", program, checksize_action)) if set(["upload", "program"]) & set(COMMAND_LINE_TARGETS): env.AddPostAction(program, checksize_action) @@ -119,38 +117,47 @@ def BuildProgram(env): return program -def ProcessFlags(env, flags): # pylint: disable=too-many-branches - if not flags: - return +def ParseFlagsExtended(env, flags): if isinstance(flags, list): flags = " ".join(flags) - parsed_flags = env.ParseFlags(str(flags)) - for flag in parsed_flags.pop("CPPDEFINES"): - if not Util.is_Sequence(flag): - env.Append(CPPDEFINES=flag) + result = env.ParseFlags(str(flags)) + + cppdefines = [] + for item in result['CPPDEFINES']: + if not Util.is_Sequence(item): + cppdefines.append(item) continue - _key, _value = flag[:2] - if '\"' in _value: - _value = _value.replace('\"', '\\\"') - elif _value.isdigit(): - _value = int(_value) - elif _value.replace(".", "", 1).isdigit(): - _value = float(_value) - env.Append(CPPDEFINES=(_key, _value)) - env.Append(**parsed_flags) + name, value = item[:2] + if '\"' in value: + value = value.replace('\"', '\\\"') + elif value.isdigit(): + value = int(value) + elif value.replace(".", "", 1).isdigit(): + value = float(value) + cppdefines.append((name, value)) + result['CPPDEFINES'] = cppdefines # fix relative CPPPATH & LIBPATH for k in ("CPPPATH", "LIBPATH"): - for i, p in enumerate(env.get(k, [])): + for i, p in enumerate(result.get(k, [])): if isdir(p): - env[k][i] = realpath(p) + result[k][i] = realpath(p) + # fix relative path for "-include" - for i, f in enumerate(env.get("CCFLAGS", [])): + for i, f in enumerate(result.get("CCFLAGS", [])): if isinstance(f, tuple) and f[0] == "-include": - env['CCFLAGS'][i] = (f[0], env.File(realpath(f[1].get_path()))) + result['CCFLAGS'][i] = (f[0], env.File(realpath(f[1].get_path()))) + + return result + + +def ProcessFlags(env, flags): # pylint: disable=too-many-branches + if not flags: + return + env.Append(**env.ParseFlagsExtended(flags)) # Cancel any previous definition of name, either built in or - # provided with a -D option // Issue #191 + # provided with a -U option // Issue #191 undefines = [ u for u in env.get("CCFLAGS", []) if isinstance(u, basestring) and u.startswith("-U") @@ -164,19 +171,16 @@ def ProcessFlags(env, flags): # pylint: disable=too-many-branches def ProcessUnFlags(env, flags): if not flags: return - if isinstance(flags, list): - flags = " ".join(flags) - parsed_flags = env.ParseFlags(str(flags)) - all_flags = [] - for items in parsed_flags.values(): - all_flags.extend(items) - all_flags = set(all_flags) - - for key in parsed_flags: - cur_flags = set(env.Flatten(env.get(key, []))) - for item in cur_flags & all_flags: - while item in env[key]: - env[key].remove(item) + for key, unflags in env.ParseFlagsExtended(flags).items(): + for unflag in unflags: + for current in env.get(key, []): + conditions = [ + unflag == current, + isinstance(current, (tuple, list)) + and unflag[0] == current[0] + ] + if any(conditions): + env[key].remove(current) def IsFileWithExt(env, file_, ext): # pylint: disable=W0613 @@ -190,8 +194,6 @@ def IsFileWithExt(env, file_, ext): # pylint: disable=W0613 def MatchSourceFiles(env, src_dir, src_filter=None): - SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>") - def _append_build_item(items, item, src_dir): if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT): items.add(item.replace(src_dir + sep, "")) @@ -281,15 +283,14 @@ def BuildFrameworks(env, frameworks): def BuildLibrary(env, variant_dir, src_dir, src_filter=None): - lib = env.Clone() - return lib.StaticLibrary( - lib.subst(variant_dir), - lib.CollectBuildFiles(variant_dir, src_dir, src_filter)) + return env.StaticLibrary( + env.subst(variant_dir), + env.CollectBuildFiles(variant_dir, src_dir, src_filter)) def BuildSources(env, variant_dir, src_dir, src_filter=None): - DefaultEnvironment().Append(PIOBUILDFILES=env.Clone().CollectBuildFiles( - variant_dir, src_dir, src_filter)) + DefaultEnvironment().Append( + PIOBUILDFILES=env.CollectBuildFiles(variant_dir, src_dir, src_filter)) def exists(_): @@ -298,6 +299,7 @@ def exists(_): def generate(env): env.AddMethod(BuildProgram) + env.AddMethod(ParseFlagsExtended) env.AddMethod(ProcessFlags) env.AddMethod(ProcessUnFlags) env.AddMethod(IsFileWithExt) diff --git a/platformio/commands/device.py b/platformio/commands/device.py index 877b62e6..78bb7cdc 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device.py @@ -165,8 +165,10 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches kwargs['environment']) monitor_options = {k: v for k, v in project_options or []} if monitor_options: - for k in ("port", "baud", "rts", "dtr"): + for k in ("port", "baud", "speed", "rts", "dtr"): k2 = "monitor_%s" % k + if k == "speed": + k = "baud" if kwargs[k] is None and k2 in monitor_options: kwargs[k] = monitor_options[k2] if k != "port": diff --git a/platformio/commands/init.py b/platformio/commands/init.py index dadf3e77..e7151287 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -139,15 +139,12 @@ def init_base_project(project_dir): join(util.get_source_dir(), "projectconftpl.ini"), join(project_dir, "platformio.ini")) - lib_dir = join(project_dir, "lib") - src_dir = join(project_dir, "src") - config = util.load_project_config(project_dir) - if config.has_option("platformio", "src_dir"): - src_dir = join(project_dir, config.get("platformio", "src_dir")) - - for d in (src_dir, lib_dir): - if not isdir(d): - makedirs(d) + with util.cd(project_dir): + lib_dir = util.get_projectlib_dir() + src_dir = util.get_projectsrc_dir() + for d in (src_dir, lib_dir): + if not isdir(d): + makedirs(d) init_lib_readme(lib_dir) init_ci_conf(project_dir) @@ -168,16 +165,21 @@ The source code of each library should be placed in separate directory, like For example, see how can be organized `Foo` and `Bar` libraries: |--lib +| | | |--Bar | | |--docs | | |--examples | | |--src | | |- Bar.c | | |- Bar.h +| | |- library.json (optional, custom build options, etc) http://docs.platformio.org/page/librarymanager/config.html +| | | |--Foo | | |- Foo.c | | |- Foo.h +| | | |- readme.txt --> THIS FILE +| |- platformio.ini |--src |- main.c diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index ee2ec01d..81400684 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -255,9 +255,10 @@ def lib_search(query, json_output, page, noninteractive, **filters): elif not click.confirm("Show next libraries?"): break result = get_api_result( - "/v2/lib/search", - {"query": " ".join(query), - "page": int(result['page']) + 1}, + "/v2/lib/search", { + "query": " ".join(query), + "page": int(result['page']) + 1 + }, cache_valid="1d") diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index f6307be5..8ec1c259 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -85,6 +85,7 @@ def _get_installed_platform_data(platform, homepage=p.homepage, repository=p.repository_url, url=p.vendor_url, + docs=p.docs_url, license=p.license, forDesktop=not p.is_embedded(), frameworks=sorted(p.frameworks.keys() if p.frameworks else []), diff --git a/platformio/commands/run.py b/platformio/commands/run.py index c0aab27f..f2d4636c 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -126,32 +126,31 @@ class EnvironmentProcessor(object): DEFAULT_DUMP_OPTIONS = ("platform", "framework", "board") - KNOWN_PLATFORMIO_OPTIONS = ("env_default", "home_dir", "lib_dir", - "libdeps_dir", "include_dir", "src_dir", - "build_dir", "data_dir", "test_dir", + KNOWN_PLATFORMIO_OPTIONS = ("description", "env_default", "home_dir", + "lib_dir", "libdeps_dir", "include_dir", + "src_dir", "build_dir", "data_dir", "test_dir", "boards_dir", "lib_extra_dirs") - KNOWN_ENV_OPTIONS = ("platform", "framework", "board", "board_mcu", - "board_f_cpu", "board_f_flash", "board_flash_mode", - "build_flags", "src_build_flags", "build_unflags", - "src_filter", "extra_scripts", "targets", - "upload_port", "upload_protocol", "upload_speed", - "upload_flags", "upload_resetmethod", "lib_deps", - "lib_ignore", "lib_extra_dirs", "lib_ldf_mode", - "lib_compat_mode", "lib_archive", "piotest", - "test_transport", "test_filter", "test_ignore", - "test_port", "test_speed", "debug_tool", "debug_port", + KNOWN_ENV_OPTIONS = ("platform", "framework", "board", "build_flags", + "src_build_flags", "build_unflags", "src_filter", + "extra_scripts", "targets", "upload_port", + "upload_protocol", "upload_speed", "upload_flags", + "upload_resetmethod", "lib_deps", "lib_ignore", + "lib_extra_dirs", "lib_ldf_mode", "lib_compat_mode", + "lib_archive", "piotest", "test_transport", + "test_filter", "test_ignore", "test_port", + "test_speed", "debug_tool", "debug_port", "debug_init_cmds", "debug_extra_cmds", "debug_server", "debug_init_break", "debug_load_cmd", - "debug_load_mode", "monitor_port", "monitor_baud", - "monitor_rts", "monitor_dtr") + "debug_load_mode", "debug_svd_path", "monitor_port", + "monitor_speed", "monitor_rts", "monitor_dtr") IGNORE_BUILD_OPTIONS = ("test_transport", "test_filter", "test_ignore", "test_port", "test_speed", "debug_port", "debug_init_cmds", "debug_extra_cmds", "debug_server", "debug_init_break", "debug_load_cmd", "debug_load_mode", - "monitor_port", "monitor_baud", "monitor_rts", + "monitor_port", "monitor_speed", "monitor_rts", "monitor_dtr") REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"} @@ -159,7 +158,12 @@ class EnvironmentProcessor(object): RENAMED_OPTIONS = { "lib_use": "lib_deps", "lib_force": "lib_deps", - "extra_script": "extra_scripts" + "extra_script": "extra_scripts", + "monitor_baud": "monitor_speed", + "board_mcu": "board_build.mcu", + "board_f_cpu": "board_build.f_cpu", + "board_f_flash": "board_build.f_flash", + "board_flash_mode": "board_build.flash_mode" } RENAMED_PLATFORMS = {"espressif": "espressif8266"} @@ -237,7 +241,11 @@ class EnvironmentProcessor(object): v = self.RENAMED_PLATFORMS[v] # warn about unknown options - if k not in self.KNOWN_ENV_OPTIONS and not k.startswith("custom_"): + unknown_conditions = [ + k not in self.KNOWN_ENV_OPTIONS, not k.startswith("custom_"), + not k.startswith("board_") + ] + if all(unknown_conditions): click.secho( "Detected non-PlatformIO `%s` option in `[env:%s]` section" % (k, self.name), @@ -411,7 +419,7 @@ def check_project_envs(config, environments=None): def calculate_project_hash(): check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") - structure = [__version__] + chunks = [__version__] for d in (util.get_projectsrc_dir(), util.get_projectlib_dir()): if not isdir(d): continue @@ -419,5 +427,10 @@ def calculate_project_hash(): for f in files: path = join(root, f) if path.endswith(check_suffixes): - structure.append(path) - return sha1(",".join(sorted(structure))).hexdigest() + chunks.append(path) + chunks_to_str = ",".join(sorted(chunks)) + if "windows" in util.get_systype(): + # Fix issue with useless project rebuilding for case insensitive FS. + # A case of disk drive can differ... + chunks_to_str = chunks_to_str.lower() + return sha1(chunks_to_str).hexdigest() diff --git a/platformio/downloader.py b/platformio/downloader.py index df9f65a2..34f597b2 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -21,7 +21,7 @@ from time import mktime import click import requests -from platformio import app, util +from platformio import util from platformio.exception import (FDSHASumMismatch, FDSizeMismatch, FDUnrecognizedStatusCode) @@ -50,7 +50,6 @@ class FileDownloader(object): else: self._fname = [p for p in url.split("/") if p][-1] - self._progressbar = None self._destination = self._fname if dest_dir: self.set_destination( @@ -70,12 +69,12 @@ class FileDownloader(object): return -1 return int(self._request.headers['content-length']) - def start(self): + def start(self, with_progress=True): label = "Downloading" itercontent = self._request.iter_content(chunk_size=self.CHUNK_SIZE) f = open(self._destination, "wb") try: - if app.is_disabled_progressbar() or self.get_size() == -1: + if not with_progress or self.get_size() == -1: click.echo("%s..." % label) for chunk in itercontent: if chunk: @@ -85,12 +84,6 @@ class FileDownloader(object): with click.progressbar(length=chunks, label=label) as pb: for _ in pb: f.write(next(itercontent)) - except IOError as e: - click.secho( - "Error: Please read http://bit.ly/package-manager-ioerror", - fg="red", - err=True) - raise e finally: f.close() self._request.close() @@ -98,6 +91,8 @@ class FileDownloader(object): if self.get_lmtime(): self._preserve_filemtime(self.get_lmtime()) + return True + def verify(self, sha1=None): _dlsize = getsize(self._destination) if self.get_size() != -1 and _dlsize != self.get_size(): diff --git a/platformio/exception.py b/platformio/exception.py index afedf5a9..8b3df0f2 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -207,6 +207,11 @@ class InvalidSettingValue(PlatformioException): MESSAGE = "Invalid value '{0}' for the setting '{1}'" +class InvalidJSONFile(PlatformioException): + + MESSAGE = "Could not load broken JSON: {0}" + + class CIBuildEnvsEmpty(PlatformioException): MESSAGE = ("Can't find PlatformIO build environments.\n" diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index cde75693..f1dcd35e 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -40,7 +40,7 @@ class ProjectGenerator(object): return sorted( [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) - @util.memoized + @util.memoized() def get_project_env(self): data = {} config = util.load_project_config(self.project_dir) @@ -54,7 +54,6 @@ class ProjectGenerator(object): data[k] = v return data - @util.memoized def get_project_build_data(self): data = { "defines": [], diff --git a/platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl b/platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl index 501ce04b..6b41950b 100644 --- a/platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl +++ b/platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl @@ -17,8 +17,8 @@ - - + + diff --git a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl index d428393d..d481de89 100644 --- a/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl @@ -1,8 +1,19 @@ { + "!!! WARNING !!!": "PLEASE DO NOT MODIFY THIS FILE! USE http://docs.platformio.org/page/projectconf/section_env_build.html#build-flags", "configurations": [ { % import platform +% from os.path import commonprefix, dirname +% % systype = platform.system().lower() +% +% cleaned_includes = [] +% for include in includes: +% if "toolchain-" not in dirname(commonprefix([include, cc_path])): +% cleaned_includes.append(include) +% end +% end +% % if systype == "windows": "name": "Win32", % elif systype == "darwin": @@ -11,7 +22,7 @@ "name": "Linux", % end "includePath": [ -% for include in includes: +% for include in cleaned_includes: "{{include.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}", % end "" @@ -20,7 +31,7 @@ "limitSymbolsToIncludedHeaders": true, "databaseFilename": "${workspaceRoot}/.vscode/.browse.c_cpp.db", "path": [ -% for include in includes: +% for include in cleaned_includes: "{{include.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}", % end "" @@ -32,7 +43,19 @@ % end "" ], - "intelliSenseMode": "clang-x64" + "intelliSenseMode": "clang-x64", +% import re +% STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)") +% cc_stds = STD_RE.findall(cc_flags) +% cxx_stds = STD_RE.findall(cxx_flags) +% +% if cc_stds: + "cStandard": "c{{ cc_stds[-1] }}", +% end +% if cxx_stds: + "cppStandard": "c++{{ cxx_stds[-1] }}", +% end + "compilerPath": "{{ cc_path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"') }}" } ] } \ No newline at end of file diff --git a/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl b/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl new file mode 100644 index 00000000..8281e64c --- /dev/null +++ b/platformio/ide/tpls/vscode/.vscode/extensions.json.tpl @@ -0,0 +1,7 @@ +{ + // See http://go.microsoft.com/fwlink/?LinkId=827846 + // for the documentation about the extensions.json format + "recommendations": [ + "platformio.platformio-ide" + ] +} \ No newline at end of file diff --git a/platformio/ide/tpls/vscode/.vscode/launch.json.tpl b/platformio/ide/tpls/vscode/.vscode/launch.json.tpl index 788edc2a..541e2485 100644 --- a/platformio/ide/tpls/vscode/.vscode/launch.json.tpl +++ b/platformio/ide/tpls/vscode/.vscode/launch.json.tpl @@ -1,17 +1,41 @@ +// AUTOMATICALLY GENERATED FILE. PLEASE DO NOT MODIFY IT MANUALLY + +// PIO Unified Debugger +// +// Documentation: http://docs.platformio.org/page/plus/debugging.html +// Configuration: http://docs.platformio.org/page/projectconf/section_env_debug.html + % from os.path import dirname, join +% +% def _escape_path(path): +% return path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"') +% end +% { "version": "0.2.0", "configurations": [ { - "type": "gdb", + "type": "platformio-debug", "request": "launch", - "cwd": "${workspaceRoot}", "name": "PlatformIO Debugger", - "target": "{{prog_path.replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}", - "gdbpath": "{{join(dirname(platformio_path), "piodebuggdb").replace('\\\\', '/').replace('\\', '/').replace('"', '\\"')}}", - "autorun": [ "source .pioinit" ], + "executable": "{{ _escape_path(prog_path) }}", + "toolchainBinDir": "{{ _escape_path(dirname(gdb_path)) }}", +% if svd_path: + "svdPath": "{{ _escape_path(svd_path) }}", +% end "preLaunchTask": "PlatformIO: Pre-Debug", "internalConsoleOptions": "openOnSessionStart" + }, + { + "type": "platformio-debug", + "request": "launch", + "name": "PlatformIO Debugger (Skip Pre-Debug)", + "executable": "{{ _escape_path(prog_path) }}", + "toolchainBinDir": "{{ _escape_path(dirname(gdb_path)) }}", +% if svd_path: + "svdPath": "{{ _escape_path(svd_path) }}", +% end + "internalConsoleOptions": "openOnSessionStart" } ] } \ No newline at end of file diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 9f3cc628..3aa37fe7 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,10 +21,10 @@ from platformio import __version__, exception, util from platformio.managers.package import PackageManager CORE_PACKAGES = { - "contrib-piohome": ">=0.7.1,<2", - "contrib-pysite": ">=0.1.5,<2", - "tool-pioplus": ">=0.14.5,<2", - "tool-unity": "~1.20302.1", + "contrib-piohome": ">=0.9.5,<2", + "contrib-pysite": ">=0.2.0,<2", + "tool-pioplus": ">=1.3.1,<2", + "tool-unity": "~1.20403.0", "tool-scons": "~2.20501.4" } @@ -69,7 +69,7 @@ class CorePackageManager(PackageManager): if manifest['name'] not in best_pkg_versions: continue if manifest['version'] != best_pkg_versions[manifest['name']]: - self.uninstall(manifest['__pkg_dir'], trigger_event=False) + self.uninstall(manifest['__pkg_dir'], after_update=True) self.cache_reset() return True diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index c1e5ab96..5613b668 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -332,7 +332,7 @@ class LibraryManager(BasePkgManager): name, requirements=None, silent=False, - trigger_event=True, + after_update=False, interactive=False, force=False): _name, _requirements, _url = self.parse_pkg_uri(name, requirements) @@ -350,7 +350,7 @@ class LibraryManager(BasePkgManager): name, requirements, silent=silent, - trigger_event=trigger_event, + after_update=after_update, force=force) if not pkg_dir: @@ -365,11 +365,20 @@ class LibraryManager(BasePkgManager): for filters in self.normalize_dependencies(manifest['dependencies']): assert "name" in filters + + # avoid circle dependencies + if not self.INSTALL_HISTORY: + self.INSTALL_HISTORY = [] + history_key = str(filters) + if history_key in self.INSTALL_HISTORY: + continue + self.INSTALL_HISTORY.append(history_key) + if any(s in filters.get("version", "") for s in ("\\", "/")): self.install( "{name}={version}".format(**filters), silent=silent, - trigger_event=trigger_event, + after_update=after_update, interactive=interactive, force=force) else: @@ -385,20 +394,20 @@ class LibraryManager(BasePkgManager): lib_id, filters.get("version"), silent=silent, - trigger_event=trigger_event, + after_update=after_update, interactive=interactive, force=force) else: self.install( lib_id, silent=silent, - trigger_event=trigger_event, + after_update=after_update, interactive=interactive, force=force) return pkg_dir -@util.memoized +@util.memoized() def get_builtin_libs(storage_names=None): items = [] storage_names = storage_names or [] @@ -417,7 +426,7 @@ def get_builtin_libs(storage_names=None): return items -@util.memoized +@util.memoized() def is_builtin_lib(name): for storage in get_builtin_libs(): if any(l.get("name") == name for l in storage['items']): diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 905b756a..3996bc06 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -177,8 +177,25 @@ class PkgInstallerMixin(object): shutil.copy(cache_path, dst_path) return dst_path - fd = FileDownloader(url, dest_dir) - fd.start() + with_progress = not app.is_disabled_progressbar() + try: + fd = FileDownloader(url, dest_dir) + fd.start(with_progress=with_progress) + except IOError as e: + raise_error = not with_progress + if with_progress: + try: + fd = FileDownloader(url, dest_dir) + fd.start(with_progress=False) + except IOError: + raise_error = True + if raise_error: + click.secho( + "Error: Please read http://bit.ly/package-manager-ioerror", + fg="red", + err=True) + raise e + if sha1: fd.verify(sha1) dst_path = fd.get_filepath() @@ -194,8 +211,15 @@ class PkgInstallerMixin(object): @staticmethod def unpack(source_path, dest_dir): - with FileUnpacker(source_path) as fu: - return fu.unpack(dest_dir) + with_progress = not app.is_disabled_progressbar() + try: + with FileUnpacker(source_path) as fu: + return fu.unpack(dest_dir, with_progress=with_progress) + except IOError as e: + if not with_progress: + raise e + with FileUnpacker(source_path) as fu: + return fu.unpack(dest_dir, with_progress=False) @staticmethod def parse_semver_spec(value, raise_exception=False): @@ -478,7 +502,7 @@ class PkgInstallerMixin(object): target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5(cur_manifest['__src_url']).hexdigest()) - os.rename(pkg_dir, join(self.package_dir, target_dirname)) + shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: target_dirname = "%s@%s" % (pkg_dirname, @@ -492,7 +516,7 @@ class PkgInstallerMixin(object): # remove previous/not-satisfied package if isdir(pkg_dir): util.rmtree_(pkg_dir) - os.rename(tmp_dir, pkg_dir) + shutil.move(tmp_dir, pkg_dir) assert isdir(pkg_dir) self.cache_reset() return pkg_dir @@ -633,7 +657,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): name, requirements=None, silent=False, - trigger_event=True, + after_update=False, force=False): name, requirements, url = self.parse_pkg_uri(name, requirements) package_dir = self.get_package_dir(name, requirements, url) @@ -676,7 +700,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): manifest = self.load_manifest(pkg_dir) assert manifest - if trigger_event: + if not after_update: telemetry.on_event( category=self.__class__.__name__, action="Install", @@ -690,7 +714,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): return pkg_dir - def uninstall(self, package, requirements=None, trigger_event=True): + def uninstall(self, package, requirements=None, after_update=False): if isdir(package): pkg_dir = package else: @@ -716,14 +740,14 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): # unfix package with the same name pkg_dir = self.get_package_dir(manifest['name']) if pkg_dir and "@" in pkg_dir: - os.rename(pkg_dir, - join(self.package_dir, - self.get_install_dirname(manifest))) + shutil.move(pkg_dir, + join(self.package_dir, + self.get_install_dirname(manifest))) self.cache_reset() click.echo("[%s]" % click.style("OK", fg="green")) - if trigger_event: + if not after_update: telemetry.on_event( category=self.__class__.__name__, action="Uninstall", @@ -769,8 +793,8 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): self._update_src_manifest( dict(version=vcs.get_current_revision()), vcs.storage_dir) else: - self.uninstall(pkg_dir, trigger_event=False) - self.install(name, latest, trigger_event=False) + self.uninstall(pkg_dir, after_update=True) + self.install(name, latest, after_update=True) telemetry.on_event( category=self.__class__.__name__, diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index f147b762..03010fdc 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -30,7 +30,7 @@ from platformio.managers.package import BasePkgManager, PackageManager class PlatformManager(BasePkgManager): - FILE_CACHE_VALID = None # disable platform caching + FILE_CACHE_VALID = None # disable platform download caching def __init__(self, package_dir=None, repositories=None): if not repositories: @@ -62,7 +62,7 @@ class PlatformManager(BasePkgManager): with_packages=None, without_packages=None, skip_default_package=False, - trigger_event=True, + after_update=False, silent=False, force=False, **_): # pylint: disable=too-many-arguments, arguments-differ @@ -70,20 +70,20 @@ class PlatformManager(BasePkgManager): self, name, requirements, silent=silent, force=force) p = PlatformFactory.newPlatform(platform_dir) - # @Hook: when 'update' operation (trigger_event is False), - # don't cleanup packages or install them - if not trigger_event: + # don't cleanup packages or install them after update + # we check packages for updates in def update() + if after_update: return True + p.install_packages( with_packages, without_packages, skip_default_package, silent=silent, force=force) - self.cleanup_packages(p.packages.keys()) - return True + return self.cleanup_packages(p.packages.keys()) - def uninstall(self, package, requirements=None, trigger_event=True): + def uninstall(self, package, requirements=None, after_update=False): if isdir(package): pkg_dir = package else: @@ -96,13 +96,12 @@ class PlatformManager(BasePkgManager): p = PlatformFactory.newPlatform(pkg_dir) BasePkgManager.uninstall(self, pkg_dir, requirements) - # @Hook: when 'update' operation (trigger_event is False), - # don't cleanup packages or install them - if not trigger_event: + # don't cleanup packages or install them after update + # we check packages for updates in def update() + if after_update: return True - self.cleanup_packages(p.packages.keys()) - return True + return self.cleanup_packages(p.packages.keys()) def update( # pylint: disable=arguments-differ self, @@ -154,11 +153,15 @@ class PlatformManager(BasePkgManager): continue if (manifest['name'] not in deppkgs or manifest['version'] not in deppkgs[manifest['name']]): - pm.uninstall(manifest['__pkg_dir'], trigger_event=False) + try: + pm.uninstall(manifest['__pkg_dir'], after_update=True) + except exception.UnknownPackage: + pass self.cache_reset() return True + @util.memoized(expire=5000) def get_installed_boards(self): boards = [] for manifest in self.get_installed(): @@ -170,7 +173,7 @@ class PlatformManager(BasePkgManager): return boards @staticmethod - @util.memoized + @util.memoized() def get_registered_boards(): return util.get_api_result("/boards", cache_valid="7d") @@ -280,21 +283,25 @@ class PlatformPackagesMixin(object): return True - def find_pkg_names(self, items): + def find_pkg_names(self, candidates): result = [] - for item in items: - candidate = item + for candidate in candidates: + found = False # lookup by package types for _name, _opts in self.packages.items(): - if _opts.get("type") == item: - candidate = _name + if _opts.get("type") == candidate: + result.append(_name) + found = True - if (self.frameworks and item.startswith("framework-") - and item[10:] in self.frameworks): - candidate = self.frameworks[item[10:]]['package'] + if (self.frameworks and candidate.startswith("framework-") + and candidate[10:] in self.frameworks): + result.append(self.frameworks[candidate[10:]]['package']) + found = True + + if not found: + result.append(candidate) - result.append(candidate) return result def update_packages(self, only_check=False): @@ -489,6 +496,10 @@ class PlatformBase( # pylint: disable=too-many-public-methods def vendor_url(self): return self._manifest.get("url") + @property + def docs_url(self): + return self._manifest.get("docs") + @property def repository_url(self): return self._manifest.get("repository", {}).get("url") @@ -654,6 +665,15 @@ class PlatformBoardConfig(object): else: raise KeyError("Invalid board option '%s'" % path) + def update(self, path, value): + newdict = None + for key in path.split(".")[::-1]: + if newdict is None: + newdict = {key: value} + else: + newdict = {key: newdict} + util.merge_dicts(self._manifest, newdict) + def __contains__(self, key): try: self.get(key) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index c3fd438f..769d58eb 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -16,6 +16,7 @@ import atexit import platform import Queue import re +import sys import threading from collections import deque from os import getenv, sep @@ -152,16 +153,22 @@ class MeasurementProtocol(TelemetryBase): cmd_path.append(sub_cmd) self['screen_name'] = " ".join([p.title() for p in cmd_path]) - def send(self, hittype): + @staticmethod + def _ignore_hit(): if not app.get_setting("enable_telemetry"): + return True + if app.get_session_var("caller_id") and \ + all(c in sys.argv for c in ("run", "idedata")): + return True + return False + + def send(self, hittype): + if self._ignore_hit(): return - self['t'] = hittype - # correct queue time if "qt" in self._params and isinstance(self['qt'], float): self['qt'] = int((time() - self['qt']) * 1000) - MPDataPusher().push(self._params) diff --git a/platformio/unpacker.py b/platformio/unpacker.py index 4e789d1c..38165c40 100644 --- a/platformio/unpacker.py +++ b/platformio/unpacker.py @@ -20,7 +20,7 @@ from zipfile import ZipFile import click -from platformio import app, util +from platformio import util from platformio.exception import UnsupportedArchiveType @@ -96,9 +96,9 @@ class FileUnpacker(object): if self._unpacker: self._unpacker.close() - def unpack(self, dest_dir="."): + def unpack(self, dest_dir=".", with_progress=True): assert self._unpacker - if app.is_disabled_progressbar(): + if not with_progress: click.echo("Unpacking...") for item in self._unpacker.get_items(): self._unpacker.extract_item(item, dest_dir) diff --git a/platformio/util.py b/platformio/util.py index 459b85e2..e37660e2 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections -import functools import json import os import platform @@ -113,40 +111,23 @@ class cd(object): class memoized(object): - ''' - Decorator. Caches a function's return value each time it is called. - If called later with the same arguments, the cached value is returned - (not reevaluated). - https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize - ''' - def __init__(self, func): - self.func = func + def __init__(self, expire=0): + self.expire = expire / 1000 # milliseconds self.cache = {} - def __call__(self, *args): - if not isinstance(args, collections.Hashable): - # uncacheable. a list, for instance. - # better to not cache than blow up. - return self.func(*args) - if args in self.cache: - return self.cache[args] - value = self.func(*args) - self.cache[args] = value - return value + def __call__(self, func): - def __repr__(self): - '''Return the function's docstring.''' - return self.func.__doc__ + @wraps(func) + def wrapper(*args, **kwargs): + key = str(args) + str(kwargs) + if (key not in self.cache + or (self.expire > 0 + and self.cache[key][0] < time.time() - self.expire)): + self.cache[key] = (time.time(), func(*args, **kwargs)) + return self.cache[key][1] - def __get__(self, obj, objtype): - '''Support instance methods.''' - fn = functools.partial(self.__call__, obj) - fn.reset = self._reset - return fn - - def _reset(self): - self.cache = {} + return wrapper class throttle(object): @@ -155,15 +136,15 @@ class throttle(object): self.threshhold = threshhold # milliseconds self.last = 0 - def __call__(self, fn): + def __call__(self, func): - @wraps(fn) + @wraps(func) def wrapper(*args, **kwargs): diff = int(round((time.time() - self.last) * 1000)) if diff < self.threshhold: time.sleep((self.threshhold - diff) * 0.001) self.last = time.time() - return fn(*args, **kwargs) + return func(*args, **kwargs) return wrapper @@ -189,8 +170,7 @@ def load_json(file_path): with open(file_path, "r") as f: return json.load(f) except ValueError: - raise exception.PlatformioException( - "Could not load broken JSON: %s" % file_path) + raise exception.InvalidJSONFile(file_path) def get_systype(): @@ -548,6 +528,14 @@ def get_mdns_services(): with mDNSListener() as mdns: time.sleep(3) for service in mdns.get_services(): + properties = None + try: + if service.properties: + json.dumps(service.properties) + properties = service.properties + except UnicodeDecodeError: + pass + items.append({ "type": service.type, @@ -558,7 +546,7 @@ def get_mdns_services(): "port": service.port, "properties": - service.properties + properties }) return items @@ -568,7 +556,7 @@ def get_request_defheaders(): return {"User-Agent": "PlatformIO/%s CI/%d %s" % data} -@memoized +@memoized(expire=10000) def _api_request_session(): return requests.Session() @@ -609,6 +597,7 @@ def _get_api_result( verify=verify_ssl) result = r.json() r.raise_for_status() + return r.text except requests.exceptions.HTTPError as e: if result and "message" in result: raise exception.APIRequestError(result['message']) @@ -622,7 +611,7 @@ def _get_api_result( finally: if r: r.close() - return result + return None def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): @@ -637,7 +626,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): if cache_key: result = cc.get(cache_key) if result is not None: - return result + return json.loads(result) # check internet before and resolve issue with 60 seconds timeout internet_on(raise_exception=True) @@ -646,7 +635,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): if cache_valid: with ContentCache() as cc: cc.set(cache_key, result, cache_valid) - return result + return json.loads(result) except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: from platformio.maintenance import in_silence @@ -670,7 +659,7 @@ PING_INTERNET_IPS = [ ] -@memoized +@memoized(expire=5000) def _internet_on(): timeout = 2 socket.setdefaulttimeout(timeout) @@ -765,6 +754,18 @@ def format_filesize(filesize): return "%d%sB" % ((base * filesize / unit), suffix) +def merge_dicts(d1, d2, path=None): + if path is None: + path = [] + for key in d2: + if (key in d1 and isinstance(d1[key], dict) + and isinstance(d2[key], dict)): + merge_dicts(d1[key], d2[key], path + [str(key)]) + else: + d1[key] = d2[key] + return d1 + + def rmtree_(path): def _onerror(_, name, __): diff --git a/scripts/docspregen.py b/scripts/docspregen.py index 9d1c2371..cd56034c 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -302,21 +302,31 @@ Stable and upstream versions You can switch between `stable releases `__ of {title} development platform and the latest upstream version using -:ref:`projectconf_env_platform` option as described below: +:ref:`projectconf_env_platform` option in :ref:`projectconf` as described below. + +Stable +~~~~~~ .. code-block:: ini - ; Custom stable version - [env:stable] - platform ={name}@x.y.z + ; Latest stable version + [env:latest_stable] + platform = {name} board = ... - ... - ; The latest upstream/development version - [env:upstream] + ; Custom stable version + [env:custom_stable] + platform = {name}@x.y.z + board = ... + +Upstream +~~~~~~~~ + +.. code-block:: ini + + [env:upstream_develop] platform = https://github.com/platformio/platform-{name}.git board = ... - ... """.format(name=p.name, title=p.title)) # diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index 12debd8c..e4db2e7e 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -62,7 +62,7 @@ def test_global_install_archive(clirunner, validate_cliresult, "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", "http://dl.platformio.org/libraries/archives/0/9540.tar.gz", - "https://github.com/adafruit/Adafruit-ST7735-Library/archive/master.zip" + "https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip" ]) validate_cliresult(result) @@ -76,7 +76,7 @@ def test_global_install_archive(clirunner, validate_cliresult, items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ "RadioHead-1.62", "ArduinoJson", "DallasTemperature_ID54", - "OneWire_ID1", "Adafruit ST7735 Library" + "OneWire_ID1", "ESP32WebServer" ] assert set(items1) >= set(items2) @@ -142,7 +142,7 @@ def test_global_lib_list(clirunner, validate_cliresult): validate_cliresult(result) assert all([ n in result.output for n in - ("Source: https://github.com/adafruit/Adafruit-ST7735-Library/archive/master.zip", + ("Source: https://github.com/Pedroalbuquerque/ESP32WebServer/archive/master.zip", "Version: 5.10.1", "Source: git+https://github.com/gioblu/PJON.git#3.0", "Version: 1fb26fd", "RadioHead-1.62") @@ -157,7 +157,7 @@ def test_global_lib_list(clirunner, validate_cliresult): ]) items1 = [i['name'] for i in json.loads(result.output)] items2 = [ - "Adafruit ST7735 Library", "ArduinoJson", "ArduinoJson", "ArduinoJson", + "ESP32WebServer", "ArduinoJson", "ArduinoJson", "ArduinoJson", "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "DallasTemperature", "ESPAsyncTCP", "NeoPixelBus", "OneWire", "PJON", "PJON", "PubSubClient", "RFcontrol", "RadioHead-1.62", "platformio-libmirror", @@ -221,9 +221,9 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, validate_cliresult(result) items = json.loads(result.output) result = clirunner.invoke(cmd_lib, - ["-g", "uninstall", items[0]['__pkg_dir']]) + ["-g", "uninstall", items[5]['__pkg_dir']]) validate_cliresult(result) - assert "Uninstalling Adafruit ST7735 Library" in result.output + assert "Uninstalling AsyncTCP" in result.output # uninstall the rest libraries result = clirunner.invoke(cmd_lib, [ @@ -238,7 +238,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "ESPAsyncTCP_ID305", "DallasTemperature_ID54", "NeoPixelBus_ID547", "PJON", "AsyncMqttClient_ID346", "ArduinoJson_ID64", - "PJON@src-79de467ebe19de18287becff0a1fb42d", "AsyncTCP_ID1826" + "PJON@src-79de467ebe19de18287becff0a1fb42d", "ESP32WebServer" ] assert set(items1) == set(items2) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 5b3053fe..d327ef37 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -61,13 +61,14 @@ def test_install_known_version(clirunner, validate_cliresult, assert len(isolated_pio_home.join("packages").listdir()) == 1 -def test_install_from_vcs(clirunner, validate_cliresult): +def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cli_platform.platform_install, [ "https://github.com/platformio/" "platform-espressif8266.git#feature/stage", "--skip-default-package" ]) validate_cliresult(result) assert "espressif8266" in result.output + assert len(isolated_pio_home.join("packages").listdir()) == 1 def test_list_json_output(clirunner, validate_cliresult): diff --git a/tests/test_builder.py b/tests/test_builder.py new file mode 100644 index 00000000..cafd1cee --- /dev/null +++ b/tests/test_builder.py @@ -0,0 +1,96 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.run import cli as cmd_run + + +def test_build_flags(clirunner, validate_cliresult, tmpdir): + build_flags = [("-D TEST_INT=13", "-DTEST_INT=13"), + ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), + ('-DTEST_STR_SPACE="Andrew Smith"', + '"-DTEST_STR_SPACE=Andrew Smith"')] + + tmpdir.join("platformio.ini").write(""" +[env:native] +platform = native +extra_scripts = extra.py +build_flags = %s + """ % " ".join([f[0] for f in build_flags])) + + tmpdir.join("extra.py").write(""" +Import("env") +env.Append(CPPDEFINES="POST_SCRIPT_MACRO") + """) + + tmpdir.mkdir("src").join("main.cpp").write(""" +#if !defined(TEST_INT) || TEST_INT != 13 +#error "TEST_INT" +#endif + +#ifndef TEST_STR_SPACE +#error "TEST_STR_SPACE" +#endif + +#ifndef POST_SCRIPT_MACRO +#error "POST_SCRIPT_MACRO" +#endif + +int main() { +} +""") + + result = clirunner.invoke( + cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) + validate_cliresult(result) + build_output = result.output[result.output.find( + "Scanning dependencies..."):] + for flag in build_flags: + assert flag[1] in build_output, flag + + +def test_build_unflags(clirunner, validate_cliresult, tmpdir): + tmpdir.join("platformio.ini").write(""" +[env:native] +platform = native +build_unflags = -DTMP_MACRO1=45 -I. -DNON_EXISTING_MACRO -lunknownLib -Os +extra_scripts = pre:extra.py +""") + + tmpdir.join("extra.py").write(""" +Import("env") +env.Append(CPPPATH="%s") +env.Append(CPPDEFINES="TMP_MACRO1") +env.Append(CPPDEFINES=["TMP_MACRO2"]) +env.Append(CPPDEFINES=("TMP_MACRO3", 13)) +env.Append(CCFLAGS=["-Os"]) +env.Append(LIBS=["unknownLib"]) + """ % str(tmpdir)) + + tmpdir.mkdir("src").join("main.c").write(""" +#ifdef TMP_MACRO1 +#error "TMP_MACRO1 should be removed" +#endif + +int main() { +} +""") + + result = clirunner.invoke( + cmd_run, ["--project-dir", str(tmpdir), "--verbose"]) + validate_cliresult(result) + build_output = result.output[result.output.find( + "Scanning dependencies..."):] + assert "-DTMP_MACRO1" not in build_output + assert "-Os" not in build_output + assert str(tmpdir) not in build_output diff --git a/tests/test_pkgmanifest.py b/tests/test_pkgmanifest.py index b7a73061..92da706f 100644 --- a/tests/test_pkgmanifest.py +++ b/tests/test_pkgmanifest.py @@ -16,11 +16,11 @@ import pytest import requests -def validate_response(req): - assert req.status_code == 200 - assert int(req.headers['Content-Length']) > 0 - assert req.headers['Content-Type'] in ("application/gzip", - "application/octet-stream") +def validate_response(r): + assert r.status_code == 200, r.url + assert int(r.headers['Content-Length']) > 0, r.url + assert r.headers['Content-Type'] in ("application/gzip", + "application/octet-stream") def test_packages():