diff --git a/.isort.cfg b/.isort.cfg index d63487e9..d094cc19 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,3 +1,3 @@ [settings] line_length=79 -known_third_party=arrow,bottle,click,lockfile,pytest,requests,SCons,semantic_version,serial +known_third_party=bottle,click,lockfile,python-dateutil,pytest,requests,SCons,semantic_version,serial diff --git a/HISTORY.rst b/HISTORY.rst index 666336b3..c9416975 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,28 @@ Release Notes PlatformIO 3.0 -------------- +3.5.1 (2018-01-18) +~~~~~~~~~~~~~~~~~~ + +* New ``test_speed`` option to control a communication baudrate/speed between + `PIO Unit Testing `__ + engine and a target device + (`issue #1273 `_) +* Show full library version in "Library Dependency Graph" including VCS + information + (`issue #1274 `_) +* Configure a custom firmware/program name in build directory (`example `__) +* Renamed ``envs_dir`` option to ``build_dir`` + in `Project Configuration File "platformio.ini" `__ +* Refactored code without "arrow" dependency (resolve issue with "ImportError: + No module named backports.functools_lru_cache") +* Improved support of PIO Unified Debugger for Eclipse Oxygen +* Improved a work in off-line mode +* Fixed project generator for CLion and Qt Creator IDE + (`issue #1299 `_) +* Fixed PIO Unified Debugger for mbed framework +* Fixed library updates when a version is declared in VCS format (not SemVer) + 3.5.0 (2017-12-28) ~~~~~~~~~~~~~~~~~~ @@ -28,7 +50,7 @@ PlatformIO 3.0 * New `include `__ folder for project's header files (`issue #1107 `_) -* Depend on development platform using VSC URL (Git, Mercurial and Subversion) +* Depend on development platform using VCS URL (Git, Mercurial and Subversion) instead of a name in `Project Configuration File "platformio.ini" `__. Drop support for ``*_stage`` dev/platform names (use VCS URL instead). * Reinstall/redownload package with a new ``-f, --force`` option for diff --git a/Makefile b/Makefile index 0031d316..6df42c42 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ yapf: yapf --recursive --in-place platformio/ test: - py.test -v -s tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py + py.test -v -s -n 3 --dist=loadscope tests --ignore tests/test_examples.py --ignore tests/test_pkgmanifest.py before-commit: isort yapf lint test diff --git a/docs b/docs index c76ccaf3..2e04299a 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit c76ccaf33786e93392c80decdb457794ac234380 +Subproject commit 2e04299a170b3654d9f4ec3c78392fb9202e829b diff --git a/examples b/examples index 2d716306..2d081875 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 2d716306f33cbaa3d9146e417d02e15747cadb2a +Subproject commit 2d08187562e83b2b3d3a2d9f4d124c25f2629506 diff --git a/platformio/__init__.py b/platformio/__init__.py index 33de06c8..db38e271 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,14 +14,16 @@ import sys -VERSION = (3, 5, 0) +VERSION = (3, 5, 1) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" -__description__ = ("An open source ecosystem for IoT development. " - "Cross-platform build system and library manager. " - "Continuous and IDE integration. " - "Arduino, ESP8266 and ARM mbed compatible") +__description__ = ( + "An open source ecosystem for IoT development. " + "Cross-platform IDE and unified debugger. " + "Remote unit testing and firmware updates. " + "Arduino, ARM mbed, Espressif (ESP8266/ESP32), STM32, PIC32, nRF51/nRF52, " + "FPGA, CMSIS, SPL, AVR, Samsung ARTIK, libOpenCM3") __url__ = "http://platformio.org" __author__ = "Ivan Kravets" @@ -36,5 +38,5 @@ if sys.version_info < (2, 7, 0) or sys.version_info >= (3, 0, 0): msg = ("PlatformIO Core v%s does not run under Python version %s.\n" "Minimum supported version is 2.7, please upgrade Python.\n" "Python 3 is not yet supported.\n") - sys.stderr.write(msg % (__version__, sys.version.split()[0])) + sys.stderr.write(msg % (__version__, sys.version)) sys.exit(1) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 6e073cce..c729e4af 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -91,8 +91,8 @@ DEFAULT_ENV_OPTIONS = dict( PROJECTSRC_DIR=util.get_projectsrc_dir(), PROJECTTEST_DIR=util.get_projecttest_dir(), PROJECTDATA_DIR=util.get_projectdata_dir(), - PROJECTPIOENVS_DIR=util.get_projectpioenvs_dir(), - BUILD_DIR=join("$PROJECTPIOENVS_DIR", "$PIOENV"), + PROJECTBUILD_DIR=util.get_projectbuild_dir(), + BUILD_DIR=join("$PROJECTBUILD_DIR", "$PIOENV"), BUILDSRC_DIR=join("$BUILD_DIR", "src"), BUILDTEST_DIR=join("$BUILD_DIR", "test"), LIBSOURCE_DIRS=[ @@ -150,7 +150,7 @@ env['LIBSOURCE_DIRS'] = [ env.LoadPioPlatform(commonvars) env.SConscriptChdir(0) -env.SConsignFile(join("$PROJECTPIOENVS_DIR", ".sconsign.dblite")) +env.SConsignFile(join("$PROJECTBUILD_DIR", ".sconsign.dblite")) for item in env.GetPreExtraScripts(): env.SConscript(item, exports="env") diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 8973bdfb..f20b6b08 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -20,6 +20,7 @@ from __future__ import absolute_import import hashlib import os import sys +from glob import glob from os.path import (basename, commonprefix, dirname, isdir, isfile, join, realpath, sep) from platform import system @@ -30,6 +31,7 @@ from SCons.Script import ARGUMENTS, COMMAND_LINE_TARGETS, DefaultEnvironment from platformio import util from platformio.builder.tools import platformio as piotool from platformio.managers.lib import LibraryManager +from platformio.managers.package import PackageManager class LibBuilderFactory(object): @@ -131,6 +133,13 @@ class LibBuilderBase(object): def version(self): return self._manifest.get("version") + @property + def vcs_info(self): + items = glob(join(self.path, ".*", PackageManager.SRC_MANIFEST_NAME)) + if not items: + return None + return util.load_json(items[0]) + @property def dependencies(self): return LibraryManager.normalize_dependencies( @@ -228,19 +237,6 @@ class LibBuilderBase(object): except (AssertionError, ValueError): return LibBuilderBase.COMPAT_MODE_DEFAULT - @staticmethod - def items_to_list(items): - if not isinstance(items, list): - items = [i.strip() for i in items.split(",")] - return [i.lower() for i in items if i] - - def items_in_list(self, items, ilist): - items = self.items_to_list(items) - ilist = self.items_to_list(ilist) - if "*" in items or "*" in ilist: - return True - return set(items) & set(ilist) - def is_platforms_compatible(self, platforms): return True @@ -273,7 +269,7 @@ class LibBuilderBase(object): if env_key not in self.env: continue if (key in item and - not self.items_in_list(self.env[env_key], item[key])): + not util.items_in_list(self.env[env_key], item[key])): if self.verbose: sys.stderr.write( "Skip %s incompatible dependency %s\n" % (key[:-1], @@ -304,9 +300,8 @@ class LibBuilderBase(object): def get_search_files(self): items = [ - join(self.src_dir, item) - for item in self.env.MatchSourceFiles(self.src_dir, - self.src_filter) + join(self.src_dir, item) for item in self.env.MatchSourceFiles( + self.src_dir, self.src_filter) ] include_dir = self.include_dir if include_dir: @@ -496,7 +491,7 @@ class ArduinoLibBuilder(LibBuilderBase): return src_filter def is_frameworks_compatible(self, frameworks): - return self.items_in_list(frameworks, ["arduino", "energia"]) + return util.items_in_list(frameworks, ["arduino", "energia"]) class MbedLibBuilder(LibBuilderBase): @@ -527,7 +522,7 @@ class MbedLibBuilder(LibBuilderBase): return include_dirs def is_frameworks_compatible(self, frameworks): - return self.items_in_list(frameworks, ["mbed"]) + return util.items_in_list(frameworks, ["mbed"]) class PlatformIOLibBuilder(LibBuilderBase): @@ -541,7 +536,7 @@ class PlatformIOLibBuilder(LibBuilderBase): if "platforms" in manifest: manifest['platforms'] = [ "espressif8266" if p == "espressif" else p - for p in self.items_to_list(manifest['platforms']) + for p in util.items_to_list(manifest['platforms']) ] return manifest @@ -610,13 +605,13 @@ class PlatformIOLibBuilder(LibBuilderBase): items = self._manifest.get("platforms") if not items: return LibBuilderBase.is_platforms_compatible(self, platforms) - return self.items_in_list(platforms, items) + return util.items_in_list(platforms, items) def is_frameworks_compatible(self, frameworks): items = self._manifest.get("frameworks") if not items: return LibBuilderBase.is_frameworks_compatible(self, frameworks) - return self.items_in_list(frameworks, items) + return util.items_in_list(frameworks, items) def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) @@ -646,7 +641,9 @@ class ProjectAsLibBuilder(LibBuilderBase): def get_include_dirs(self): include_dirs = LibBuilderBase.get_include_dirs(self) - include_dirs.append(self.env.subst("$PROJECTINCLUDE_DIR")) + project_include_dir = self.env.subst("$PROJECTINCLUDE_DIR") + if isdir(project_include_dir): + include_dirs.append(project_include_dir) return include_dirs def get_search_files(self): @@ -655,9 +652,9 @@ class ProjectAsLibBuilder(LibBuilderBase): # test files if "__test" in COMMAND_LINE_TARGETS: items.extend([ - join("$PROJECTTEST_DIR", item) - for item in self.env.MatchSourceFiles("$PROJECTTEST_DIR", - "$PIOTEST_SRC_FILTER") + join("$PROJECTTEST_DIR", + item) for item in self.env.MatchSourceFiles( + "$PROJECTTEST_DIR", "$PIOTEST_SRC_FILTER") ]) return items @@ -803,10 +800,15 @@ def BuildProjectLibraries(env): margin = "| " * (level) for lb in root.depbuilders: title = "<%s>" % lb.name + vcs_info = lb.vcs_info if lb.version: title += " v%s" % lb.version + if vcs_info: + title += " #%s" % vcs_info.get("version") sys.stdout.write("%s|-- %s" % (margin, title)) if int(ARGUMENTS.get("PIOVERBOSE", 0)): + if vcs_info: + sys.stdout.write(" [%s]" % vcs_info.get("url")) sys.stdout.write(" (") sys.stdout.write(lb.path) sys.stdout.write(")") @@ -815,7 +817,7 @@ def BuildProjectLibraries(env): print_deps_tree(lb, level + 1) print "Collected %d compatible libraries" % len(lib_builders) - print "Looking for dependencies..." + print "Scanning dependencies..." project = ProjectAsLibBuilder(env, "$PROJECT_DIR") project.env = env diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index dd6be6a4..f9dd5386 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -46,6 +46,9 @@ def BuildProgram(env): if not case_sensitive_suffixes(".s", ".S"): env.Replace(AS="$CC", ASCOM="$ASPPCOM") + if "__debug" in COMMAND_LINE_TARGETS: + env.ProcessDebug() + # process extra flags from board if "BOARD" in env and "build.extra_flags" in env.BoardConfig(): env.ProcessFlags(env.BoardConfig().get("build.extra_flags")) @@ -60,18 +63,6 @@ def BuildProgram(env): # restore PIO macros if it was deleted by framework _append_pio_macros() - # Search for project source files - env.Append( - LIBPATH=["$BUILD_DIR"], - PIOBUILDFILES=env.CollectBuildFiles( - "$BUILDSRC_DIR", "$PROJECTSRC_DIR", "$SRC_FILTER", - duplicate=False)) - - if "__debug" in COMMAND_LINE_TARGETS: - env.ProcessDebug() - if "__test" in COMMAND_LINE_TARGETS: - env.Append(PIOBUILDFILES=env.ProcessTest()) - # build dependent libs env.Append(LIBS=env.BuildProjectLibraries()) @@ -88,7 +79,18 @@ def BuildProgram(env): # Handle SRC_BUILD_FLAGS env.ProcessFlags(env.get("SRC_BUILD_FLAGS")) - if not env.get("PIOBUILDFILES") and not COMMAND_LINE_TARGETS: + env.Append( + LIBPATH=["$BUILD_DIR"], + PIOBUILDFILES=env.CollectBuildFiles( + "$BUILDSRC_DIR", + "$PROJECTSRC_DIR", + src_filter=env.get("SRC_FILTER"), + duplicate=False)) + + if "__test" in COMMAND_LINE_TARGETS: + env.Append(PIOBUILDFILES=env.ProcessTest()) + + if not env['PIOBUILDFILES'] and not COMMAND_LINE_TARGETS: sys.stderr.write( "Error: Nothing to build. Please put your source code files " "to '%s' folder\n" % env.subst("$PROJECTSRC_DIR")) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 051720c6..87f87d7c 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -15,16 +15,14 @@ # pylint: disable=too-many-branches, too-many-locals import json +import time from os.path import isdir, join -from time import sleep from urllib import quote -import arrow import click from platformio import exception, util -from platformio.managers.lib import LibraryManager -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.lib import LibraryManager, get_builtin_libs from platformio.util import get_api_result @@ -99,7 +97,7 @@ def cli(ctx, **options): help="Reinstall/redownload library if exists") @click.pass_obj def lib_install(lm, libraries, silent, interactive, force): - # @TODO "save" option + # @TODO: "save" option for library in libraries: lm.install( library, silent=silent, interactive=interactive, force=force) @@ -252,7 +250,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): result['perpage'], fg="yellow") click.echo() - sleep(5) + time.sleep(5) elif not click.confirm("Show next libraries?"): break result = get_api_result( @@ -280,25 +278,6 @@ def lib_list(lm, json_output): return True -@util.memoized -def get_builtin_libs(storage_names=None): - items = [] - storage_names = storage_names or [] - pm = PlatformManager() - for manifest in pm.get_installed(): - p = PlatformFactory.newPlatform(manifest['__pkg_dir']) - for storage in p.get_lib_storages(): - if storage_names and storage['name'] not in storage_names: - continue - lm = LibraryManager(storage['path']) - items.append({ - "name": storage['name'], - "path": storage['path'], - "items": lm.get_installed() - }) - return items - - @cli.command("builtin", short_help="List built-in libraries") @click.option("--storage", multiple=True) @click.option("--json-output", is_flag=True) @@ -326,8 +305,13 @@ def lib_builtin(storage, json_output): def lib_show(library, json_output): lm = LibraryManager() name, requirements, _ = lm.parse_pkg_uri(library) - lib_id = lm.get_pkg_id_by_name( - name, requirements, silent=json_output, interactive=not json_output) + lib_id = lm.search_lib_id( + { + "name": name, + "requirements": requirements + }, + silent=json_output, + interactive=not json_output) lib = get_api_result("/lib/info/%d" % lib_id, cache_valid="1d") if json_output: return click.echo(json.dumps(lib)) @@ -338,9 +322,10 @@ def lib_show(library, json_output): click.echo(lib['description']) click.echo() - click.echo("Version: %s, released %s" % - (lib['version']['name'], - arrow.get(lib['version']['released']).humanize())) + click.echo( + "Version: %s, released %s" % + (lib['version']['name'], + time.strftime("%c", util.parse_date(lib['version']['released'])))) click.echo("Manifest: %s" % lib['confurl']) for key in ("homepage", "repository", "license"): if key not in lib or not lib[key]: @@ -376,7 +361,8 @@ def lib_show(library, json_output): blocks.append(("Headers", lib['headers'])) blocks.append(("Examples", lib['examples'])) blocks.append(("Versions", [ - "%s, released %s" % (v['name'], arrow.get(v['released']).humanize()) + "%s, released %s" % + (v['name'], time.strftime("%c", util.parse_date(v['released']))) for v in lib['versions'] ])) blocks.append(("Unique Downloads", [ @@ -439,7 +425,7 @@ def lib_stats(json_output): if "date" in item else printitem_tpl).format( name=click.style(item['name'], fg="cyan"), date=str( - arrow.get(item['date']).humanize() + time.strftime("%c", util.parse_date(item['date'])) if "date" in item else ""), url=click.style( "http://platformio.org/lib/show/%s/%s" % diff --git a/platformio/commands/run.py b/platformio/commands/run.py index 59654de5..f75a07c9 100644 --- a/platformio/commands/run.py +++ b/platformio/commands/run.py @@ -23,10 +23,9 @@ import click from platformio import __version__, exception, telemetry, util from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.commands.lib import lib_install as cmd_lib_install -from platformio.commands.lib import get_builtin_libs from platformio.commands.platform import \ platform_install as cmd_platform_install -from platformio.managers.lib import LibraryManager +from platformio.managers.lib import LibraryManager, is_builtin_lib from platformio.managers.platform import PlatformFactory # pylint: disable=too-many-arguments,too-many-locals,too-many-branches @@ -60,15 +59,15 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose, raise exception.NotPlatformIOProject(project_dir) with util.cd(project_dir): - # clean obsolete .pioenvs dir + # clean obsolete build dir if not disable_auto_clean: try: - _clean_pioenvs_dir(util.get_projectpioenvs_dir()) + _clean_build_dir(util.get_projectbuild_dir()) except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " - "`.pioenvs` directory from the project manually to avoid " - "build issues" % util.get_projectpioenvs_dir(force=True), + "it manually to avoid build issues" % + util.get_projectbuild_dir(force=True), fg="yellow") config = util.load_project_config() @@ -133,17 +132,19 @@ class EnvironmentProcessor(object): "upload_resetmethod", "lib_deps", "lib_ignore", "lib_extra_dirs", "lib_ldf_mode", "lib_compat_mode", "lib_archive", "piotest", "test_transport", "test_filter", - "test_ignore", "test_port", "debug_tool", "debug_port", - "debug_init_cmds", "debug_extra_cmds", "debug_server", - "debug_init_break", "debug_load_cmd", "monitor_port", - "monitor_baud", "monitor_rts", "monitor_dtr") + "test_ignore", "test_port", "test_speed", "debug_tool", + "debug_port", "debug_init_cmds", "debug_extra_cmds", + "debug_server", "debug_init_break", "debug_load_cmd", + "monitor_port", "monitor_baud", "monitor_rts", + "monitor_dtr") IGNORE_BUILD_OPTIONS = ("test_transport", "test_filter", "test_ignore", - "test_port", "debug_tool", "debug_port", - "debug_init_cmds", "debug_extra_cmds", - "debug_server", "debug_init_break", - "debug_load_cmd", "monitor_port", "monitor_baud", - "monitor_rts", "monitor_dtr") + "test_port", "test_speed", "debug_tool", + "debug_port", "debug_init_cmds", + "debug_extra_cmds", "debug_server", + "debug_init_break", "debug_load_cmd", + "monitor_port", "monitor_baud", "monitor_rts", + "monitor_dtr") REMAPED_OPTIONS = {"framework": "pioframework", "platform": "pioplatform"} @@ -307,36 +308,31 @@ def _autoinstall_libdeps(ctx, libraries, verbose=False): try: ctx.invoke(cmd_lib_install, libraries=[lib], silent=not verbose) except exception.LibNotFound as e: - if not _is_builtin_lib(lib): + if verbose or not is_builtin_lib(lib): click.secho("Warning! %s" % e, fg="yellow") + except exception.InternetIsOffline as e: + click.secho(str(e), fg="yellow") -def _is_builtin_lib(lib_name): - for storage in get_builtin_libs(): - if any([l.get("name") == lib_name for l in storage['items']]): - return True - return False - - -def _clean_pioenvs_dir(pioenvs_dir): - structhash_file = join(pioenvs_dir, "structure.hash") +def _clean_build_dir(build_dir): + structhash_file = join(build_dir, "structure.hash") proj_hash = calculate_project_hash() # if project's config is modified - if (isdir(pioenvs_dir) + if (isdir(build_dir) and getmtime(join(util.get_project_dir(), - "platformio.ini")) > getmtime(pioenvs_dir)): - util.rmtree_(pioenvs_dir) + "platformio.ini")) > getmtime(build_dir)): + util.rmtree_(build_dir) # check project structure - if isdir(pioenvs_dir) and isfile(structhash_file): + if isdir(build_dir) and isfile(structhash_file): with open(structhash_file) as f: if f.read() == proj_hash: return - util.rmtree_(pioenvs_dir) + util.rmtree_(build_dir) - if not isdir(pioenvs_dir): - makedirs(pioenvs_dir) + if not isdir(build_dir): + makedirs(build_dir) with open(structhash_file, "w") as f: f.write(proj_hash) @@ -384,13 +380,13 @@ def check_project_defopts(config): if not config.has_section("platformio"): return True known = ("env_default", "home_dir", "lib_dir", "libdeps_dir", "src_dir", - "envs_dir", "data_dir", "test_dir", "boards_dir", + "build_dir", "data_dir", "test_dir", "boards_dir", "lib_extra_dirs") unknown = set([k for k, _ in config.items("platformio")]) - set(known) if not unknown: return True click.secho( - "Warning! Ignore unknown `%s` option from `[platformio]` section" % + "Warning! Ignore unknown `%s` option in `[platformio]` section" % ", ".join(unknown), fg="yellow") return False diff --git a/platformio/exception.py b/platformio/exception.py index e477bd30..073ea4f7 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -97,10 +97,9 @@ class UndefinedPackageVersion(PlatformioException): class PackageInstallError(PlatformioException): - MESSAGE = ( - "Could not install '{0}' with version requirements '{1}' for your " - "system '{2}'.\n If you use Antivirus, it can block PlatformIO " - "Package Manager. Try to disable it for a while.") + MESSAGE = ("Could not install '{0}' with version requirements '{1}' " + "for your system '{2}'.\n" + "More details: http://bit.ly/faq-package-manager") class FDUnrecognizedStatusCode(PlatformioException): diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 63274c43..075bc805 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -51,4 +51,18 @@ add_custom_target( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) +if(IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/lib) + add_custom_target( + CODE_COMPLETION_PIOLIB + SOURCES lib + ) +endif() + +if(IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.piolibdeps) + add_custom_target( + CODE_COMPLETION_PIOLIBDEPS + SOURCES .piolibdeps + ) +endif() + add_executable(${PROJECT_NAME} ${SRC_LIST}) diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index ca27ad43..6efdb036 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -7,8 +7,9 @@ SET(CMAKE_CXX_FLAGS_DISTRIBUTION "{{cxx_flags}}") SET(CMAKE_C_FLAGS_DISTRIBUTION "{{cc_flags}}") set(CMAKE_CXX_STANDARD 11) +% import re % for define in defines: -add_definitions(-D{{!define}}) +add_definitions(-D'{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}') % end % for include in includes: diff --git a/platformio/ide/tpls/eclipse/.cproject.tpl b/platformio/ide/tpls/eclipse/.cproject.tpl index 3060a7bb..29b76ebc 100644 --- a/platformio/ide/tpls/eclipse/.cproject.tpl +++ b/platformio/ide/tpls/eclipse/.cproject.tpl @@ -5,13 +5,13 @@ + - @@ -99,6 +99,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl b/platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl index e27a3061..501ce04b 100644 --- a/platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl +++ b/platformio/ide/tpls/eclipse/.settings/PlatformIO Debugger.launch.tpl @@ -2,7 +2,7 @@ - + @@ -12,17 +12,17 @@ - + - + - + diff --git a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl index 53c9c72f..19740737 100644 --- a/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl +++ b/platformio/ide/tpls/eclipse/.settings/language.settings.xml.tpl @@ -15,4 +15,19 @@ + + + + + + % if "windows" in systype: + + % else: + + % end + + + + + diff --git a/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl b/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl index 8cac7fbd..e87cf90b 100644 --- a/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl +++ b/platformio/ide/tpls/eclipse/.settings/org.eclipse.cdt.core.prefs.tpl @@ -3,4 +3,9 @@ environment/project/0.910961921/PATH/delimiter={{env_pathsep.replace(":", "\\:") environment/project/0.910961921/PATH/operation=replace environment/project/0.910961921/PATH/value={{env_path.replace(":", "\\:")}} environment/project/0.910961921/append=true -environment/project/0.910961921/appendContributed=true \ No newline at end of file +environment/project/0.910961921/appendContributed=true +environment/project/0.910961921.1363900502/PATH/delimiter={{env_pathsep.replace(":", "\\:")}} +environment/project/0.910961921.1363900502/PATH/operation=replace +environment/project/0.910961921.1363900502/PATH/value={{env_path.replace(":", "\\:")}} +environment/project/0.910961921.1363900502/append=true +environment/project/0.910961921.1363900502/appendContributed=true \ No newline at end of file diff --git a/platformio/ide/tpls/qtcreator/platformio.pro.tpl b/platformio/ide/tpls/qtcreator/platformio.pro.tpl index 2ae9d49d..1ef5f8eb 100644 --- a/platformio/ide/tpls/qtcreator/platformio.pro.tpl +++ b/platformio/ide/tpls/qtcreator/platformio.pro.tpl @@ -14,7 +14,7 @@ INCLUDEPATH += "{{include}}" % end % for define in defines: -DEFINES += "{{define}}" +DEFINES += {{!define}} % end OTHER_FILES += platformio.ini diff --git a/platformio/ide/tpls/vscode/.gitignore.tpl b/platformio/ide/tpls/vscode/.gitignore.tpl index f39f7c9b..f92113ad 100644 --- a/platformio/ide/tpls/vscode/.gitignore.tpl +++ b/platformio/ide/tpls/vscode/.gitignore.tpl @@ -1,3 +1,4 @@ .pioenvs .piolibdeps .vscode/c_cpp_properties.json +.vscode/launch.json diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 42a6c775..3d1e8721 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -21,9 +21,9 @@ from platformio import __version__, exception, util from platformio.managers.package import PackageManager CORE_PACKAGES = { - "contrib-piohome": ">=0.6.0,<2", + "contrib-piohome": ">=0.6.1,<2", "contrib-pysite": ">=0.1.2,<2", - "tool-pioplus": ">=0.12.1,<2", + "tool-pioplus": ">=0.13.3,<2", "tool-unity": "~1.20302.1", "tool-scons": "~3.20501.2" } @@ -101,8 +101,7 @@ def pioplus_call(args, **kwargs): raise exception.PlatformioException( "PlatformIO Core Plus v%s does not run under Python version %s.\n" "Minimum supported version is 2.7.6, please upgrade Python.\n" - "Python 3 is not yet supported.\n" % (__version__, - sys.version.split()[0])) + "Python 3 is not yet supported.\n" % (__version__, sys.version)) pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus") pythonexe_path = util.get_pythonexe_path() diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index ef00787b..09c5ef12 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -13,17 +13,18 @@ # limitations under the License. # pylint: disable=too-many-arguments, too-many-locals, too-many-branches +# pylint: disable=too-many-return-statements import json import re from glob import glob from os.path import isdir, join -import arrow import click from platformio import app, commands, exception, util from platformio.managers.package import BasePkgManager +from platformio.managers.platform import PlatformFactory, PlatformManager class LibraryManager(BasePkgManager): @@ -155,8 +156,8 @@ class LibraryManager(BasePkgManager): def max_satisfying_repo_version(self, versions, requirements=None): def _cmp_dates(datestr1, datestr2): - date1 = arrow.get(datestr1) - date2 = arrow.get(datestr2) + date1 = util.parse_date(datestr1) + date2 = util.parse_date(datestr2) if date1 == date2: return 0 return -1 if date1 < date2 else 1 @@ -186,29 +187,15 @@ class LibraryManager(BasePkgManager): def get_latest_repo_version(self, name, requirements, silent=False): item = self.max_satisfying_repo_version( util.get_api_result( - "/lib/info/%d" % self.get_pkg_id_by_name( - name, requirements, silent=silent), + "/lib/info/%d" % self.search_lib_id( + { + "name": name, + "requirements": requirements + }, + silent=silent), cache_valid="1h")['versions'], requirements) return item['name'] if item else None - def get_pkg_id_by_name(self, - name, - requirements, - silent=False, - interactive=False): - if name.startswith("id="): - return int(name[3:]) - # try to find ID from installed packages - package_dir = self.get_package_dir(name, requirements) - if package_dir: - manifest = self.load_manifest(package_dir) - if "id" in manifest: - return int(manifest['id']) - return int( - self.search_for_library({ - "name": name - }, silent, interactive)['id']) - def _install_from_piorepo(self, name, requirements): assert name.startswith("id="), name version = self.get_latest_repo_version(name, requirements) @@ -225,88 +212,20 @@ class LibraryManager(BasePkgManager): "http://", "https://") if app.get_setting("enable_ssl") else dl_data['url'], requirements) - def install( # pylint: disable=arguments-differ + def search_lib_id( # pylint: disable=too-many-branches self, - name, - requirements=None, - silent=False, - trigger_event=True, - interactive=False, - force=False): - pkg_dir = None - try: - _name, _requirements, _url = self.parse_pkg_uri(name, requirements) - if not _url: - name = "id=%d" % self.get_pkg_id_by_name( - _name, - _requirements, - silent=silent, - interactive=interactive) - requirements = _requirements - pkg_dir = BasePkgManager.install( - self, - name, - requirements, - silent=silent, - trigger_event=trigger_event, - force=force) - except exception.InternetIsOffline as e: - if not silent: - click.secho(str(e), fg="yellow") - return None - - if not pkg_dir: - return None - - manifest = self.load_manifest(pkg_dir) - if "dependencies" not in manifest: - return pkg_dir - - if not silent: - click.secho("Installing dependencies", fg="yellow") - - for filters in self.normalize_dependencies(manifest['dependencies']): - assert "name" in filters - if any([s in filters.get("version", "") for s in ("\\", "/")]): - self.install( - "{name}={version}".format(**filters), - silent=silent, - trigger_event=trigger_event, - interactive=interactive, - force=force) - else: - try: - lib_info = self.search_for_library(filters, silent, - interactive) - except exception.LibNotFound as e: - if not silent: - click.secho("Warning! %s" % e, fg="yellow") - continue - - if filters.get("version"): - self.install( - lib_info['id'], - filters.get("version"), - silent=silent, - trigger_event=trigger_event, - interactive=interactive, - force=force) - else: - self.install( - lib_info['id'], - silent=silent, - trigger_event=trigger_event, - interactive=interactive, - force=force) - return pkg_dir - - @staticmethod - def search_for_library( # pylint: disable=too-many-branches filters, silent=False, interactive=False): assert isinstance(filters, dict) assert "name" in filters + + # try to find ID within installed packages + lib_id = self._get_lib_id_from_installed(filters) + if lib_id: + return lib_id + + # looking in PIO Library Registry if not silent: click.echo("Looking for %s library in registry" % click.style( filters['name'], fg="cyan")) @@ -366,4 +285,141 @@ class LibraryManager(BasePkgManager): "http://platformio.org/lib/show/{id}/{name}".format( **lib_info), fg="blue")) - return lib_info + return int(lib_info['id']) + + def _get_lib_id_from_installed(self, filters): + if filters['name'].startswith("id="): + return int(filters['name'][3:]) + package_dir = self.get_package_dir(filters['name'], + filters.get("requirements", + filters.get("version"))) + if not package_dir: + return None + manifest = self.load_manifest(package_dir) + if "id" not in manifest: + return None + + for key in ("frameworks", "platforms"): + if key not in filters: + continue + if key not in manifest: + return None + if not util.items_in_list( + util.items_to_list(filters[key]), + util.items_to_list(manifest[key])): + return None + + if "authors" in filters: + if "authors" not in manifest: + return None + manifest_authors = manifest['authors'] + if not isinstance(manifest_authors, list): + manifest_authors = [manifest_authors] + manifest_authors = [ + a['name'] for a in manifest_authors + if isinstance(a, dict) and "name" in a + ] + filter_authors = filters['authors'] + if not isinstance(filter_authors, list): + filter_authors = [filter_authors] + if not set(filter_authors) <= set(manifest_authors): + return None + + return int(manifest['id']) + + def install( # pylint: disable=arguments-differ + self, + name, + requirements=None, + silent=False, + trigger_event=True, + interactive=False, + force=False): + _name, _requirements, _url = self.parse_pkg_uri(name, requirements) + if not _url: + name = "id=%d" % self.search_lib_id( + { + "name": _name, + "requirements": _requirements + }, + silent=silent, + interactive=interactive) + requirements = _requirements + pkg_dir = BasePkgManager.install( + self, + name, + requirements, + silent=silent, + trigger_event=trigger_event, + force=force) + + if not pkg_dir: + return None + + manifest = self.load_manifest(pkg_dir) + if "dependencies" not in manifest: + return pkg_dir + + if not silent: + click.secho("Installing dependencies", fg="yellow") + + for filters in self.normalize_dependencies(manifest['dependencies']): + assert "name" in filters + if any([s in filters.get("version", "") for s in ("\\", "/")]): + self.install( + "{name}={version}".format(**filters), + silent=silent, + trigger_event=trigger_event, + interactive=interactive, + force=force) + else: + try: + lib_id = self.search_lib_id(filters, silent, interactive) + except exception.LibNotFound as e: + if not silent or is_builtin_lib(filters['name']): + click.secho("Warning! %s" % e, fg="yellow") + continue + + if filters.get("version"): + self.install( + lib_id, + filters.get("version"), + silent=silent, + trigger_event=trigger_event, + interactive=interactive, + force=force) + else: + self.install( + lib_id, + silent=silent, + trigger_event=trigger_event, + interactive=interactive, + force=force) + return pkg_dir + + +@util.memoized +def get_builtin_libs(storage_names=None): + items = [] + storage_names = storage_names or [] + pm = PlatformManager() + for manifest in pm.get_installed(): + p = PlatformFactory.newPlatform(manifest['__pkg_dir']) + for storage in p.get_lib_storages(): + if storage_names and storage['name'] not in storage_names: + continue + lm = LibraryManager(storage['path']) + items.append({ + "name": storage['name'], + "path": storage['path'], + "items": lm.get_installed() + }) + return items + + +@util.memoized +def is_builtin_lib(name): + for storage in get_builtin_libs(): + if any([l.get("name") == name for l in storage['items']]): + return True + return False diff --git a/platformio/managers/package.py b/platformio/managers/package.py index cc3e444c..0b1e0ad1 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -134,6 +134,7 @@ class PkgRepoMixin(object): class PkgInstallerMixin(object): SRC_MANIFEST_NAME = ".piopkgmanager.json" + TMP_FOLDER_PREFIX = "_tmp_installing-" FILE_CACHE_VALID = "1m" # 1 month FILE_CACHE_MAX_SIZE = 1024 * 1024 @@ -211,6 +212,8 @@ class PkgInstallerMixin(object): try: return semantic_version.Version(value) except ValueError: + if "." not in str(value) and not str(value).isdigit(): + raise ValueError("Invalid SemVer version %s" % value) return semantic_version.Version.coerce(value) except ValueError as e: if raise_exception: @@ -292,6 +295,8 @@ class PkgInstallerMixin(object): def get_installed(self): items = [] for pkg_dir in self.read_dirs(self.package_dir): + if self.TMP_FOLDER_PREFIX in pkg_dir: + continue manifest = self.load_manifest(pkg_dir) if not manifest: continue @@ -361,7 +366,7 @@ class PkgInstallerMixin(object): break except Exception as e: # pylint: disable=broad-except click.secho("Warning! Package Mirror: %s" % e, fg="yellow") - click.secho("Looking for other mirror...", fg="yellow") + click.secho("Looking for another mirror...", fg="yellow") if versions is None: raise exception.UnknownPackage(name) @@ -376,7 +381,7 @@ class PkgInstallerMixin(object): requirements=None, sha1=None, track=False): - tmp_dir = mkdtemp("-package", "_tmp_installing-", self.package_dir) + tmp_dir = mkdtemp("-package", self.TMP_FOLDER_PREFIX, self.package_dir) src_manifest_dir = None src_manifest = {"name": name, "url": url, "requirements": requirements} @@ -661,6 +666,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): name, url, requirements, track=True) else: pkg_dir = self._install_from_piorepo(name, requirements) + if not pkg_dir or not self.manifest_exists(pkg_dir): raise exception.PackageInstallError(name, requirements or "*", util.get_systype()) diff --git a/platformio/util.py b/platformio/util.py index a040380f..53c23918 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -22,13 +22,13 @@ import socket import stat import subprocess import sys +import time from functools import wraps from glob import glob from os.path import (abspath, basename, dirname, expanduser, isdir, isfile, join, normpath, splitdrive) from shutil import rmtree from threading import Thread -from time import sleep, time import click import requests @@ -159,10 +159,10 @@ class throttle(object): @wraps(fn) def wrapper(*args, **kwargs): - diff = int(round((time() - self.last) * 1000)) + diff = int(round((time.time() - self.last) * 1000)) if diff < self.threshhold: - sleep((self.threshhold - diff) * 0.001) - self.last = time() + time.sleep((self.threshhold - diff) * 0.001) + self.last = time.time() return fn(*args, **kwargs) return wrapper @@ -311,8 +311,8 @@ def get_projectboards_dir(): join(get_project_dir(), "boards")) -def get_projectpioenvs_dir(force=False): - path = get_project_optional_dir("envs_dir", +def get_projectbuild_dir(force=False): + path = get_project_optional_dir("build_dir", join(get_project_dir(), ".pioenvs")) try: if not isdir(path): @@ -322,7 +322,7 @@ def get_projectpioenvs_dir(force=False): with open(dontmod_path, "w") as fp: fp.write(""" [InternetShortcut] -URL=http://docs.platformio.org/page/projectconf.html#envs-dir +URL=http://docs.platformio.org/page/projectconf/section_platformio.html#build-dir """) except Exception as e: # pylint: disable=broad-except if not force: @@ -330,6 +330,10 @@ URL=http://docs.platformio.org/page/projectconf.html#envs-dir return path +# compatibility with PIO Core+ +get_projectpioenvs_dir = get_projectbuild_dir + + def get_projectdata_dir(): return get_project_optional_dir("data_dir", join(get_project_dir(), "data")) @@ -445,6 +449,10 @@ def get_serial_ports(filter_hwid=False): return result +# Backward compatibility for PIO Core <3.5 +get_serialports = get_serial_ports + + def get_logical_devices(): items = [] if platform.system() == "Windows": @@ -483,14 +491,6 @@ def get_logical_devices(): return items -### Backward compatibility for PIO Core <3.5 -get_serialports = get_serial_ports -get_logicaldisks = lambda: [{ - "disk": d['path'], - "name": d['name'] -} for d in get_logical_devices()] - - def get_mdns_services(): try: import zeroconf @@ -541,7 +541,7 @@ def get_mdns_services(): items = [] with mDNSListener() as mdns: - sleep(3) + time.sleep(3) for service in mdns.get_services(): items.append({ "type": @@ -621,7 +621,6 @@ def _get_api_result( def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): - internet_on(raise_exception=True) from platformio.app import ContentCache total = 0 max_retries = 5 @@ -634,6 +633,10 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): result = cc.get(cache_key) if result is not None: return result + + # check internet before and resolve issue with 60 seconds timeout + internet_on(raise_exception=True) + result = _get_api_result(url, params, data) if cache_valid: with ContentCache() as cc: @@ -648,7 +651,7 @@ def get_api_result(url, params=None, data=None, auth=None, cache_valid=None): "[API] ConnectionError: {0} (incremented retry: max={1}, " "total={2})".format(e, max_retries, total), fg="yellow") - sleep(2 * total) + time.sleep(2 * total) raise exception.APIRequestError( "Could not connect to PlatformIO API Service. " @@ -720,6 +723,26 @@ def pepver_to_semver(pepver): return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) +def items_to_list(items): + if not isinstance(items, list): + items = [i.strip() for i in items.split(",")] + return [i.lower() for i in items if i] + + +def items_in_list(needle, haystack): + needle = items_to_list(needle) + haystack = items_to_list(haystack) + if "*" in needle or "*" in haystack: + return True + return set(needle) & set(haystack) + + +def parse_date(datestr): + if "T" in datestr and "Z" in datestr: + return time.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") + return time.strptime(datestr) + + def rmtree_(path): def _onerror(_, name, __): diff --git a/setup.py b/setup.py index df14dd1d..b4b8b1aa 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ from platformio import (__author__, __description__, __email__, __license__, __title__, __url__, __version__) install_requires = [ - "arrow>=0.10.0,!=0.11.0", "bottle<0.13", "click>=5,<6", "colorama", @@ -37,6 +36,7 @@ setup( author_email=__email__, url=__url__, license=__license__, + python_requires='>=2.7, <3', install_requires=install_requires, packages=find_packages(), package_data={ diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 9c77582b..2ab76544 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -45,7 +45,7 @@ def test_ci_lib_and_board(clirunner, validate_cliresult): example_dir = join("examples", "atmelavr", "arduino-external-libs") result = clirunner.invoke(cmd_ci, [ join(example_dir, "lib", "OneWire", "examples", "DS2408_Switch", - "DS2408_Switch.pde"), "-l", join(example_dir, "lib", "OneWire"), - "-b", "uno" + "DS2408_Switch.pde"), "-l", + join(example_dir, "lib", "OneWire"), "-b", "uno" ]) validate_cliresult(result) diff --git a/tests/commands/test_init.py b/tests/commands/test_init.py index 4b7c0a94..eecb25f3 100644 --- a/tests/commands/test_init.py +++ b/tests/commands/test_init.py @@ -54,7 +54,7 @@ def test_init_duplicated_boards(clirunner, validate_cliresult, tmpdir): assert set(config.sections()) == set(["env:uno"]) -def test_init_ide_without_board(clirunner, validate_cliresult, tmpdir): +def test_init_ide_without_board(clirunner, tmpdir): with tmpdir.as_cwd(): result = clirunner.invoke(cmd_init, ["--ide", "atom"]) assert result.exit_code == -1 @@ -67,13 +67,15 @@ def test_init_ide_atom(clirunner, validate_cliresult, tmpdir): cmd_init, ["--ide", "atom", "-b", "uno", "-b", "teensy31"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) - assert all([tmpdir.join(f).check() - for f in (".clang_complete", ".gcc-flags.json")]) + assert all([ + tmpdir.join(f).check() + for f in (".clang_complete", ".gcc-flags.json") + ]) assert "arduinoavr" in tmpdir.join(".clang_complete").read() # switch to NodeMCU - result = clirunner.invoke( - cmd_init, ["--ide", "atom", "-b", "nodemcuv2"]) + result = clirunner.invoke(cmd_init, + ["--ide", "atom", "-b", "nodemcuv2"]) validate_cliresult(result) validate_pioproject(str(tmpdir)) assert "arduinoespressif" in tmpdir.join(".clang_complete").read() @@ -104,15 +106,13 @@ def test_init_special_board(clirunner, validate_cliresult): boards = json.loads(result.output) config = util.load_project_config() - expected_result = [ - ("platform", str(boards[0]['platform'])), - ("framework", str(boards[0]['frameworks'][0])), ("board", "uno") - ] + expected_result = [("platform", str(boards[0]['platform'])), + ("framework", + str(boards[0]['frameworks'][0])), ("board", "uno")] assert config.has_section("env:uno") - assert len( - set(expected_result).symmetric_difference( - set(config.items("env:uno")))) == 0 + assert not set(expected_result).symmetric_difference( + set(config.items("env:uno"))) def test_init_enable_auto_uploading(clirunner, validate_cliresult): @@ -122,14 +122,11 @@ def test_init_enable_auto_uploading(clirunner, validate_cliresult): validate_cliresult(result) validate_pioproject(getcwd()) config = util.load_project_config() - expected_result = [ - ("platform", "atmelavr"), ("framework", "arduino"), - ("board", "uno"), ("targets", "upload") - ] + expected_result = [("platform", "atmelavr"), ("framework", "arduino"), + ("board", "uno"), ("targets", "upload")] assert config.has_section("env:uno") - assert len( - set(expected_result).symmetric_difference( - set(config.items("env:uno")))) == 0 + assert not set(expected_result).symmetric_difference( + set(config.items("env:uno"))) def test_init_custom_framework(clirunner, validate_cliresult): @@ -139,14 +136,11 @@ def test_init_custom_framework(clirunner, validate_cliresult): validate_cliresult(result) validate_pioproject(getcwd()) config = util.load_project_config() - expected_result = [ - ("platform", "teensy"), ("framework", "mbed"), - ("board", "teensy31") - ] + expected_result = [("platform", "teensy"), ("framework", "mbed"), + ("board", "teensy31")] assert config.has_section("env:teensy31") - assert len( - set(expected_result).symmetric_difference( - set(config.items("env:teensy31")))) == 0 + assert not set(expected_result).symmetric_difference( + set(config.items("env:teensy31"))) def test_init_incorrect_board(clirunner): diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index f2b8d725..8513d401 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -15,8 +15,7 @@ import json import re -from platformio import exception, util -from platformio.commands.init import cli as cmd_init +from platformio import exception from platformio.commands.lib import cli as cmd_lib @@ -36,25 +35,11 @@ def test_search(clirunner, validate_cliresult): def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_lib, [ - "-g", "install", "58", "547@2.2.4", "DallasTemperature", - "http://dl.platformio.org/libraries/archives/3/5174.tar.gz", - "ArduinoJson@5.6.7", "ArduinoJson@~5.7.0", "168@00589a3250" + "-g", "install", "64", "ArduinoJson@~5.10.0", "547@2.2.4", + "AsyncMqttClient@<=0.8.2", "999@77d4eb3f8a" ]) validate_cliresult(result) - # check lib with duplicate URL - result = clirunner.invoke(cmd_lib, [ - "-g", "install", - "http://dl.platformio.org/libraries/archives/3/5174.tar.gz" - ]) - validate_cliresult(result) - assert "is already installed" in result.output - - # check lib with duplicate ID - result = clirunner.invoke(cmd_lib, ["-g", "install", "305"]) - validate_cliresult(result) - assert "is already installed" in result.output - # install unknown library result = clirunner.invoke(cmd_lib, ["-g", "install", "Unknown"]) assert result.exit_code != 0 @@ -62,9 +47,9 @@ def test_global_install_registry(clirunner, validate_cliresult, items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ - "ArduinoJson_ID64", "ArduinoJson_ID64@5.6.7", "DallasTemperature_ID54", - "DHT22_ID58", "ESPAsyncTCP_ID305", "NeoPixelBus_ID547", "OneWire_ID1", - "EspSoftwareSerial_ID168" + "ArduinoJson_ID64", "ArduinoJson_ID64@5.10.1", "NeoPixelBus_ID547", + "AsyncMqttClient_ID346", "ESPAsyncTCP_ID305", "AsyncTCP_ID1826", + "RFcontrol_ID999" ] assert set(items1) == set(items2) @@ -72,11 +57,12 @@ def test_global_install_registry(clirunner, validate_cliresult, def test_global_install_archive(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_lib, [ - "-g", "install", "https://github.com/adafruit/Adafruit-ST7735-Library/" - "archive/master.zip", + "-g", "install", "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip", "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2" + "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip@5.8.2", + "http://dl.platformio.org/libraries/archives/0/9540.tar.gz", + "https://github.com/adafruit/Adafruit-ST7735-Library/archive/master.zip" ]) validate_cliresult(result) @@ -87,16 +73,11 @@ def test_global_install_archive(clirunner, validate_cliresult, ]) assert result.exit_code != 0 - # check lib with duplicate URL - result = clirunner.invoke(cmd_lib, [ - "-g", "install", - "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip" - ]) - validate_cliresult(result) - assert "is already installed" in result.output - items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] - items2 = ["Adafruit ST7735 Library", "RadioHead-1.62"] + items2 = [ + "RadioHead-1.62", "ArduinoJson", "DallasTemperature_ID54", + "OneWire_ID1", "Adafruit ST7735 Library" + ] assert set(items1) >= set(items2) @@ -113,18 +94,41 @@ def test_global_install_repository(clirunner, validate_cliresult, "https://gitlab.com/ivankravets/rs485-nodeproto.git", "https://github.com/platformio/platformio-libmirror.git", # "https://developer.mbed.org/users/simon/code/TextLCD/", - "knolleary/pubsubclient" + "knolleary/pubsubclient#bef58148582f956dfa772687db80c44e2279a163" ]) validate_cliresult(result) items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ "PJON", "PJON@src-79de467ebe19de18287becff0a1fb42d", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "rs485-nodeproto", - "PubSubClient" + "platformio-libmirror", "PubSubClient" ] assert set(items1) >= set(items2) - # check lib with duplicate URL + +def test_install_duplicates(clirunner, validate_cliresult, without_internet): + # registry + result = clirunner.invoke(cmd_lib, [ + "-g", "install", + "http://dl.platformio.org/libraries/archives/0/9540.tar.gz" + ]) + validate_cliresult(result) + assert "is already installed" in result.output + + # by ID + result = clirunner.invoke(cmd_lib, ["-g", "install", "999"]) + validate_cliresult(result) + assert "is already installed" in result.output + + # archive + result = clirunner.invoke(cmd_lib, [ + "-g", "install", + "http://www.airspayce.com/mikem/arduino/RadioHead/RadioHead-1.62.zip" + ]) + validate_cliresult(result) + assert "is already installed" in result.output + + # repository result = clirunner.invoke(cmd_lib, [ "-g", "install", "https://github.com/platformio/platformio-libmirror.git" @@ -136,23 +140,42 @@ def test_global_install_repository(clirunner, validate_cliresult, def test_global_lib_list(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "list"]) validate_cliresult(result) - assert all([n in result.output for n in ("OneWire", "DHT22", "64")]) + assert all([ + n in result.output for n in + ("Source: https://github.com/adafruit/Adafruit-ST7735-Library/archive/master.zip", + "Version: 5.10.1", + "Source: git+https://github.com/gioblu/PJON.git#3.0", + "Version: 1fb26fd", "RadioHead-1.62") + ]) result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) assert all([ - n in result.output - for n in ( - "PJON", "git+https://github.com/knolleary/pubsubclient", - "https://github.com/bblanchon/ArduinoJson/archive/v5.8.2.zip") + n in result.output for n in + ("__pkg_dir", + '"__src_url": "git+https://gitlab.com/ivankravets/rs485-nodeproto.git"', + '"version": "5.10.1"') ]) items1 = [i['name'] for i in json.loads(result.output)] items2 = [ - "OneWire", "DHT22", "PJON", "ESPAsyncTCP", "ArduinoJson", - "PubSubClient", "rs485-nodeproto", "Adafruit ST7735 Library", - "RadioHead-1.62", "DallasTemperature", "NeoPixelBus", - "EspSoftwareSerial", "platformio-libmirror" + "Adafruit ST7735 Library", "ArduinoJson", "ArduinoJson", "ArduinoJson", + "ArduinoJson", "AsyncMqttClient", "AsyncTCP", "DallasTemperature", + "ESPAsyncTCP", "NeoPixelBus", "OneWire", "PJON", "PJON", + "PubSubClient", "RFcontrol", "RadioHead-1.62", "platformio-libmirror", + "rs485-nodeproto" ] - assert set(items1) == set(items2) + assert sorted(items1) == sorted(items2) + + versions1 = [ + "{name}@{version}".format(**item) + for item in json.loads(result.output) + ] + versions2 = [ + 'ArduinoJson@5.8.2', 'ArduinoJson@5.10.1', 'AsyncMqttClient@0.8.2', + 'AsyncTCP@1.0.1', 'ESPAsyncTCP@1.1.3', 'NeoPixelBus@2.2.4', + 'PJON@07fe9aa', 'PJON@1fb26fd', 'PubSubClient@bef5814', + 'RFcontrol@77d4eb3f8a', 'RadioHead-1.62@0.0.0' + ] + assert set(versions1) >= set(versions2) def test_global_lib_update_check(clirunner, validate_cliresult): @@ -160,7 +183,7 @@ def test_global_lib_update_check(clirunner, validate_cliresult): cmd_lib, ["-g", "update", "--only-check", "--json-output"]) validate_cliresult(result) output = json.loads(result.output) - assert set(["ArduinoJson", "EspSoftwareSerial", + assert set(["RFcontrol", "NeoPixelBus"]) == set([l['name'] for l in output]) @@ -181,11 +204,9 @@ def test_global_lib_update(clirunner, validate_cliresult): # update rest libraries result = clirunner.invoke(cmd_lib, ["-g", "update"]) validate_cliresult(result) - validate_cliresult(result) - assert result.output.count("[Fixed]") == 5 - assert result.output.count("[Up-to-date]") == 10 - assert "Uninstalling ArduinoJson @ 5.7.3" in result.output - assert "Uninstalling EspSoftwareSerial @ 00589a3250" in result.output + assert result.output.count("[Fixed]") == 6 + assert result.output.count("[Up-to-date]") == 11 + assert "Uninstalling RFcontrol @ 77d4eb3f8a" in result.output # update unknown library result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) @@ -207,16 +228,17 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, # uninstall the rest libraries result = clirunner.invoke(cmd_lib, [ "-g", "uninstall", "1", "https://github.com/bblanchon/ArduinoJson.git", - "ArduinoJson@!=5.6.7", "EspSoftwareSerial@>=3.3.1" + "ArduinoJson@!=5.6.7", "RFcontrol" ]) validate_cliresult(result) items1 = [d.basename for d in isolated_pio_home.join("lib").listdir()] items2 = [ - "ArduinoJson_ID64", "ArduinoJson_ID64@5.6.7", "DallasTemperature_ID54", - "DHT22_ID58", "ESPAsyncTCP_ID305", "NeoPixelBus_ID547", "PJON", - "PJON@src-79de467ebe19de18287becff0a1fb42d", "PubSubClient", - "RadioHead-1.62", "rs485-nodeproto", "platformio-libmirror" + "RadioHead-1.62", "rs485-nodeproto", "platformio-libmirror", + "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", + "ESPAsyncTCP_ID305", "DallasTemperature_ID54", "NeoPixelBus_ID547", + "PJON", "AsyncMqttClient_ID346", "ArduinoJson_ID64", + "PJON@src-79de467ebe19de18287becff0a1fb42d", "AsyncTCP_ID1826" ] assert set(items1) == set(items2) @@ -248,7 +270,7 @@ def test_lib_stats(clirunner, validate_cliresult): validate_cliresult(result) assert all([ s in result.output - for s in ("UPDATED", "ago", "http://platformio.org/lib/show") + for s in ("UPDATED", "POPULAR", "http://platformio.org/lib/show") ]) result = clirunner.invoke(cmd_lib, ["stats", "--json-output"]) diff --git a/tests/commands/test_platform.py b/tests/commands/test_platform.py index 2bb747fd..5b3053fe 100644 --- a/tests/commands/test_platform.py +++ b/tests/commands/test_platform.py @@ -24,27 +24,25 @@ def test_search_json_output(clirunner, validate_cliresult, isolated_pio_home): validate_cliresult(result) search_result = json.loads(result.output) assert isinstance(search_result, list) - assert len(search_result) + assert search_result platforms = [item['name'] for item in search_result] assert "atmelsam" in platforms -def test_search_raw_output(clirunner, validate_cliresult, isolated_pio_home): +def test_search_raw_output(clirunner, validate_cliresult): result = clirunner.invoke(cli_platform.platform_search, ["arduino"]) validate_cliresult(result) assert "teensy" in result.output -def test_install_unknown_version(clirunner, validate_cliresult, - isolated_pio_home): +def test_install_unknown_version(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["atmelavr@99.99.99"]) assert result.exit_code == -1 assert isinstance(result.exception, exception.UndefinedPackageVersion) -def test_install_unknown_from_registry(clirunner, validate_cliresult, - isolated_pio_home): +def test_install_unknown_from_registry(clirunner): result = clirunner.invoke(cli_platform.platform_install, ["unknown-platform"]) assert result.exit_code == -1 @@ -63,7 +61,7 @@ def test_install_known_version(clirunner, validate_cliresult, assert len(isolated_pio_home.join("packages").listdir()) == 1 -def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home): +def test_install_from_vcs(clirunner, validate_cliresult): result = clirunner.invoke(cli_platform.platform_install, [ "https://github.com/platformio/" "platform-espressif8266.git#feature/stage", "--skip-default-package" @@ -72,17 +70,17 @@ def test_install_from_vcs(clirunner, validate_cliresult, isolated_pio_home): assert "espressif8266" in result.output -def test_list_json_output(clirunner, validate_cliresult, isolated_pio_home): +def test_list_json_output(clirunner, validate_cliresult): result = clirunner.invoke(cli_platform.platform_list, ["--json-output"]) validate_cliresult(result) list_result = json.loads(result.output) assert isinstance(list_result, list) - assert len(list_result) + assert list_result platforms = [item['name'] for item in list_result] assert set(["atmelavr", "espressif8266"]) == set(platforms) -def test_list_raw_output(clirunner, validate_cliresult, isolated_pio_home): +def test_list_raw_output(clirunner, validate_cliresult): result = clirunner.invoke(cli_platform.platform_list) validate_cliresult(result) assert all( @@ -111,4 +109,4 @@ def test_uninstall(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cli_platform.platform_uninstall, ["atmelavr", "espressif8266"]) validate_cliresult(result) - assert len(isolated_pio_home.join("platforms").listdir()) == 0 + assert not isolated_pio_home.join("platforms").listdir() diff --git a/tests/commands/test_settings.py b/tests/commands/test_settings.py index c6cd33da..e846ecc6 100644 --- a/tests/commands/test_settings.py +++ b/tests/commands/test_settings.py @@ -19,6 +19,6 @@ from platformio.commands.settings import cli def test_settings_check(clirunner, validate_cliresult): result = clirunner.invoke(cli, ["get"]) validate_cliresult(result) - assert len(result.output) + assert result.output for item in app.DEFAULT_SETTINGS.items(): assert item[0] in result.output diff --git a/tests/commands/test_update.py b/tests/commands/test_update.py index b8309fb8..9325e501 100644 --- a/tests/commands/test_update.py +++ b/tests/commands/test_update.py @@ -16,11 +16,7 @@ from platformio.commands.update import cli as cmd_update def test_update(clirunner, validate_cliresult): - matches = ( - "Platform Manager", - "Up-to-date", - "Library Manager" - ) + matches = ("Platform Manager", "Up-to-date", "Library Manager") result = clirunner.invoke(cmd_update, ["--only-check"]) validate_cliresult(result) assert all([m in result.output for m in matches]) diff --git a/tests/conftest.py b/tests/conftest.py index b407b328..5b8103c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,10 +17,7 @@ import os import pytest from click.testing import CliRunner - -@pytest.fixture(scope="module") -def clirunner(): - return CliRunner() +from platformio import util @pytest.fixture(scope="session") @@ -33,6 +30,11 @@ def validate_cliresult(): return decorator +@pytest.fixture(scope="module") +def clirunner(): + return CliRunner() + + @pytest.fixture(scope="module") def isolated_pio_home(request, tmpdir_factory): home_dir = tmpdir_factory.mktemp(".platformio") @@ -43,3 +45,8 @@ def isolated_pio_home(request, tmpdir_factory): request.addfinalizer(fin) return home_dir + + +@pytest.fixture(scope="function") +def without_internet(monkeypatch): + monkeypatch.setattr(util, "_internet_on", lambda: False) diff --git a/tests/test_examples.py b/tests/test_examples.py index 86bc7741..4d2a1a52 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -36,26 +36,27 @@ def pytest_generate_tests(metafunc): @pytest.mark.examples def test_run(pioproject_dir): - if isdir(join(pioproject_dir, ".pioenvs")): - util.rmtree_(join(pioproject_dir, ".pioenvs")) + with util.cd(pioproject_dir): + build_dir = util.get_projectbuild_dir() + if isdir(build_dir): + util.rmtree_(build_dir) - result = util.exec_command( - ["platformio", "--force", "run", "--project-dir", pioproject_dir] - ) - if result['returncode'] != 0: - pytest.fail(result) + result = util.exec_command(["platformio", "--force", "run"]) + if result['returncode'] != 0: + pytest.fail(result) - # check .elf file - pioenvs_dir = join(pioproject_dir, ".pioenvs") - for item in listdir(pioenvs_dir): - if not isdir(item): - continue - assert isfile(join(pioenvs_dir, item, "firmware.elf")) - # check .hex or .bin files - firmwares = [] - for ext in ("bin", "hex"): - firmwares += glob(join(pioenvs_dir, item, "firmware*.%s" % ext)) - if not firmwares: - pytest.fail("Missed firmware file") - for firmware in firmwares: - assert getsize(firmware) > 0 + assert isdir(build_dir) + + # check .elf file + for item in listdir(build_dir): + if not isdir(item): + continue + assert isfile(join(build_dir, item, "firmware.elf")) + # check .hex or .bin files + firmwares = [] + for ext in ("bin", "hex"): + firmwares += glob(join(build_dir, item, "firmware*.%s" % ext)) + if not firmwares: + pytest.fail("Missed firmware file") + for firmware in firmwares: + assert getsize(firmware) > 0 diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 9fd7fd6b..667c1563 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -57,8 +57,7 @@ def test_after_upgrade_2_to_3(clirunner, validate_cliresult, assert board_ids == set([b['id'] for b in json.loads(result.output)]) -def test_after_upgrade_silence(clirunner, validate_cliresult, - isolated_pio_home): +def test_after_upgrade_silence(clirunner, validate_cliresult): app.set_state_item("last_version", "2.11.2") result = clirunner.invoke(cli_pio, ["boards", "--json-output"]) validate_cliresult(result) @@ -66,7 +65,7 @@ def test_after_upgrade_silence(clirunner, validate_cliresult, assert any([b['id'] == "uno" for b in boards]) -def test_check_pio_upgrade(clirunner, validate_cliresult, isolated_pio_home): +def test_check_pio_upgrade(clirunner, validate_cliresult): def _patch_pio_version(version): maintenance.__version__ = version @@ -96,7 +95,7 @@ def test_check_pio_upgrade(clirunner, validate_cliresult, isolated_pio_home): _patch_pio_version(origin_version) -def test_check_lib_updates(clirunner, validate_cliresult, isolated_pio_home): +def test_check_lib_updates(clirunner, validate_cliresult): # install obsolete library result = clirunner.invoke(cli_pio, ["lib", "-g", "install", "ArduinoJson@<5.7"]) @@ -113,8 +112,7 @@ def test_check_lib_updates(clirunner, validate_cliresult, isolated_pio_home): result.output) -def test_check_and_update_libraries(clirunner, validate_cliresult, - isolated_pio_home): +def test_check_and_update_libraries(clirunner, validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_libraries", "Yes"]) @@ -168,8 +166,7 @@ def test_check_platform_updates(clirunner, validate_cliresult, assert "There are the new updates for platforms (native)" in result.output -def test_check_and_update_platforms(clirunner, validate_cliresult, - isolated_pio_home): +def test_check_and_update_platforms(clirunner, validate_cliresult): # enable library auto-updates result = clirunner.invoke( cli_pio, ["settings", "set", "auto_update_platforms", "Yes"]) @@ -190,8 +187,7 @@ def test_check_and_update_platforms(clirunner, validate_cliresult, validate_cliresult(result) assert "There are the new updates for platforms (native)" in result.output assert "Please wait while updating platforms" in result.output - assert re.search(r"Updating native\s+@ 0.0.0\s+\[[\d\.]+\]", - result.output) + assert re.search(r"Updating native\s+@ 0.0.0\s+\[[\d\.]+\]", result.output) # check updated version result = clirunner.invoke(cli_pio, ["platform", "list", "--json-output"]) diff --git a/tests/test_managers.py b/tests/test_managers.py index 559c9d29..6fe2f273 100644 --- a/tests/test_managers.py +++ b/tests/test_managers.py @@ -186,7 +186,7 @@ def test_install_packages(isolated_pio_home, tmpdir): "packages").listdir()]) == set(pkg_dirnames) -def test_get_package(isolated_pio_home): +def test_get_package(): tests = [ [("unknown", ), None], [("1", ), None], diff --git a/tests/test_misc.py b/tests/test_misc.py index e8121c99..b918767b 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -12,11 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest import requests -from platformio import util +from platformio import exception, util def test_ping_internet_ips(): for ip in util.PING_INTERNET_IPS: requests.get("http://%s" % ip, allow_redirects=False, timeout=2) + + +def test_api_internet_offline(without_internet, isolated_pio_home): + with pytest.raises(exception.InternetIsOffline): + util.get_api_result("/stats") + + +def test_api_cache(monkeypatch, isolated_pio_home): + api_kwargs = {"url": "/stats", "cache_valid": "10s"} + result = util.get_api_result(**api_kwargs) + assert result and "boards" in result + monkeypatch.setattr(util, '_internet_on', lambda: False) + assert util.get_api_result(**api_kwargs) == result diff --git a/tox.ini b/tox.ini index ad3288c7..f1196266 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,7 @@ deps = yapf pylint pytest + pytest-xdist commands = python --version [testenv:docs]