diff --git a/.appveyor.yml b/.appveyor.yml index 12d10edf..955b9aca 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,13 +7,15 @@ environment: matrix: - TOXENV: "py27" PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P2_{build} + PYTHON_DIRS: C:\Python27-x64;C:\Python27-x64\Scripts - TOXENV: "py36" PLATFORMIO_BUILD_CACHE_DIR: C:\Temp\PIO_Build_Cache_P3_{build} + PYTHON_DIRS: C:\Python36-x64;C:\Python36-x64\Scripts install: - cmd: git submodule update --init --recursive - - cmd: SET PATH=C:\MinGW\bin;%PATH% + - cmd: SET PATH=%PYTHON_DIRS%;C:\MinGW\bin;%PATH% - cmd: SET PLATFORMIO_CORE_DIR=C:\.pio - cmd: pip install --force-reinstall tox diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index 7bc85964..00000000 --- a/.style.yapf +++ /dev/null @@ -1,3 +0,0 @@ -[style] -blank_line_before_nested_class_or_def = true -allow_multiline_lambdas = true diff --git a/HISTORY.rst b/HISTORY.rst index 5660956c..4771a3c0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,38 @@ Release Notes PlatformIO Core 4 ----------------- +4.3.0 (2020-03-19) +~~~~~~~~~~~~~~~~~~ + +* Initial support for an official `PlatformIO for CLion IDE `__ plugin: + + - Smart C and C++ editor + - Code refactoring + - On-the-fly code analysis + - "New PlatformIO Project" wizard + - Building, Uploading, Testing + - Integrated debugger (inline variable view, conditional breakpoints, expressions, watchpoints, peripheral registers, multi-thread support, etc.) + +* `Device Monitor 2.0 `__ + + - Added **PlatformIO Device Monitor Filter API** (dev-platforms can extend base device monitor with a custom functionality, such as exception decoding) (`pull #3383 `_) + - Configure project device monitor with `monitor_filters `__ option + - `Capture device monitor output to a file `__ with ``log2file`` filter (`issue #670 `_) + - Show a timestamp for each new line with ``time`` filter (`issue #981 `_) + - Send a text to device on ENTER with ``send_on_enter`` filter (`issue #926 `_) + - Show a hexadecimal representation of the data (code point of each character) with ``hexlify`` filter + +* New standalone (1-script) `PlatformIO Core Installer `_ +* Initial support for `Renode `__ simulation framework (`issue #3401 `_) +* Added support for Arm Mbed "module.json" ``dependencies`` field (`issue #3400 `_) +* Improved support for Arduino "library.properties" ``depends`` field +* Fixed an issue when quitting from PlatformIO IDE does not shutdown PIO Home server +* Fixed an issue "the JSON object must be str, not 'bytes'" when PIO Home is used with Python 3.5 (`issue #3396 `_) +* Fixed an issue when Python 2 does not keep encoding when converting ".ino" (`issue #3393 `_) +* Fixed an issue when ``"libArchive": false`` in "library.json" does not work (`issue #3403 `_) +* Fixed an issue when not all commands in `compilation database "compile_commands.json" `__ use absolute paths (`pull #3415 `_) +* Fixed an issue when unknown transport is used for `PIO Unit Testing `__ engine (`issue #3422 `_) + 4.2.1 (2020-02-17) ~~~~~~~~~~~~~~~~~~ @@ -34,7 +66,7 @@ PlatformIO Core 4 - Show computed project configuration with a new `platformio project config `_ command or dump to JSON with ``platformio project config --json-output`` (`issue #3335 `_) - Moved ``platformio init`` command to `platformio project init `_ -* Generate `compilation database "compile_commands.json" `_ (`issue #2990 `_) +* Generate `compilation database "compile_commands.json" `__ (`issue #2990 `_) * Control debug flags and optimization level with a new `debug_build_flags `__ option * Install a dev-platform with ALL declared packages using a new ``--with-all-packages`` option for `pio platform install `__ command (`issue #3345 `_) * Added support for "pythonPackages" in `platform.json `__ manifest (PlatformIO Package Manager will install dependent Python packages from PyPi registry automatically when dev-platform is installed) diff --git a/README.rst b/README.rst index 992b7c0c..0122741d 100644 --- a/README.rst +++ b/README.rst @@ -58,8 +58,8 @@ Instruments * `Continuous Integration `_ * `Advanced Scripting API `_ -PIO Plus --------- +Professional +------------ * `PIO Check `_ * `PIO Remote `_ diff --git a/docs b/docs index 4b50528d..51b7dd49 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 4b50528d78ad3c5e36d149e275f20b5a7154e3a8 +Subproject commit 51b7dd49b703fd6156375a93d4bbb6e0890ecb09 diff --git a/examples b/examples index e1d64112..370c2c41 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit e1d641126d5fb8855da3070a029946fec25118b6 +Subproject commit 370c2c41a10fe02aafe0f4910f827a00b0f24ad8 diff --git a/platformio/__init__.py b/platformio/__init__.py index 120df28d..c5dd8dcc 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION = (4, 2, 1) +VERSION = (4, 3, 0) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py index 2226f825..52463e83 100644 --- a/platformio/builder/tools/compilation_db.py +++ b/platformio/builder/tools/compilation_db.py @@ -76,6 +76,16 @@ def makeEmitCompilationDbEntry(comstr): :return: target(s), source(s) """ + # Resolve absolute path of toolchain + for cmd in ("CC", "CXX", "AS"): + if cmd not in env: + continue + if os.path.isabs(env[cmd]): + continue + env[cmd] = where_is_program( + env.subst("$%s" % cmd), env.subst("${ENV['PATH']}") + ) + dbtarget = __CompilationDbNode(source) entry = env.__COMPILATIONDB_Entry( @@ -195,14 +205,6 @@ def generate(env, **kwargs): ) def CompilationDatabase(env, target): - # Resolve absolute path of toolchain - for cmd in ("CC", "CXX", "AS"): - if cmd not in env: - continue - env[cmd] = where_is_program( - env.subst("$%s" % cmd), env.subst("${ENV['PATH']}") - ) - result = env.__COMPILATIONDB_Database(target=target, source=[]) env.AlwaysBuild(result) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index b55c492e..c9c97e28 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -18,6 +18,7 @@ from __future__ import absolute_import import hashlib +import io import os import re import sys @@ -82,7 +83,8 @@ class LibBuilderFactory(object): fname, piotool.SRC_BUILD_EXT + piotool.SRC_HEADER_EXT ): continue - content = fs.get_file_contents(join(root, fname)) + with io.open(join(root, fname), errors="ignore") as fp: + content = fp.read() if not content: continue if "Arduino.h" in content and include_re.search(content): @@ -716,9 +718,11 @@ class PlatformIOLibBuilder(LibBuilderBase): @property def lib_archive(self): - unique_value = "_not_declared_%s" % id(self) - global_value = self.env.GetProjectOption("lib_archive", unique_value) - if global_value != unique_value: + missing = object() + global_value = self.env.GetProjectConfig().getraw( + "env:" + self.env["PIOENV"], "lib_archive", missing + ) + if global_value != missing: return global_value return self._manifest.get("build", {}).get( "libArchive", LibBuilderBase.lib_archive.fget(self) diff --git a/platformio/builder/tools/piomaxlen.py b/platformio/builder/tools/piomaxlen.py index 3bf258db..04f1304f 100644 --- a/platformio/builder/tools/piomaxlen.py +++ b/platformio/builder/tools/piomaxlen.py @@ -18,7 +18,6 @@ from hashlib import md5 from os import makedirs from os.path import isdir, isfile, join -from platformio import fs from platformio.compat import WINDOWS, hashlib_encode_data # Windows CLI has limit with command length to 8192 @@ -67,7 +66,8 @@ def _file_long_data(env, data): ) if isfile(tmp_file): return tmp_file - fs.write_file_contents(tmp_file, data) + with open(tmp_file, "w") as fp: + fp.write(data) return tmp_file diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index ba019ec3..1079f402 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -15,17 +15,19 @@ from __future__ import absolute_import import atexit +import io import re import sys from os import environ, remove, walk from os.path import basename, isdir, isfile, join, realpath, relpath, sep from tempfile import mkstemp +import click from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import fs, util -from platformio.compat import glob_escape +from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape from platformio.managers.core import get_core_package_dir from platformio.proc import exec_command @@ -48,6 +50,39 @@ class InoToCPPConverter(object): def __init__(self, env): self.env = env self._main_ino = None + self._safe_encoding = None + + def read_safe_contents(self, path): + error_reported = False + for encoding in ( + "utf-8", + None, + get_filesystem_encoding(), + get_locale_encoding(), + "latin-1", + ): + try: + with io.open(path, encoding=encoding) as fp: + contents = fp.read() + self._safe_encoding = encoding + return contents + except UnicodeDecodeError: + if not error_reported: + error_reported = True + click.secho( + "Unicode decode error has occurred, please remove invalid " + "(non-ASCII or non-UTF8) characters from %s file or convert it to UTF-8" + % path, + fg="yellow", + err=True, + ) + return "" + + def write_safe_contents(self, path, contents): + with io.open( + path, "w", encoding=self._safe_encoding, errors="backslashreplace" + ) as fp: + return fp.write(contents) def is_main_node(self, contents): return self.DETECTMAIN_RE.search(contents) @@ -62,7 +97,7 @@ class InoToCPPConverter(object): assert nodes lines = [] for node in nodes: - contents = fs.get_file_contents(node.get_path()) + contents = self.read_safe_contents(node.get_path()) _lines = ['# 1 "%s"' % node.get_path().replace("\\", "/"), contents] if self.is_main_node(contents): lines = _lines + lines @@ -78,16 +113,14 @@ class InoToCPPConverter(object): def process(self, contents): out_file = self._main_ino + ".cpp" assert self._gcc_preprocess(contents, out_file) - contents = fs.get_file_contents(out_file) + contents = self.read_safe_contents(out_file) contents = self._join_multiline_strings(contents) - fs.write_file_contents( - out_file, self.append_prototypes(contents), errors="backslashreplace" - ) + self.write_safe_contents(out_file, self.append_prototypes(contents)) return out_file def _gcc_preprocess(self, contents, out_file): tmp_path = mkstemp()[1] - fs.write_file_contents(tmp_path, contents, errors="backslashreplace") + self.write_safe_contents(tmp_path, contents) self.env.Execute( self.env.VerboseAction( '$CXX -o "{0}" -x c++ -fpreprocessed -dD -E "{1}"'.format( diff --git a/platformio/builder/tools/pioproject.py b/platformio/builder/tools/pioproject.py index 4a42b3d8..4bf848d9 100644 --- a/platformio/builder/tools/pioproject.py +++ b/platformio/builder/tools/pioproject.py @@ -14,7 +14,7 @@ from __future__ import absolute_import -from platformio.project.config import ProjectConfig, ProjectOptions +from platformio.project.config import MISSING, ProjectConfig, ProjectOptions def GetProjectConfig(env): @@ -25,7 +25,7 @@ def GetProjectOptions(env, as_dict=False): return env.GetProjectConfig().items(env=env["PIOENV"], as_dict=as_dict) -def GetProjectOption(env, option, default=None): +def GetProjectOption(env, option, default=MISSING): return env.GetProjectConfig().get("env:" + env["PIOENV"], option, default) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index fa468bba..4e2298f9 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -20,21 +20,21 @@ from hashlib import sha1 from os.path import basename, dirname, isdir, join, realpath, splitext from tempfile import mkdtemp +from twisted.internet import defer # pylint: disable=import-error from twisted.internet import protocol # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error from twisted.internet import stdio # pylint: disable=import-error from twisted.internet import task # pylint: disable=import-error from platformio import app, fs, proc, telemetry, util -from platformio.commands.debug import helpers, initcfgs +from platformio.commands.debug import helpers from platformio.commands.debug.exception import DebugInvalidOptionsError +from platformio.commands.debug.initcfgs import get_gdb_init_config from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer from platformio.compat import hashlib_encode_data, is_bytes from platformio.project.helpers import get_project_cache_dir -LOG_FILE = None - class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes @@ -42,6 +42,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" def __init__(self, project_dir, args, debug_options, env_options): + super(GDBClient, self).__init__() self.project_dir = project_dir self.args = list(args) self.debug_options = debug_options @@ -55,10 +56,10 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes self._gdbsrc_dir = mkdtemp(dir=get_project_cache_dir(), prefix=".piodebug-") self._target_is_run = False - self._last_server_activity = 0 self._auto_continue_timer = None self._errors_buffer = b"" + @defer.inlineCallbacks def spawn(self, gdb_path, prog_path): session_hash = gdb_path + prog_path self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest() @@ -75,10 +76,10 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []), } - self._debug_server.spawn(patterns) - + yield self._debug_server.spawn(patterns) if not patterns["DEBUG_PORT"]: patterns["DEBUG_PORT"] = self._debug_server.get_debug_port() + self.generate_pioinit(self._gdbsrc_dir, patterns) # start GDB client @@ -100,9 +101,10 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes args.extend(["--data-directory", gdb_data_dir]) args.append(patterns["PROG_PATH"]) - return reactor.spawnProcess( + transport = reactor.spawnProcess( self, gdb_path, args, path=self.project_dir, env=os.environ ) + defer.returnValue(transport) @staticmethod def _get_data_dir(gdb_path): @@ -112,22 +114,8 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes return gdb_data_dir if isdir(gdb_data_dir) else None def generate_pioinit(self, dst_dir, patterns): - server_exe = ( - (self.debug_options.get("server") or {}).get("executable", "").lower() - ) - if "jlink" in server_exe: - cfg = initcfgs.GDB_JLINK_INIT_CONFIG - elif "st-util" in server_exe: - cfg = initcfgs.GDB_STUTIL_INIT_CONFIG - elif "mspdebug" in server_exe: - cfg = initcfgs.GDB_MSPDEBUG_INIT_CONFIG - elif "qemu" in server_exe: - cfg = initcfgs.GDB_QEMU_INIT_CONFIG - elif self.debug_options["require_debug_port"]: - cfg = initcfgs.GDB_BLACKMAGIC_INIT_CONFIG - else: - cfg = initcfgs.GDB_DEFAULT_INIT_CONFIG - commands = cfg.split("\n") + # default GDB init commands depending on debug tool + commands = get_gdb_init_config(self.debug_options).split("\n") if self.debug_options["init_cmds"]: commands = self.debug_options["init_cmds"] @@ -175,12 +163,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes stdio.StandardIO(p) def onStdInData(self, data): - if LOG_FILE: - with open(LOG_FILE, "ab") as fp: - fp.write(data) - - self._last_server_activity = time.time() - + super(GDBClient, self).onStdInData(data) if b"-exec-run" in data: if self._target_is_run: token, _ = data.split(b"-", 1) @@ -206,11 +189,6 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes reactor.stop() def outReceived(self, data): - if LOG_FILE: - with open(LOG_FILE, "ab") as fp: - fp.write(data) - - self._last_server_activity = time.time() super(GDBClient, self).outReceived(data) self._handle_error(data) # go to init break automatically @@ -232,7 +210,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes def _auto_exec_continue(self): auto_exec_delay = 0.5 # in seconds - if self._last_server_activity > (time.time() - auto_exec_delay): + if self._last_activity > (time.time() - auto_exec_delay): return if self._auto_continue_timer: self._auto_continue_timer.stop() diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 5edba11a..4604a861 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -125,7 +125,8 @@ def validate_debug_options(cmd_ctx, env_options): server_options["executable"] = server_options["arguments"][0] server_options["arguments"] = server_options["arguments"][1:] elif "server" in tool_settings: - server_package = tool_settings["server"].get("package") + server_options = tool_settings["server"] + server_package = server_options.get("package") server_package_dir = ( platform.get_package_dir(server_package) if server_package else None ) @@ -134,15 +135,17 @@ def validate_debug_options(cmd_ctx, env_options): with_packages=[server_package], skip_default_package=True, silent=True ) server_package_dir = platform.get_package_dir(server_package) - server_options = dict( - cwd=server_package_dir if server_package else None, - executable=tool_settings["server"].get("executable"), - arguments=[ - a.replace("$PACKAGE_DIR", server_package_dir) - if server_package_dir - else a - for a in tool_settings["server"].get("arguments", []) - ], + server_options.update( + dict( + cwd=server_package_dir if server_package else None, + executable=server_options.get("executable"), + arguments=[ + a.replace("$PACKAGE_DIR", server_package_dir) + if server_package_dir + else a + for a in server_options.get("arguments", []) + ], + ) ) extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds")) @@ -252,12 +255,14 @@ def is_prog_obsolete(prog_path): break shasum.update(data) new_digest = shasum.hexdigest() - old_digest = ( - fs.get_file_contents(prog_hash_path) if isfile(prog_hash_path) else None - ) + old_digest = None + if isfile(prog_hash_path): + with open(prog_hash_path) as fp: + old_digest = fp.read() if new_digest == old_digest: return False - fs.write_file_contents(prog_hash_path, new_digest) + with open(prog_hash_path, "w") as fp: + fp.write(new_digest) return True diff --git a/platformio/commands/debug/initcfgs.py b/platformio/commands/debug/initcfgs.py index 50da1779..55e7c34f 100644 --- a/platformio/commands/debug/initcfgs.py +++ b/platformio/commands/debug/initcfgs.py @@ -123,3 +123,39 @@ $LOAD_CMDS pio_reset_halt_target $INIT_BREAK """ + +GDB_RENODE_INIT_CONFIG = """ +define pio_reset_halt_target + monitor machine Reset + $LOAD_CMDS + monitor start +end + +define pio_reset_run_target + pio_reset_halt_target +end + +target extended-remote $DEBUG_PORT +$LOAD_CMDS +$INIT_BREAK +monitor start +""" + + +TOOL_TO_CONFIG = { + "jlink": GDB_JLINK_INIT_CONFIG, + "mspdebug": GDB_MSPDEBUG_INIT_CONFIG, + "qemu": GDB_QEMU_INIT_CONFIG, + "blackmagic": GDB_BLACKMAGIC_INIT_CONFIG, + "renode": GDB_RENODE_INIT_CONFIG, +} + + +def get_gdb_init_config(debug_options): + tool = debug_options.get("tool") + if tool and tool in TOOL_TO_CONFIG: + return TOOL_TO_CONFIG[tool] + server_exe = (debug_options.get("server") or {}).get("executable", "").lower() + if "st-util" in server_exe: + return GDB_STUTIL_INIT_CONFIG + return GDB_DEFAULT_INIT_CONFIG diff --git a/platformio/commands/debug/process.py b/platformio/commands/debug/process.py index d23f26b3..67557f3d 100644 --- a/platformio/commands/debug/process.py +++ b/platformio/commands/debug/process.py @@ -13,6 +13,7 @@ # limitations under the License. import signal +import time import click from twisted.internet import protocol # pylint: disable=import-error @@ -22,12 +23,11 @@ from platformio.compat import string_types from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_core_dir -LOG_FILE = None - class BaseProcess(protocol.ProcessProtocol, object): STDOUT_CHUNK_SIZE = 2048 + LOG_FILE = None COMMON_PATTERNS = { "PLATFORMIO_HOME_DIR": get_project_core_dir(), @@ -35,6 +35,9 @@ class BaseProcess(protocol.ProcessProtocol, object): "PYTHONEXE": get_pythonexe_path(), } + def __init__(self): + self._last_activity = 0 + def apply_patterns(self, source, patterns=None): _patterns = self.COMMON_PATTERNS.copy() _patterns.update(patterns or {}) @@ -61,23 +64,30 @@ class BaseProcess(protocol.ProcessProtocol, object): return source + def onStdInData(self, data): + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: + fp.write(data) + def outReceived(self, data): - if LOG_FILE: - with open(LOG_FILE, "ab") as fp: + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: fp.write(data) while data: chunk = data[: self.STDOUT_CHUNK_SIZE] click.echo(chunk, nl=False) data = data[self.STDOUT_CHUNK_SIZE :] - @staticmethod - def errReceived(data): - if LOG_FILE: - with open(LOG_FILE, "ab") as fp: + def errReceived(self, data): + self._last_activity = time.time() + if self.LOG_FILE: + with open(self.LOG_FILE, "ab") as fp: fp.write(data) click.echo(data, nl=False, err=True) - @staticmethod - def processEnded(_): + def processEnded(self, _): + self._last_activity = time.time() # Allow terminating via SIGINT/CTRL+C signal.signal(signal.SIGINT, signal.default_int_handler) diff --git a/platformio/commands/debug/server.py b/platformio/commands/debug/server.py index 855628c3..29a00678 100644 --- a/platformio/commands/debug/server.py +++ b/platformio/commands/debug/server.py @@ -13,8 +13,10 @@ # limitations under the License. import os +import time from os.path import isdir, isfile, join +from twisted.internet import defer # pylint: disable=import-error from twisted.internet import reactor # pylint: disable=import-error from platformio import fs, util @@ -26,13 +28,16 @@ from platformio.proc import where_is_program class DebugServer(BaseProcess): def __init__(self, debug_options, env_options): + super(DebugServer, self).__init__() self.debug_options = debug_options self.env_options = env_options - self._debug_port = None + self._debug_port = ":3333" self._transport = None self._process_ended = False + self._ready = False + @defer.inlineCallbacks def spawn(self, patterns): # pylint: disable=too-many-branches systype = util.get_systype() server = self.debug_options.get("server") @@ -62,7 +67,6 @@ class DebugServer(BaseProcess): % server_executable ) - self._debug_port = ":3333" openocd_pipe_allowed = all( [not self.debug_options["port"], "openocd" in server_executable] ) @@ -79,43 +83,62 @@ class DebugServer(BaseProcess): ) self._debug_port = '| "%s" %s' % (server_executable, str_args) self._debug_port = fs.to_unix_path(self._debug_port) - else: - env = os.environ.copy() - # prepend server "lib" folder to LD path - if ( - "windows" not in systype - and server["cwd"] - and isdir(join(server["cwd"], "lib")) - ): - ld_key = ( - "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH" - ) - env[ld_key] = join(server["cwd"], "lib") - if os.environ.get(ld_key): - env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key)) - # prepend BIN to PATH - if server["cwd"] and isdir(join(server["cwd"], "bin")): - env["PATH"] = "%s%s%s" % ( - join(server["cwd"], "bin"), - os.pathsep, - os.environ.get("PATH", os.environ.get("Path", "")), - ) + return self._debug_port - self._transport = reactor.spawnProcess( - self, - server_executable, - [server_executable] + server["arguments"], - path=server["cwd"], - env=env, + env = os.environ.copy() + # prepend server "lib" folder to LD path + if ( + "windows" not in systype + and server["cwd"] + and isdir(join(server["cwd"], "lib")) + ): + ld_key = "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH" + env[ld_key] = join(server["cwd"], "lib") + if os.environ.get(ld_key): + env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key)) + # prepend BIN to PATH + if server["cwd"] and isdir(join(server["cwd"], "bin")): + env["PATH"] = "%s%s%s" % ( + join(server["cwd"], "bin"), + os.pathsep, + os.environ.get("PATH", os.environ.get("Path", "")), ) - if "mspdebug" in server_executable.lower(): - self._debug_port = ":2000" - elif "jlink" in server_executable.lower(): - self._debug_port = ":2331" - elif "qemu" in server_executable.lower(): - self._debug_port = ":1234" - return self._transport + self._transport = reactor.spawnProcess( + self, + server_executable, + [server_executable] + server["arguments"], + path=server["cwd"], + env=env, + ) + if "mspdebug" in server_executable.lower(): + self._debug_port = ":2000" + elif "jlink" in server_executable.lower(): + self._debug_port = ":2331" + elif "qemu" in server_executable.lower(): + self._debug_port = ":1234" + + yield self._wait_until_ready() + + return self._debug_port + + @defer.inlineCallbacks + def _wait_until_ready(self): + timeout = 10 + elapsed = 0 + delay = 0.5 + auto_ready_delay = 0.5 + while not self._ready and not self._process_ended and elapsed < timeout: + yield self.async_sleep(delay) + if not self.debug_options.get("server", {}).get("ready_pattern"): + self._ready = self._last_activity < (time.time() - auto_ready_delay) + elapsed += delay + + @staticmethod + def async_sleep(secs): + d = defer.Deferred() + reactor.callLater(secs, d.callback, None) + return d def get_debug_port(self): return self._debug_port @@ -124,6 +147,11 @@ class DebugServer(BaseProcess): super(DebugServer, self).outReceived( escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data ) + if self._ready: + return + ready_pattern = self.debug_options.get("server", {}).get("ready_pattern") + if ready_pattern: + self._ready = ready_pattern.encode() in data def processEnded(self, reason): self._process_ended = True diff --git a/platformio/commands/device/__init__.py b/platformio/commands/device/__init__.py new file mode 100644 index 00000000..bcee03cc --- /dev/null +++ b/platformio/commands/device/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.device.filters.base import DeviceMonitorFilter diff --git a/platformio/commands/device.py b/platformio/commands/device/command.py similarity index 76% rename from platformio/commands/device.py rename to platformio/commands/device/command.py index 24e7d3ca..e93b1214 100644 --- a/platformio/commands/device.py +++ b/platformio/commands/device/command.py @@ -12,17 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import sys from fnmatch import fnmatch -from os import getcwd import click from serial.tools import miniterm from platformio import exception, fs, util +from platformio.commands.device import helpers as device_helpers from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformFactory -from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError @@ -135,7 +135,7 @@ def device_list( # pylint: disable=too-many-branches help="Set the encoding for the serial port (e.g. hexlify, " "Latin1, UTF-8), default: UTF-8", ) -@click.option("--filter", "-f", multiple=True, help="Add text transformation") +@click.option("--filter", "-f", multiple=True, help="Add filters/text transformations") @click.option( "--eol", default="CRLF", @@ -165,7 +165,7 @@ def device_list( # pylint: disable=too-many-branches @click.option( "-d", "--project-dir", - default=getcwd, + default=os.getcwd, type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), ) @click.option( @@ -174,27 +174,36 @@ def device_list( # pylint: disable=too-many-branches help="Load configuration from `platformio.ini` and specified environment", ) def device_monitor(**kwargs): # pylint: disable=too-many-branches - click.echo( - "Looking for advanced Serial Monitor with UI? " - "Check http://bit.ly/pio-advanced-monitor" - ) + # load default monitor filters + filters_dir = os.path.join(fs.get_source_dir(), "commands", "device", "filters") + for name in os.listdir(filters_dir): + if not name.endswith(".py"): + continue + device_helpers.load_monitor_filter(os.path.join(filters_dir, name)) + project_options = {} try: with fs.cd(kwargs["project_dir"]): - project_options = get_project_options(kwargs["environment"]) - kwargs = apply_project_monitor_options(kwargs, project_options) + project_options = device_helpers.get_project_options(kwargs["environment"]) + kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options) except NotPlatformIOProjectError: pass + platform = None + if "platform" in project_options: + with fs.cd(kwargs["project_dir"]): + platform = PlatformFactory.newPlatform(project_options["platform"]) + device_helpers.register_platform_filters( + platform, kwargs["project_dir"], kwargs["environment"] + ) + if not kwargs["port"]: ports = util.get_serial_ports(filter_hwid=True) if len(ports) == 1: kwargs["port"] = ports[0]["port"] elif "platform" in project_options and "board" in project_options: - board_hwids = get_board_hwids( - kwargs["project_dir"], - project_options["platform"], - project_options["board"], + board_hwids = device_helpers.get_board_hwids( + kwargs["project_dir"], platform, project_options["board"], ) for item in ports: for hwid in board_hwids: @@ -211,12 +220,18 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches break # override system argv with patched options - sys.argv = ["monitor"] + options_to_argv( + sys.argv = ["monitor"] + device_helpers.options_to_argv( kwargs, project_options, ignore=("port", "baud", "rts", "dtr", "environment", "project_dir"), ) + if not kwargs["quiet"]: + click.echo( + "--- Available filters and text transformations: %s" + % ", ".join(sorted(miniterm.TRANSFORMATIONS.keys())) + ) + click.echo("--- More details at http://bit.ly/pio-monitor-filters") try: miniterm.main( default_port=kwargs["port"], @@ -226,55 +241,3 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches ) except Exception as e: raise exception.MinitermException(e) - - -def apply_project_monitor_options(cli_options, project_options): - for k in ("port", "speed", "rts", "dtr"): - k2 = "monitor_%s" % k - if k == "speed": - k = "baud" - if cli_options[k] is None and k2 in project_options: - cli_options[k] = project_options[k2] - if k != "port": - cli_options[k] = int(cli_options[k]) - return cli_options - - -def options_to_argv(cli_options, project_options, ignore=None): - result = project_options.get("monitor_flags", []) - for k, v in cli_options.items(): - if v is None or (ignore and k in ignore): - continue - k = "--" + k.replace("_", "-") - if k in project_options.get("monitor_flags", []): - continue - if isinstance(v, bool): - if v: - result.append(k) - elif isinstance(v, tuple): - for i in v: - result.extend([k, i]) - else: - result.extend([k, str(v)]) - return result - - -def get_project_options(environment=None): - config = ProjectConfig.get_instance() - config.validate(envs=[environment] if environment else None) - if not environment: - default_envs = config.default_envs() - if default_envs: - environment = default_envs[0] - else: - environment = config.envs()[0] - return config.items(env=environment, as_dict=True) - - -def get_board_hwids(project_dir, platform, board): - with fs.cd(project_dir): - return ( - PlatformFactory.newPlatform(platform) - .board_config(board) - .get("build.hwids", []) - ) diff --git a/platformio/commands/device/filters/__init__.py b/platformio/commands/device/filters/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/commands/device/filters/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/commands/device/filters/base.py b/platformio/commands/device/filters/base.py new file mode 100644 index 00000000..bc0880b3 --- /dev/null +++ b/platformio/commands/device/filters/base.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from serial.tools import miniterm + +from platformio.project.config import ProjectConfig + + +class DeviceMonitorFilter(miniterm.Transform): + def __init__(self, project_dir=None, environment=None): + """ Called by PlatformIO to pass context """ + super(DeviceMonitorFilter, self).__init__() + + self.project_dir = project_dir + self.environment = environment + + self.config = ProjectConfig.get_instance() + if not self.environment: + default_envs = self.config.default_envs() + if default_envs: + self.environment = default_envs[0] + elif self.config.envs(): + self.environment = self.config.envs()[0] + + def __call__(self): + """ Called by the miniterm library when the filter is actually used """ + return self + + @property + def NAME(self): + raise NotImplementedError("Please declare NAME attribute for the filter class") diff --git a/platformio/commands/device/filters/hexlify.py b/platformio/commands/device/filters/hexlify.py new file mode 100644 index 00000000..1023b573 --- /dev/null +++ b/platformio/commands/device/filters/hexlify.py @@ -0,0 +1,38 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import serial + +from platformio.commands.device import DeviceMonitorFilter + + +class Hexlify(DeviceMonitorFilter): + NAME = "hexlify" + + def __init__(self, *args, **kwargs): + super(Hexlify, self).__init__(*args, **kwargs) + self._counter = 0 + + def rx(self, text): + result = "" + for b in serial.iterbytes(text): + if (self._counter % 16) == 0: + result += "\n{:04X} | ".format(self._counter) + asciicode = ord(b) + if asciicode <= 255: + result += "{:02X} ".format(asciicode) + else: + result += "?? " + self._counter += 1 + return result diff --git a/platformio/commands/device/filters/log2file.py b/platformio/commands/device/filters/log2file.py new file mode 100644 index 00000000..69118510 --- /dev/null +++ b/platformio/commands/device/filters/log2file.py @@ -0,0 +1,44 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os.path +from datetime import datetime + +from platformio.commands.device import DeviceMonitorFilter + + +class LogToFile(DeviceMonitorFilter): + NAME = "log2file" + + def __init__(self, *args, **kwargs): + super(LogToFile, self).__init__(*args, **kwargs) + self._log_fp = None + + def __call__(self): + log_file_name = "platformio-device-monitor-%s.log" % datetime.now().strftime( + "%y%m%d-%H%M%S" + ) + print("--- Logging an output to %s" % os.path.abspath(log_file_name)) + self._log_fp = io.open(log_file_name, "w", encoding="utf-8") + return self + + def __del__(self): + if self._log_fp: + self._log_fp.close() + + def rx(self, text): + self._log_fp.write(text) + self._log_fp.flush() + return text diff --git a/platformio/commands/device/filters/send_on_enter.py b/platformio/commands/device/filters/send_on_enter.py new file mode 100644 index 00000000..10ca2103 --- /dev/null +++ b/platformio/commands/device/filters/send_on_enter.py @@ -0,0 +1,31 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.commands.device import DeviceMonitorFilter + + +class SendOnEnter(DeviceMonitorFilter): + NAME = "send_on_enter" + + def __init__(self, *args, **kwargs): + super(SendOnEnter, self).__init__(*args, **kwargs) + self._buffer = "" + + def tx(self, text): + self._buffer += text + if self._buffer.endswith("\r\n"): + text = self._buffer[:-2] + self._buffer = "" + return text + return "" diff --git a/platformio/commands/device/filters/time.py b/platformio/commands/device/filters/time.py new file mode 100644 index 00000000..203e6aa9 --- /dev/null +++ b/platformio/commands/device/filters/time.py @@ -0,0 +1,34 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +from platformio.commands.device import DeviceMonitorFilter + + +class Timestamp(DeviceMonitorFilter): + NAME = "time" + + def __init__(self, *args, **kwargs): + super(Timestamp, self).__init__(*args, **kwargs) + self._first_text_received = False + + def rx(self, text): + if self._first_text_received and "\n" not in text: + return text + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + if not self._first_text_received: + self._first_text_received = True + return "%s > %s" % (timestamp, text) + return text.replace("\n", "\n%s > " % timestamp) diff --git a/platformio/commands/device/helpers.py b/platformio/commands/device/helpers.py new file mode 100644 index 00000000..3bfe8fc6 --- /dev/null +++ b/platformio/commands/device/helpers.py @@ -0,0 +1,106 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import os + +from serial.tools import miniterm + +from platformio import fs +from platformio.commands.device import DeviceMonitorFilter +from platformio.compat import get_object_members, load_python_module +from platformio.project.config import ProjectConfig + + +def apply_project_monitor_options(cli_options, project_options): + for k in ("port", "speed", "rts", "dtr"): + k2 = "monitor_%s" % k + if k == "speed": + k = "baud" + if cli_options[k] is None and k2 in project_options: + cli_options[k] = project_options[k2] + if k != "port": + cli_options[k] = int(cli_options[k]) + return cli_options + + +def options_to_argv(cli_options, project_options, ignore=None): + confmon_flags = project_options.get("monitor_flags", []) + result = confmon_flags[::] + + for f in project_options.get("monitor_filters", []): + result.extend(["--filter", f]) + + for k, v in cli_options.items(): + if v is None or (ignore and k in ignore): + continue + k = "--" + k.replace("_", "-") + if k in confmon_flags: + continue + if isinstance(v, bool): + if v: + result.append(k) + elif isinstance(v, tuple): + for i in v: + result.extend([k, i]) + else: + result.extend([k, str(v)]) + return result + + +def get_project_options(environment=None): + config = ProjectConfig.get_instance() + config.validate(envs=[environment] if environment else None) + if not environment: + default_envs = config.default_envs() + if default_envs: + environment = default_envs[0] + else: + environment = config.envs()[0] + return config.items(env=environment, as_dict=True) + + +def get_board_hwids(project_dir, platform, board): + with fs.cd(project_dir): + return platform.board_config(board).get("build.hwids", []) + + +def load_monitor_filter(path, project_dir=None, environment=None): + name = os.path.basename(path) + name = name[: name.find(".")] + module = load_python_module("platformio.commands.device.filters.%s" % name, path) + for cls in get_object_members(module).values(): + if ( + not inspect.isclass(cls) + or not issubclass(cls, DeviceMonitorFilter) + or cls == DeviceMonitorFilter + ): + continue + obj = cls(project_dir, environment) + miniterm.TRANSFORMATIONS[obj.NAME] = obj + return True + + +def register_platform_filters(platform, project_dir, environment): + monitor_dir = os.path.join(platform.get_dir(), "monitor") + if not os.path.isdir(monitor_dir): + return + + for name in os.listdir(monitor_dir): + if not name.startswith("filter_") or not name.endswith(".py"): + continue + path = os.path.join(monitor_dir, name) + if not os.path.isfile(path): + continue + load_monitor_filter(path, project_dir, environment) diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 208354bf..229c875e 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=too-many-locals +# pylint: disable=too-many-locals,too-many-statements import mimetypes import socket @@ -63,6 +63,7 @@ def cli(port, host, no_open, shutdown_timeout): from twisted.internet import reactor from twisted.web import server + from twisted.internet.error import CannotListenError from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.ide import IDERPC @@ -121,6 +122,12 @@ def cli(port, host, no_open, shutdown_timeout): click.echo("") click.echo("Open PlatformIO Home in your browser by this URL => %s" % home_url) + try: + reactor.listenTCP(port, site, interface=host) + except CannotListenError as e: + click.secho(str(e), fg="red", err=True) + already_started = True + if already_started: click.secho( "PlatformIO Home server is already started in another process.", fg="yellow" @@ -129,7 +136,6 @@ def cli(port, host, no_open, shutdown_timeout): click.echo("PIO Home has been started. Press Ctrl+C to shutdown.") - reactor.listenTCP(port, site, interface=host) reactor.run() diff --git a/platformio/commands/home/rpc/handlers/os.py b/platformio/commands/home/rpc/handlers/os.py index cefb0630..745ae817 100644 --- a/platformio/commands/home/rpc/handlers/os.py +++ b/platformio/commands/home/rpc/handlers/os.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import glob +import io import os import shutil from functools import cmp_to_key @@ -66,7 +67,8 @@ class OSRPC(object): if uri.startswith("http"): return self.fetch_content(uri, data, headers, cache_valid) if os.path.isfile(uri): - return fs.get_file_contents(uri, encoding="utf8") + with io.open(uri, encoding="utf-8") as fp: + return fp.read() return None @staticmethod diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index 9ef39a03..41009c4e 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -146,6 +146,8 @@ class PIOCoreRPC(object): raise Exception(text) if not to_json: return text + if is_bytes(out): + out = out.decode() try: return json.loads(out) except ValueError as e: diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 51e47dcb..77a04646 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -244,7 +244,8 @@ class ProjectRPC(object): return project_dir if not os.path.isdir(src_dir): os.makedirs(src_dir) - fs.write_file_contents(main_path, main_content.strip()) + with open(main_path, "w") as fp: + fp.write(main_content.strip()) return project_dir def import_arduino(self, board, use_arduino_libs, arduino_project_dir): diff --git a/platformio/commands/home/web.py b/platformio/commands/home/web.py index 45b6c37d..32bf0692 100644 --- a/platformio/commands/home/web.py +++ b/platformio/commands/home/web.py @@ -18,7 +18,7 @@ from twisted.web import static # pylint: disable=import-error class WebRoot(static.File): def render_GET(self, request): - if request.args.get("__shutdown__", False): + if request.args.get(b"__shutdown__", False): reactor.stop() return "Server has been stopped" diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index d7c42d4c..e3b415d5 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -21,7 +21,7 @@ import click import semantic_version from tabulate import tabulate -from platformio import exception, util +from platformio import exception, fs, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import LibraryManager, get_builtin_libs, is_builtin_lib @@ -106,17 +106,20 @@ def cli(ctx, **options): if not is_platformio_project(storage_dir): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue - config = ProjectConfig.get_instance(os.path.join(storage_dir, "platformio.ini")) - config.validate(options["environment"], silent=in_silence) - libdeps_dir = config.get_optional_dir("libdeps") - for env in config.envs(): - if options["environment"] and env not in options["environment"]: - continue - storage_dir = os.path.join(libdeps_dir, env) - ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) - ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( - "env:" + env, "lib_deps", [] + with fs.cd(storage_dir): + config = ProjectConfig.get_instance( + os.path.join(storage_dir, "platformio.ini") ) + config.validate(options["environment"], silent=in_silence) + libdeps_dir = config.get_optional_dir("libdeps") + for env in config.envs(): + if options["environment"] and env not in options["environment"]: + continue + storage_dir = os.path.join(libdeps_dir, env) + ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) + ctx.meta[CTX_META_STORAGE_LIBDEPS_KEY][storage_dir] = config.get( + "env:" + env, "lib_deps", [] + ) @cli.command("install", short_help="Install library") diff --git a/platformio/commands/project.py b/platformio/commands/project.py index 33d35006..3d73f4ff 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -187,9 +187,9 @@ def init_base_project(project_dir): def init_include_readme(include_dir): - fs.write_file_contents( - os.path.join(include_dir, "README"), - """ + with open(os.path.join(include_dir, "README"), "w") as fp: + fp.write( + """ This directory is intended for project header files. A header file is a file containing C declarations and macro definitions @@ -229,14 +229,14 @@ Read more about using header files in official GCC documentation: https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html """, - ) + ) def init_lib_readme(lib_dir): # pylint: disable=line-too-long - fs.write_file_contents( - os.path.join(lib_dir, "README"), - """ + with open(os.path.join(lib_dir, "README"), "w") as fp: + fp.write( + """ This directory is intended for project specific (private) libraries. PlatformIO will compile them to static libraries and link into executable file. @@ -283,13 +283,13 @@ libraries scanning project source files. More information about PlatformIO Library Dependency Finder - https://docs.platformio.org/page/librarymanager/ldf.html """, - ) + ) def init_test_readme(test_dir): - fs.write_file_contents( - os.path.join(test_dir, "README"), - """ + with open(os.path.join(test_dir, "README"), "w") as fp: + fp.write( + """ This directory is intended for PIO Unit Testing and project tests. Unit Testing is a software testing method by which individual units of @@ -301,16 +301,16 @@ in the development cycle. More information about PIO Unit Testing: - https://docs.platformio.org/page/plus/unit-testing.html """, - ) + ) def init_ci_conf(project_dir): conf_path = os.path.join(project_dir, ".travis.yml") if os.path.isfile(conf_path): return - fs.write_file_contents( - conf_path, - """# Continuous Integration (CI) is the practice, in software + with open(conf_path, "w") as fp: + fp.write( + """# Continuous Integration (CI) is the practice, in software # engineering, of merging all developer working copies with a shared mainline # several times a day < https://docs.platformio.org/page/ci/index.html > # @@ -378,14 +378,15 @@ def init_ci_conf(project_dir): # script: # - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N """, - ) + ) def init_cvs_ignore(project_dir): conf_path = os.path.join(project_dir, ".gitignore") if os.path.isfile(conf_path): return - fs.write_file_contents(conf_path, ".pio\n") + with open(conf_path, "w") as fp: + fp.write(".pio\n") def fill_project_envs( diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index 0ebbf856..ca296b69 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -21,7 +21,8 @@ from time import sleep import click from platformio import exception, fs -from platformio.commands import device +from platformio.commands.device import helpers as device_helpers +from platformio.commands.device.command import device_monitor as cmd_device_monitor from platformio.managers.core import pioplus_call from platformio.project.exception import NotPlatformIOProjectError @@ -197,8 +198,8 @@ def device_monitor(ctx, **kwargs): project_options = {} try: with fs.cd(kwargs["project_dir"]): - project_options = device.get_project_options(kwargs["environment"]) - kwargs = device.apply_project_monitor_options(kwargs, project_options) + project_options = device_helpers.get_project_options(kwargs["environment"]) + kwargs = device_helpers.apply_project_monitor_options(kwargs, project_options) except NotPlatformIOProjectError: pass @@ -206,7 +207,7 @@ def device_monitor(ctx, **kwargs): def _tx_target(sock_dir): pioplus_argv = ["remote", "device", "monitor"] - pioplus_argv.extend(device.options_to_argv(kwargs, project_options)) + pioplus_argv.extend(device_helpers.options_to_argv(kwargs, project_options)) pioplus_argv.extend(["--sock", sock_dir]) try: pioplus_call(pioplus_argv) @@ -222,8 +223,9 @@ def device_monitor(ctx, **kwargs): sleep(0.1) if not t.is_alive(): return - kwargs["port"] = fs.get_file_contents(sock_file) - ctx.invoke(device.device_monitor, **kwargs) + with open(sock_file) as fp: + kwargs["port"] = fp.read() + ctx.invoke(cmd_device_monitor, **kwargs) t.join(2) finally: fs.rmtree(sock_dir) diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 5058c159..378eaf0d 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -21,7 +21,7 @@ import click from tabulate import tabulate from platformio import app, exception, fs, util -from platformio.commands.device import device_monitor as cmd_device_monitor +from platformio.commands.device.command import device_monitor as cmd_device_monitor from platformio.commands.run.helpers import clean_build_dir, handle_legacy_libdeps from platformio.commands.run.processor import EnvironmentProcessor from platformio.commands.test.processor import CTX_META_TEST_IS_RUNNING diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index ec038e84..ab362c24 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -53,9 +53,12 @@ def clean_build_dir(build_dir, config): if isdir(build_dir): # check project structure - if isfile(checksum_file) and fs.get_file_contents(checksum_file) == checksum: - return + if isfile(checksum_file): + with open(checksum_file) as fp: + if fp.read() == checksum: + return fs.rmtree(build_dir) makedirs(build_dir) - fs.write_file_contents(checksum_file, checksum) + with open(checksum_file, "w") as fp: + fp.write(checksum) diff --git a/platformio/commands/test/processor.py b/platformio/commands/test/processor.py index 7d9704b8..5291b9f7 100644 --- a/platformio/commands/test/processor.py +++ b/platformio/commands/test/processor.py @@ -19,7 +19,7 @@ from string import Template import click -from platformio import exception, fs +from platformio import exception TRANSPORT_OPTIONS = { "arduino": { @@ -83,6 +83,7 @@ class TestProcessorBase(object): self._outputcpp_generated = False def get_transport(self): + transport = None if self.env_options.get("platform") == "native": transport = "native" elif "framework" in self.env_options: @@ -91,7 +92,9 @@ class TestProcessorBase(object): transport = self.env_options["test_transport"] if transport not in TRANSPORT_OPTIONS: raise exception.PlatformioException( - "Unknown Unit Test transport `%s`" % transport + "Unknown Unit Test transport `%s`. Please check a documentation how " + "to create an own 'Test Transport':\n" + "- https://docs.platformio.org/page/plus/unit-testing.html" % transport ) return transport.lower() @@ -195,6 +198,7 @@ class TestProcessorBase(object): data = Template(tpl).substitute(baudrate=self.get_baudrate()) tmp_file = join(test_dir, "output_export.cpp") - fs.write_file_contents(tmp_file, data) + with open(tmp_file, "w") as fp: + fp.write(data) atexit.register(delete_tmptest_file, tmp_file) diff --git a/platformio/compat.py b/platformio/compat.py index 9107f8b1..c812e98d 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -38,12 +38,14 @@ def get_locale_encoding(): return None -def get_class_attributes(cls): - attributes = inspect.getmembers(cls, lambda a: not inspect.isroutine(a)) +def get_object_members(obj, ignore_private=True): + members = inspect.getmembers(obj, lambda a: not inspect.isroutine(a)) + if not ignore_private: + return members return { - a[0]: a[1] - for a in attributes - if not (a[0].startswith("__") and a[0].endswith("__")) + item[0]: item[1] + for item in members + if not (item[0].startswith("__") and item[0].endswith("__")) } @@ -58,7 +60,7 @@ if PY2: def path_to_unicode(path): if isinstance(path, unicode): return path - return path.decode(get_filesystem_encoding()).encode("utf-8") + return path.decode(get_filesystem_encoding()) def hashlib_encode_data(data): if is_bytes(data): diff --git a/platformio/fs.py b/platformio/fs.py index ed0102cd..575a14e5 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import io import json import os import re @@ -49,30 +48,6 @@ def get_source_dir(): return os.path.dirname(curpath) -def get_file_contents(path, encoding=None): - try: - with io.open(path, encoding=encoding) as fp: - return fp.read() - except UnicodeDecodeError: - click.secho( - "Unicode decode error has occurred, please remove invalid " - "(non-ASCII or non-UTF8) characters from %s file" % path, - fg="yellow", - err=True, - ) - with io.open(path, encoding="latin-1") as fp: - return fp.read() - - -def write_file_contents(path, contents, errors=None): - try: - with open(path, "w") as fp: - return fp.write(contents) - except UnicodeEncodeError: - with io.open(path, "w", encoding="latin-1", errors=errors) as fp: - return fp.write(contents) - - def load_json(file_path): try: with open(file_path, "r") as f: @@ -102,11 +77,14 @@ def ensure_udev_rules(): from platformio.util import get_systype # pylint: disable=import-outside-toplevel def _rules_to_set(rules_path): - return set( - l.strip() - for l in get_file_contents(rules_path).split("\n") - if l.strip() and not l.startswith("#") - ) + result = set() + with open(rules_path) as fp: + for line in fp.readlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + result.add(line) + return result if "linux" not in get_systype(): return None diff --git a/platformio/ide/tpls/clion/.gitignore.tpl b/platformio/ide/tpls/clion/.gitignore.tpl index ff1a5181..3fe18ad4 100644 --- a/platformio/ide/tpls/clion/.gitignore.tpl +++ b/platformio/ide/tpls/clion/.gitignore.tpl @@ -1,2 +1,3 @@ .pio CMakeListsPrivate.txt +cmake-build-*/ diff --git a/platformio/ide/tpls/clion/.idea/clion.iml.tpl b/platformio/ide/tpls/clion/.idea/clion.iml.tpl deleted file mode 100644 index bc2cd874..00000000 --- a/platformio/ide/tpls/clion/.idea/clion.iml.tpl +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/misc.xml.tpl b/platformio/ide/tpls/clion/.idea/misc.xml.tpl deleted file mode 100644 index 3463fba1..00000000 --- a/platformio/ide/tpls/clion/.idea/misc.xml.tpl +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/modules.xml.tpl b/platformio/ide/tpls/clion/.idea/modules.xml.tpl deleted file mode 100644 index 9ce81f04..00000000 --- a/platformio/ide/tpls/clion/.idea/modules.xml.tpl +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/platformio.iml.tpl b/platformio/ide/tpls/clion/.idea/platformio.iml.tpl deleted file mode 100644 index bc2cd874..00000000 --- a/platformio/ide/tpls/clion/.idea/platformio.iml.tpl +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl b/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl deleted file mode 100644 index fcf8b23d..00000000 --- a/platformio/ide/tpls/clion/.idea/watcherTasks.xml.tpl +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/platformio/ide/tpls/clion/.idea/workspace.xml.tpl b/platformio/ide/tpls/clion/.idea/workspace.xml.tpl deleted file mode 100644 index d9cf6a8f..00000000 --- a/platformio/ide/tpls/clion/.idea/workspace.xml.tpl +++ /dev/null @@ -1,259 +0,0 @@ - - - - - - - - - - - - - - - - - - - -% envs = config.envs() -% if len(envs) > 1: -% for env in envs: - -% end - -% else: - -% end - - - - - - - - - - - - - - - - - - - - % for file in src_files: - - - - % end - - - - - true - - - - - - - - - - - - - - - - - C/C++ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1435919971910 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/platformio/ide/tpls/clion/CMakeLists.txt.tpl b/platformio/ide/tpls/clion/CMakeLists.txt.tpl index 52848378..3153e616 100644 --- a/platformio/ide/tpls/clion/CMakeLists.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeLists.txt.tpl @@ -5,9 +5,12 @@ # please create `CMakeListsUser.txt` in the root of project. # The `CMakeListsUser.txt` will not be overwritten by PlatformIO. -cmake_minimum_required(VERSION 3.2) +cmake_minimum_required(VERSION 3.13) +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_C_COMPILER_WORKS 1) +set(CMAKE_CXX_COMPILER_WORKS 1) -project("{{project_name}}") +project("{{project_name}}" C CXX) include(CMakeListsPrivate.txt) @@ -16,74 +19,14 @@ include(CMakeListsUser.txt) endif() add_custom_target( - PLATFORMIO_BUILD ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run "$<$>:-e${CMAKE_BUILD_TYPE}>" + Production ALL + COMMAND platformio -c clion run "$<$>:-e${CMAKE_BUILD_TYPE}>" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) add_custom_target( - PLATFORMIO_BUILD_VERBOSE ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --verbose "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_UPLOAD ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --target upload "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_CLEAN ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --target clean "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_MONITOR ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion device monitor "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_TEST ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion test "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_PROGRAM ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --target program "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_UPLOADFS ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --target uploadfs "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_BUILD_DEBUG ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion run --target debug "$<$>:-e${CMAKE_BUILD_TYPE}>" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_UPDATE_ALL ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion update - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_REBUILD_PROJECT_INDEX ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion init --ide clion - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) - -add_custom_target( - PLATFORMIO_DEVICE_LIST ALL - COMMAND ${PLATFORMIO_CMD} -f -c clion device list + Debug ALL + COMMAND platformio -c clion run --target debug "$<$>:-e${CMAKE_BUILD_TYPE}>" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl index 0c6eda0c..df8171fa 100644 --- a/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl +++ b/platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl @@ -29,14 +29,13 @@ % envs = config.envs() % if len(envs) > 1: -set(CMAKE_CONFIGURATION_TYPES "{{ ";".join(envs) }};" CACHE STRING "" FORCE) +set(CMAKE_CONFIGURATION_TYPES "{{ ";".join(envs) }};" CACHE STRING "Build Types reflect PlatformIO Environments" FORCE) % else: -set(CMAKE_CONFIGURATION_TYPES "{{ env_name }}" CACHE STRING "" FORCE) +set(CMAKE_CONFIGURATION_TYPES "{{ env_name }}" CACHE STRING "Build Types reflect PlatformIO Environments" FORCE) % end -set(PLATFORMIO_CMD "{{ _normalize_path(platformio_path) }}") % if svd_path: -set(SVD_PATH "{{ _normalize_path(svd_path) }}") +set(CLION_SVD_FILE_PATH "{{ _normalize_path(svd_path) }}" CACHE FILEPATH "Peripheral Registers Definitions File" FORCE) % end SET(CMAKE_C_COMPILER "{{ _normalize_path(cc_path) }}") diff --git a/platformio/maintenance.py b/platformio/maintenance.py index c9b3d742..d2e7ea1c 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -28,7 +28,7 @@ from platformio.commands.upgrade import get_latest_version from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager from platformio.managers.platform import PlatformFactory, PlatformManager -from platformio.proc import is_ci, is_container +from platformio.proc import is_container def on_platformio_start(ctx, force, caller): @@ -186,14 +186,6 @@ def after_upgrade(ctx): click.style("https://platformio.org/platformio-ide", fg="cyan"), ) ) - if not is_ci(): - click.echo( - "- %s us with PlatformIO Plus > %s" - % ( - click.style("support", fg="cyan"), - click.style("https://pioplus.com", fg="cyan"), - ) - ) click.echo("*" * terminal_width) click.echo("") diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 55a0ece4..64d48085 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -190,7 +190,7 @@ def get_contrib_pysite_deps(): def pioplus_call(args, **kwargs): if WINDOWS and sys.version_info < (2, 7, 6): raise exception.PlatformioException( - "PlatformIO Core Plus v%s does not run under Python version %s.\n" + "PlatformIO Remote v%s does not run under Python version %s.\n" "Minimum supported version is 2.7.6, please upgrade Python.\n" "Python 3 is not yet supported.\n" % (__version__, sys.version) ) diff --git a/platformio/managers/lib.py b/platformio/managers/lib.py index b006d8af..d85f0fa0 100644 --- a/platformio/managers/lib.py +++ b/platformio/managers/lib.py @@ -26,6 +26,8 @@ from platformio import app, exception, util from platformio.compat import glob_escape from platformio.managers.package import BasePkgManager from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.package.exception import ManifestException +from platformio.package.manifest.parser import ManifestParserFactory from platformio.project.config import ProjectConfig @@ -281,8 +283,12 @@ class LibraryManager(BasePkgManager): if not pkg_dir: return None - manifest = self.load_manifest(pkg_dir) - if "dependencies" not in manifest: + manifest = None + try: + manifest = ManifestParserFactory.new_from_dir(pkg_dir).as_dict() + except ManifestException: + pass + if not manifest or not manifest.get("dependencies"): return pkg_dir if not silent: diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8f23f1bd..8e107c1f 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -436,16 +436,23 @@ class PlatformRunMixin(object): for key, value in variables.items(): args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - def _write_and_flush(stream, data): - try: - stream.write(data) - stream.flush() - except IOError: - pass - proc.copy_pythonpath_to_osenv() + + if targets and "menuconfig" in targets: + return proc.exec_command( + args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin + ) + if click._compat.isatty(sys.stdout): - result = proc.exec_command( + + def _write_and_flush(stream, data): + try: + stream.write(data) + stream.flush() + except IOError: + pass + + return proc.exec_command( args, stdout=proc.BuildAsyncPipe( line_callback=self._on_stdout_line, @@ -456,13 +463,12 @@ class PlatformRunMixin(object): data_callback=lambda data: _write_and_flush(sys.stderr, data), ), ) - else: - result = proc.exec_command( - args, - stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), - stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), - ) - return result + + return proc.exec_command( + args, + stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), + stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), + ) def _on_stdout_line(self, line): if "`buildprog' is up to date." in line: diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 35a00386..e8ec5929 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -13,6 +13,7 @@ # limitations under the License. import inspect +import io import json import os import re @@ -20,8 +21,7 @@ import re import requests from platformio import util -from platformio.compat import get_class_attributes, string_types -from platformio.fs import get_file_contents +from platformio.compat import get_object_members, string_types from platformio.package.exception import ManifestParserError, UnknownManifestError from platformio.project.helpers import is_platformio_project @@ -40,7 +40,7 @@ class ManifestFileType(object): @classmethod def items(cls): - return get_class_attributes(ManifestFileType) + return get_object_members(ManifestFileType) @classmethod def from_uri(cls, uri): @@ -59,24 +59,29 @@ class ManifestFileType(object): class ManifestParserFactory(object): @staticmethod - def new_from_file(path, remote_url=False): + def read_manifest_contents(path): + with io.open(path, encoding="utf-8") as fp: + return fp.read() + + @classmethod + def new_from_file(cls, path, remote_url=False): if not path or not os.path.isfile(path): raise UnknownManifestError("Manifest file does not exist %s" % path) type_from_uri = ManifestFileType.from_uri(path) if not type_from_uri: raise UnknownManifestError("Unknown manifest file type %s" % path) return ManifestParserFactory.new( - get_file_contents(path, encoding="utf8"), type_from_uri, remote_url + cls.read_manifest_contents(path), type_from_uri, remote_url ) - @staticmethod - def new_from_dir(path, remote_url=None): + @classmethod + def new_from_dir(cls, path, remote_url=None): assert os.path.isdir(path), "Invalid directory %s" % path type_from_uri = ManifestFileType.from_uri(remote_url) if remote_url else None if type_from_uri and os.path.isfile(os.path.join(path, type_from_uri)): return ManifestParserFactory.new( - get_file_contents(os.path.join(path, type_from_uri), encoding="utf8"), + cls.read_manifest_contents(os.path.join(path, type_from_uri)), type_from_uri, remote_url=remote_url, package_dir=path, @@ -88,7 +93,7 @@ class ManifestParserFactory(object): "Unknown manifest file type in %s directory" % path ) return ManifestParserFactory.new( - get_file_contents(os.path.join(path, type_from_dir), encoding="utf8"), + cls.read_manifest_contents(os.path.join(path, type_from_dir)), type_from_dir, remote_url=remote_url, package_dir=path, @@ -363,13 +368,15 @@ class LibraryJsonManifestParser(BaseManifestParser): return [dict(name=name, version=version) for name, version in raw.items()] if isinstance(raw, list): for i, dependency in enumerate(raw): - assert isinstance(dependency, dict) - for k, v in dependency.items(): - if k not in ("platforms", "frameworks", "authors"): - continue - if "*" in v: - del raw[i][k] - raw[i][k] = util.items_to_list(v) + if isinstance(dependency, dict): + for k, v in dependency.items(): + if k not in ("platforms", "frameworks", "authors"): + continue + if "*" in v: + del raw[i][k] + raw[i][k] = util.items_to_list(v) + else: + raw[i] = {"name": dependency} return raw raise ManifestParserError( "Invalid dependencies format, should be list or dictionary" @@ -390,6 +397,8 @@ class ModuleJsonManifestParser(BaseManifestParser): if "licenses" in data: data["license"] = self._parse_license(data.get("licenses")) del data["licenses"] + if "dependencies" in data: + data["dependencies"] = self._parse_dependencies(data["dependencies"]) return data def _parse_authors(self, raw): @@ -409,6 +418,15 @@ class ModuleJsonManifestParser(BaseManifestParser): return None return raw[0].get("type") + @staticmethod + def _parse_dependencies(raw): + if isinstance(raw, dict): + return [ + dict(name=name, version=version, frameworks=["mbed"]) + for name, version in raw.items() + ] + raise ManifestParserError("Invalid dependencies format, should be a dictionary") + class LibraryPropertiesManifestParser(BaseManifestParser): manifest_type = ManifestFileType.LIBRARY_PROPERTIES diff --git a/platformio/project/config.py b/platformio/project/config.py index 4dc3c38c..23d089bf 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -280,7 +280,7 @@ class ProjectConfigBase(object): value = envvar_value if value == MISSING: - value = option_meta.default or default + value = default if default != MISSING else option_meta.default if value == MISSING: return None diff --git a/platformio/project/options.py b/platformio/project/options.py index d38ed566..ecf030b8 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -445,6 +445,14 @@ ProjectOptions = OrderedDict( oldnames=["monitor_baud"], default=9600, ), + ConfigEnvOption( + group="monitor", + name="monitor_filters", + description=( + "Apply the filters and text transformations to monitor output" + ), + multiple=True, + ), ConfigEnvOption( group="monitor", name="monitor_rts", diff --git a/scripts/get-platformio.py b/scripts/get-platformio.py index 988f6054..9d99ea97 100644 --- a/scripts/get-platformio.py +++ b/scripts/get-platformio.py @@ -12,173 +12,93 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import subprocess -import site +import tempfile +import io import sys -from platform import system -from tempfile import NamedTemporaryFile +import subprocess -CURINTERPRETER_PATH = os.path.normpath(sys.executable) -IS_WINDOWS = system().lower() == "windows" +MAIN_SCRIPT_URL = "https://raw.githubusercontent.com/platformio/platformio-core-installer/master/get-platformio.py" -def fix_winpython_pathenv(): - """ - Add Python & Python Scripts to the search path on Windows - """ - try: - import _winreg as winreg - except ImportError: - import winreg +def download_with_requests(url, dst): + import requests - # took these lines from the native "win_add2path.py" - pythonpath = os.path.dirname(os.path.normpath(sys.executable)) - scripts = os.path.join(pythonpath, "Scripts") - appdata = os.environ["APPDATA"] - if hasattr(site, "USER_SITE"): - userpath = site.USER_SITE.replace(appdata, "%APPDATA%") - userscripts = os.path.join(userpath, "Scripts") - else: - userscripts = None - - with winreg.CreateKey(winreg.HKEY_CURRENT_USER, "Environment") as key: - try: - envpath = winreg.QueryValueEx(key, "PATH")[0] - except WindowsError: - envpath = u"%PATH%" - - paths = [envpath] - for path in (pythonpath, scripts, userscripts): - if path and path not in envpath and os.path.isdir(path): - paths.append(path) - - envpath = os.pathsep.join(paths) - winreg.SetValueEx(key, "PATH", 0, winreg.REG_EXPAND_SZ, envpath) - return True + resp = requests.get(url, stream=True) + itercontent = resp.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) + with open(dst, "wb") as fp: + for chunk in itercontent: + fp.write(chunk) + return dst -def exec_command(*args, **kwargs): - result = {"out": None, "err": None, "returncode": None} +def download_with_urllib3(url, dst): + import urllib3 - kwargs['stdout'] = subprocess.PIPE - kwargs['stderr'] = subprocess.PIPE - kwargs['shell'] = IS_WINDOWS + http = urllib3.PoolManager() + r = http.request("GET", url, preload_content=False) - p = subprocess.Popen(*args, **kwargs) - result['out'], result['err'] = p.communicate() - result['returncode'] = p.returncode + with open(dst, "wb") as out: + while True: + data = r.read(io.DEFAULT_BUFFER_SIZE) + if not data: + break + out.write(data) - for k, v in result.items(): - if v and isinstance(v, str): - result[k].strip() - - return result + r.release_conn() + return dst -def print_exec_result(result): - if result['returncode'] == 0: - print(result['out']) - else: - raise Exception("\n".join([result['out'], result['err']])) - - -def exec_python_cmd(args): - return exec_command([CURINTERPRETER_PATH] + args) - - -def install_pip(): - r = exec_python_cmd(["-m", "pip", "--version"]) - if r['returncode'] == 0: - print(r['out']) - return - try: - from urllib2 import urlopen - except ImportError: +def download_with_urllib(url, dst): + if sys.version_info[0] == 3: from urllib.request import urlopen + else: + from urllib import urlopen - f = NamedTemporaryFile(delete=False) - response = urlopen("https://bootstrap.pypa.io/get-pip.py") - f.write(response.read()) - f.close() + response = urlopen(url) + CHUNK = 16 * 1024 + with open(dst, "wb") as f: + while True: + chunk = response.read(CHUNK) + if not chunk: + break + f.write(chunk) - try: - r = exec_python_cmd([f.name]) - finally: - os.unlink(f.name) - - print_exec_result(r) + return dst -def install_platformio(): - r = None - cmd = ["-m", "pip", "install", "-U", "platformio"] - # cmd = [ - # "-m", "pip", "install", "-U", - # "https://github.com/platformio/platformio-core/archive/develop.zip" - # ] - try: - r = exec_python_cmd(cmd) - assert r['returncode'] == 0 - except AssertionError: - cmd.insert(2, "--no-cache-dir") - r = exec_python_cmd(cmd) - if r: - print_exec_result(r) +def download_with_curl(url, dst): + subprocess.check_output(["curl", "-o", dst, url]) + return dst + + +def download_with_wget(url, dst): + subprocess.check_output(["wget", "-O", dst, url]) + return dst + + +def download_file(url, dst): + methods = [ + download_with_requests, + download_with_urllib3, + download_with_urllib, + download_with_curl, + download_with_wget, + ] + for method in methods: + try: + method(url, dst) + return dst + except: + pass + raise Exception("Could not download file '%s' to '%s' " % (url, dst)) def main(): - steps = [("Fixing Windows %PATH% Environment", fix_winpython_pathenv), - ("Installing Python Package Manager", install_pip), - ("Installing PlatformIO and dependencies", install_platformio)] - - if not IS_WINDOWS: - del steps[0] - - is_error = False - for s in steps: - if is_error: - break - print("\n==> %s ..." % s[0]) - try: - s[1]() - print("[SUCCESS]") - except Exception as e: - is_error = True - print(str(e)) - print("[FAILURE]") - - permission_errors = ("permission denied", "not permitted") - if (any([m in str(e).lower() for m in permission_errors]) and - not IS_WINDOWS): - print(""" ------------------ -Permission denied ------------------ - -You need the `sudo` permission to install Python packages. Try - -$ sudo python -c "$(curl -fsSL -https://raw.githubusercontent.com/platformio/platformio/develop/scripts/get-platformio.py)" -""") - - if is_error: - print("The installation process has been FAILED!\n" - "Please report about this problem here\n" - "< https://github.com/platformio/platformio-core/issues >") - return - else: - print("\n ==> Installation process has been " - "successfully FINISHED! <==\n") - print(""" - ----------------------------------------- -Please RESTART your Terminal Application ----------------------------------------- - -Then run `platformio --help` command. - -""") + with tempfile.NamedTemporaryFile() as tmp_file: + dst = download_file(MAIN_SCRIPT_URL, str(tmp_file.name)) + command = [sys.executable, dst] + command.extend(sys.argv[1:]) + subprocess.check_call(command) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 06a915a6..2d3cd72f 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ from platformio.compat import PY2 install_requires = [ "bottle<0.13", - "click>=5,<8", + "click>=5,<8,!=7.1,!=7.1.1", "colorama", "pyserial>=3,<4,!=3.3", "requests>=2.4.0,<3", diff --git a/tests/commands/test_lib.py b/tests/commands/test_lib.py index dc5dc00a..752c2c30 100644 --- a/tests/commands/test_lib.py +++ b/tests/commands/test_lib.py @@ -44,7 +44,7 @@ def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_hom "ArduinoJson@~5.10.0", "547@2.2.4", "AsyncMqttClient@<=0.8.2", - "999@77d4eb3f8a", + "Adafruit PN532@1.2.0", ], ) validate_cliresult(result) @@ -62,7 +62,8 @@ def test_global_install_registry(clirunner, validate_cliresult, isolated_pio_hom "AsyncMqttClient_ID346", "ESPAsyncTCP_ID305", "AsyncTCP_ID1826", - "RFcontrol_ID999", + "Adafruit PN532_ID29", + "Adafruit BusIO_ID6214", ] assert set(items1) == set(items2) @@ -135,7 +136,7 @@ def test_install_duplicates(clirunner, validate_cliresult, without_internet): assert "is already installed" in result.output # by ID - result = clirunner.invoke(cmd_lib, ["-g", "install", "999"]) + result = clirunner.invoke(cmd_lib, ["-g", "install", "29"]) validate_cliresult(result) assert "is already installed" in result.output @@ -202,7 +203,8 @@ def test_global_lib_list(clirunner, validate_cliresult): "PJON", "PJON", "PubSubClient", - "RFcontrol", + "Adafruit PN532", + "Adafruit BusIO", "platformio-libmirror", "rs485-nodeproto", ] @@ -219,7 +221,7 @@ def test_global_lib_list(clirunner, validate_cliresult): "PJON@07fe9aa", "PJON@1fb26fd", "PubSubClient@bef5814", - "RFcontrol@77d4eb3f8a", + "Adafruit PN532@1.2.0", ] assert set(versions1) >= set(versions2) @@ -230,9 +232,7 @@ def test_global_lib_update_check(clirunner, validate_cliresult): ) validate_cliresult(result) output = json.loads(result.output) - assert set(["RFcontrol", "ESPAsyncTCP", "NeoPixelBus"]) == set( - [l["name"] for l in output] - ) + assert set(["ESPAsyncTCP", "NeoPixelBus"]) == set([l["name"] for l in output]) def test_global_lib_update(clirunner, validate_cliresult): @@ -252,8 +252,7 @@ def test_global_lib_update(clirunner, validate_cliresult): result = clirunner.invoke(cmd_lib, ["-g", "update"]) validate_cliresult(result) assert result.output.count("[Detached]") == 5 - assert result.output.count("[Up-to-date]") == 10 - assert "Uninstalling RFcontrol @ 77d4eb3f8a" in result.output + assert result.output.count("[Up-to-date]") == 12 # update unknown library result = clirunner.invoke(cmd_lib, ["-g", "update", "Unknown"]) @@ -266,9 +265,10 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): result = clirunner.invoke(cmd_lib, ["-g", "list", "--json-output"]) validate_cliresult(result) items = json.loads(result.output) - result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[5]["__pkg_dir"]]) + items = sorted(items, key=lambda item: item["__pkg_dir"]) + result = clirunner.invoke(cmd_lib, ["-g", "uninstall", items[0]["__pkg_dir"]]) validate_cliresult(result) - assert "Uninstalling AsyncTCP" in result.output + assert ("Uninstalling %s" % items[0]["name"]) in result.output # uninstall the rest libraries result = clirunner.invoke( @@ -279,7 +279,7 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): "1", "https://github.com/bblanchon/ArduinoJson.git", "ArduinoJson@!=5.6.7", - "RFcontrol", + "Adafruit PN532", ], ) validate_cliresult(result) @@ -291,13 +291,14 @@ def test_global_lib_uninstall(clirunner, validate_cliresult, isolated_pio_home): "PubSubClient", "ArduinoJson@src-69ebddd821f771debe7ee734d3c7fa81", "ESPAsyncTCP_ID305", - "SomeLib_ID54", + "ESP32WebServer", "NeoPixelBus_ID547", "PJON", "AsyncMqttClient_ID346", "ArduinoJson_ID64", + "SomeLib_ID54", "PJON@src-79de467ebe19de18287becff0a1fb42d", - "ESP32WebServer", + "AsyncTCP_ID1826", ] assert set(items1) == set(items2) diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 22f9691d..1ad66a75 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -112,6 +112,21 @@ def test_library_json_parser(): }, ) + raw_data = parser.LibraryJsonManifestParser( + '{"dependencies": ["dep1", "dep2", "@owner/dep3"]}' + ).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) + assert not jsondiff.diff( + raw_data, + { + "dependencies": [ + {"name": "@owner/dep3"}, + {"name": "dep1"}, + {"name": "dep2"}, + ], + }, + ) + # broken dependencies with pytest.raises(parser.ManifestParserError): parser.LibraryJsonManifestParser({"dependencies": ["deps1", "deps2"]}) @@ -139,13 +154,18 @@ def test_module_json_parser(): "url": "git@github.com:username/repo.git" }, "version": "1.2.3", + "dependencies": { + "usefulmodule": "^1.2.3", + "simplelog": "ARMmbed/simplelog#~0.0.1" + }, "customField": "Custom Value" } """ - mp = parser.ModuleJsonManifestParser(contents) + raw_data = parser.ModuleJsonManifestParser(contents).as_dict() + raw_data["dependencies"] = sorted(raw_data["dependencies"], key=lambda a: a["name"]) assert not jsondiff.diff( - mp.as_dict(), + raw_data, { "name": "YottaLibrary", "description": "This is Yotta library", @@ -158,6 +178,14 @@ def test_module_json_parser(): "authors": [{"email": "name@surname.com", "name": "Name Surname"}], "version": "1.2.3", "repository": {"type": "git", "url": "git@github.com:username/repo.git"}, + "dependencies": [ + { + "name": "simplelog", + "version": "ARMmbed/simplelog#~0.0.1", + "frameworks": ["mbed"], + }, + {"name": "usefulmodule", "version": "^1.2.3", "frameworks": ["mbed"]}, + ], "customField": "Custom Value", }, ) diff --git a/tests/test_projectconf.py b/tests/test_projectconf.py index 8172e692..0e68f2d3 100644 --- a/tests/test_projectconf.py +++ b/tests/test_projectconf.py @@ -123,6 +123,8 @@ def test_defaults(config): ) assert config.get("env:extra_2", "lib_compat_mode") == "soft" assert config.get("env:extra_2", "build_type") == "release" + assert config.get("env:extra_2", "build_type", None) is None + assert config.get("env:extra_2", "lib_archive", "no") is False def test_sections(config):