diff --git a/HISTORY.rst b/HISTORY.rst index 5f0dd973..65b23388 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,7 +18,9 @@ PlatformIO 4.0 * Override default source and include directories for a library via `library.json `__ manifest using ``includeDir`` and ``srcDir`` fields * Added support for the latest Python "Click" package (CLI Builder) (`issue #349 `_) -* Deprecated ``--only-check`` CLI option for "update" sub-commands, please use ``--dry-run`` instead +* Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead +* Removed line-buffering from `platformio run `__ command which was leading to omitting progress bar from upload tools + (`issue #856 `_) PlatformIO 3.0 -------------- diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index f6dac646..b5537937 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -15,6 +15,7 @@ import base64 import os import re +import sys from imp import load_source from multiprocessing import cpu_count from os.path import basename, dirname, isdir, isfile, join @@ -26,8 +27,8 @@ from platformio import __version__, app, exception, util from platformio.compat import PY2 from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager -from platformio.proc import (copy_pythonpath_to_osenv, exec_command, - get_pythonexe_path) +from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv, + exec_command, get_pythonexe_path) from platformio.project.helpers import get_projectboards_dir try: @@ -399,19 +400,27 @@ class PlatformRunMixin(object): "%s=%s" % (key.upper(), base64.b64encode( value.encode()).decode())) + def _write_and_flush(stream, data): + stream.write(data) + stream.flush() + copy_pythonpath_to_osenv() result = exec_command( cmd, - stdout=util.AsyncPipe(self.on_run_out), - stderr=util.AsyncPipe(self.on_run_err)) + stdout=BuildAsyncPipe( + line_callback=self._on_stdout_line, + data_callback=lambda data: _write_and_flush(sys.stdout, data)), + stderr=BuildAsyncPipe( + line_callback=self._on_stderr_line, + data_callback=lambda data: _write_and_flush(sys.stderr, data))) return result - def on_run_out(self, line): + def _on_stdout_line(self, line): if "`buildprog' is up to date." in line: return self._echo_line(line, level=1) - def on_run_err(self, line): + def _on_stderr_line(self, line): is_error = self.LINE_ERROR_RE.search(line) is not None self._echo_line(line, level=3 if is_error else 2) @@ -430,7 +439,7 @@ class PlatformRunMixin(object): fg = (None, "yellow", "red")[level - 1] if level == 1 and "is up to date" in line: fg = "green" - click.secho(line, fg=fg, err=level > 1) + click.secho(line, fg=fg, err=level > 1, nl=False) @staticmethod def _echo_missed_dependency(filename): diff --git a/platformio/proc.py b/platformio/proc.py index a0432458..7cc3d8a7 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -22,17 +22,14 @@ from platformio import exception from platformio.compat import PY2, WINDOWS, string_types -class AsyncPipe(Thread): - - def __init__(self, outcallback=None): - super(AsyncPipe, self).__init__() - self.outcallback = outcallback +class AsyncPipeBase(object): + def __init__(self): self._fd_read, self._fd_write = os.pipe() self._pipe_reader = os.fdopen(self._fd_read) - self._buffer = [] - - self.start() + self._buffer = "" + self._thread = Thread(target=self.run) + self._thread.start() def get_buffer(self): return self._buffer @@ -41,18 +38,67 @@ class AsyncPipe(Thread): return self._fd_write def run(self): - for line in iter(self._pipe_reader.readline, ""): - line = line.strip() - self._buffer.append(line) - if self.outcallback: - self.outcallback(line) - else: - print(line) - self._pipe_reader.close() + try: + self.do_reading() + except (KeyboardInterrupt, SystemExit, IOError): + self.close() + + def do_reading(self): + raise NotImplementedError() def close(self): + self._buffer = "" os.close(self._fd_write) - self.join() + self._thread.join() + + +class BuildAsyncPipe(AsyncPipeBase): + + def __init__(self, line_callback, data_callback): + self.line_callback = line_callback + self.data_callback = data_callback + super(BuildAsyncPipe, self).__init__() + + def do_reading(self): + line = "" + print_immediately = False + + for byte in iter(lambda: self._pipe_reader.read(1), ""): + self._buffer += byte + + if line and line[-3:] == (line[-1] * 3): + print_immediately = True + + if print_immediately: + # leftover bytes + if line: + self.data_callback(line) + line = "" + self.data_callback(byte) + if byte == "\n": + print_immediately = False + else: + line += byte + if byte != "\n": + continue + self.line_callback(line) + line = "" + + self._pipe_reader.close() + + +class LineBufferedAsyncPipe(AsyncPipeBase): + + def __init__(self, line_callback): + self.line_callback = line_callback + super(LineBufferedAsyncPipe, self).__init__() + + def do_reading(self): + for line in iter(self._pipe_reader.readline, ""): + self._buffer += line + # FIXME: Remove striping + self.line_callback(line.strip()) + self._pipe_reader.close() def exec_command(*args, **kwargs): @@ -70,12 +116,12 @@ def exec_command(*args, **kwargs): raise exception.AbortedByUser() finally: for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipe): + if isinstance(kwargs[s], AsyncPipeBase): kwargs[s].close() for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipe): - result[s[3:]] = "\n".join(kwargs[s].get_buffer()) + if isinstance(kwargs[s], AsyncPipeBase): + result[s[3:]] = kwargs[s].get_buffer() for k, v in result.items(): if not PY2 and isinstance(result[k], bytes): diff --git a/platformio/util.py b/platformio/util.py index 4e0979fc..83552d6b 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -34,7 +34,8 @@ import requests from platformio import __apiurl__, __version__, exception from platformio.compat import PY2, WINDOWS, path_to_unicode -from platformio.proc import AsyncPipe, exec_command, is_ci, where_is_program +from platformio.proc import LineBufferedAsyncPipe as AsyncPipe +from platformio.proc import exec_command, is_ci, where_is_program from platformio.project.config import ProjectConfig from platformio.project.helpers import ( get_project_dir, get_project_optional_dir, get_projectboards_dir,