Remove line-buffering from "platformio run" command which was leading to omitting progress bar from upload tools // Resolve #856, Resolve #857

HISTORY.rst

@@ -18,7 +18,9 @@ PlatformIO 4.0
 * Override default source and include directories for a library via `library.json <http://docs.platformio.org/page/librarymanager/config.html>`__ manifest using ``includeDir`` and ``srcDir`` fields
 * Added support for the latest Python "Click" package (CLI Builder)
   (`issue #349 <https://github.com/platformio/platformio-core/issues/349>`_)
-* Deprecated ``--only-check`` CLI option for "update" sub-commands, please use ``--dry-run`` instead
+* Deprecated ``--only-check`` PlatformIO Core CLI option for "update" sub-commands, please use ``--dry-run`` instead
+* Removed line-buffering from `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__ command which was leading to omitting progress bar from upload tools
+  (`issue #856 <https://github.com/platformio/platformio-core/issues/856>`_)
 
 PlatformIO 3.0
 --------------

platformio/managers/platform.py

@@ -15,6 +15,7 @@
 import base64
 import os
 import re
+import sys
 from imp import load_source
 from multiprocessing import cpu_count
 from os.path import basename, dirname, isdir, isfile, join
@@ -26,8 +27,8 @@ from platformio import __version__, app, exception, util
 from platformio.compat import PY2
 from platformio.managers.core import get_core_package_dir
 from platformio.managers.package import BasePkgManager, PackageManager
-from platformio.proc import (copy_pythonpath_to_osenv, exec_command,
+from platformio.proc import (BuildAsyncPipe, copy_pythonpath_to_osenv,
-                             get_pythonexe_path)
+                             exec_command, get_pythonexe_path)
 from platformio.project.helpers import get_projectboards_dir
 
 try:
@@ -399,19 +400,27 @@ class PlatformRunMixin(object):
                     "%s=%s" % (key.upper(), base64.b64encode(
                         value.encode()).decode()))
 
+        def _write_and_flush(stream, data):
+            stream.write(data)
+            stream.flush()
+
         copy_pythonpath_to_osenv()
         result = exec_command(
             cmd,
-            stdout=util.AsyncPipe(self.on_run_out),
+            stdout=BuildAsyncPipe(
-            stderr=util.AsyncPipe(self.on_run_err))
+                line_callback=self._on_stdout_line,
+                data_callback=lambda data: _write_and_flush(sys.stdout, data)),
+            stderr=BuildAsyncPipe(
+                line_callback=self._on_stderr_line,
+                data_callback=lambda data: _write_and_flush(sys.stderr, data)))
         return result
 
-    def on_run_out(self, line):
+    def _on_stdout_line(self, line):
         if "`buildprog' is up to date." in line:
             return
         self._echo_line(line, level=1)
 
-    def on_run_err(self, line):
+    def _on_stderr_line(self, line):
         is_error = self.LINE_ERROR_RE.search(line) is not None
         self._echo_line(line, level=3 if is_error else 2)
 
@@ -430,7 +439,7 @@ class PlatformRunMixin(object):
         fg = (None, "yellow", "red")[level - 1]
         if level == 1 and "is up to date" in line:
             fg = "green"
-        click.secho(line, fg=fg, err=level > 1)
+        click.secho(line, fg=fg, err=level > 1, nl=False)
 
     @staticmethod
     def _echo_missed_dependency(filename):

platformio/proc.py

@@ -22,17 +22,14 @@ from platformio import exception
 from platformio.compat import PY2, WINDOWS, string_types
 
 
-class AsyncPipe(Thread):
+class AsyncPipeBase(object):
-
-    def __init__(self, outcallback=None):
-        super(AsyncPipe, self).__init__()
-        self.outcallback = outcallback
 
+    def __init__(self):
         self._fd_read, self._fd_write = os.pipe()
         self._pipe_reader = os.fdopen(self._fd_read)
-        self._buffer = []
+        self._buffer = ""
-
+        self._thread = Thread(target=self.run)
-        self.start()
+        self._thread.start()
 
     def get_buffer(self):
         return self._buffer
@@ -41,18 +38,67 @@ class AsyncPipe(Thread):
         return self._fd_write
 
     def run(self):
-        for line in iter(self._pipe_reader.readline, ""):
+        try:
-            line = line.strip()
+            self.do_reading()
-            self._buffer.append(line)
+        except (KeyboardInterrupt, SystemExit, IOError):
-            if self.outcallback:
+            self.close()
-                self.outcallback(line)
+
-            else:
+    def do_reading(self):
-                print(line)
+        raise NotImplementedError()
-        self._pipe_reader.close()
 
     def close(self):
+        self._buffer = ""
         os.close(self._fd_write)
-        self.join()
+        self._thread.join()
+
+
+class BuildAsyncPipe(AsyncPipeBase):
+
+    def __init__(self, line_callback, data_callback):
+        self.line_callback = line_callback
+        self.data_callback = data_callback
+        super(BuildAsyncPipe, self).__init__()
+
+    def do_reading(self):
+        line = ""
+        print_immediately = False
+
+        for byte in iter(lambda: self._pipe_reader.read(1), ""):
+            self._buffer += byte
+
+            if line and line[-3:] == (line[-1] * 3):
+                print_immediately = True
+
+            if print_immediately:
+                # leftover bytes
+                if line:
+                    self.data_callback(line)
+                    line = ""
+                self.data_callback(byte)
+                if byte == "\n":
+                    print_immediately = False
+            else:
+                line += byte
+                if byte != "\n":
+                    continue
+                self.line_callback(line)
+                line = ""
+
+        self._pipe_reader.close()
+
+
+class LineBufferedAsyncPipe(AsyncPipeBase):
+
+    def __init__(self, line_callback):
+        self.line_callback = line_callback
+        super(LineBufferedAsyncPipe, self).__init__()
+
+    def do_reading(self):
+        for line in iter(self._pipe_reader.readline, ""):
+            self._buffer += line
+            # FIXME: Remove striping
+            self.line_callback(line.strip())
+        self._pipe_reader.close()
 
 
 def exec_command(*args, **kwargs):
@@ -70,12 +116,12 @@ def exec_command(*args, **kwargs):
         raise exception.AbortedByUser()
     finally:
         for s in ("stdout", "stderr"):
-            if isinstance(kwargs[s], AsyncPipe):
+            if isinstance(kwargs[s], AsyncPipeBase):
                 kwargs[s].close()
 
     for s in ("stdout", "stderr"):
-        if isinstance(kwargs[s], AsyncPipe):
+        if isinstance(kwargs[s], AsyncPipeBase):
-            result[s[3:]] = "\n".join(kwargs[s].get_buffer())
+            result[s[3:]] = kwargs[s].get_buffer()
 
     for k, v in result.items():
         if not PY2 and isinstance(result[k], bytes):

platformio/util.py

@@ -34,7 +34,8 @@ import requests
 
 from platformio import __apiurl__, __version__, exception
 from platformio.compat import PY2, WINDOWS, path_to_unicode
-from platformio.proc import AsyncPipe, exec_command, is_ci, where_is_program
+from platformio.proc import LineBufferedAsyncPipe as AsyncPipe
+from platformio.proc import exec_command, is_ci, where_is_program
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import (
     get_project_dir, get_project_optional_dir, get_projectboards_dir,