Move process related helpers to "proc" module

platformio/app.py

@@ -26,6 +26,7 @@ import requests
 
 from platformio import exception, lockfile, util
 from platformio.compat import PY2, WINDOWS
+from platformio.proc import is_ci
 
 
 def projects_dir_validate(projects_dir):
@@ -339,7 +340,7 @@ def set_session_var(name, value):
 def is_disabled_progressbar():
     return any([
         get_session_var("force_option"),
-        util.is_ci(),
+        is_ci(),
         getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true"
     ])
 

platformio/builder/main.py

@@ -30,6 +30,7 @@ from SCons.Script import Variables  # pylint: disable=import-error
 
 from platformio import util
 from platformio.compat import PY2, path_to_unicode
+from platformio.proc import get_pythonexe_path
 from platformio.project.helpers import (
     get_project_dir, get_project_optional_dir, get_projectbuild_dir,
     get_projectdata_dir, get_projectinclude_dir, get_projectlib_dir,
@@ -123,7 +124,7 @@ DEFAULT_ENV_OPTIONS = dict(
     ],
     PROGNAME="program",
     PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
-    PYTHONEXE=util.get_pythonexe_path())
+    PYTHONEXE=get_pythonexe_path())
 
 if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
     DEFAULT_ENV_OPTIONS['ARCOMSTR'] = "Archiving $TARGET"

platformio/builder/tools/pioide.py

@@ -22,6 +22,7 @@ from SCons.Defaults import processDefines  # pylint: disable=import-error
 
 from platformio import util
 from platformio.managers.core import get_core_package_dir
+from platformio.proc import exec_command
 
 
 def _dump_includes(env):
@@ -71,7 +72,7 @@ def _get_gcc_defines(env):
     try:
         sysenv = environ.copy()
         sysenv['PATH'] = str(env['ENV']['PATH'])
-        result = util.exec_command(
+        result = exec_command(
             "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True)
     except OSError:
         return items

platformio/builder/tools/piomisc.py

@@ -26,6 +26,7 @@ from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 
 from platformio import util
 from platformio.managers.core import get_core_package_dir
+from platformio.proc import exec_command
 
 
 class InoToCPPConverter(object):
@@ -211,7 +212,7 @@ def _get_compiler_type(env):
     try:
         sysenv = environ.copy()
         sysenv['PATH'] = str(env['ENV']['PATH'])
-        result = util.exec_command([env.subst("$CC"), "-v"], env=sysenv)
+        result = exec_command([env.subst("$CC"), "-v"], env=sysenv)
     except OSError:
         return None
     if result['returncode'] != 0:

platformio/builder/tools/pioupload.py

@@ -27,6 +27,7 @@ from serial import Serial, SerialException
 
 from platformio import exception, util
 from platformio.compat import WINDOWS
+from platformio.proc import exec_command
 
 # pylint: disable=unused-argument
 
@@ -211,7 +212,7 @@ def CheckUploadSize(_, target, source, env):
         cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
         sysenv = environ.copy()
         sysenv['PATH'] = str(env['ENV']['PATH'])
-        result = util.exec_command(env.subst(cmd), env=sysenv)
+        result = exec_command(env.subst(cmd), env=sysenv)
         if result['returncode'] != 0:
             return None
         return result['out'].strip()

platformio/commands/lib.py

@@ -23,6 +23,7 @@ import click
 from platformio import exception, util
 from platformio.commands import PlatformioCLI
 from platformio.managers.lib import LibraryManager, get_builtin_libs
+from platformio.proc import is_ci
 from platformio.project.helpers import (
     get_project_dir, get_projectlibdeps_dir, is_platformio_project)
 
@@ -62,7 +63,7 @@ def cli(ctx, **options):
             storage_dir = join(util.get_home_dir(), "lib")
         elif is_platformio_project():
             storage_dir = get_projectlibdeps_dir()
-        elif util.is_ci():
+        elif is_ci():
             storage_dir = join(util.get_home_dir(), "lib")
             click.secho(
                 "Warning! Global library storage is used automatically. "

platformio/commands/platform.py

@@ -335,8 +335,8 @@ def platform_uninstall(platforms):
     is_flag=True,
     help="Do not update, only check for the new versions")
 @click.option("--json-output", is_flag=True)
-def platform_update(platforms, only_packages, only_check, dry_run,
+def platform_update(  # pylint: disable=too-many-locals
-                    json_output):
+        platforms, only_packages, only_check, dry_run, json_output):
     pm = PlatformManager()
     pkg_dir_to_name = {}
     if not platforms:

platformio/commands/upgrade.py

@@ -22,6 +22,7 @@ import requests
 from platformio import VERSION, __version__, exception, util
 from platformio.compat import WINDOWS
 from platformio.managers.core import shutdown_piohome_servers
+from platformio.proc import exec_command, get_pythonexe_path
 
 
 @click.command(
@@ -47,13 +48,13 @@ def cli(dev):
     r = {}
     try:
         for cmd in cmds:
-            cmd = [util.get_pythonexe_path(), "-m"] + cmd
+            cmd = [get_pythonexe_path(), "-m"] + cmd
-            r = util.exec_command(cmd)
+            r = exec_command(cmd)
 
             # try pip with disabled cache
             if r['returncode'] != 0 and cmd[2] == "pip":
                 cmd.insert(3, "--no-cache-dir")
-                r = util.exec_command(cmd)
+                r = exec_command(cmd)
 
             assert r['returncode'] == 0
         assert "version" in r['out']
@@ -100,9 +101,9 @@ def get_pip_package(to_develop):
     pkg_name = os.path.join(cache_dir, "piocoredevelop.zip")
     try:
         with open(pkg_name, "w") as fp:
-            r = util.exec_command(["curl", "-fsSL", dl_url],
+            r = exec_command(["curl", "-fsSL", dl_url],
-                                  stdout=fp,
+                             stdout=fp,
-                                  universal_newlines=True)
+                             universal_newlines=True)
             assert r['returncode'] == 0
         # check ZIP structure
         with ZipFile(pkg_name) as zp:

platformio/downloader.py

@@ -25,6 +25,7 @@ from platformio import util
 from platformio.compat import PY2, get_filesystem_encoding
 from platformio.exception import (FDSHASumMismatch, FDSizeMismatch,
                                   FDUnrecognizedStatusCode)
+from platformio.proc import exec_command
 
 
 class FileDownloader(object):
@@ -105,12 +106,11 @@ class FileDownloader(object):
 
         dlsha1 = None
         try:
-            result = util.exec_command(["sha1sum", self._destination])
+            result = exec_command(["sha1sum", self._destination])
             dlsha1 = result['out']
         except (OSError, ValueError):
             try:
-                result = util.exec_command(
+                result = exec_command(["shasum", "-a", "1", self._destination])
-                    ["shasum", "-a", "1", self._destination])
                 dlsha1 = result['out']
             except (OSError, ValueError):
                 pass

platformio/maintenance.py

@@ -33,6 +33,7 @@ from platformio.commands.upgrade import get_latest_version
 from platformio.managers.core import update_core_packages
 from platformio.managers.lib import LibraryManager
 from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.proc import is_ci, is_container
 
 
 def on_platformio_start(ctx, force, caller):
@@ -79,7 +80,7 @@ def set_caller(caller=None):
             caller = getenv("PLATFORMIO_CALLER")
         elif getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"):
             caller = "vscode"
-        elif util.is_container():
+        elif is_container():
             if getenv("C9_UID"):
                 caller = "C9"
             elif getenv("USER") == "cabox":
@@ -222,7 +223,7 @@ def after_upgrade(ctx):
             "- %s PlatformIO IDE for IoT development > %s" %
             (click.style("try", fg="cyan"),
              click.style("https://platformio.org/platformio-ide", fg="cyan")))
-    if not util.is_ci():
+    if not is_ci():
         click.echo("- %s us with PlatformIO Plus > %s" % (click.style(
             "support", fg="cyan"), click.style(
                 "https://pioplus.com", fg="cyan")))

platformio/managers/core.py

@@ -23,6 +23,7 @@ import requests
 from platformio import __version__, exception, util
 from platformio.compat import PY2, WINDOWS
 from platformio.managers.package import PackageManager
+from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
 
 CORE_PACKAGES = {
     "contrib-piohome": "^2.0.1",
@@ -125,13 +126,13 @@ def pioplus_call(args, **kwargs):
             "Python 3 is not yet supported.\n" % (__version__, sys.version))
 
     pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus")
-    pythonexe_path = util.get_pythonexe_path()
+    pythonexe_path = get_pythonexe_path()
     os.environ['PYTHONEXEPATH'] = pythonexe_path
     os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite")
     os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "")
     os.environ['PATH'] = (os.pathsep).join(
         [dirname(pythonexe_path), os.environ['PATH']])
-    util.copy_pythonpath_to_osenv()
+    copy_pythonpath_to_osenv()
     code = subprocess.call([pioplus_path] + args, **kwargs)
 
     # handle remote update request

platformio/managers/platform.py

@@ -26,6 +26,8 @@ from platformio import __version__, app, exception, util
 from platformio.compat import PY2
 from platformio.managers.core import get_core_package_dir
 from platformio.managers.package import BasePkgManager, PackageManager
+from platformio.proc import (copy_pythonpath_to_osenv, exec_command,
+                             get_pythonexe_path)
 from platformio.project.helpers import get_projectboards_dir
 
 try:
@@ -379,7 +381,7 @@ class PlatformRunMixin(object):
 
     def _run_scons(self, variables, targets):
         cmd = [
-            util.get_pythonexe_path(),
+            get_pythonexe_path(),
             join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q",
             "-j %d" % self.get_job_nums(), "--warn=no-no-parallel-support",
             "-f",
@@ -397,8 +399,8 @@ class PlatformRunMixin(object):
                     "%s=%s" % (key.upper(), base64.b64encode(
                         value.encode()).decode()))
 
-        util.copy_pythonpath_to_osenv()
+        copy_pythonpath_to_osenv()
-        result = util.exec_command(
+        result = exec_command(
             cmd,
             stdout=util.AsyncPipe(self.on_run_out),
             stderr=util.AsyncPipe(self.on_run_err))

platformio/proc.py

@@ -0,0 +1,143 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+from os.path import isdir, isfile, join, normpath
+from threading import Thread
+
+from platformio import exception
+from platformio.compat import PY2, WINDOWS, string_types
+
+
+class AsyncPipe(Thread):
+
+    def __init__(self, outcallback=None):
+        super(AsyncPipe, self).__init__()
+        self.outcallback = outcallback
+
+        self._fd_read, self._fd_write = os.pipe()
+        self._pipe_reader = os.fdopen(self._fd_read)
+        self._buffer = []
+
+        self.start()
+
+    def get_buffer(self):
+        return self._buffer
+
+    def fileno(self):
+        return self._fd_write
+
+    def run(self):
+        for line in iter(self._pipe_reader.readline, ""):
+            line = line.strip()
+            self._buffer.append(line)
+            if self.outcallback:
+                self.outcallback(line)
+            else:
+                print(line)
+        self._pipe_reader.close()
+
+    def close(self):
+        os.close(self._fd_write)
+        self.join()
+
+
+def exec_command(*args, **kwargs):
+    result = {"out": None, "err": None, "returncode": None}
+
+    default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    default.update(kwargs)
+    kwargs = default
+
+    p = subprocess.Popen(*args, **kwargs)
+    try:
+        result['out'], result['err'] = p.communicate()
+        result['returncode'] = p.returncode
+    except KeyboardInterrupt:
+        raise exception.AbortedByUser()
+    finally:
+        for s in ("stdout", "stderr"):
+            if isinstance(kwargs[s], AsyncPipe):
+                kwargs[s].close()
+
+    for s in ("stdout", "stderr"):
+        if isinstance(kwargs[s], AsyncPipe):
+            result[s[3:]] = "\n".join(kwargs[s].get_buffer())
+
+    for k, v in result.items():
+        if not PY2 and isinstance(result[k], bytes):
+            result[k] = result[k].decode()
+        if v and isinstance(v, string_types):
+            result[k] = result[k].strip()
+
+    return result
+
+
+def is_ci():
+    return os.getenv("CI", "").lower() == "true"
+
+
+def is_container():
+    if not isfile("/proc/1/cgroup"):
+        return False
+    with open("/proc/1/cgroup") as fp:
+        for line in fp:
+            line = line.strip()
+            if ":" in line and not line.endswith(":/"):
+                return True
+    return False
+
+
+def get_pythonexe_path():
+    return os.environ.get("PYTHONEXEPATH", normpath(sys.executable))
+
+
+def copy_pythonpath_to_osenv():
+    _PYTHONPATH = []
+    if "PYTHONPATH" in os.environ:
+        _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep)
+    for p in os.sys.path:
+        conditions = [p not in _PYTHONPATH]
+        if not WINDOWS:
+            conditions.append(
+                isdir(join(p, "click")) or isdir(join(p, "platformio")))
+        if all(conditions):
+            _PYTHONPATH.append(p)
+    os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH)
+
+
+def where_is_program(program, envpath=None):
+    env = os.environ
+    if envpath:
+        env['PATH'] = envpath
+
+    # try OS's built-in commands
+    try:
+        result = exec_command(["where" if WINDOWS else "which", program],
+                              env=env)
+        if result['returncode'] == 0 and isfile(result['out'].strip()):
+            return result['out'].strip()
+    except OSError:
+        pass
+
+    # look up in $PATH
+    for bin_dir in env.get("PATH", "").split(os.pathsep):
+        if isfile(join(bin_dir, program)):
+            return join(bin_dir, program)
+        if isfile(join(bin_dir, "%s.exe" % program)):
+            return join(bin_dir, "%s.exe" % program)
+
+    return program

platformio/telemetry.py

@@ -28,6 +28,7 @@ import requests
 
 from platformio import __version__, app, exception, util
 from platformio.commands import PlatformioCLI
+from platformio.proc import is_ci, is_container
 
 try:
     import queue
@@ -122,7 +123,7 @@ class MeasurementProtocol(TelemetryBase):
                                         platform.platform())
         # self['cd3'] = " ".join(_filter_args(sys.argv[1:]))
         self['cd4'] = 1 if (not util.is_ci() and
-                            (caller_id or not util.is_container())) else 0
+                            (caller_id or not is_container())) else 0
         if caller_id:
             self['cd5'] = caller_id.lower()
 
@@ -273,7 +274,7 @@ def on_command():
     mp = MeasurementProtocol()
     mp.send("screenview")
 
-    if util.is_ci():
+    if is_ci():
         measure_ci()
 
 

platformio/util.py

@@ -12,69 +12,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# FIXME: Remove line below before 4.0 release
+# pylint: disable=unused-import
+
 import json
 import os
 import platform
 import re
 import socket
 import stat
-import subprocess
 import sys
 import time
 from functools import wraps
 from glob import glob
 from os.path import (abspath, basename, dirname, expanduser, isdir, isfile,
-                     join, normpath, splitdrive)
+                     join, splitdrive)
 from shutil import rmtree
-from threading import Thread
 
 import click
 import requests
 
 from platformio import __apiurl__, __version__, exception
-from platformio.compat import path_to_unicode  # pylint: disable=unused-import
+from platformio.compat import PY2, WINDOWS, path_to_unicode
-from platformio.compat import PY2, WINDOWS, string_types
+from platformio.proc import AsyncPipe, exec_command, is_ci, where_is_program
 from platformio.project.config import ProjectConfig
-from platformio.project.helpers import (  # pylint: disable=unused-import
+from platformio.project.helpers import (
     get_project_dir, get_project_optional_dir, get_projectboards_dir,
     get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir,
     get_projectsrc_dir, get_projecttest_dir, is_platformio_project)
 
-# FIXME: remove import of path_to_unicode
-
-
-class AsyncPipe(Thread):
-
-    def __init__(self, outcallback=None):
-        super(AsyncPipe, self).__init__()
-        self.outcallback = outcallback
-
-        self._fd_read, self._fd_write = os.pipe()
-        self._pipe_reader = os.fdopen(self._fd_read)
-        self._buffer = []
-
-        self.start()
-
-    def get_buffer(self):
-        return self._buffer
-
-    def fileno(self):
-        return self._fd_write
-
-    def run(self):
-        for line in iter(self._pipe_reader.readline, ""):
-            line = line.strip()
-            self._buffer.append(line)
-            if self.outcallback:
-                self.outcallback(line)
-            else:
-                print(line)
-        self._pipe_reader.close()
-
-    def close(self):
-        os.close(self._fd_write)
-        self.join()
-
 
 class cd(object):
 
@@ -219,66 +185,6 @@ def change_filemtime(path, mtime):
     os.utime(path, (mtime, mtime))
 
 
-def is_ci():
-    return os.getenv("CI", "").lower() == "true"
-
-
-def is_container():
-    if not isfile("/proc/1/cgroup"):
-        return False
-    with open("/proc/1/cgroup") as fp:
-        for line in fp:
-            line = line.strip()
-            if ":" in line and not line.endswith(":/"):
-                return True
-    return False
-
-
-def exec_command(*args, **kwargs):
-    result = {"out": None, "err": None, "returncode": None}
-
-    default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    default.update(kwargs)
-    kwargs = default
-
-    p = subprocess.Popen(*args, **kwargs)
-    try:
-        result['out'], result['err'] = p.communicate()
-        result['returncode'] = p.returncode
-    except KeyboardInterrupt:
-        raise exception.AbortedByUser()
-    finally:
-        for s in ("stdout", "stderr"):
-            if isinstance(kwargs[s], AsyncPipe):
-                kwargs[s].close()
-
-    for s in ("stdout", "stderr"):
-        if isinstance(kwargs[s], AsyncPipe):
-            result[s[3:]] = "\n".join(kwargs[s].get_buffer())
-
-    for k, v in result.items():
-        if not PY2 and isinstance(result[k], bytes):
-            result[k] = result[k].decode()
-        if v and isinstance(v, string_types):
-            result[k] = result[k].strip()
-
-    return result
-
-
-def copy_pythonpath_to_osenv():
-    _PYTHONPATH = []
-    if "PYTHONPATH" in os.environ:
-        _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep)
-    for p in os.sys.path:
-        conditions = [p not in _PYTHONPATH]
-        if not WINDOWS:
-            conditions.append(
-                isdir(join(p, "click")) or isdir(join(p, "platformio")))
-        if all(conditions):
-            _PYTHONPATH.append(p)
-    os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH)
-
-
 def get_serial_ports(filter_hwid=False):
     try:
         from serial.tools.list_ports import comports
@@ -563,34 +469,6 @@ def internet_on(raise_exception=False):
     return result
 
 
-def get_pythonexe_path():
-    return os.environ.get("PYTHONEXEPATH", normpath(sys.executable))
-
-
-def where_is_program(program, envpath=None):
-    env = os.environ
-    if envpath:
-        env['PATH'] = envpath
-
-    # try OS's built-in commands
-    try:
-        result = exec_command(["where" if WINDOWS else "which", program],
-                              env=env)
-        if result['returncode'] == 0 and isfile(result['out'].strip()):
-            return result['out'].strip()
-    except OSError:
-        pass
-
-    # look up in $PATH
-    for bin_dir in env.get("PATH", "").split(os.pathsep):
-        if isfile(join(bin_dir, program)):
-            return join(bin_dir, program)
-        if isfile(join(bin_dir, "%s.exe" % program)):
-            return join(bin_dir, "%s.exe" % program)
-
-    return program
-
-
 def pepver_to_semver(pepver):
     return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1)
 

platformio/vcsclient.py

@@ -17,8 +17,8 @@ from os.path import join
 from subprocess import CalledProcessError, check_call
 from sys import modules
 
-from platformio import util
 from platformio.exception import PlatformioException, UserSideException
+from platformio.proc import exec_command
 
 try:
     from urllib.parse import urlparse
@@ -109,7 +109,7 @@ class VCSClientBase(object):
         args = [self.command] + args
         if "cwd" not in kwargs:
             kwargs['cwd'] = self.src_dir
-        result = util.exec_command(args, **kwargs)
+        result = exec_command(args, **kwargs)
         if result['returncode'] == 0:
             return result['out'].strip()
         raise PlatformioException(