From 971049b41cb137cb092d165364f32331f8096a49 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 16 May 2019 21:03:15 +0300 Subject: [PATCH] Move process related helpers to "proc" module --- platformio/app.py | 3 +- platformio/builder/main.py | 3 +- platformio/builder/tools/pioide.py | 3 +- platformio/builder/tools/piomisc.py | 3 +- platformio/builder/tools/pioupload.py | 3 +- platformio/commands/lib.py | 3 +- platformio/commands/platform.py | 4 +- platformio/commands/upgrade.py | 13 +-- platformio/downloader.py | 6 +- platformio/maintenance.py | 5 +- platformio/managers/core.py | 5 +- platformio/managers/platform.py | 8 +- platformio/proc.py | 143 ++++++++++++++++++++++++++ platformio/telemetry.py | 5 +- platformio/util.py | 136 ++---------------------- platformio/vcsclient.py | 4 +- 16 files changed, 190 insertions(+), 157 deletions(-) create mode 100644 platformio/proc.py diff --git a/platformio/app.py b/platformio/app.py index c8a071fc..b8e3b691 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -26,6 +26,7 @@ import requests from platformio import exception, lockfile, util from platformio.compat import PY2, WINDOWS +from platformio.proc import is_ci def projects_dir_validate(projects_dir): @@ -339,7 +340,7 @@ def set_session_var(name, value): def is_disabled_progressbar(): return any([ get_session_var("force_option"), - util.is_ci(), + is_ci(), getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true" ]) diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 95c4d4ca..287c0e84 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -30,6 +30,7 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import util from platformio.compat import PY2, path_to_unicode +from platformio.proc import get_pythonexe_path from platformio.project.helpers import ( get_project_dir, get_project_optional_dir, get_projectbuild_dir, get_projectdata_dir, get_projectinclude_dir, get_projectlib_dir, @@ -123,7 +124,7 @@ DEFAULT_ENV_OPTIONS = dict( ], PROGNAME="program", PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), - PYTHONEXE=util.get_pythonexe_path()) + PYTHONEXE=get_pythonexe_path()) if not int(ARGUMENTS.get("PIOVERBOSE", 0)): DEFAULT_ENV_OPTIONS['ARCOMSTR'] = "Archiving $TARGET" diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index db496741..311b0203 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -22,6 +22,7 @@ from SCons.Defaults import processDefines # pylint: disable=import-error from platformio import util from platformio.managers.core import get_core_package_dir +from platformio.proc import exec_command def _dump_includes(env): @@ -71,7 +72,7 @@ def _get_gcc_defines(env): try: sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) - result = util.exec_command( + result = exec_command( "echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True) except OSError: return items diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 01a797ea..19768146 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -26,6 +26,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from platformio import util from platformio.managers.core import get_core_package_dir +from platformio.proc import exec_command class InoToCPPConverter(object): @@ -211,7 +212,7 @@ def _get_compiler_type(env): try: sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) - result = util.exec_command([env.subst("$CC"), "-v"], env=sysenv) + result = exec_command([env.subst("$CC"), "-v"], env=sysenv) except OSError: return None if result['returncode'] != 0: diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index a7b2685d..6db94bfc 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -27,6 +27,7 @@ from serial import Serial, SerialException from platformio import exception, util from platformio.compat import WINDOWS +from platformio.proc import exec_command # pylint: disable=unused-argument @@ -211,7 +212,7 @@ def CheckUploadSize(_, target, source, env): cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg] sysenv = environ.copy() sysenv['PATH'] = str(env['ENV']['PATH']) - result = util.exec_command(env.subst(cmd), env=sysenv) + result = exec_command(env.subst(cmd), env=sysenv) if result['returncode'] != 0: return None return result['out'].strip() diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index dde2aa0b..81a4640a 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -23,6 +23,7 @@ import click from platformio import exception, util from platformio.commands import PlatformioCLI from platformio.managers.lib import LibraryManager, get_builtin_libs +from platformio.proc import is_ci from platformio.project.helpers import ( get_project_dir, get_projectlibdeps_dir, is_platformio_project) @@ -62,7 +63,7 @@ def cli(ctx, **options): storage_dir = join(util.get_home_dir(), "lib") elif is_platformio_project(): storage_dir = get_projectlibdeps_dir() - elif util.is_ci(): + elif is_ci(): storage_dir = join(util.get_home_dir(), "lib") click.secho( "Warning! Global library storage is used automatically. " diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index db0149f3..70612f51 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -335,8 +335,8 @@ def platform_uninstall(platforms): is_flag=True, help="Do not update, only check for the new versions") @click.option("--json-output", is_flag=True) -def platform_update(platforms, only_packages, only_check, dry_run, - json_output): +def platform_update( # pylint: disable=too-many-locals + platforms, only_packages, only_check, dry_run, json_output): pm = PlatformManager() pkg_dir_to_name = {} if not platforms: diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 6acb83ae..18e49cb3 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -22,6 +22,7 @@ import requests from platformio import VERSION, __version__, exception, util from platformio.compat import WINDOWS from platformio.managers.core import shutdown_piohome_servers +from platformio.proc import exec_command, get_pythonexe_path @click.command( @@ -47,13 +48,13 @@ def cli(dev): r = {} try: for cmd in cmds: - cmd = [util.get_pythonexe_path(), "-m"] + cmd - r = util.exec_command(cmd) + cmd = [get_pythonexe_path(), "-m"] + cmd + r = exec_command(cmd) # try pip with disabled cache if r['returncode'] != 0 and cmd[2] == "pip": cmd.insert(3, "--no-cache-dir") - r = util.exec_command(cmd) + r = exec_command(cmd) assert r['returncode'] == 0 assert "version" in r['out'] @@ -100,9 +101,9 @@ def get_pip_package(to_develop): pkg_name = os.path.join(cache_dir, "piocoredevelop.zip") try: with open(pkg_name, "w") as fp: - r = util.exec_command(["curl", "-fsSL", dl_url], - stdout=fp, - universal_newlines=True) + r = exec_command(["curl", "-fsSL", dl_url], + stdout=fp, + universal_newlines=True) assert r['returncode'] == 0 # check ZIP structure with ZipFile(pkg_name) as zp: diff --git a/platformio/downloader.py b/platformio/downloader.py index f86b723d..ab84e518 100644 --- a/platformio/downloader.py +++ b/platformio/downloader.py @@ -25,6 +25,7 @@ from platformio import util from platformio.compat import PY2, get_filesystem_encoding from platformio.exception import (FDSHASumMismatch, FDSizeMismatch, FDUnrecognizedStatusCode) +from platformio.proc import exec_command class FileDownloader(object): @@ -105,12 +106,11 @@ class FileDownloader(object): dlsha1 = None try: - result = util.exec_command(["sha1sum", self._destination]) + result = exec_command(["sha1sum", self._destination]) dlsha1 = result['out'] except (OSError, ValueError): try: - result = util.exec_command( - ["shasum", "-a", "1", self._destination]) + result = exec_command(["shasum", "-a", "1", self._destination]) dlsha1 = result['out'] except (OSError, ValueError): pass diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 4bebcb7b..4b208385 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -33,6 +33,7 @@ from platformio.commands.upgrade import get_latest_version from platformio.managers.core import update_core_packages from platformio.managers.lib import LibraryManager from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.proc import is_ci, is_container def on_platformio_start(ctx, force, caller): @@ -79,7 +80,7 @@ def set_caller(caller=None): caller = getenv("PLATFORMIO_CALLER") elif getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"): caller = "vscode" - elif util.is_container(): + elif is_container(): if getenv("C9_UID"): caller = "C9" elif getenv("USER") == "cabox": @@ -222,7 +223,7 @@ def after_upgrade(ctx): "- %s PlatformIO IDE for IoT development > %s" % (click.style("try", fg="cyan"), click.style("https://platformio.org/platformio-ide", fg="cyan"))) - if not util.is_ci(): + if not is_ci(): click.echo("- %s us with PlatformIO Plus > %s" % (click.style( "support", fg="cyan"), click.style( "https://pioplus.com", fg="cyan"))) diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 62492bcf..3c66592c 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -23,6 +23,7 @@ import requests from platformio import __version__, exception, util from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager +from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path CORE_PACKAGES = { "contrib-piohome": "^2.0.1", @@ -125,13 +126,13 @@ def pioplus_call(args, **kwargs): "Python 3 is not yet supported.\n" % (__version__, sys.version)) pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus") - pythonexe_path = util.get_pythonexe_path() + pythonexe_path = get_pythonexe_path() os.environ['PYTHONEXEPATH'] = pythonexe_path os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite") os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "") os.environ['PATH'] = (os.pathsep).join( [dirname(pythonexe_path), os.environ['PATH']]) - util.copy_pythonpath_to_osenv() + copy_pythonpath_to_osenv() code = subprocess.call([pioplus_path] + args, **kwargs) # handle remote update request diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8c0b7de9..f6dac646 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -26,6 +26,8 @@ from platformio import __version__, app, exception, util from platformio.compat import PY2 from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager +from platformio.proc import (copy_pythonpath_to_osenv, exec_command, + get_pythonexe_path) from platformio.project.helpers import get_projectboards_dir try: @@ -379,7 +381,7 @@ class PlatformRunMixin(object): def _run_scons(self, variables, targets): cmd = [ - util.get_pythonexe_path(), + get_pythonexe_path(), join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q", "-j %d" % self.get_job_nums(), "--warn=no-no-parallel-support", "-f", @@ -397,8 +399,8 @@ class PlatformRunMixin(object): "%s=%s" % (key.upper(), base64.b64encode( value.encode()).decode())) - util.copy_pythonpath_to_osenv() - result = util.exec_command( + copy_pythonpath_to_osenv() + result = exec_command( cmd, stdout=util.AsyncPipe(self.on_run_out), stderr=util.AsyncPipe(self.on_run_err)) diff --git a/platformio/proc.py b/platformio/proc.py new file mode 100644 index 00000000..a0432458 --- /dev/null +++ b/platformio/proc.py @@ -0,0 +1,143 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys +from os.path import isdir, isfile, join, normpath +from threading import Thread + +from platformio import exception +from platformio.compat import PY2, WINDOWS, string_types + + +class AsyncPipe(Thread): + + def __init__(self, outcallback=None): + super(AsyncPipe, self).__init__() + self.outcallback = outcallback + + self._fd_read, self._fd_write = os.pipe() + self._pipe_reader = os.fdopen(self._fd_read) + self._buffer = [] + + self.start() + + def get_buffer(self): + return self._buffer + + def fileno(self): + return self._fd_write + + def run(self): + for line in iter(self._pipe_reader.readline, ""): + line = line.strip() + self._buffer.append(line) + if self.outcallback: + self.outcallback(line) + else: + print(line) + self._pipe_reader.close() + + def close(self): + os.close(self._fd_write) + self.join() + + +def exec_command(*args, **kwargs): + result = {"out": None, "err": None, "returncode": None} + + default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) + default.update(kwargs) + kwargs = default + + p = subprocess.Popen(*args, **kwargs) + try: + result['out'], result['err'] = p.communicate() + result['returncode'] = p.returncode + except KeyboardInterrupt: + raise exception.AbortedByUser() + finally: + for s in ("stdout", "stderr"): + if isinstance(kwargs[s], AsyncPipe): + kwargs[s].close() + + for s in ("stdout", "stderr"): + if isinstance(kwargs[s], AsyncPipe): + result[s[3:]] = "\n".join(kwargs[s].get_buffer()) + + for k, v in result.items(): + if not PY2 and isinstance(result[k], bytes): + result[k] = result[k].decode() + if v and isinstance(v, string_types): + result[k] = result[k].strip() + + return result + + +def is_ci(): + return os.getenv("CI", "").lower() == "true" + + +def is_container(): + if not isfile("/proc/1/cgroup"): + return False + with open("/proc/1/cgroup") as fp: + for line in fp: + line = line.strip() + if ":" in line and not line.endswith(":/"): + return True + return False + + +def get_pythonexe_path(): + return os.environ.get("PYTHONEXEPATH", normpath(sys.executable)) + + +def copy_pythonpath_to_osenv(): + _PYTHONPATH = [] + if "PYTHONPATH" in os.environ: + _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) + for p in os.sys.path: + conditions = [p not in _PYTHONPATH] + if not WINDOWS: + conditions.append( + isdir(join(p, "click")) or isdir(join(p, "platformio"))) + if all(conditions): + _PYTHONPATH.append(p) + os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH) + + +def where_is_program(program, envpath=None): + env = os.environ + if envpath: + env['PATH'] = envpath + + # try OS's built-in commands + try: + result = exec_command(["where" if WINDOWS else "which", program], + env=env) + if result['returncode'] == 0 and isfile(result['out'].strip()): + return result['out'].strip() + except OSError: + pass + + # look up in $PATH + for bin_dir in env.get("PATH", "").split(os.pathsep): + if isfile(join(bin_dir, program)): + return join(bin_dir, program) + if isfile(join(bin_dir, "%s.exe" % program)): + return join(bin_dir, "%s.exe" % program) + + return program diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 987d5899..6bdb48a1 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -28,6 +28,7 @@ import requests from platformio import __version__, app, exception, util from platformio.commands import PlatformioCLI +from platformio.proc import is_ci, is_container try: import queue @@ -122,7 +123,7 @@ class MeasurementProtocol(TelemetryBase): platform.platform()) # self['cd3'] = " ".join(_filter_args(sys.argv[1:])) self['cd4'] = 1 if (not util.is_ci() and - (caller_id or not util.is_container())) else 0 + (caller_id or not is_container())) else 0 if caller_id: self['cd5'] = caller_id.lower() @@ -273,7 +274,7 @@ def on_command(): mp = MeasurementProtocol() mp.send("screenview") - if util.is_ci(): + if is_ci(): measure_ci() diff --git a/platformio/util.py b/platformio/util.py index 6e5db848..4e0979fc 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -12,69 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. +# FIXME: Remove line below before 4.0 release +# pylint: disable=unused-import + import json import os import platform import re import socket import stat -import subprocess import sys import time from functools import wraps from glob import glob from os.path import (abspath, basename, dirname, expanduser, isdir, isfile, - join, normpath, splitdrive) + join, splitdrive) from shutil import rmtree -from threading import Thread import click import requests from platformio import __apiurl__, __version__, exception -from platformio.compat import path_to_unicode # pylint: disable=unused-import -from platformio.compat import PY2, WINDOWS, string_types +from platformio.compat import PY2, WINDOWS, path_to_unicode +from platformio.proc import AsyncPipe, exec_command, is_ci, where_is_program from platformio.project.config import ProjectConfig -from platformio.project.helpers import ( # pylint: disable=unused-import +from platformio.project.helpers import ( get_project_dir, get_project_optional_dir, get_projectboards_dir, get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir, get_projectsrc_dir, get_projecttest_dir, is_platformio_project) -# FIXME: remove import of path_to_unicode - - -class AsyncPipe(Thread): - - def __init__(self, outcallback=None): - super(AsyncPipe, self).__init__() - self.outcallback = outcallback - - self._fd_read, self._fd_write = os.pipe() - self._pipe_reader = os.fdopen(self._fd_read) - self._buffer = [] - - self.start() - - def get_buffer(self): - return self._buffer - - def fileno(self): - return self._fd_write - - def run(self): - for line in iter(self._pipe_reader.readline, ""): - line = line.strip() - self._buffer.append(line) - if self.outcallback: - self.outcallback(line) - else: - print(line) - self._pipe_reader.close() - - def close(self): - os.close(self._fd_write) - self.join() - class cd(object): @@ -219,66 +185,6 @@ def change_filemtime(path, mtime): os.utime(path, (mtime, mtime)) -def is_ci(): - return os.getenv("CI", "").lower() == "true" - - -def is_container(): - if not isfile("/proc/1/cgroup"): - return False - with open("/proc/1/cgroup") as fp: - for line in fp: - line = line.strip() - if ":" in line and not line.endswith(":/"): - return True - return False - - -def exec_command(*args, **kwargs): - result = {"out": None, "err": None, "returncode": None} - - default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) - default.update(kwargs) - kwargs = default - - p = subprocess.Popen(*args, **kwargs) - try: - result['out'], result['err'] = p.communicate() - result['returncode'] = p.returncode - except KeyboardInterrupt: - raise exception.AbortedByUser() - finally: - for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipe): - kwargs[s].close() - - for s in ("stdout", "stderr"): - if isinstance(kwargs[s], AsyncPipe): - result[s[3:]] = "\n".join(kwargs[s].get_buffer()) - - for k, v in result.items(): - if not PY2 and isinstance(result[k], bytes): - result[k] = result[k].decode() - if v and isinstance(v, string_types): - result[k] = result[k].strip() - - return result - - -def copy_pythonpath_to_osenv(): - _PYTHONPATH = [] - if "PYTHONPATH" in os.environ: - _PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep) - for p in os.sys.path: - conditions = [p not in _PYTHONPATH] - if not WINDOWS: - conditions.append( - isdir(join(p, "click")) or isdir(join(p, "platformio"))) - if all(conditions): - _PYTHONPATH.append(p) - os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH) - - def get_serial_ports(filter_hwid=False): try: from serial.tools.list_ports import comports @@ -563,34 +469,6 @@ def internet_on(raise_exception=False): return result -def get_pythonexe_path(): - return os.environ.get("PYTHONEXEPATH", normpath(sys.executable)) - - -def where_is_program(program, envpath=None): - env = os.environ - if envpath: - env['PATH'] = envpath - - # try OS's built-in commands - try: - result = exec_command(["where" if WINDOWS else "which", program], - env=env) - if result['returncode'] == 0 and isfile(result['out'].strip()): - return result['out'].strip() - except OSError: - pass - - # look up in $PATH - for bin_dir in env.get("PATH", "").split(os.pathsep): - if isfile(join(bin_dir, program)): - return join(bin_dir, program) - if isfile(join(bin_dir, "%s.exe" % program)): - return join(bin_dir, "%s.exe" % program) - - return program - - def pepver_to_semver(pepver): return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1) diff --git a/platformio/vcsclient.py b/platformio/vcsclient.py index 67ef29d2..222ba0e5 100644 --- a/platformio/vcsclient.py +++ b/platformio/vcsclient.py @@ -17,8 +17,8 @@ from os.path import join from subprocess import CalledProcessError, check_call from sys import modules -from platformio import util from platformio.exception import PlatformioException, UserSideException +from platformio.proc import exec_command try: from urllib.parse import urlparse @@ -109,7 +109,7 @@ class VCSClientBase(object): args = [self.command] + args if "cwd" not in kwargs: kwargs['cwd'] = self.src_dir - result = util.exec_command(args, **kwargs) + result = exec_command(args, **kwargs) if result['returncode'] == 0: return result['out'].strip() raise PlatformioException(