platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-29 17:47:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Move process related helpers to "proc" module

Browse Source
This commit is contained in:
Ivan Kravets
2019-05-16 21:03:15 +03:00
parent aaf61082c1
commit 971049b41c
16 changed files with 190 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
platformio/app.py
View File

@ -26,6 +26,7 @@ import requests
from platformio import exception, lockfile, util
from platformio.compat import PY2, WINDOWS
from platformio.proc import is_ci
def projects_dir_validate(projects_dir):
@ -339,7 +340,7 @@ def set_session_var(name, value):
def is_disabled_progressbar():
return any([
get_session_var("force_option"),
util.is_ci(),
is_ci(),
getenv("PLATFORMIO_DISABLE_PROGRESSBAR") == "true"
])

3
platformio/builder/main.py
View File

@ -30,6 +30,7 @@ from SCons.Script import Variables # pylint: disable=import-error
from platformio import util
from platformio.compat import PY2, path_to_unicode
from platformio.proc import get_pythonexe_path
from platformio.project.helpers import (
get_project_dir, get_project_optional_dir, get_projectbuild_dir,
get_projectdata_dir, get_projectinclude_dir, get_projectlib_dir,
@ -123,7 +124,7 @@ DEFAULT_ENV_OPTIONS = dict(
],
PROGNAME="program",
PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
PYTHONEXE=util.get_pythonexe_path())
PYTHONEXE=get_pythonexe_path())
if not int(ARGUMENTS.get("PIOVERBOSE", 0)):
DEFAULT_ENV_OPTIONS['ARCOMSTR'] = "Archiving $TARGET"

3
platformio/builder/tools/pioide.py
View File

@ -22,6 +22,7 @@ from SCons.Defaults import processDefines # pylint: disable=import-error
from platformio import util
from platformio.managers.core import get_core_package_dir
from platformio.proc import exec_command
def _dump_includes(env):
@ -71,7 +72,7 @@ def _get_gcc_defines(env):
try:
sysenv = environ.copy()
sysenv['PATH'] = str(env['ENV']['PATH'])
result = util.exec_command(
result = exec_command(
"echo | %s -dM -E -" % env.subst("$CC"), env=sysenv, shell=True)
except OSError:
return items

3
platformio/builder/tools/piomisc.py
View File

@ -26,6 +26,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error
from platformio import util
from platformio.managers.core import get_core_package_dir
from platformio.proc import exec_command
class InoToCPPConverter(object):
@ -211,7 +212,7 @@ def _get_compiler_type(env):
try:
sysenv = environ.copy()
sysenv['PATH'] = str(env['ENV']['PATH'])
result = util.exec_command([env.subst("$CC"), "-v"], env=sysenv)
result = exec_command([env.subst("$CC"), "-v"], env=sysenv)
except OSError:
return None
if result['returncode'] != 0:

3
platformio/builder/tools/pioupload.py
View File

@ -27,6 +27,7 @@ from serial import Serial, SerialException
from platformio import exception, util
from platformio.compat import WINDOWS
from platformio.proc import exec_command
# pylint: disable=unused-argument
@ -211,7 +212,7 @@ def CheckUploadSize(_, target, source, env):
cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
sysenv = environ.copy()
sysenv['PATH'] = str(env['ENV']['PATH'])
result = util.exec_command(env.subst(cmd), env=sysenv)
result = exec_command(env.subst(cmd), env=sysenv)
if result['returncode'] != 0:
return None
return result['out'].strip()

3
platformio/commands/lib.py
View File

@ -23,6 +23,7 @@ import click
from platformio import exception, util
from platformio.commands import PlatformioCLI
from platformio.managers.lib import LibraryManager, get_builtin_libs
from platformio.proc import is_ci
from platformio.project.helpers import (
get_project_dir, get_projectlibdeps_dir, is_platformio_project)
@ -62,7 +63,7 @@ def cli(ctx, **options):
storage_dir = join(util.get_home_dir(), "lib")
elif is_platformio_project():
storage_dir = get_projectlibdeps_dir()
elif util.is_ci():
elif is_ci():
storage_dir = join(util.get_home_dir(), "lib")
click.secho(
"Warning! Global library storage is used automatically. "

4
platformio/commands/platform.py
View File

@ -335,8 +335,8 @@ def platform_uninstall(platforms):
is_flag=True,
help="Do not update, only check for the new versions")
@click.option("--json-output", is_flag=True)
def platform_update(platforms, only_packages, only_check, dry_run,
json_output):
def platform_update( # pylint: disable=too-many-locals
platforms, only_packages, only_check, dry_run, json_output):
pm = PlatformManager()
pkg_dir_to_name = {}
if not platforms:

13
platformio/commands/upgrade.py
View File

@ -22,6 +22,7 @@ import requests
from platformio import VERSION, __version__, exception, util
from platformio.compat import WINDOWS
from platformio.managers.core import shutdown_piohome_servers
from platformio.proc import exec_command, get_pythonexe_path
@click.command(
@ -47,13 +48,13 @@ def cli(dev):
r = {}
try:
for cmd in cmds:
cmd = [util.get_pythonexe_path(), "-m"] + cmd
r = util.exec_command(cmd)
cmd = [get_pythonexe_path(), "-m"] + cmd
r = exec_command(cmd)
# try pip with disabled cache
if r['returncode'] != 0 and cmd[2] == "pip":
cmd.insert(3, "--no-cache-dir")
r = util.exec_command(cmd)
r = exec_command(cmd)
assert r['returncode'] == 0
assert "version" in r['out']
@ -100,9 +101,9 @@ def get_pip_package(to_develop):
pkg_name = os.path.join(cache_dir, "piocoredevelop.zip")
try:
with open(pkg_name, "w") as fp:
r = util.exec_command(["curl", "-fsSL", dl_url],
stdout=fp,
universal_newlines=True)
r = exec_command(["curl", "-fsSL", dl_url],
stdout=fp,
universal_newlines=True)
assert r['returncode'] == 0
# check ZIP structure
with ZipFile(pkg_name) as zp:

6
platformio/downloader.py
View File

@ -25,6 +25,7 @@ from platformio import util
from platformio.compat import PY2, get_filesystem_encoding
from platformio.exception import (FDSHASumMismatch, FDSizeMismatch,
FDUnrecognizedStatusCode)
from platformio.proc import exec_command
class FileDownloader(object):
@ -105,12 +106,11 @@ class FileDownloader(object):
dlsha1 = None
try:
result = util.exec_command(["sha1sum", self._destination])
result = exec_command(["sha1sum", self._destination])
dlsha1 = result['out']
except (OSError, ValueError):
try:
result = util.exec_command(
["shasum", "-a", "1", self._destination])
result = exec_command(["shasum", "-a", "1", self._destination])
dlsha1 = result['out']
except (OSError, ValueError):
pass

5
platformio/maintenance.py
View File

@ -33,6 +33,7 @@ from platformio.commands.upgrade import get_latest_version
from platformio.managers.core import update_core_packages
from platformio.managers.lib import LibraryManager
from platformio.managers.platform import PlatformFactory, PlatformManager
from platformio.proc import is_ci, is_container
def on_platformio_start(ctx, force, caller):
@ -79,7 +80,7 @@ def set_caller(caller=None):
caller = getenv("PLATFORMIO_CALLER")
elif getenv("VSCODE_PID") or getenv("VSCODE_NLS_CONFIG"):
caller = "vscode"
elif util.is_container():
elif is_container():
if getenv("C9_UID"):
caller = "C9"
elif getenv("USER") == "cabox":
@ -222,7 +223,7 @@ def after_upgrade(ctx):
"- %s PlatformIO IDE for IoT development > %s" %
(click.style("try", fg="cyan"),
click.style("https://platformio.org/platformio-ide", fg="cyan")))
if not util.is_ci():
if not is_ci():
click.echo("- %s us with PlatformIO Plus > %s" % (click.style(
"support", fg="cyan"), click.style(
"https://pioplus.com", fg="cyan")))

5
platformio/managers/core.py
View File

@ -23,6 +23,7 @@ import requests
from platformio import __version__, exception, util
from platformio.compat import PY2, WINDOWS
from platformio.managers.package import PackageManager
from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
CORE_PACKAGES = {
"contrib-piohome": "^2.0.1",
@ -125,13 +126,13 @@ def pioplus_call(args, **kwargs):
"Python 3 is not yet supported.\n" % (__version__, sys.version))
pioplus_path = join(get_core_package_dir("tool-pioplus"), "pioplus")
pythonexe_path = util.get_pythonexe_path()
pythonexe_path = get_pythonexe_path()
os.environ['PYTHONEXEPATH'] = pythonexe_path
os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite")
os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "")
os.environ['PATH'] = (os.pathsep).join(
[dirname(pythonexe_path), os.environ['PATH']])
util.copy_pythonpath_to_osenv()
copy_pythonpath_to_osenv()
code = subprocess.call([pioplus_path] + args, **kwargs)
# handle remote update request

8
platformio/managers/platform.py
View File

@ -26,6 +26,8 @@ from platformio import __version__, app, exception, util
from platformio.compat import PY2
from platformio.managers.core import get_core_package_dir
from platformio.managers.package import BasePkgManager, PackageManager
from platformio.proc import (copy_pythonpath_to_osenv, exec_command,
get_pythonexe_path)
from platformio.project.helpers import get_projectboards_dir
try:
@ -379,7 +381,7 @@ class PlatformRunMixin(object):
def _run_scons(self, variables, targets):
cmd = [
util.get_pythonexe_path(),
get_pythonexe_path(),
join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q",
"-j %d" % self.get_job_nums(), "--warn=no-no-parallel-support",
"-f",
@ -397,8 +399,8 @@ class PlatformRunMixin(object):
"%s=%s" % (key.upper(), base64.b64encode(
value.encode()).decode()))
util.copy_pythonpath_to_osenv()
result = util.exec_command(
copy_pythonpath_to_osenv()
result = exec_command(
cmd,
stdout=util.AsyncPipe(self.on_run_out),
stderr=util.AsyncPipe(self.on_run_err))

143
platformio/proc.py Normal file
View File

@ -0,0 +1,143 @@
# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import subprocess
import sys
from os.path import isdir, isfile, join, normpath
from threading import Thread
from platformio import exception
from platformio.compat import PY2, WINDOWS, string_types
class AsyncPipe(Thread):
def __init__(self, outcallback=None):
super(AsyncPipe, self).__init__()
self.outcallback = outcallback
self._fd_read, self._fd_write = os.pipe()
self._pipe_reader = os.fdopen(self._fd_read)
self._buffer = []
self.start()
def get_buffer(self):
return self._buffer
def fileno(self):
return self._fd_write
def run(self):
for line in iter(self._pipe_reader.readline, ""):
line = line.strip()
self._buffer.append(line)
if self.outcallback:
self.outcallback(line)
else:
print(line)
self._pipe_reader.close()
def close(self):
os.close(self._fd_write)
self.join()
def exec_command(*args, **kwargs):
result = {"out": None, "err": None, "returncode": None}
default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
default.update(kwargs)
kwargs = default
p = subprocess.Popen(*args, **kwargs)
try:
result['out'], result['err'] = p.communicate()
result['returncode'] = p.returncode
except KeyboardInterrupt:
raise exception.AbortedByUser()
finally:
for s in ("stdout", "stderr"):
if isinstance(kwargs[s], AsyncPipe):
kwargs[s].close()
for s in ("stdout", "stderr"):
if isinstance(kwargs[s], AsyncPipe):
result[s[3:]] = "\n".join(kwargs[s].get_buffer())
for k, v in result.items():
if not PY2 and isinstance(result[k], bytes):
result[k] = result[k].decode()
if v and isinstance(v, string_types):
result[k] = result[k].strip()
return result
def is_ci():
return os.getenv("CI", "").lower() == "true"
def is_container():
if not isfile("/proc/1/cgroup"):
return False
with open("/proc/1/cgroup") as fp:
for line in fp:
line = line.strip()
if ":" in line and not line.endswith(":/"):
return True
return False
def get_pythonexe_path():
return os.environ.get("PYTHONEXEPATH", normpath(sys.executable))
def copy_pythonpath_to_osenv():
_PYTHONPATH = []
if "PYTHONPATH" in os.environ:
_PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep)
for p in os.sys.path:
conditions = [p not in _PYTHONPATH]
if not WINDOWS:
conditions.append(
isdir(join(p, "click")) or isdir(join(p, "platformio")))
if all(conditions):
_PYTHONPATH.append(p)
os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH)
def where_is_program(program, envpath=None):
env = os.environ
if envpath:
env['PATH'] = envpath
# try OS's built-in commands
try:
result = exec_command(["where" if WINDOWS else "which", program],
env=env)
if result['returncode'] == 0 and isfile(result['out'].strip()):
return result['out'].strip()
except OSError:
pass
# look up in $PATH
for bin_dir in env.get("PATH", "").split(os.pathsep):
if isfile(join(bin_dir, program)):
return join(bin_dir, program)
if isfile(join(bin_dir, "%s.exe" % program)):
return join(bin_dir, "%s.exe" % program)
return program

5
platformio/telemetry.py
View File

@ -28,6 +28,7 @@ import requests
from platformio import __version__, app, exception, util
from platformio.commands import PlatformioCLI
from platformio.proc import is_ci, is_container
try:
import queue
@ -122,7 +123,7 @@ class MeasurementProtocol(TelemetryBase):
platform.platform())
# self['cd3'] = " ".join(_filter_args(sys.argv[1:]))
self['cd4'] = 1 if (not util.is_ci() and
(caller_id or not util.is_container())) else 0
(caller_id or not is_container())) else 0
if caller_id:
self['cd5'] = caller_id.lower()
@ -273,7 +274,7 @@ def on_command():
mp = MeasurementProtocol()
mp.send("screenview")
if util.is_ci():
if is_ci():
measure_ci()

136
platformio/util.py
View File

@ -12,69 +12,35 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# FIXME: Remove line below before 4.0 release
# pylint: disable=unused-import
import json
import os
import platform
import re
import socket
import stat
import subprocess
import sys
import time
from functools import wraps
from glob import glob
from os.path import (abspath, basename, dirname, expanduser, isdir, isfile,
join, normpath, splitdrive)
join, splitdrive)
from shutil import rmtree
from threading import Thread
import click
import requests
from platformio import __apiurl__, __version__, exception
from platformio.compat import path_to_unicode # pylint: disable=unused-import
from platformio.compat import PY2, WINDOWS, string_types
from platformio.compat import PY2, WINDOWS, path_to_unicode
from platformio.proc import AsyncPipe, exec_command, is_ci, where_is_program
from platformio.project.config import ProjectConfig
from platformio.project.helpers import ( # pylint: disable=unused-import
from platformio.project.helpers import (
get_project_dir, get_project_optional_dir, get_projectboards_dir,
get_projectbuild_dir, get_projectdata_dir, get_projectlib_dir,
get_projectsrc_dir, get_projecttest_dir, is_platformio_project)
# FIXME: remove import of path_to_unicode
class AsyncPipe(Thread):
def __init__(self, outcallback=None):
super(AsyncPipe, self).__init__()
self.outcallback = outcallback
self._fd_read, self._fd_write = os.pipe()
self._pipe_reader = os.fdopen(self._fd_read)
self._buffer = []
self.start()
def get_buffer(self):
return self._buffer
def fileno(self):
return self._fd_write
def run(self):
for line in iter(self._pipe_reader.readline, ""):
line = line.strip()
self._buffer.append(line)
if self.outcallback:
self.outcallback(line)
else:
print(line)
self._pipe_reader.close()
def close(self):
os.close(self._fd_write)
self.join()
class cd(object):
@ -219,66 +185,6 @@ def change_filemtime(path, mtime):
os.utime(path, (mtime, mtime))
def is_ci():
return os.getenv("CI", "").lower() == "true"
def is_container():
if not isfile("/proc/1/cgroup"):
return False
with open("/proc/1/cgroup") as fp:
for line in fp:
line = line.strip()
if ":" in line and not line.endswith(":/"):
return True
return False
def exec_command(*args, **kwargs):
result = {"out": None, "err": None, "returncode": None}
default = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
default.update(kwargs)
kwargs = default
p = subprocess.Popen(*args, **kwargs)
try:
result['out'], result['err'] = p.communicate()
result['returncode'] = p.returncode
except KeyboardInterrupt:
raise exception.AbortedByUser()
finally:
for s in ("stdout", "stderr"):
if isinstance(kwargs[s], AsyncPipe):
kwargs[s].close()
for s in ("stdout", "stderr"):
if isinstance(kwargs[s], AsyncPipe):
result[s[3:]] = "\n".join(kwargs[s].get_buffer())
for k, v in result.items():
if not PY2 and isinstance(result[k], bytes):
result[k] = result[k].decode()
if v and isinstance(v, string_types):
result[k] = result[k].strip()
return result
def copy_pythonpath_to_osenv():
_PYTHONPATH = []
if "PYTHONPATH" in os.environ:
_PYTHONPATH = os.environ.get("PYTHONPATH").split(os.pathsep)
for p in os.sys.path:
conditions = [p not in _PYTHONPATH]
if not WINDOWS:
conditions.append(
isdir(join(p, "click")) or isdir(join(p, "platformio")))
if all(conditions):
_PYTHONPATH.append(p)
os.environ['PYTHONPATH'] = os.pathsep.join(_PYTHONPATH)
def get_serial_ports(filter_hwid=False):
try:
from serial.tools.list_ports import comports
@ -563,34 +469,6 @@ def internet_on(raise_exception=False):
return result
def get_pythonexe_path():
return os.environ.get("PYTHONEXEPATH", normpath(sys.executable))
def where_is_program(program, envpath=None):
env = os.environ
if envpath:
env['PATH'] = envpath
# try OS's built-in commands
try:
result = exec_command(["where" if WINDOWS else "which", program],
env=env)
if result['returncode'] == 0 and isfile(result['out'].strip()):
return result['out'].strip()
except OSError:
pass
# look up in $PATH
for bin_dir in env.get("PATH", "").split(os.pathsep):
if isfile(join(bin_dir, program)):
return join(bin_dir, program)
if isfile(join(bin_dir, "%s.exe" % program)):
return join(bin_dir, "%s.exe" % program)
return program
def pepver_to_semver(pepver):
return re.sub(r"(\.\d+)\.?(dev|a|b|rc|post)", r"\1-\2.", pepver, 1)

4
platformio/vcsclient.py
View File

@ -17,8 +17,8 @@ from os.path import join
from subprocess import CalledProcessError, check_call
from sys import modules
from platformio import util
from platformio.exception import PlatformioException, UserSideException
from platformio.proc import exec_command
try:
from urllib.parse import urlparse
@ -109,7 +109,7 @@ class VCSClientBase(object):
args = [self.command] + args
if "cwd" not in kwargs:
kwargs['cwd'] = self.src_dir
result = util.exec_command(args, **kwargs)
result = exec_command(args, **kwargs)
if result['returncode'] == 0:
return result['out'].strip()
raise PlatformioException(