diff --git a/HISTORY.rst b/HISTORY.rst index 05fca2d8..d3f412fd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,7 +14,7 @@ PlatformIO Core 5 * **PlatformIO Debugging** - Boosted `PlatformIO Debugging `__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack - - Support debugging on Windows using Windows CMD (CLI) (`issue #3793 `_) + - Support debugging on Windows using Windows CMD/CLI (`pio debug `__) (`issue #3793 `_) 5.1.1 (2021-03-17) ~~~~~~~~~~~~~~~~~~ diff --git a/platformio/__main__.py b/platformio/__main__.py index 537b1d0a..dbf2215b 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=import-outside-toplevel + import os import sys from traceback import format_exc import click -from platformio import __version__, exception, maintenance, util +from platformio import __version__, exception from platformio.commands import PlatformioCLI -from platformio.compat import CYGWIN +from platformio.compat import CYGWIN, PY2, ensure_python3 try: import click_completion # pylint: disable=import-error @@ -60,16 +62,19 @@ def cli(ctx, force, caller, no_ansi): except: # pylint: disable=bare-except pass + from platformio import maintenance + maintenance.on_platformio_start(ctx, force, caller) @cli.resultcallback() @click.pass_context def process_result(ctx, result, *_, **__): + from platformio import maintenance + maintenance.on_platformio_end(ctx, result) -@util.memoized() def configure(): if CYGWIN: raise exception.CygwinEnvDetected() @@ -105,6 +110,7 @@ def main(argv=None): assert isinstance(argv, list) sys.argv = argv try: + ensure_python3(raise_exception=True) configure() cli() # pylint: disable=no-value-for-parameter except SystemExit as e: @@ -112,7 +118,10 @@ def main(argv=None): exit_code = int(e.code) except Exception as e: # pylint: disable=broad-except if not isinstance(e, exception.ReturnErrorCode): - maintenance.on_platformio_exception(e) + if not PY2: + from platformio import maintenance + + maintenance.on_platformio_exception(e) error_str = "Error: " if isinstance(e, exception.PlatformioException): error_str += str(e) diff --git a/platformio/app.py b/platformio/app.py index 04d02c39..9d64994c 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -24,7 +24,7 @@ import uuid from os.path import dirname, isdir, isfile, join, realpath from platformio import __version__, exception, fs, proc -from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data +from platformio.compat import WINDOWS, hashlib_encode_data from platformio.package.lockfile import LockFile from platformio.project.helpers import get_default_projects_dir, get_project_core_dir @@ -115,7 +115,7 @@ class State(object): if self.modified: try: with open(self.path, "w") as fp: - fp.write(dump_json_to_unicode(self._storage)) + fp.write(json.dumps(self._storage)) except IOError: raise exception.HomeDirPermissionsError(get_project_core_dir()) self._unlock_state_file() diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 6a060dd1..208cb0e2 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import os import sys -from os import environ, makedirs -from os.path import isdir, join from time import time import click @@ -29,7 +29,6 @@ from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error from platformio import compat, fs -from platformio.compat import dump_json_to_unicode from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_dir @@ -65,18 +64,18 @@ DEFAULT_ENV_OPTIONS = dict( "pioide", "piosize", ], - toolpath=[join(fs.get_source_dir(), "builder", "tools")], + toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")], variables=clivars, # Propagating External Environment - ENV=environ, + ENV=os.environ, UNIX_TIME=int(time()), - BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"), - BUILD_SRC_DIR=join("$BUILD_DIR", "src"), - BUILD_TEST_DIR=join("$BUILD_DIR", "test"), - COMPILATIONDB_PATH=join("$BUILD_DIR", "compile_commands.json"), + BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"), + BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"), + BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"), + COMPILATIONDB_PATH=os.path.join("$BUILD_DIR", "compile_commands.json"), LIBPATH=["$BUILD_DIR"], PROGNAME="program", - PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), + PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"), PYTHONEXE=get_pythonexe_path(), IDE_EXTRA_DATA={}, ) @@ -124,7 +123,7 @@ env.Replace( BUILD_CACHE_DIR=config.get_optional_dir("build_cache"), LIBSOURCE_DIRS=[ config.get_optional_dir("lib"), - join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), + os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"), config.get_optional_dir("globallib"), ], ) @@ -142,8 +141,8 @@ if ( ) if env.subst("$BUILD_CACHE_DIR"): - if not isdir(env.subst("$BUILD_CACHE_DIR")): - makedirs(env.subst("$BUILD_CACHE_DIR")) + if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")): + os.makedirs(env.subst("$BUILD_CACHE_DIR")) env.CacheDir("$BUILD_CACHE_DIR") if int(ARGUMENTS.get("ISATTY", 0)): @@ -160,15 +159,17 @@ elif not int(ARGUMENTS.get("PIOVERBOSE", 0)): if "compiledb" in COMMAND_LINE_TARGETS: env.Tool("compilation_db") -if not isdir(env.subst("$BUILD_DIR")): - makedirs(env.subst("$BUILD_DIR")) +if not os.path.isdir(env.subst("$BUILD_DIR")): + os.makedirs(env.subst("$BUILD_DIR")) env.LoadProjectOptions() env.LoadPioPlatform() env.SConscriptChdir(0) env.SConsignFile( - join("$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1])) + os.path.join( + "$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]) + ) ) for item in env.GetExtraScripts("pre"): @@ -225,9 +226,7 @@ if "idedata" in COMMAND_LINE_TARGETS: projenv = env click.echo( "\n%s\n" - % dump_json_to_unicode( - projenv.DumpIDEData(env) # pylint: disable=undefined-variable - ) + % json.dumps(projenv.DumpIDEData(env)) # pylint: disable=undefined-variable ) env.Exit(0) diff --git a/platformio/builder/tools/pioide.py b/platformio/builder/tools/pioide.py index de7e0cc8..332ec6ed 100644 --- a/platformio/builder/tools/pioide.py +++ b/platformio/builder/tools/pioide.py @@ -14,13 +14,12 @@ from __future__ import absolute_import +import glob import os -from glob import glob import SCons.Defaults # pylint: disable=import-error import SCons.Subst # pylint: disable=import-error -from platformio.compat import glob_escape from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command, where_is_program @@ -49,7 +48,7 @@ def _dump_includes(env): for pkg in p.get_installed_packages(): if p.get_package_type(pkg.metadata.name) != "toolchain": continue - toolchain_dir = glob_escape(pkg.path) + toolchain_dir = glob.escape(pkg.path) toolchain_incglobs = [ os.path.join(toolchain_dir, "*", "include", "c++", "*"), os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"), @@ -57,7 +56,9 @@ def _dump_includes(env): os.path.join(toolchain_dir, "*", "include*"), ] for g in toolchain_incglobs: - includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)]) + includes["toolchain"].extend( + [os.path.realpath(inc) for inc in glob.glob(g)] + ) # include Unity framework if there are tests in project includes["unity"] = [] diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 799b192f..dbc39012 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -15,16 +15,17 @@ from __future__ import absolute_import import atexit +import glob import io import os import re import sys -from tempfile import mkstemp +import tempfile import click from platformio import fs, util -from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape +from platformio.compat import get_filesystem_encoding, get_locale_encoding from platformio.package.manager.core import get_core_package_dir from platformio.proc import exec_command @@ -116,7 +117,7 @@ class InoToCPPConverter(object): return out_file def _gcc_preprocess(self, contents, out_file): - tmp_path = mkstemp()[1] + tmp_path = tempfile.mkstemp()[1] self.write_safe_contents(tmp_path, contents) self.env.Execute( self.env.VerboseAction( @@ -229,7 +230,7 @@ class InoToCPPConverter(object): def ConvertInoToCpp(env): - src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR")) + src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR")) ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob( os.path.join(src_dir, "*.pde") ) diff --git a/platformio/builder/tools/piosize.py b/platformio/builder/tools/piosize.py index 83b3a3f5..eec4c87c 100644 --- a/platformio/builder/tools/piosize.py +++ b/platformio/builder/tools/piosize.py @@ -16,6 +16,7 @@ from __future__ import absolute_import +import json import sys from os import environ, makedirs, remove from os.path import isdir, join, splitdrive @@ -23,7 +24,6 @@ from os.path import isdir, join, splitdrive from elftools.elf.descriptions import describe_sh_flags from elftools.elf.elffile import ELFFile -from platformio.compat import dump_json_to_unicode from platformio.proc import exec_command from platformio.util import get_systype @@ -242,7 +242,7 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument data["memory"]["files"].append(file_data) with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp: - fp.write(dump_json_to_unicode(data)) + fp.write(json.dumps(data)) def exists(_): diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 4170b32f..f93607d4 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -18,7 +18,6 @@ import click from tabulate import tabulate from platformio import fs -from platformio.compat import dump_json_to_unicode from platformio.package.manager.platform import PlatformPackageManager @@ -83,4 +82,4 @@ def _print_boards_json(query, installed=False): if query.lower() not in search_data.lower(): continue result.append(board) - click.echo(dump_json_to_unicode(result)) + click.echo(json.dumps(result)) diff --git a/platformio/commands/check/command.py b/platformio/commands/check/command.py index 8f9a6dca..082373f3 100644 --- a/platformio/commands/check/command.py +++ b/platformio/commands/check/command.py @@ -15,6 +15,7 @@ # pylint: disable=too-many-arguments,too-many-locals,too-many-branches # pylint: disable=redefined-builtin,too-many-statements +import json import os from collections import Counter from os.path import dirname, isfile @@ -26,7 +27,6 @@ from tabulate import tabulate from platformio import app, exception, fs, util from platformio.commands.check.defect import DefectItem from platformio.commands.check.tools import CheckToolFactory -from platformio.compat import dump_json_to_unicode from platformio.project.config import ProjectConfig from platformio.project.helpers import find_project_dir_above, get_project_dir @@ -163,7 +163,7 @@ def cli( print_processing_footer(result) if json_output: - click.echo(dump_json_to_unicode(results_to_json(results))) + click.echo(json.dumps(results_to_json(results))) elif not silent: print_check_summary(results) diff --git a/platformio/commands/check/tools/base.py b/platformio/commands/check/tools/base.py index 7eda6936..da38c97e 100644 --- a/platformio/commands/check/tools/base.py +++ b/platformio/commands/check/tools/base.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import glob import os -from tempfile import NamedTemporaryFile +import tempfile import click -from platformio import compat, fs, proc +from platformio import fs, proc from platformio.commands.check.defect import DefectItem from platformio.project.helpers import load_project_ide_data @@ -104,7 +105,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes return {lang: _extract_defines(lang, incflags_file) for lang in ("c", "c++")} def _create_tmp_file(self, data): - with NamedTemporaryFile("w", delete=False) as fp: + with tempfile.NamedTemporaryFile("w", delete=False) as fp: fp.write(data) self._tmp_files.append(fp.name) return fp.name @@ -207,7 +208,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes result["c++"].append(os.path.realpath(path)) for pattern in patterns: - for item in compat.glob_recursive(pattern): + for item in glob.glob(pattern, recursive=True): if not os.path.isdir(item): _add_file(item) for root, _, files in os.walk(item, followlinks=True): diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index e72ddf76..5bc5b38b 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from os import getenv, makedirs, remove -from os.path import basename, isdir, isfile, join, realpath -from shutil import copyfile, copytree -from tempfile import mkdtemp +import glob +import os +import shutil +import tempfile import click -from platformio import app, compat, fs +from platformio import app, fs from platformio.commands.project import project_init as cmd_project_init from platformio.commands.project import validate_boards from platformio.commands.run.command import cli as cmd_run @@ -33,8 +33,8 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument for i, p in enumerate(value): if p.startswith("~"): value[i] = fs.expanduser(p) - value[i] = realpath(value[i]) - if not compat.glob_recursive(value[i]): + value[i] = os.path.realpath(value[i]) + if not glob.glob(value[i], recursive=True): invalid_path = p break try: @@ -51,7 +51,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument @click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) @click.option( "--build-dir", - default=mkdtemp, + default=tempfile.mkdtemp, type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), ) @click.option("--keep-build-dir", is_flag=True) @@ -78,28 +78,28 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches verbose, ): - if not src and getenv("PLATFORMIO_CI_SRC"): - src = validate_path(ctx, None, getenv("PLATFORMIO_CI_SRC").split(":")) + if not src and os.getenv("PLATFORMIO_CI_SRC"): + src = validate_path(ctx, None, os.getenv("PLATFORMIO_CI_SRC").split(":")) if not src: raise click.BadParameter("Missing argument 'src'") try: app.set_session_var("force_option", True) - if not keep_build_dir and isdir(build_dir): + if not keep_build_dir and os.path.isdir(build_dir): fs.rmtree(build_dir) - if not isdir(build_dir): - makedirs(build_dir) + if not os.path.isdir(build_dir): + os.makedirs(build_dir) for dir_name, patterns in dict(lib=lib, src=src).items(): if not patterns: continue contents = [] for p in patterns: - contents += compat.glob_recursive(p) - _copy_contents(join(build_dir, dir_name), contents) + contents += glob.glob(p, recursive=True) + _copy_contents(os.path.join(build_dir, dir_name), contents) - if project_conf and isfile(project_conf): + if project_conf and os.path.isfile(project_conf): _copy_project_conf(build_dir, project_conf) elif not board: raise CIBuildEnvsEmpty() @@ -126,48 +126,50 @@ def _copy_contents(dst_dir, contents): items = {"dirs": set(), "files": set()} for path in contents: - if isdir(path): + if os.path.isdir(path): items["dirs"].add(path) - elif isfile(path): + elif os.path.isfile(path): items["files"].add(path) - dst_dir_name = basename(dst_dir) + dst_dir_name = os.path.basename(dst_dir) if dst_dir_name == "src" and len(items["dirs"]) == 1: - copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True) + shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True) else: - if not isdir(dst_dir): - makedirs(dst_dir) + if not os.path.isdir(dst_dir): + os.makedirs(dst_dir) for d in items["dirs"]: - copytree(d, join(dst_dir, basename(d)), symlinks=True) + shutil.copytree( + d, os.path.join(dst_dir, os.path.basename(d)), symlinks=True + ) if not items["files"]: return if dst_dir_name == "lib": - dst_dir = join(dst_dir, mkdtemp(dir=dst_dir)) + dst_dir = os.path.join(dst_dir, tempfile.mkdtemp(dir=dst_dir)) for f in items["files"]: - dst_file = join(dst_dir, basename(f)) + dst_file = os.path.join(dst_dir, os.path.basename(f)) if f == dst_file: continue - copyfile(f, dst_file) + shutil.copyfile(f, dst_file) def _exclude_contents(dst_dir, patterns): contents = [] for p in patterns: - contents += compat.glob_recursive(join(compat.glob_escape(dst_dir), p)) + contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True) for path in contents: - path = realpath(path) - if isdir(path): + path = os.path.realpath(path) + if os.path.isdir(path): fs.rmtree(path) - elif isfile(path): - remove(path) + elif os.path.isfile(path): + os.remove(path) def _copy_project_conf(build_dir, project_conf): config = ProjectConfig(project_conf, parse_extra=False) if config.has_section("platformio"): config.remove_section("platformio") - config.save(join(build_dir, "platformio.ini")) + config.save(os.path.join(build_dir, "platformio.ini")) diff --git a/platformio/commands/debug.py b/platformio/commands/debug.py index 85e44116..11e25248 100644 --- a/platformio/commands/debug.py +++ b/platformio/commands/debug.py @@ -167,7 +167,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro loop.run_until_complete(coro) if WINDOWS: # an issue with asyncio executor and STIDIN, it cannot be closed gracefully - os._exit(0) # pylint: disable=protected-access + proc.force_exit() loop.close() return True diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index fd385a46..1f386438 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import sys from fnmatch import fnmatch @@ -21,7 +22,6 @@ from serial.tools import miniterm from platformio import exception, fs, util from platformio.commands.device import helpers as device_helpers -from platformio.compat import dump_json_to_unicode from platformio.platform.factory import PlatformFactory from platformio.project.exception import NotPlatformIOProjectError @@ -52,9 +52,7 @@ def device_list( # pylint: disable=too-many-branches single_key = list(data)[0] if len(list(data)) == 1 else None if json_output: - return click.echo( - dump_json_to_unicode(data[single_key] if single_key else data) - ) + return click.echo(json.dumps(data[single_key] if single_key else data)) titles = { "serial": "Serial Ports", diff --git a/platformio/commands/home/command.py b/platformio/commands/home/command.py index 2973bdd2..81764a54 100644 --- a/platformio/commands/home/command.py +++ b/platformio/commands/home/command.py @@ -17,6 +17,7 @@ import mimetypes import click from platformio.commands.home.helpers import is_port_used +from platformio.commands.home.run import run_server from platformio.compat import ensure_python3 @@ -87,9 +88,6 @@ def cli(port, host, no_open, shutdown_timeout, session_id): click.launch(home_url) return - # pylint: disable=import-outside-toplevel - from platformio.commands.home.run import run_server - run_server( host=host, port=port, diff --git a/platformio/commands/home/rpc/handlers/ide.py b/platformio/commands/home/rpc/handlers/ide.py index bacf8391..59a52df5 100644 --- a/platformio/commands/home/rpc/handlers/ide.py +++ b/platformio/commands/home/rpc/handlers/ide.py @@ -16,7 +16,7 @@ import time from ajsonrpc.core import JSONRPC20DispatchException -from platformio.compat import get_running_loop +from platformio.compat import aio_get_running_loop class IDERPC: @@ -36,7 +36,7 @@ class IDERPC: async def listen_commands(self, sid=0): if sid not in self._queue: self._queue[sid] = [] - self._queue[sid].append(get_running_loop().create_future()) + self._queue[sid].append(aio_get_running_loop().create_future()) return await self._queue[sid][-1] def open_project(self, sid, project_dir): diff --git a/platformio/commands/home/rpc/handlers/misc.py b/platformio/commands/home/rpc/handlers/misc.py index c16a6cc9..7626456a 100644 --- a/platformio/commands/home/rpc/handlers/misc.py +++ b/platformio/commands/home/rpc/handlers/misc.py @@ -17,7 +17,7 @@ import time from platformio.cache import ContentCache from platformio.commands.home.rpc.handlers.os import OSRPC -from platformio.compat import create_task +from platformio.compat import aio_create_task class MiscRPC: @@ -30,7 +30,7 @@ class MiscRPC: cache_data = json.loads(cache_data) # automatically update cache in background every 12 hours if cache_data["time"] < (time.time() - (3600 * 12)): - create_task( + aio_create_task( self._preload_latest_tweets(data_url, cache_key, cache_valid) ) return cache_data["result"] diff --git a/platformio/commands/home/rpc/server.py b/platformio/commands/home/rpc/server.py index 6aef10e3..8e0dd44f 100644 --- a/platformio/commands/home/rpc/server.py +++ b/platformio/commands/home/rpc/server.py @@ -17,7 +17,7 @@ from ajsonrpc.dispatcher import Dispatcher from ajsonrpc.manager import AsyncJSONRPCResponseManager from starlette.endpoints import WebSocketEndpoint -from platformio.compat import create_task, get_running_loop +from platformio.compat import aio_create_task, aio_get_running_loop from platformio.proc import force_exit @@ -63,7 +63,7 @@ class JSONRPCServerFactoryBase: click.echo("Automatically shutdown server on timeout") force_exit() - self.shutdown_timer = get_running_loop().call_later( + self.shutdown_timer = aio_get_running_loop().call_later( self.shutdown_timeout, _auto_shutdown_server ) @@ -84,7 +84,7 @@ class WebSocketJSONRPCServer(WebSocketEndpoint): self.factory.on_client_connect() # pylint: disable=no-member async def on_receive(self, websocket, data): - create_task(self._handle_rpc(websocket, data)) + aio_create_task(self._handle_rpc(websocket, data)) async def on_disconnect(self, websocket, close_code): self.factory.on_client_disconnect() # pylint: disable=no-member diff --git a/platformio/commands/home/run.py b/platformio/commands/home/run.py index 4e225720..b923cd99 100644 --- a/platformio/commands/home/run.py +++ b/platformio/commands/home/run.py @@ -32,7 +32,7 @@ from platformio.commands.home.rpc.handlers.os import OSRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.commands.home.rpc.handlers.project import ProjectRPC from platformio.commands.home.rpc.server import WebSocketJSONRPCServerFactory -from platformio.compat import get_running_loop +from platformio.compat import aio_get_running_loop from platformio.exception import PlatformioException from platformio.package.manager.core import get_core_package_dir from platformio.proc import force_exit @@ -49,7 +49,7 @@ class ShutdownMiddleware: async def shutdown_server(_=None): - get_running_loop().call_later(0.5, force_exit) + aio_get_running_loop().call_later(0.5, force_exit) return PlainTextResponse("Server has been shutdown!") diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index d49e6cb6..dac03816 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -14,6 +14,7 @@ # pylint: disable=too-many-branches, too-many-locals +import json import os import time @@ -23,7 +24,6 @@ from tabulate import tabulate from platformio import exception, fs, util from platformio.commands import PlatformioCLI from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps -from platformio.compat import dump_json_to_unicode from platformio.package.exception import NotGlobalLibDir, UnknownPackageError from platformio.package.manager.library import LibraryPackageManager from platformio.package.meta import PackageItem, PackageSpec @@ -286,7 +286,7 @@ def lib_update( # pylint: disable=too-many-arguments if json_output: return click.echo( - dump_json_to_unicode( + json.dumps( json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result ) ) @@ -315,7 +315,7 @@ def lib_list(ctx, json_output): if json_output: return click.echo( - dump_json_to_unicode( + json.dumps( json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result ) ) @@ -359,7 +359,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): ) if json_output: - click.echo(dump_json_to_unicode(result)) + click.echo(json.dumps(result)) return if result["total"] == 0: @@ -418,7 +418,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): def lib_builtin(storage, json_output): items = get_builtin_libs(storage) if json_output: - return click.echo(dump_json_to_unicode(items)) + return click.echo(json.dumps(items)) for storage_ in items: if not storage_["items"]: @@ -442,7 +442,7 @@ def lib_show(library, json_output): regclient = lm.get_registry_client_instance() lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h") if json_output: - return click.echo(dump_json_to_unicode(lib)) + return click.echo(json.dumps(lib)) title = "{ownername}/{name}".format(**lib) click.secho(title, fg="cyan") @@ -538,7 +538,7 @@ def lib_stats(json_output): result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h") if json_output: - return click.echo(dump_json_to_unicode(result)) + return click.echo(json.dumps(result)) for key in ("updated", "added"): tabular_data = [ diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 054a7a12..f03f8833 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import click from platformio.cache import cleanup_content_cache from platformio.commands.boards import print_boards -from platformio.compat import dump_json_to_unicode from platformio.package.manager.platform import PlatformPackageManager from platformio.package.meta import PackageItem, PackageSpec from platformio.package.version import get_original_version @@ -172,7 +172,7 @@ def platform_search(query, json_output): for platform in _get_registry_platforms(): if query == "all": query = "" - search_data = dump_json_to_unicode(platform) + search_data = json.dumps(platform) if query and query.lower() not in search_data.lower(): continue platforms.append( @@ -182,7 +182,7 @@ def platform_search(query, json_output): ) if json_output: - click.echo(dump_json_to_unicode(platforms)) + click.echo(json.dumps(platforms)) else: _print_platforms(platforms) @@ -198,7 +198,7 @@ def platform_frameworks(query, json_output): ): if query == "all": query = "" - search_data = dump_json_to_unicode(framework) + search_data = json.dumps(framework) if query and query.lower() not in search_data.lower(): continue framework["homepage"] = "https://platformio.org/frameworks/" + framework["name"] @@ -211,7 +211,7 @@ def platform_frameworks(query, json_output): frameworks = sorted(frameworks, key=lambda manifest: manifest["name"]) if json_output: - click.echo(dump_json_to_unicode(frameworks)) + click.echo(json.dumps(frameworks)) else: _print_platforms(frameworks) @@ -228,7 +228,7 @@ def platform_list(json_output): platforms = sorted(platforms, key=lambda manifest: manifest["name"]) if json_output: - click.echo(dump_json_to_unicode(platforms)) + click.echo(json.dumps(platforms)) else: _print_platforms(platforms) @@ -241,7 +241,7 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches if not data: raise UnknownPlatform(platform) if json_output: - return click.echo(dump_json_to_unicode(data)) + return click.echo(json.dumps(data)) dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"] click.echo( @@ -401,7 +401,7 @@ def platform_update( # pylint: disable=too-many-locals, too-many-arguments str(outdated.latest) if outdated.latest else None ) result.append(data) - return click.echo(dump_json_to_unicode(result)) + return click.echo(json.dumps(result)) # cleanup cached board and platform lists cleanup_content_cache("http") diff --git a/platformio/compat.py b/platformio/compat.py index 53c1507c..95bbd5df 100644 --- a/platformio/compat.py +++ b/platformio/compat.py @@ -12,23 +12,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=unused-import, no-name-in-module, import-error, -# pylint: disable=no-member, undefined-variable, unexpected-keyword-arg +# pylint: disable=unused-import -import glob import inspect -import json import locale -import os -import re import sys from platformio.exception import UserSideException +if sys.version_info >= (3,): + if sys.version_info >= (3, 7): + from asyncio import create_task as aio_create_task + from asyncio import get_running_loop as aio_get_running_loop + else: + from asyncio import ensure_future as aio_create_task + from asyncio import get_event_loop as aio_get_running_loop + + PY2 = sys.version_info[0] == 2 CYGWIN = sys.platform.startswith("cygwin") WINDOWS = sys.platform.startswith("win") MACOS = sys.platform.startswith("darwin") +string_types = (str,) + + +def is_bytes(x): + return isinstance(x, (bytes, memoryview, bytearray)) + + +def ci_strings_are_equal(a, b): + if a == b: + return True + if not a or not b: + return False + return a.strip().lower() == b.strip().lower() + + +def hashlib_encode_data(data): + if is_bytes(data): + return data + if not isinstance(data, string_types): + data = str(data) + return data.encode() + + +def load_python_module(name, pathname): + import importlib.util # pylint: disable=import-outside-toplevel + + spec = importlib.util.spec_from_file_location(name, pathname) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module def get_filesystem_encoding(): @@ -53,14 +87,6 @@ def get_object_members(obj, ignore_private=True): } -def ci_strings_are_equal(a, b): - if a == b: - return True - if not a or not b: - return False - return a.strip().lower() == b.strip().lower() - - def ensure_python3(raise_exception=True): compatible = sys.version_info >= (3, 6) if not raise_exception or compatible: @@ -71,101 +97,3 @@ def ensure_python3(raise_exception=True): "https://docs.platformio.org/en/latest/core/migration.html" "#drop-support-for-python-2-and-3-5" ) - - -if PY2: - import imp - - string_types = (str, unicode) - - def create_task(coro, name=None): - raise NotImplementedError - - def get_running_loop(): - raise NotImplementedError - - def is_bytes(x): - return isinstance(x, (buffer, bytearray)) - - def path_to_unicode(path): - if isinstance(path, unicode): - return path - return path.decode(get_filesystem_encoding()) - - def hashlib_encode_data(data): - if is_bytes(data): - return data - if isinstance(data, unicode): - data = data.encode(get_filesystem_encoding()) - elif not isinstance(data, string_types): - data = str(data) - return data - - def dump_json_to_unicode(obj): - if isinstance(obj, unicode): - return obj - return json.dumps( - obj, encoding=get_filesystem_encoding(), ensure_ascii=False - ).encode("utf8") - - _magic_check = re.compile("([*?[])") - _magic_check_bytes = re.compile(b"([*?[])") - - def glob_recursive(pathname): - return glob.glob(pathname) - - def glob_escape(pathname): - """Escape all special characters.""" - # https://github.com/python/cpython/blob/master/Lib/glob.py#L161 - # Escaping is done by wrapping any of "*?[" between square brackets. - # Metacharacters do not work in the drive part and shouldn't be - # escaped. - drive, pathname = os.path.splitdrive(pathname) - if isinstance(pathname, bytes): - pathname = _magic_check_bytes.sub(br"[\1]", pathname) - else: - pathname = _magic_check.sub(r"[\1]", pathname) - return drive + pathname - - def load_python_module(name, pathname): - return imp.load_source(name, pathname) - - -else: - import importlib.util - from glob import escape as glob_escape - - if sys.version_info >= (3, 7): - from asyncio import create_task, get_running_loop - else: - from asyncio import ensure_future as create_task - from asyncio import get_event_loop as get_running_loop - - string_types = (str,) - - def is_bytes(x): - return isinstance(x, (bytes, memoryview, bytearray)) - - def path_to_unicode(path): - return path - - def hashlib_encode_data(data): - if is_bytes(data): - return data - if not isinstance(data, string_types): - data = str(data) - return data.encode() - - def dump_json_to_unicode(obj): - if isinstance(obj, string_types): - return obj - return json.dumps(obj) - - def glob_recursive(pathname): - return glob.glob(pathname, recursive=True) - - def load_python_module(name, pathname): - spec = importlib.util.spec_from_file_location(name, pathname) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module diff --git a/platformio/debug/process/base.py b/platformio/debug/process/base.py index f7b38dff..a97d5446 100644 --- a/platformio/debug/process/base.py +++ b/platformio/debug/process/base.py @@ -21,9 +21,9 @@ import time from platformio import fs from platformio.compat import ( WINDOWS, - create_task, + aio_create_task, + aio_get_running_loop, get_locale_encoding, - get_running_loop, string_types, ) from platformio.proc import get_pythonexe_path @@ -83,7 +83,7 @@ class DebugBaseProcess: for pipe in ("stdin", "stdout", "stderr"): if pipe not in kwargs: kwargs[pipe] = subprocess.PIPE - loop = get_running_loop() + loop = aio_get_running_loop() await loop.subprocess_exec( lambda: DebugSubprocessProtocol(self), *args, **kwargs ) @@ -99,10 +99,10 @@ class DebugBaseProcess: self.transport = transport def connect_stdin_pipe(self): - self._stdin_read_task = create_task(self._read_stdin_pipe()) + self._stdin_read_task = aio_create_task(self._read_stdin_pipe()) async def _read_stdin_pipe(self): - loop = get_running_loop() + loop = aio_get_running_loop() if WINDOWS: while True: self.stdin_data_received( diff --git a/platformio/debug/process/client.py b/platformio/debug/process/client.py index a5765b39..e8b05715 100644 --- a/platformio/debug/process/client.py +++ b/platformio/debug/process/client.py @@ -21,7 +21,7 @@ import time from platformio import fs, proc, telemetry, util from platformio.cache import ContentCache -from platformio.compat import get_running_loop, hashlib_encode_data, is_bytes +from platformio.compat import aio_get_running_loop, hashlib_encode_data, is_bytes from platformio.debug import helpers from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.initcfgs import get_gdb_init_config @@ -191,7 +191,7 @@ class DebugClientProcess( def _auto_exec_continue(self): auto_exec_delay = 0.5 # in seconds if self._last_activity > (time.time() - auto_exec_delay): - get_running_loop().call_later(0.1, self._auto_exec_continue) + aio_get_running_loop().call_later(0.1, self._auto_exec_continue) return if not self.debug_options["init_break"] or self._target_is_running: diff --git a/platformio/fs.py b/platformio/fs.py index da2101c5..3aefdb6e 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import glob import hashlib import io import json @@ -24,7 +25,7 @@ import sys import click from platformio import exception -from platformio.compat import WINDOWS, glob_escape, glob_recursive +from platformio.compat import WINDOWS class cd(object): @@ -158,7 +159,9 @@ def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True): src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter): items = set() - for item in glob_recursive(os.path.join(glob_escape(src_dir), pattern)): + for item in glob.glob( + os.path.join(glob.escape(src_dir), pattern), recursive=True + ): if os.path.isdir(item): for root, _, files in os.walk(item, followlinks=followlinks): for f in files: diff --git a/platformio/maintenance.py b/platformio/maintenance.py index e22f8407..3ef47015 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -28,7 +28,6 @@ from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.system.prune import calculate_unnecessary_system_data from platformio.commands.upgrade import get_latest_version -from platformio.compat import ensure_python3 from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.platform import PlatformPackageManager @@ -40,8 +39,6 @@ from platformio.proc import is_container def on_platformio_start(ctx, force, caller): - ensure_python3(raise_exception=True) - app.set_session_var("command_ctx", ctx) app.set_session_var("force_option", force) set_caller(caller) diff --git a/platformio/proc.py b/platformio/proc.py index 24640c38..b4ef4505 100644 --- a/platformio/proc.py +++ b/platformio/proc.py @@ -24,7 +24,6 @@ from platformio.compat import ( WINDOWS, get_filesystem_encoding, get_locale_encoding, - get_running_loop, string_types, ) @@ -221,9 +220,4 @@ def append_env_path(name, value): def force_exit(code=0): - try: - get_running_loop().stop() - except: # pylint: disable=bare-except - pass - finally: - sys.exit(code) + os._exit(code) # pylint: disable=protected-access diff --git a/platformio/project/config.py b/platformio/project/config.py index 786f080a..9f78a67b 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -12,21 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import glob +import hashlib import json import os import re -from hashlib import sha1 import click from platformio import fs -from platformio.compat import ( - PY2, - WINDOWS, - glob_recursive, - hashlib_encode_data, - string_types, -) +from platformio.compat import PY2, WINDOWS, hashlib_encode_data, string_types from platformio.project import exception from platformio.project.options import ProjectOptions @@ -122,7 +117,7 @@ class ProjectConfigBase(object): for pattern in self.get("platformio", "extra_configs", []): if pattern.startswith("~"): pattern = fs.expanduser(pattern) - for item in glob_recursive(pattern): + for item in glob.glob(pattern, recursive=True): self.read(item) def _maintain_renaimed_options(self): @@ -402,7 +397,7 @@ class ProjectConfigDirsMixin(object): "%s-%s" % ( os.path.basename(project_dir), - sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], + hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], ), ) diff --git a/tox.ini b/tox.ini index c8c18db9..be832555 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ known_third_party=OpenSSL, SCons, jsonrpc, twisted, zope passenv = * usedevelop = True deps = - black + py36,py37,py38,py39: black isort pylint pytest