Remove Python 2 code

HISTORY.rst

@@ -14,7 +14,7 @@ PlatformIO Core 5
 * **PlatformIO Debugging**
 
   - Boosted `PlatformIO Debugging <https://docs.platformio.org/page/plus/debugging.html>`__  performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
-  - Support debugging on Windows using Windows CMD (CLI) (`issue #3793 <https://github.com/platformio/platformio-core/issues/3793>`_)
+  - Support debugging on Windows using Windows CMD/CLI (`pio debug <https://docs.platformio.org/page/core/userguide/cmd_debug.html>`__) (`issue #3793 <https://github.com/platformio/platformio-core/issues/3793>`_)
 
 5.1.1 (2021-03-17)
 ~~~~~~~~~~~~~~~~~~

platformio/__main__.py

@@ -12,15 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=import-outside-toplevel
+
 import os
 import sys
 from traceback import format_exc
 
 import click
 
-from platformio import __version__, exception, maintenance, util
+from platformio import __version__, exception
 from platformio.commands import PlatformioCLI
-from platformio.compat import CYGWIN
+from platformio.compat import CYGWIN, PY2, ensure_python3
 
 try:
     import click_completion  # pylint: disable=import-error
@@ -60,16 +62,19 @@ def cli(ctx, force, caller, no_ansi):
     except:  # pylint: disable=bare-except
         pass
 
+    from platformio import maintenance
+
     maintenance.on_platformio_start(ctx, force, caller)
 
 
 @cli.resultcallback()
 @click.pass_context
 def process_result(ctx, result, *_, **__):
+    from platformio import maintenance
+
     maintenance.on_platformio_end(ctx, result)
 
 
-@util.memoized()
 def configure():
     if CYGWIN:
         raise exception.CygwinEnvDetected()
@@ -105,6 +110,7 @@ def main(argv=None):
         assert isinstance(argv, list)
         sys.argv = argv
     try:
+        ensure_python3(raise_exception=True)
         configure()
         cli()  # pylint: disable=no-value-for-parameter
     except SystemExit as e:
@@ -112,7 +118,10 @@ def main(argv=None):
             exit_code = int(e.code)
     except Exception as e:  # pylint: disable=broad-except
         if not isinstance(e, exception.ReturnErrorCode):
-            maintenance.on_platformio_exception(e)
+            if not PY2:
+                from platformio import maintenance
+
+                maintenance.on_platformio_exception(e)
             error_str = "Error: "
             if isinstance(e, exception.PlatformioException):
                 error_str += str(e)

platformio/app.py

@@ -24,7 +24,7 @@ import uuid
 from os.path import dirname, isdir, isfile, join, realpath
 
 from platformio import __version__, exception, fs, proc
-from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data
+from platformio.compat import WINDOWS, hashlib_encode_data
 from platformio.package.lockfile import LockFile
 from platformio.project.helpers import get_default_projects_dir, get_project_core_dir
 
@@ -115,7 +115,7 @@ class State(object):
         if self.modified:
             try:
                 with open(self.path, "w") as fp:
-                    fp.write(dump_json_to_unicode(self._storage))
+                    fp.write(json.dumps(self._storage))
             except IOError:
                 raise exception.HomeDirPermissionsError(get_project_core_dir())
         self._unlock_state_file()

platformio/builder/main.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+import os
 import sys
-from os import environ, makedirs
-from os.path import isdir, join
 from time import time
 
 import click
@@ -29,7 +29,6 @@ from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
 from platformio import compat, fs
-from platformio.compat import dump_json_to_unicode
 from platformio.platform.base import PlatformBase
 from platformio.proc import get_pythonexe_path
 from platformio.project.helpers import get_project_dir
@@ -65,18 +64,18 @@ DEFAULT_ENV_OPTIONS = dict(
         "pioide",
         "piosize",
     ],
-    toolpath=[join(fs.get_source_dir(), "builder", "tools")],
+    toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")],
     variables=clivars,
     # Propagating External Environment
-    ENV=environ,
+    ENV=os.environ,
     UNIX_TIME=int(time()),
-    BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"),
+    BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
-    BUILD_SRC_DIR=join("$BUILD_DIR", "src"),
+    BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
-    BUILD_TEST_DIR=join("$BUILD_DIR", "test"),
+    BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
-    COMPILATIONDB_PATH=join("$BUILD_DIR", "compile_commands.json"),
+    COMPILATIONDB_PATH=os.path.join("$BUILD_DIR", "compile_commands.json"),
     LIBPATH=["$BUILD_DIR"],
     PROGNAME="program",
-    PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
+    PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
     PYTHONEXE=get_pythonexe_path(),
     IDE_EXTRA_DATA={},
 )
@@ -124,7 +123,7 @@ env.Replace(
     BUILD_CACHE_DIR=config.get_optional_dir("build_cache"),
     LIBSOURCE_DIRS=[
         config.get_optional_dir("lib"),
-        join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
+        os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
         config.get_optional_dir("globallib"),
     ],
 )
@@ -142,8 +141,8 @@ if (
     )
 
 if env.subst("$BUILD_CACHE_DIR"):
-    if not isdir(env.subst("$BUILD_CACHE_DIR")):
+    if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")):
-        makedirs(env.subst("$BUILD_CACHE_DIR"))
+        os.makedirs(env.subst("$BUILD_CACHE_DIR"))
     env.CacheDir("$BUILD_CACHE_DIR")
 
 if int(ARGUMENTS.get("ISATTY", 0)):
@@ -160,15 +159,17 @@ elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
 if "compiledb" in COMMAND_LINE_TARGETS:
     env.Tool("compilation_db")
 
-if not isdir(env.subst("$BUILD_DIR")):
+if not os.path.isdir(env.subst("$BUILD_DIR")):
-    makedirs(env.subst("$BUILD_DIR"))
+    os.makedirs(env.subst("$BUILD_DIR"))
 
 env.LoadProjectOptions()
 env.LoadPioPlatform()
 
 env.SConscriptChdir(0)
 env.SConsignFile(
-    join("$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]))
+    os.path.join(
+        "$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1])
+    )
 )
 
 for item in env.GetExtraScripts("pre"):
@@ -225,9 +226,7 @@ if "idedata" in COMMAND_LINE_TARGETS:
         projenv = env
     click.echo(
         "\n%s\n"
-        % dump_json_to_unicode(
+        % json.dumps(projenv.DumpIDEData(env))  # pylint: disable=undefined-variable
-            projenv.DumpIDEData(env)  # pylint: disable=undefined-variable
-        )
     )
     env.Exit(0)
 

platformio/builder/tools/pioide.py

@@ -14,13 +14,12 @@
 
 from __future__ import absolute_import
 
+import glob
 import os
-from glob import glob
 
 import SCons.Defaults  # pylint: disable=import-error
 import SCons.Subst  # pylint: disable=import-error
 
-from platformio.compat import glob_escape
 from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import exec_command, where_is_program
 
@@ -49,7 +48,7 @@ def _dump_includes(env):
     for pkg in p.get_installed_packages():
         if p.get_package_type(pkg.metadata.name) != "toolchain":
             continue
-        toolchain_dir = glob_escape(pkg.path)
+        toolchain_dir = glob.escape(pkg.path)
         toolchain_incglobs = [
             os.path.join(toolchain_dir, "*", "include", "c++", "*"),
             os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"),
@@ -57,7 +56,9 @@ def _dump_includes(env):
             os.path.join(toolchain_dir, "*", "include*"),
         ]
         for g in toolchain_incglobs:
-            includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)])
+            includes["toolchain"].extend(
+                [os.path.realpath(inc) for inc in glob.glob(g)]
+            )
 
     # include Unity framework if there are tests in project
     includes["unity"] = []

platformio/builder/tools/piomisc.py

@@ -15,16 +15,17 @@
 from __future__ import absolute_import
 
 import atexit
+import glob
 import io
 import os
 import re
 import sys
-from tempfile import mkstemp
+import tempfile
 
 import click
 
 from platformio import fs, util
-from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape
+from platformio.compat import get_filesystem_encoding, get_locale_encoding
 from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import exec_command
 
@@ -116,7 +117,7 @@ class InoToCPPConverter(object):
         return out_file
 
     def _gcc_preprocess(self, contents, out_file):
-        tmp_path = mkstemp()[1]
+        tmp_path = tempfile.mkstemp()[1]
         self.write_safe_contents(tmp_path, contents)
         self.env.Execute(
             self.env.VerboseAction(
@@ -229,7 +230,7 @@ class InoToCPPConverter(object):
 
 
 def ConvertInoToCpp(env):
-    src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR"))
+    src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
     ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
         os.path.join(src_dir, "*.pde")
     )

platformio/builder/tools/piosize.py

@@ -16,6 +16,7 @@
 
 from __future__ import absolute_import
 
+import json
 import sys
 from os import environ, makedirs, remove
 from os.path import isdir, join, splitdrive
@@ -23,7 +24,6 @@ from os.path import isdir, join, splitdrive
 from elftools.elf.descriptions import describe_sh_flags
 from elftools.elf.elffile import ELFFile
 
-from platformio.compat import dump_json_to_unicode
 from platformio.proc import exec_command
 from platformio.util import get_systype
 
@@ -242,7 +242,7 @@ def DumpSizeData(_, target, source, env):  # pylint: disable=unused-argument
             data["memory"]["files"].append(file_data)
 
     with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp:
-        fp.write(dump_json_to_unicode(data))
+        fp.write(json.dumps(data))
 
 
 def exists(_):

platformio/commands/boards.py

@@ -18,7 +18,6 @@ import click
 from tabulate import tabulate
 
 from platformio import fs
-from platformio.compat import dump_json_to_unicode
 from platformio.package.manager.platform import PlatformPackageManager
 
 
@@ -83,4 +82,4 @@ def _print_boards_json(query, installed=False):
             if query.lower() not in search_data.lower():
                 continue
         result.append(board)
-    click.echo(dump_json_to_unicode(result))
+    click.echo(json.dumps(result))

platformio/commands/check/command.py

@@ -15,6 +15,7 @@
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
 # pylint: disable=redefined-builtin,too-many-statements
 
+import json
 import os
 from collections import Counter
 from os.path import dirname, isfile
@@ -26,7 +27,6 @@ from tabulate import tabulate
 from platformio import app, exception, fs, util
 from platformio.commands.check.defect import DefectItem
 from platformio.commands.check.tools import CheckToolFactory
-from platformio.compat import dump_json_to_unicode
 from platformio.project.config import ProjectConfig
 from platformio.project.helpers import find_project_dir_above, get_project_dir
 
@@ -163,7 +163,7 @@ def cli(
                     print_processing_footer(result)
 
         if json_output:
-            click.echo(dump_json_to_unicode(results_to_json(results)))
+            click.echo(json.dumps(results_to_json(results)))
         elif not silent:
             print_check_summary(results)
 

platformio/commands/check/tools/base.py

@@ -12,12 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import glob
 import os
-from tempfile import NamedTemporaryFile
+import tempfile
 
 import click
 
-from platformio import compat, fs, proc
+from platformio import fs, proc
 from platformio.commands.check.defect import DefectItem
 from platformio.project.helpers import load_project_ide_data
 
@@ -104,7 +105,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
         return {lang: _extract_defines(lang, incflags_file) for lang in ("c", "c++")}
 
     def _create_tmp_file(self, data):
-        with NamedTemporaryFile("w", delete=False) as fp:
+        with tempfile.NamedTemporaryFile("w", delete=False) as fp:
             fp.write(data)
             self._tmp_files.append(fp.name)
             return fp.name
@@ -207,7 +208,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
                 result["c++"].append(os.path.realpath(path))
 
         for pattern in patterns:
-            for item in compat.glob_recursive(pattern):
+            for item in glob.glob(pattern, recursive=True):
                 if not os.path.isdir(item):
                     _add_file(item)
                 for root, _, files in os.walk(item, followlinks=True):

platformio/commands/ci.py

@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from os import getenv, makedirs, remove
+import glob
-from os.path import basename, isdir, isfile, join, realpath
+import os
-from shutil import copyfile, copytree
+import shutil
-from tempfile import mkdtemp
+import tempfile
 
 import click
 
-from platformio import app, compat, fs
+from platformio import app, fs
 from platformio.commands.project import project_init as cmd_project_init
 from platformio.commands.project import validate_boards
 from platformio.commands.run.command import cli as cmd_run
@@ -33,8 +33,8 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
     for i, p in enumerate(value):
         if p.startswith("~"):
             value[i] = fs.expanduser(p)
-        value[i] = realpath(value[i])
+        value[i] = os.path.realpath(value[i])
-        if not compat.glob_recursive(value[i]):
+        if not glob.glob(value[i], recursive=True):
             invalid_path = p
             break
     try:
@@ -51,7 +51,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
 @click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
 @click.option(
     "--build-dir",
-    default=mkdtemp,
+    default=tempfile.mkdtemp,
     type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
 )
 @click.option("--keep-build-dir", is_flag=True)
@@ -78,28 +78,28 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
     verbose,
 ):
 
-    if not src and getenv("PLATFORMIO_CI_SRC"):
+    if not src and os.getenv("PLATFORMIO_CI_SRC"):
-        src = validate_path(ctx, None, getenv("PLATFORMIO_CI_SRC").split(":"))
+        src = validate_path(ctx, None, os.getenv("PLATFORMIO_CI_SRC").split(":"))
     if not src:
         raise click.BadParameter("Missing argument 'src'")
 
     try:
         app.set_session_var("force_option", True)
 
-        if not keep_build_dir and isdir(build_dir):
+        if not keep_build_dir and os.path.isdir(build_dir):
             fs.rmtree(build_dir)
-        if not isdir(build_dir):
+        if not os.path.isdir(build_dir):
-            makedirs(build_dir)
+            os.makedirs(build_dir)
 
         for dir_name, patterns in dict(lib=lib, src=src).items():
             if not patterns:
                 continue
             contents = []
             for p in patterns:
-                contents += compat.glob_recursive(p)
+                contents += glob.glob(p, recursive=True)
-            _copy_contents(join(build_dir, dir_name), contents)
+            _copy_contents(os.path.join(build_dir, dir_name), contents)
 
-        if project_conf and isfile(project_conf):
+        if project_conf and os.path.isfile(project_conf):
             _copy_project_conf(build_dir, project_conf)
         elif not board:
             raise CIBuildEnvsEmpty()
@@ -126,48 +126,50 @@ def _copy_contents(dst_dir, contents):
     items = {"dirs": set(), "files": set()}
 
     for path in contents:
-        if isdir(path):
+        if os.path.isdir(path):
             items["dirs"].add(path)
-        elif isfile(path):
+        elif os.path.isfile(path):
             items["files"].add(path)
 
-    dst_dir_name = basename(dst_dir)
+    dst_dir_name = os.path.basename(dst_dir)
 
     if dst_dir_name == "src" and len(items["dirs"]) == 1:
-        copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
+        shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
     else:
-        if not isdir(dst_dir):
+        if not os.path.isdir(dst_dir):
-            makedirs(dst_dir)
+            os.makedirs(dst_dir)
         for d in items["dirs"]:
-            copytree(d, join(dst_dir, basename(d)), symlinks=True)
+            shutil.copytree(
+                d, os.path.join(dst_dir, os.path.basename(d)), symlinks=True
+            )
 
     if not items["files"]:
         return
 
     if dst_dir_name == "lib":
-        dst_dir = join(dst_dir, mkdtemp(dir=dst_dir))
+        dst_dir = os.path.join(dst_dir, tempfile.mkdtemp(dir=dst_dir))
 
     for f in items["files"]:
-        dst_file = join(dst_dir, basename(f))
+        dst_file = os.path.join(dst_dir, os.path.basename(f))
         if f == dst_file:
             continue
-        copyfile(f, dst_file)
+        shutil.copyfile(f, dst_file)
 
 
 def _exclude_contents(dst_dir, patterns):
     contents = []
     for p in patterns:
-        contents += compat.glob_recursive(join(compat.glob_escape(dst_dir), p))
+        contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True)
     for path in contents:
-        path = realpath(path)
+        path = os.path.realpath(path)
-        if isdir(path):
+        if os.path.isdir(path):
             fs.rmtree(path)
-        elif isfile(path):
+        elif os.path.isfile(path):
-            remove(path)
+            os.remove(path)
 
 
 def _copy_project_conf(build_dir, project_conf):
     config = ProjectConfig(project_conf, parse_extra=False)
     if config.has_section("platformio"):
         config.remove_section("platformio")
-    config.save(join(build_dir, "platformio.ini"))
+    config.save(os.path.join(build_dir, "platformio.ini"))

platformio/commands/debug.py

@@ -167,7 +167,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro
     loop.run_until_complete(coro)
     if WINDOWS:
         # an issue with asyncio executor and STIDIN, it cannot be closed gracefully
-        os._exit(0)  # pylint: disable=protected-access
+        proc.force_exit()
     loop.close()
 
     return True

platformio/commands/device/command.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import sys
 from fnmatch import fnmatch
@@ -21,7 +22,6 @@ from serial.tools import miniterm
 
 from platformio import exception, fs, util
 from platformio.commands.device import helpers as device_helpers
-from platformio.compat import dump_json_to_unicode
 from platformio.platform.factory import PlatformFactory
 from platformio.project.exception import NotPlatformIOProjectError
 
@@ -52,9 +52,7 @@ def device_list(  # pylint: disable=too-many-branches
     single_key = list(data)[0] if len(list(data)) == 1 else None
 
     if json_output:
-        return click.echo(
+        return click.echo(json.dumps(data[single_key] if single_key else data))
-            dump_json_to_unicode(data[single_key] if single_key else data)
-        )
 
     titles = {
         "serial": "Serial Ports",

platformio/commands/home/command.py

@@ -17,6 +17,7 @@ import mimetypes
 import click
 
 from platformio.commands.home.helpers import is_port_used
+from platformio.commands.home.run import run_server
 from platformio.compat import ensure_python3
 
 
@@ -87,9 +88,6 @@ def cli(port, host, no_open, shutdown_timeout, session_id):
             click.launch(home_url)
         return
 
-    # pylint: disable=import-outside-toplevel
-    from platformio.commands.home.run import run_server
-
     run_server(
         host=host,
         port=port,

platformio/commands/home/rpc/handlers/ide.py

@@ -16,7 +16,7 @@ import time
 
 from ajsonrpc.core import JSONRPC20DispatchException
 
-from platformio.compat import get_running_loop
+from platformio.compat import aio_get_running_loop
 
 
 class IDERPC:
@@ -36,7 +36,7 @@ class IDERPC:
     async def listen_commands(self, sid=0):
         if sid not in self._queue:
             self._queue[sid] = []
-        self._queue[sid].append(get_running_loop().create_future())
+        self._queue[sid].append(aio_get_running_loop().create_future())
         return await self._queue[sid][-1]
 
     def open_project(self, sid, project_dir):

platformio/commands/home/rpc/handlers/misc.py

@@ -17,7 +17,7 @@ import time
 
 from platformio.cache import ContentCache
 from platformio.commands.home.rpc.handlers.os import OSRPC
-from platformio.compat import create_task
+from platformio.compat import aio_create_task
 
 
 class MiscRPC:
@@ -30,7 +30,7 @@ class MiscRPC:
                 cache_data = json.loads(cache_data)
                 # automatically update cache in background every 12 hours
                 if cache_data["time"] < (time.time() - (3600 * 12)):
-                    create_task(
+                    aio_create_task(
                         self._preload_latest_tweets(data_url, cache_key, cache_valid)
                     )
                 return cache_data["result"]

platformio/commands/home/rpc/server.py

@@ -17,7 +17,7 @@ from ajsonrpc.dispatcher import Dispatcher
 from ajsonrpc.manager import AsyncJSONRPCResponseManager
 from starlette.endpoints import WebSocketEndpoint
 
-from platformio.compat import create_task, get_running_loop
+from platformio.compat import aio_create_task, aio_get_running_loop
 from platformio.proc import force_exit
 
 
@@ -63,7 +63,7 @@ class JSONRPCServerFactoryBase:
             click.echo("Automatically shutdown server on timeout")
             force_exit()
 
-        self.shutdown_timer = get_running_loop().call_later(
+        self.shutdown_timer = aio_get_running_loop().call_later(
             self.shutdown_timeout, _auto_shutdown_server
         )
 
@@ -84,7 +84,7 @@ class WebSocketJSONRPCServer(WebSocketEndpoint):
         self.factory.on_client_connect()  # pylint: disable=no-member
 
     async def on_receive(self, websocket, data):
-        create_task(self._handle_rpc(websocket, data))
+        aio_create_task(self._handle_rpc(websocket, data))
 
     async def on_disconnect(self, websocket, close_code):
         self.factory.on_client_disconnect()  # pylint: disable=no-member

platformio/commands/home/run.py

@@ -32,7 +32,7 @@ from platformio.commands.home.rpc.handlers.os import OSRPC
 from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
 from platformio.commands.home.rpc.handlers.project import ProjectRPC
 from platformio.commands.home.rpc.server import WebSocketJSONRPCServerFactory
-from platformio.compat import get_running_loop
+from platformio.compat import aio_get_running_loop
 from platformio.exception import PlatformioException
 from platformio.package.manager.core import get_core_package_dir
 from platformio.proc import force_exit
@@ -49,7 +49,7 @@ class ShutdownMiddleware:
 
 
 async def shutdown_server(_=None):
-    get_running_loop().call_later(0.5, force_exit)
+    aio_get_running_loop().call_later(0.5, force_exit)
     return PlainTextResponse("Server has been shutdown!")
 
 

platformio/commands/lib/command.py

@@ -14,6 +14,7 @@
 
 # pylint: disable=too-many-branches, too-many-locals
 
+import json
 import os
 import time
 
@@ -23,7 +24,6 @@ from tabulate import tabulate
 from platformio import exception, fs, util
 from platformio.commands import PlatformioCLI
 from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps
-from platformio.compat import dump_json_to_unicode
 from platformio.package.exception import NotGlobalLibDir, UnknownPackageError
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.meta import PackageItem, PackageSpec
@@ -286,7 +286,7 @@ def lib_update(  # pylint: disable=too-many-arguments
 
     if json_output:
         return click.echo(
-            dump_json_to_unicode(
+            json.dumps(
                 json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
             )
         )
@@ -315,7 +315,7 @@ def lib_list(ctx, json_output):
 
     if json_output:
         return click.echo(
-            dump_json_to_unicode(
+            json.dumps(
                 json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
             )
         )
@@ -359,7 +359,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
     )
 
     if json_output:
-        click.echo(dump_json_to_unicode(result))
+        click.echo(json.dumps(result))
         return
 
     if result["total"] == 0:
@@ -418,7 +418,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
 def lib_builtin(storage, json_output):
     items = get_builtin_libs(storage)
     if json_output:
-        return click.echo(dump_json_to_unicode(items))
+        return click.echo(json.dumps(items))
 
     for storage_ in items:
         if not storage_["items"]:
@@ -442,7 +442,7 @@ def lib_show(library, json_output):
     regclient = lm.get_registry_client_instance()
     lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h")
     if json_output:
-        return click.echo(dump_json_to_unicode(lib))
+        return click.echo(json.dumps(lib))
 
     title = "{ownername}/{name}".format(**lib)
     click.secho(title, fg="cyan")
@@ -538,7 +538,7 @@ def lib_stats(json_output):
     result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h")
 
     if json_output:
-        return click.echo(dump_json_to_unicode(result))
+        return click.echo(json.dumps(result))
 
     for key in ("updated", "added"):
         tabular_data = [

platformio/commands/platform.py

@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 
 import click
 
 from platformio.cache import cleanup_content_cache
 from platformio.commands.boards import print_boards
-from platformio.compat import dump_json_to_unicode
 from platformio.package.manager.platform import PlatformPackageManager
 from platformio.package.meta import PackageItem, PackageSpec
 from platformio.package.version import get_original_version
@@ -172,7 +172,7 @@ def platform_search(query, json_output):
     for platform in _get_registry_platforms():
         if query == "all":
             query = ""
-        search_data = dump_json_to_unicode(platform)
+        search_data = json.dumps(platform)
         if query and query.lower() not in search_data.lower():
             continue
         platforms.append(
@@ -182,7 +182,7 @@ def platform_search(query, json_output):
         )
 
     if json_output:
-        click.echo(dump_json_to_unicode(platforms))
+        click.echo(json.dumps(platforms))
     else:
         _print_platforms(platforms)
 
@@ -198,7 +198,7 @@ def platform_frameworks(query, json_output):
     ):
         if query == "all":
             query = ""
-        search_data = dump_json_to_unicode(framework)
+        search_data = json.dumps(framework)
         if query and query.lower() not in search_data.lower():
             continue
         framework["homepage"] = "https://platformio.org/frameworks/" + framework["name"]
@@ -211,7 +211,7 @@ def platform_frameworks(query, json_output):
 
     frameworks = sorted(frameworks, key=lambda manifest: manifest["name"])
     if json_output:
-        click.echo(dump_json_to_unicode(frameworks))
+        click.echo(json.dumps(frameworks))
     else:
         _print_platforms(frameworks)
 
@@ -228,7 +228,7 @@ def platform_list(json_output):
 
     platforms = sorted(platforms, key=lambda manifest: manifest["name"])
     if json_output:
-        click.echo(dump_json_to_unicode(platforms))
+        click.echo(json.dumps(platforms))
     else:
         _print_platforms(platforms)
 
@@ -241,7 +241,7 @@ def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     if not data:
         raise UnknownPlatform(platform)
     if json_output:
-        return click.echo(dump_json_to_unicode(data))
+        return click.echo(json.dumps(data))
 
     dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"]
     click.echo(
@@ -401,7 +401,7 @@ def platform_update(  # pylint: disable=too-many-locals, too-many-arguments
                     str(outdated.latest) if outdated.latest else None
                 )
             result.append(data)
-        return click.echo(dump_json_to_unicode(result))
+        return click.echo(json.dumps(result))
 
     # cleanup cached board and platform lists
     cleanup_content_cache("http")

platformio/compat.py

@@ -12,23 +12,57 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pylint: disable=unused-import, no-name-in-module, import-error,
+# pylint: disable=unused-import
-# pylint: disable=no-member, undefined-variable, unexpected-keyword-arg
 
-import glob
 import inspect
-import json
 import locale
-import os
-import re
 import sys
 
 from platformio.exception import UserSideException
 
+if sys.version_info >= (3,):
+    if sys.version_info >= (3, 7):
+        from asyncio import create_task as aio_create_task
+        from asyncio import get_running_loop as aio_get_running_loop
+    else:
+        from asyncio import ensure_future as aio_create_task
+        from asyncio import get_event_loop as aio_get_running_loop
+
+
 PY2 = sys.version_info[0] == 2
 CYGWIN = sys.platform.startswith("cygwin")
 WINDOWS = sys.platform.startswith("win")
 MACOS = sys.platform.startswith("darwin")
+string_types = (str,)
+
+
+def is_bytes(x):
+    return isinstance(x, (bytes, memoryview, bytearray))
+
+
+def ci_strings_are_equal(a, b):
+    if a == b:
+        return True
+    if not a or not b:
+        return False
+    return a.strip().lower() == b.strip().lower()
+
+
+def hashlib_encode_data(data):
+    if is_bytes(data):
+        return data
+    if not isinstance(data, string_types):
+        data = str(data)
+    return data.encode()
+
+
+def load_python_module(name, pathname):
+    import importlib.util  # pylint: disable=import-outside-toplevel
+
+    spec = importlib.util.spec_from_file_location(name, pathname)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
 
 
 def get_filesystem_encoding():
@@ -53,14 +87,6 @@ def get_object_members(obj, ignore_private=True):
     }
 
 
-def ci_strings_are_equal(a, b):
-    if a == b:
-        return True
-    if not a or not b:
-        return False
-    return a.strip().lower() == b.strip().lower()
-
-
 def ensure_python3(raise_exception=True):
     compatible = sys.version_info >= (3, 6)
     if not raise_exception or compatible:
@@ -71,101 +97,3 @@ def ensure_python3(raise_exception=True):
         "https://docs.platformio.org/en/latest/core/migration.html"
         "#drop-support-for-python-2-and-3-5"
     )
-
-
-if PY2:
-    import imp
-
-    string_types = (str, unicode)
-
-    def create_task(coro, name=None):
-        raise NotImplementedError
-
-    def get_running_loop():
-        raise NotImplementedError
-
-    def is_bytes(x):
-        return isinstance(x, (buffer, bytearray))
-
-    def path_to_unicode(path):
-        if isinstance(path, unicode):
-            return path
-        return path.decode(get_filesystem_encoding())
-
-    def hashlib_encode_data(data):
-        if is_bytes(data):
-            return data
-        if isinstance(data, unicode):
-            data = data.encode(get_filesystem_encoding())
-        elif not isinstance(data, string_types):
-            data = str(data)
-        return data
-
-    def dump_json_to_unicode(obj):
-        if isinstance(obj, unicode):
-            return obj
-        return json.dumps(
-            obj, encoding=get_filesystem_encoding(), ensure_ascii=False
-        ).encode("utf8")
-
-    _magic_check = re.compile("([*?[])")
-    _magic_check_bytes = re.compile(b"([*?[])")
-
-    def glob_recursive(pathname):
-        return glob.glob(pathname)
-
-    def glob_escape(pathname):
-        """Escape all special characters."""
-        # https://github.com/python/cpython/blob/master/Lib/glob.py#L161
-        # Escaping is done by wrapping any of "*?[" between square brackets.
-        # Metacharacters do not work in the drive part and shouldn't be
-        # escaped.
-        drive, pathname = os.path.splitdrive(pathname)
-        if isinstance(pathname, bytes):
-            pathname = _magic_check_bytes.sub(br"[\1]", pathname)
-        else:
-            pathname = _magic_check.sub(r"[\1]", pathname)
-        return drive + pathname
-
-    def load_python_module(name, pathname):
-        return imp.load_source(name, pathname)
-
-
-else:
-    import importlib.util
-    from glob import escape as glob_escape
-
-    if sys.version_info >= (3, 7):
-        from asyncio import create_task, get_running_loop
-    else:
-        from asyncio import ensure_future as create_task
-        from asyncio import get_event_loop as get_running_loop
-
-    string_types = (str,)
-
-    def is_bytes(x):
-        return isinstance(x, (bytes, memoryview, bytearray))
-
-    def path_to_unicode(path):
-        return path
-
-    def hashlib_encode_data(data):
-        if is_bytes(data):
-            return data
-        if not isinstance(data, string_types):
-            data = str(data)
-        return data.encode()
-
-    def dump_json_to_unicode(obj):
-        if isinstance(obj, string_types):
-            return obj
-        return json.dumps(obj)
-
-    def glob_recursive(pathname):
-        return glob.glob(pathname, recursive=True)
-
-    def load_python_module(name, pathname):
-        spec = importlib.util.spec_from_file_location(name, pathname)
-        module = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(module)
-        return module

platformio/debug/process/base.py

@@ -21,9 +21,9 @@ import time
 from platformio import fs
 from platformio.compat import (
     WINDOWS,
-    create_task,
+    aio_create_task,
+    aio_get_running_loop,
     get_locale_encoding,
-    get_running_loop,
     string_types,
 )
 from platformio.proc import get_pythonexe_path
@@ -83,7 +83,7 @@ class DebugBaseProcess:
         for pipe in ("stdin", "stdout", "stderr"):
             if pipe not in kwargs:
                 kwargs[pipe] = subprocess.PIPE
-        loop = get_running_loop()
+        loop = aio_get_running_loop()
         await loop.subprocess_exec(
             lambda: DebugSubprocessProtocol(self), *args, **kwargs
         )
@@ -99,10 +99,10 @@ class DebugBaseProcess:
         self.transport = transport
 
     def connect_stdin_pipe(self):
-        self._stdin_read_task = create_task(self._read_stdin_pipe())
+        self._stdin_read_task = aio_create_task(self._read_stdin_pipe())
 
     async def _read_stdin_pipe(self):
-        loop = get_running_loop()
+        loop = aio_get_running_loop()
         if WINDOWS:
             while True:
                 self.stdin_data_received(

platformio/debug/process/client.py

@@ -21,7 +21,7 @@ import time
 
 from platformio import fs, proc, telemetry, util
 from platformio.cache import ContentCache
-from platformio.compat import get_running_loop, hashlib_encode_data, is_bytes
+from platformio.compat import aio_get_running_loop, hashlib_encode_data, is_bytes
 from platformio.debug import helpers
 from platformio.debug.exception import DebugInvalidOptionsError
 from platformio.debug.initcfgs import get_gdb_init_config
@@ -191,7 +191,7 @@ class DebugClientProcess(
     def _auto_exec_continue(self):
         auto_exec_delay = 0.5  # in seconds
         if self._last_activity > (time.time() - auto_exec_delay):
-            get_running_loop().call_later(0.1, self._auto_exec_continue)
+            aio_get_running_loop().call_later(0.1, self._auto_exec_continue)
             return
 
         if not self.debug_options["init_break"] or self._target_is_running:

platformio/fs.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import glob
 import hashlib
 import io
 import json
@@ -24,7 +25,7 @@ import sys
 import click
 
 from platformio import exception
-from platformio.compat import WINDOWS, glob_escape, glob_recursive
+from platformio.compat import WINDOWS
 
 
 class cd(object):
@@ -158,7 +159,9 @@ def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True):
     src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep)
     for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter):
         items = set()
-        for item in glob_recursive(os.path.join(glob_escape(src_dir), pattern)):
+        for item in glob.glob(
+            os.path.join(glob.escape(src_dir), pattern), recursive=True
+        ):
             if os.path.isdir(item):
                 for root, _, files in os.walk(item, followlinks=followlinks):
                     for f in files:

platformio/maintenance.py

@@ -28,7 +28,6 @@ from platformio.commands.lib.command import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.commands.system.prune import calculate_unnecessary_system_data
 from platformio.commands.upgrade import get_latest_version
-from platformio.compat import ensure_python3
 from platformio.package.manager.core import update_core_packages
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manager.platform import PlatformPackageManager
@@ -40,8 +39,6 @@ from platformio.proc import is_container
 
 
 def on_platformio_start(ctx, force, caller):
-    ensure_python3(raise_exception=True)
-
     app.set_session_var("command_ctx", ctx)
     app.set_session_var("force_option", force)
     set_caller(caller)

platformio/proc.py

@@ -24,7 +24,6 @@ from platformio.compat import (
     WINDOWS,
     get_filesystem_encoding,
     get_locale_encoding,
-    get_running_loop,
     string_types,
 )
 
@@ -221,9 +220,4 @@ def append_env_path(name, value):
 
 
 def force_exit(code=0):
-    try:
+    os._exit(code)  # pylint: disable=protected-access
-        get_running_loop().stop()
-    except:  # pylint: disable=bare-except
-        pass
-    finally:
-        sys.exit(code)

platformio/project/config.py

@@ -12,21 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import glob
+import hashlib
 import json
 import os
 import re
-from hashlib import sha1
 
 import click
 
 from platformio import fs
-from platformio.compat import (
+from platformio.compat import PY2, WINDOWS, hashlib_encode_data, string_types
-    PY2,
-    WINDOWS,
-    glob_recursive,
-    hashlib_encode_data,
-    string_types,
-)
 from platformio.project import exception
 from platformio.project.options import ProjectOptions
 
@@ -122,7 +117,7 @@ class ProjectConfigBase(object):
         for pattern in self.get("platformio", "extra_configs", []):
             if pattern.startswith("~"):
                 pattern = fs.expanduser(pattern)
-            for item in glob_recursive(pattern):
+            for item in glob.glob(pattern, recursive=True):
                 self.read(item)
 
     def _maintain_renaimed_options(self):
@@ -402,7 +397,7 @@ class ProjectConfigDirsMixin(object):
                 "%s-%s"
                 % (
                     os.path.basename(project_dir),
-                    sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
+                    hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
                 ),
             )
 

tox.ini

@@ -23,7 +23,7 @@ known_third_party=OpenSSL, SCons, jsonrpc, twisted, zope
 passenv = *
 usedevelop = True
 deps =
-    black
+    py36,py37,py38,py39: black
     isort
     pylint
     pytest