platformio/platformio-core
1
0
Fork 1
mirror of https://github.com/platformio/platformio-core.git synced 2025-07-29 17:47:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Remove Python 2 code

Browse Source
This commit is contained in:
Ivan Kravets
2021-03-17 21:08:06 +02:00
parent 064fa6027d
commit c0357daf01
28 changed files with 175 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
HISTORY.rst
View File

@ -14,7 +14,7 @@ PlatformIO Core 5
* **PlatformIO Debugging**
- Boosted `PlatformIO Debugging <https://docs.platformio.org/page/plus/debugging.html>`__ performance thanks to migrating the codebase to the pure Python 3 Asynchronous I/O stack
- Support debugging on Windows using Windows CMD (CLI) (`issue #3793 <https://github.com/platformio/platformio-core/issues/3793>`_)
- Support debugging on Windows using Windows CMD/CLI (`pio debug <https://docs.platformio.org/page/core/userguide/cmd_debug.html>`__) (`issue #3793 <https://github.com/platformio/platformio-core/issues/3793>`_)
5.1.1 (2021-03-17)
~~~~~~~~~~~~~~~~~~

17
platformio/__main__.py
View File

@ -12,15 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=import-outside-toplevel
import os
import sys
from traceback import format_exc
import click
from platformio import __version__, exception, maintenance, util
from platformio import __version__, exception
from platformio.commands import PlatformioCLI
from platformio.compat import CYGWIN
from platformio.compat import CYGWIN, PY2, ensure_python3
try:
import click_completion # pylint: disable=import-error
@ -60,16 +62,19 @@ def cli(ctx, force, caller, no_ansi):
except: # pylint: disable=bare-except
pass
from platformio import maintenance
maintenance.on_platformio_start(ctx, force, caller)
@cli.resultcallback()
@click.pass_context
def process_result(ctx, result, *_, **__):
from platformio import maintenance
maintenance.on_platformio_end(ctx, result)
@util.memoized()
def configure():
if CYGWIN:
raise exception.CygwinEnvDetected()
@ -105,6 +110,7 @@ def main(argv=None):
assert isinstance(argv, list)
sys.argv = argv
try:
ensure_python3(raise_exception=True)
configure()
cli() # pylint: disable=no-value-for-parameter
except SystemExit as e:
@ -112,7 +118,10 @@ def main(argv=None):
exit_code = int(e.code)
except Exception as e: # pylint: disable=broad-except
if not isinstance(e, exception.ReturnErrorCode):
maintenance.on_platformio_exception(e)
if not PY2:
from platformio import maintenance
maintenance.on_platformio_exception(e)
error_str = "Error: "
if isinstance(e, exception.PlatformioException):
error_str += str(e)

4
platformio/app.py
View File

@ -24,7 +24,7 @@ import uuid
from os.path import dirname, isdir, isfile, join, realpath
from platformio import __version__, exception, fs, proc
from platformio.compat import WINDOWS, dump_json_to_unicode, hashlib_encode_data
from platformio.compat import WINDOWS, hashlib_encode_data
from platformio.package.lockfile import LockFile
from platformio.project.helpers import get_default_projects_dir, get_project_core_dir
@ -115,7 +115,7 @@ class State(object):
if self.modified:
try:
with open(self.path, "w") as fp:
fp.write(dump_json_to_unicode(self._storage))
fp.write(json.dumps(self._storage))
except IOError:
raise exception.HomeDirPermissionsError(get_project_core_dir())
self._unlock_state_file()

37
platformio/builder/main.py
View File

@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import sys
from os import environ, makedirs
from os.path import isdir, join
from time import time
import click
@ -29,7 +29,6 @@ from SCons.Script import Import # pylint: disable=import-error
from SCons.Script import Variables # pylint: disable=import-error
from platformio import compat, fs
from platformio.compat import dump_json_to_unicode
from platformio.platform.base import PlatformBase
from platformio.proc import get_pythonexe_path
from platformio.project.helpers import get_project_dir
@ -65,18 +64,18 @@ DEFAULT_ENV_OPTIONS = dict(
"pioide",
"piosize",
],
toolpath=[join(fs.get_source_dir(), "builder", "tools")],
toolpath=[os.path.join(fs.get_source_dir(), "builder", "tools")],
variables=clivars,
# Propagating External Environment
ENV=environ,
ENV=os.environ,
UNIX_TIME=int(time()),
BUILD_DIR=join("$PROJECT_BUILD_DIR", "$PIOENV"),
BUILD_SRC_DIR=join("$BUILD_DIR", "src"),
BUILD_TEST_DIR=join("$BUILD_DIR", "test"),
COMPILATIONDB_PATH=join("$BUILD_DIR", "compile_commands.json"),
BUILD_DIR=os.path.join("$PROJECT_BUILD_DIR", "$PIOENV"),
BUILD_SRC_DIR=os.path.join("$BUILD_DIR", "src"),
BUILD_TEST_DIR=os.path.join("$BUILD_DIR", "test"),
COMPILATIONDB_PATH=os.path.join("$BUILD_DIR", "compile_commands.json"),
LIBPATH=["$BUILD_DIR"],
PROGNAME="program",
PROG_PATH=join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
PROG_PATH=os.path.join("$BUILD_DIR", "$PROGNAME$PROGSUFFIX"),
PYTHONEXE=get_pythonexe_path(),
IDE_EXTRA_DATA={},
)
@ -124,7 +123,7 @@ env.Replace(
BUILD_CACHE_DIR=config.get_optional_dir("build_cache"),
LIBSOURCE_DIRS=[
config.get_optional_dir("lib"),
join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
os.path.join("$PROJECT_LIBDEPS_DIR", "$PIOENV"),
config.get_optional_dir("globallib"),
],
)
@ -142,8 +141,8 @@ if (
)
if env.subst("$BUILD_CACHE_DIR"):
if not isdir(env.subst("$BUILD_CACHE_DIR")):
makedirs(env.subst("$BUILD_CACHE_DIR"))
if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")):
os.makedirs(env.subst("$BUILD_CACHE_DIR"))
env.CacheDir("$BUILD_CACHE_DIR")
if int(ARGUMENTS.get("ISATTY", 0)):
@ -160,15 +159,17 @@ elif not int(ARGUMENTS.get("PIOVERBOSE", 0)):
if "compiledb" in COMMAND_LINE_TARGETS:
env.Tool("compilation_db")
if not isdir(env.subst("$BUILD_DIR")):
makedirs(env.subst("$BUILD_DIR"))
if not os.path.isdir(env.subst("$BUILD_DIR")):
os.makedirs(env.subst("$BUILD_DIR"))
env.LoadProjectOptions()
env.LoadPioPlatform()
env.SConscriptChdir(0)
env.SConsignFile(
join("$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]))
os.path.join(
"$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1])
)
)
for item in env.GetExtraScripts("pre"):
@ -225,9 +226,7 @@ if "idedata" in COMMAND_LINE_TARGETS:
projenv = env
click.echo(
"\n%s\n"
% dump_json_to_unicode(
projenv.DumpIDEData(env) # pylint: disable=undefined-variable
)
% json.dumps(projenv.DumpIDEData(env)) # pylint: disable=undefined-variable
)
env.Exit(0)

9
platformio/builder/tools/pioide.py
View File

@ -14,13 +14,12 @@
from __future__ import absolute_import
import glob
import os
from glob import glob
import SCons.Defaults # pylint: disable=import-error
import SCons.Subst # pylint: disable=import-error
from platformio.compat import glob_escape
from platformio.package.manager.core import get_core_package_dir
from platformio.proc import exec_command, where_is_program
@ -49,7 +48,7 @@ def _dump_includes(env):
for pkg in p.get_installed_packages():
if p.get_package_type(pkg.metadata.name) != "toolchain":
continue
toolchain_dir = glob_escape(pkg.path)
toolchain_dir = glob.escape(pkg.path)
toolchain_incglobs = [
os.path.join(toolchain_dir, "*", "include", "c++", "*"),
os.path.join(toolchain_dir, "*", "include", "c++", "*", "*-*-*"),
@ -57,7 +56,9 @@ def _dump_includes(env):
os.path.join(toolchain_dir, "*", "include*"),
]
for g in toolchain_incglobs:
includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)])
includes["toolchain"].extend(
[os.path.realpath(inc) for inc in glob.glob(g)]
)
# include Unity framework if there are tests in project
includes["unity"] = []

9
platformio/builder/tools/piomisc.py
View File

@ -15,16 +15,17 @@
from __future__ import absolute_import
import atexit
import glob
import io
import os
import re
import sys
from tempfile import mkstemp
import tempfile
import click
from platformio import fs, util
from platformio.compat import get_filesystem_encoding, get_locale_encoding, glob_escape
from platformio.compat import get_filesystem_encoding, get_locale_encoding
from platformio.package.manager.core import get_core_package_dir
from platformio.proc import exec_command
@ -116,7 +117,7 @@ class InoToCPPConverter(object):
return out_file
def _gcc_preprocess(self, contents, out_file):
tmp_path = mkstemp()[1]
tmp_path = tempfile.mkstemp()[1]
self.write_safe_contents(tmp_path, contents)
self.env.Execute(
self.env.VerboseAction(
@ -229,7 +230,7 @@ class InoToCPPConverter(object):
def ConvertInoToCpp(env):
src_dir = glob_escape(env.subst("$PROJECT_SRC_DIR"))
src_dir = glob.escape(env.subst("$PROJECT_SRC_DIR"))
ino_nodes = env.Glob(os.path.join(src_dir, "*.ino")) + env.Glob(
os.path.join(src_dir, "*.pde")
)

4
platformio/builder/tools/piosize.py
View File

@ -16,6 +16,7 @@
from __future__ import absolute_import
import json
import sys
from os import environ, makedirs, remove
from os.path import isdir, join, splitdrive
@ -23,7 +24,6 @@ from os.path import isdir, join, splitdrive
from elftools.elf.descriptions import describe_sh_flags
from elftools.elf.elffile import ELFFile
from platformio.compat import dump_json_to_unicode
from platformio.proc import exec_command
from platformio.util import get_systype
@ -242,7 +242,7 @@ def DumpSizeData(_, target, source, env): # pylint: disable=unused-argument
data["memory"]["files"].append(file_data)
with open(join(env.subst("$BUILD_DIR"), "sizedata.json"), "w") as fp:
fp.write(dump_json_to_unicode(data))
fp.write(json.dumps(data))
def exists(_):

3
platformio/commands/boards.py
View File

@ -18,7 +18,6 @@ import click
from tabulate import tabulate
from platformio import fs
from platformio.compat import dump_json_to_unicode
from platformio.package.manager.platform import PlatformPackageManager
@ -83,4 +82,4 @@ def _print_boards_json(query, installed=False):
if query.lower() not in search_data.lower():
continue
result.append(board)
click.echo(dump_json_to_unicode(result))
click.echo(json.dumps(result))

4
platformio/commands/check/command.py
View File

@ -15,6 +15,7 @@
# pylint: disable=too-many-arguments,too-many-locals,too-many-branches
# pylint: disable=redefined-builtin,too-many-statements
import json
import os
from collections import Counter
from os.path import dirname, isfile
@ -26,7 +27,6 @@ from tabulate import tabulate
from platformio import app, exception, fs, util
from platformio.commands.check.defect import DefectItem
from platformio.commands.check.tools import CheckToolFactory
from platformio.compat import dump_json_to_unicode
from platformio.project.config import ProjectConfig
from platformio.project.helpers import find_project_dir_above, get_project_dir
@ -163,7 +163,7 @@ def cli(
print_processing_footer(result)
if json_output:
click.echo(dump_json_to_unicode(results_to_json(results)))
click.echo(json.dumps(results_to_json(results)))
elif not silent:
print_check_summary(results)

9
platformio/commands/check/tools/base.py
View File

@ -12,12 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import glob
import os
from tempfile import NamedTemporaryFile
import tempfile
import click
from platformio import compat, fs, proc
from platformio import fs, proc
from platformio.commands.check.defect import DefectItem
from platformio.project.helpers import load_project_ide_data
@ -104,7 +105,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes
return {lang: _extract_defines(lang, incflags_file) for lang in ("c", "c++")}
def _create_tmp_file(self, data):
with NamedTemporaryFile("w", delete=False) as fp:
with tempfile.NamedTemporaryFile("w", delete=False) as fp:
fp.write(data)
self._tmp_files.append(fp.name)
return fp.name
@ -207,7 +208,7 @@ class CheckToolBase(object): # pylint: disable=too-many-instance-attributes
result["c++"].append(os.path.realpath(path))
for pattern in patterns:
for item in compat.glob_recursive(pattern):
for item in glob.glob(pattern, recursive=True):
if not os.path.isdir(item):
_add_file(item)
for root, _, files in os.walk(item, followlinks=True):

66
platformio/commands/ci.py
View File

@ -12,14 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from os import getenv, makedirs, remove
from os.path import basename, isdir, isfile, join, realpath
from shutil import copyfile, copytree
from tempfile import mkdtemp
import glob
import os
import shutil
import tempfile
import click
from platformio import app, compat, fs
from platformio import app, fs
from platformio.commands.project import project_init as cmd_project_init
from platformio.commands.project import validate_boards
from platformio.commands.run.command import cli as cmd_run
@ -33,8 +33,8 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument
for i, p in enumerate(value):
if p.startswith("~"):
value[i] = fs.expanduser(p)
value[i] = realpath(value[i])
if not compat.glob_recursive(value[i]):
value[i] = os.path.realpath(value[i])
if not glob.glob(value[i], recursive=True):
invalid_path = p
break
try:
@ -51,7 +51,7 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument
@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards)
@click.option(
"--build-dir",
default=mkdtemp,
default=tempfile.mkdtemp,
type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True),
)
@click.option("--keep-build-dir", is_flag=True)
@ -78,28 +78,28 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches
verbose,
):
if not src and getenv("PLATFORMIO_CI_SRC"):
src = validate_path(ctx, None, getenv("PLATFORMIO_CI_SRC").split(":"))
if not src and os.getenv("PLATFORMIO_CI_SRC"):
src = validate_path(ctx, None, os.getenv("PLATFORMIO_CI_SRC").split(":"))
if not src:
raise click.BadParameter("Missing argument 'src'")
try:
app.set_session_var("force_option", True)
if not keep_build_dir and isdir(build_dir):
if not keep_build_dir and os.path.isdir(build_dir):
fs.rmtree(build_dir)
if not isdir(build_dir):
makedirs(build_dir)
if not os.path.isdir(build_dir):
os.makedirs(build_dir)
for dir_name, patterns in dict(lib=lib, src=src).items():
if not patterns:
continue
contents = []
for p in patterns:
contents += compat.glob_recursive(p)
_copy_contents(join(build_dir, dir_name), contents)
contents += glob.glob(p, recursive=True)
_copy_contents(os.path.join(build_dir, dir_name), contents)
if project_conf and isfile(project_conf):
if project_conf and os.path.isfile(project_conf):
_copy_project_conf(build_dir, project_conf)
elif not board:
raise CIBuildEnvsEmpty()
@ -126,48 +126,50 @@ def _copy_contents(dst_dir, contents):
items = {"dirs": set(), "files": set()}
for path in contents:
if isdir(path):
if os.path.isdir(path):
items["dirs"].add(path)
elif isfile(path):
elif os.path.isfile(path):
items["files"].add(path)
dst_dir_name = basename(dst_dir)
dst_dir_name = os.path.basename(dst_dir)
if dst_dir_name == "src" and len(items["dirs"]) == 1:
copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
shutil.copytree(list(items["dirs"]).pop(), dst_dir, symlinks=True)
else:
if not isdir(dst_dir):
makedirs(dst_dir)
if not os.path.isdir(dst_dir):
os.makedirs(dst_dir)
for d in items["dirs"]:
copytree(d, join(dst_dir, basename(d)), symlinks=True)
shutil.copytree(
d, os.path.join(dst_dir, os.path.basename(d)), symlinks=True
)
if not items["files"]:
return
if dst_dir_name == "lib":
dst_dir = join(dst_dir, mkdtemp(dir=dst_dir))
dst_dir = os.path.join(dst_dir, tempfile.mkdtemp(dir=dst_dir))
for f in items["files"]:
dst_file = join(dst_dir, basename(f))
dst_file = os.path.join(dst_dir, os.path.basename(f))
if f == dst_file:
continue
copyfile(f, dst_file)
shutil.copyfile(f, dst_file)
def _exclude_contents(dst_dir, patterns):
contents = []
for p in patterns:
contents += compat.glob_recursive(join(compat.glob_escape(dst_dir), p))
contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True)
for path in contents:
path = realpath(path)
if isdir(path):
path = os.path.realpath(path)
if os.path.isdir(path):
fs.rmtree(path)
elif isfile(path):
remove(path)
elif os.path.isfile(path):
os.remove(path)
def _copy_project_conf(build_dir, project_conf):
config = ProjectConfig(project_conf, parse_extra=False)
if config.has_section("platformio"):
config.remove_section("platformio")
config.save(join(build_dir, "platformio.ini"))
config.save(os.path.join(build_dir, "platformio.ini"))

2
platformio/commands/debug.py
View File

@ -167,7 +167,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, __unpro
loop.run_until_complete(coro)
if WINDOWS:
# an issue with asyncio executor and STIDIN, it cannot be closed gracefully
os._exit(0) # pylint: disable=protected-access
proc.force_exit()
loop.close()
return True

6
platformio/commands/device/command.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import sys
from fnmatch import fnmatch
@ -21,7 +22,6 @@ from serial.tools import miniterm
from platformio import exception, fs, util
from platformio.commands.device import helpers as device_helpers
from platformio.compat import dump_json_to_unicode
from platformio.platform.factory import PlatformFactory
from platformio.project.exception import NotPlatformIOProjectError
@ -52,9 +52,7 @@ def device_list( # pylint: disable=too-many-branches
single_key = list(data)[0] if len(list(data)) == 1 else None
if json_output:
return click.echo(
dump_json_to_unicode(data[single_key] if single_key else data)
)
return click.echo(json.dumps(data[single_key] if single_key else data))
titles = {
"serial": "Serial Ports",

4
platformio/commands/home/command.py
View File

@ -17,6 +17,7 @@ import mimetypes
import click
from platformio.commands.home.helpers import is_port_used
from platformio.commands.home.run import run_server
from platformio.compat import ensure_python3
@ -87,9 +88,6 @@ def cli(port, host, no_open, shutdown_timeout, session_id):
click.launch(home_url)
return
# pylint: disable=import-outside-toplevel
from platformio.commands.home.run import run_server
run_server(
host=host,
port=port,

4
platformio/commands/home/rpc/handlers/ide.py
View File

@ -16,7 +16,7 @@ import time
from ajsonrpc.core import JSONRPC20DispatchException
from platformio.compat import get_running_loop
from platformio.compat import aio_get_running_loop
class IDERPC:
@ -36,7 +36,7 @@ class IDERPC:
async def listen_commands(self, sid=0):
if sid not in self._queue:
self._queue[sid] = []
self._queue[sid].append(get_running_loop().create_future())
self._queue[sid].append(aio_get_running_loop().create_future())
return await self._queue[sid][-1]
def open_project(self, sid, project_dir):

4
platformio/commands/home/rpc/handlers/misc.py
View File

@ -17,7 +17,7 @@ import time
from platformio.cache import ContentCache
from platformio.commands.home.rpc.handlers.os import OSRPC
from platformio.compat import create_task
from platformio.compat import aio_create_task
class MiscRPC:
@ -30,7 +30,7 @@ class MiscRPC:
cache_data = json.loads(cache_data)
# automatically update cache in background every 12 hours
if cache_data["time"] < (time.time() - (3600 * 12)):
create_task(
aio_create_task(
self._preload_latest_tweets(data_url, cache_key, cache_valid)
)
return cache_data["result"]

6
platformio/commands/home/rpc/server.py
View File

@ -17,7 +17,7 @@ from ajsonrpc.dispatcher import Dispatcher
from ajsonrpc.manager import AsyncJSONRPCResponseManager
from starlette.endpoints import WebSocketEndpoint
from platformio.compat import create_task, get_running_loop
from platformio.compat import aio_create_task, aio_get_running_loop
from platformio.proc import force_exit
@ -63,7 +63,7 @@ class JSONRPCServerFactoryBase:
click.echo("Automatically shutdown server on timeout")
force_exit()
self.shutdown_timer = get_running_loop().call_later(
self.shutdown_timer = aio_get_running_loop().call_later(
self.shutdown_timeout, _auto_shutdown_server
)
@ -84,7 +84,7 @@ class WebSocketJSONRPCServer(WebSocketEndpoint):
self.factory.on_client_connect() # pylint: disable=no-member
async def on_receive(self, websocket, data):
create_task(self._handle_rpc(websocket, data))
aio_create_task(self._handle_rpc(websocket, data))
async def on_disconnect(self, websocket, close_code):
self.factory.on_client_disconnect() # pylint: disable=no-member

4
platformio/commands/home/run.py
View File

@ -32,7 +32,7 @@ from platformio.commands.home.rpc.handlers.os import OSRPC
from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
from platformio.commands.home.rpc.handlers.project import ProjectRPC
from platformio.commands.home.rpc.server import WebSocketJSONRPCServerFactory
from platformio.compat import get_running_loop
from platformio.compat import aio_get_running_loop
from platformio.exception import PlatformioException
from platformio.package.manager.core import get_core_package_dir
from platformio.proc import force_exit
@ -49,7 +49,7 @@ class ShutdownMiddleware:
async def shutdown_server(_=None):
get_running_loop().call_later(0.5, force_exit)
aio_get_running_loop().call_later(0.5, force_exit)
return PlainTextResponse("Server has been shutdown!")

14
platformio/commands/lib/command.py
View File

@ -14,6 +14,7 @@
# pylint: disable=too-many-branches, too-many-locals
import json
import os
import time
@ -23,7 +24,6 @@ from tabulate import tabulate
from platformio import exception, fs, util
from platformio.commands import PlatformioCLI
from platformio.commands.lib.helpers import get_builtin_libs, save_project_libdeps
from platformio.compat import dump_json_to_unicode
from platformio.package.exception import NotGlobalLibDir, UnknownPackageError
from platformio.package.manager.library import LibraryPackageManager
from platformio.package.meta import PackageItem, PackageSpec
@ -286,7 +286,7 @@ def lib_update( # pylint: disable=too-many-arguments
if json_output:
return click.echo(
dump_json_to_unicode(
json.dumps(
json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
)
)
@ -315,7 +315,7 @@ def lib_list(ctx, json_output):
if json_output:
return click.echo(
dump_json_to_unicode(
json.dumps(
json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
)
)
@ -359,7 +359,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
)
if json_output:
click.echo(dump_json_to_unicode(result))
click.echo(json.dumps(result))
return
if result["total"] == 0:
@ -418,7 +418,7 @@ def lib_search(query, json_output, page, noninteractive, **filters):
def lib_builtin(storage, json_output):
items = get_builtin_libs(storage)
if json_output:
return click.echo(dump_json_to_unicode(items))
return click.echo(json.dumps(items))
for storage_ in items:
if not storage_["items"]:
@ -442,7 +442,7 @@ def lib_show(library, json_output):
regclient = lm.get_registry_client_instance()
lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h")
if json_output:
return click.echo(dump_json_to_unicode(lib))
return click.echo(json.dumps(lib))
title = "{ownername}/{name}".format(**lib)
click.secho(title, fg="cyan")
@ -538,7 +538,7 @@ def lib_stats(json_output):
result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h")
if json_output:
return click.echo(dump_json_to_unicode(result))
return click.echo(json.dumps(result))
for key in ("updated", "added"):
tabular_data = [

16
platformio/commands/platform.py
View File

@ -12,13 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import click
from platformio.cache import cleanup_content_cache
from platformio.commands.boards import print_boards
from platformio.compat import dump_json_to_unicode
from platformio.package.manager.platform import PlatformPackageManager
from platformio.package.meta import PackageItem, PackageSpec
from platformio.package.version import get_original_version
@ -172,7 +172,7 @@ def platform_search(query, json_output):
for platform in _get_registry_platforms():
if query == "all":
query = ""
search_data = dump_json_to_unicode(platform)
search_data = json.dumps(platform)
if query and query.lower() not in search_data.lower():
continue
platforms.append(
@ -182,7 +182,7 @@ def platform_search(query, json_output):
)
if json_output:
click.echo(dump_json_to_unicode(platforms))
click.echo(json.dumps(platforms))
else:
_print_platforms(platforms)
@ -198,7 +198,7 @@ def platform_frameworks(query, json_output):
):
if query == "all":
query = ""
search_data = dump_json_to_unicode(framework)
search_data = json.dumps(framework)
if query and query.lower() not in search_data.lower():
continue
framework["homepage"] = "https://platformio.org/frameworks/" + framework["name"]
@ -211,7 +211,7 @@ def platform_frameworks(query, json_output):
frameworks = sorted(frameworks, key=lambda manifest: manifest["name"])
if json_output:
click.echo(dump_json_to_unicode(frameworks))
click.echo(json.dumps(frameworks))
else:
_print_platforms(frameworks)
@ -228,7 +228,7 @@ def platform_list(json_output):
platforms = sorted(platforms, key=lambda manifest: manifest["name"])
if json_output:
click.echo(dump_json_to_unicode(platforms))
click.echo(json.dumps(platforms))
else:
_print_platforms(platforms)
@ -241,7 +241,7 @@ def platform_show(platform, json_output): # pylint: disable=too-many-branches
if not data:
raise UnknownPlatform(platform)
if json_output:
return click.echo(dump_json_to_unicode(data))
return click.echo(json.dumps(data))
dep = "{ownername}/{name}".format(**data) if "ownername" in data else data["name"]
click.echo(
@ -401,7 +401,7 @@ def platform_update( # pylint: disable=too-many-locals, too-many-arguments
str(outdated.latest) if outdated.latest else None
)
result.append(data)
return click.echo(dump_json_to_unicode(result))
return click.echo(json.dumps(result))
# cleanup cached board and platform lists
cleanup_content_cache("http")

152
platformio/compat.py
View File

@ -12,23 +12,57 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=unused-import, no-name-in-module, import-error,
# pylint: disable=no-member, undefined-variable, unexpected-keyword-arg
# pylint: disable=unused-import
import glob
import inspect
import json
import locale
import os
import re
import sys
from platformio.exception import UserSideException
if sys.version_info >= (3,):
if sys.version_info >= (3, 7):
from asyncio import create_task as aio_create_task
from asyncio import get_running_loop as aio_get_running_loop
else:
from asyncio import ensure_future as aio_create_task
from asyncio import get_event_loop as aio_get_running_loop
PY2 = sys.version_info[0] == 2
CYGWIN = sys.platform.startswith("cygwin")
WINDOWS = sys.platform.startswith("win")
MACOS = sys.platform.startswith("darwin")
string_types = (str,)
def is_bytes(x):
return isinstance(x, (bytes, memoryview, bytearray))
def ci_strings_are_equal(a, b):
if a == b:
return True
if not a or not b:
return False
return a.strip().lower() == b.strip().lower()
def hashlib_encode_data(data):
if is_bytes(data):
return data
if not isinstance(data, string_types):
data = str(data)
return data.encode()
def load_python_module(name, pathname):
import importlib.util # pylint: disable=import-outside-toplevel
spec = importlib.util.spec_from_file_location(name, pathname)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def get_filesystem_encoding():
@ -53,14 +87,6 @@ def get_object_members(obj, ignore_private=True):
}
def ci_strings_are_equal(a, b):
if a == b:
return True
if not a or not b:
return False
return a.strip().lower() == b.strip().lower()
def ensure_python3(raise_exception=True):
compatible = sys.version_info >= (3, 6)
if not raise_exception or compatible:
@ -71,101 +97,3 @@ def ensure_python3(raise_exception=True):
"https://docs.platformio.org/en/latest/core/migration.html"
"#drop-support-for-python-2-and-3-5"
)
if PY2:
import imp
string_types = (str, unicode)
def create_task(coro, name=None):
raise NotImplementedError
def get_running_loop():
raise NotImplementedError
def is_bytes(x):
return isinstance(x, (buffer, bytearray))
def path_to_unicode(path):
if isinstance(path, unicode):
return path
return path.decode(get_filesystem_encoding())
def hashlib_encode_data(data):
if is_bytes(data):
return data
if isinstance(data, unicode):
data = data.encode(get_filesystem_encoding())
elif not isinstance(data, string_types):
data = str(data)
return data
def dump_json_to_unicode(obj):
if isinstance(obj, unicode):
return obj
return json.dumps(
obj, encoding=get_filesystem_encoding(), ensure_ascii=False
).encode("utf8")
_magic_check = re.compile("([*?[])")
_magic_check_bytes = re.compile(b"([*?[])")
def glob_recursive(pathname):
return glob.glob(pathname)
def glob_escape(pathname):
"""Escape all special characters."""
# https://github.com/python/cpython/blob/master/Lib/glob.py#L161
# Escaping is done by wrapping any of "*?[" between square brackets.
# Metacharacters do not work in the drive part and shouldn't be
# escaped.
drive, pathname = os.path.splitdrive(pathname)
if isinstance(pathname, bytes):
pathname = _magic_check_bytes.sub(br"[\1]", pathname)
else:
pathname = _magic_check.sub(r"[\1]", pathname)
return drive + pathname
def load_python_module(name, pathname):
return imp.load_source(name, pathname)
else:
import importlib.util
from glob import escape as glob_escape
if sys.version_info >= (3, 7):
from asyncio import create_task, get_running_loop
else:
from asyncio import ensure_future as create_task
from asyncio import get_event_loop as get_running_loop
string_types = (str,)
def is_bytes(x):
return isinstance(x, (bytes, memoryview, bytearray))
def path_to_unicode(path):
return path
def hashlib_encode_data(data):
if is_bytes(data):
return data
if not isinstance(data, string_types):
data = str(data)
return data.encode()
def dump_json_to_unicode(obj):
if isinstance(obj, string_types):
return obj
return json.dumps(obj)
def glob_recursive(pathname):
return glob.glob(pathname, recursive=True)
def load_python_module(name, pathname):
spec = importlib.util.spec_from_file_location(name, pathname)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module

10
platformio/debug/process/base.py
View File

@ -21,9 +21,9 @@ import time
from platformio import fs
from platformio.compat import (
WINDOWS,
create_task,
aio_create_task,
aio_get_running_loop,
get_locale_encoding,
get_running_loop,
string_types,
)
from platformio.proc import get_pythonexe_path
@ -83,7 +83,7 @@ class DebugBaseProcess:
for pipe in ("stdin", "stdout", "stderr"):
if pipe not in kwargs:
kwargs[pipe] = subprocess.PIPE
loop = get_running_loop()
loop = aio_get_running_loop()
await loop.subprocess_exec(
lambda: DebugSubprocessProtocol(self), *args, **kwargs
)
@ -99,10 +99,10 @@ class DebugBaseProcess:
self.transport = transport
def connect_stdin_pipe(self):
self._stdin_read_task = create_task(self._read_stdin_pipe())
self._stdin_read_task = aio_create_task(self._read_stdin_pipe())
async def _read_stdin_pipe(self):
loop = get_running_loop()
loop = aio_get_running_loop()
if WINDOWS:
while True:
self.stdin_data_received(

4
platformio/debug/process/client.py
View File

@ -21,7 +21,7 @@ import time
from platformio import fs, proc, telemetry, util
from platformio.cache import ContentCache
from platformio.compat import get_running_loop, hashlib_encode_data, is_bytes
from platformio.compat import aio_get_running_loop, hashlib_encode_data, is_bytes
from platformio.debug import helpers
from platformio.debug.exception import DebugInvalidOptionsError
from platformio.debug.initcfgs import get_gdb_init_config
@ -191,7 +191,7 @@ class DebugClientProcess(
def _auto_exec_continue(self):
auto_exec_delay = 0.5 # in seconds
if self._last_activity > (time.time() - auto_exec_delay):
get_running_loop().call_later(0.1, self._auto_exec_continue)
aio_get_running_loop().call_later(0.1, self._auto_exec_continue)
return
if not self.debug_options["init_break"] or self._target_is_running:

7
platformio/fs.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import glob
import hashlib
import io
import json
@ -24,7 +25,7 @@ import sys
import click
from platformio import exception
from platformio.compat import WINDOWS, glob_escape, glob_recursive
from platformio.compat import WINDOWS
class cd(object):
@ -158,7 +159,9 @@ def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True):
src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep)
for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter):
items = set()
for item in glob_recursive(os.path.join(glob_escape(src_dir), pattern)):
for item in glob.glob(
os.path.join(glob.escape(src_dir), pattern), recursive=True
):
if os.path.isdir(item):
for root, _, files in os.walk(item, followlinks=followlinks):
for f in files:

3
platformio/maintenance.py
View File

@ -28,7 +28,6 @@ from platformio.commands.lib.command import lib_update as cmd_lib_update
from platformio.commands.platform import platform_update as cmd_platform_update
from platformio.commands.system.prune import calculate_unnecessary_system_data
from platformio.commands.upgrade import get_latest_version
from platformio.compat import ensure_python3
from platformio.package.manager.core import update_core_packages
from platformio.package.manager.library import LibraryPackageManager
from platformio.package.manager.platform import PlatformPackageManager
@ -40,8 +39,6 @@ from platformio.proc import is_container
def on_platformio_start(ctx, force, caller):
ensure_python3(raise_exception=True)
app.set_session_var("command_ctx", ctx)
app.set_session_var("force_option", force)
set_caller(caller)

8
platformio/proc.py
View File

@ -24,7 +24,6 @@ from platformio.compat import (
WINDOWS,
get_filesystem_encoding,
get_locale_encoding,
get_running_loop,
string_types,
)
@ -221,9 +220,4 @@ def append_env_path(name, value):
def force_exit(code=0):
try:
get_running_loop().stop()
except: # pylint: disable=bare-except
pass
finally:
sys.exit(code)
os._exit(code) # pylint: disable=protected-access

15
platformio/project/config.py
View File

@ -12,21 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import glob
import hashlib
import json
import os
import re
from hashlib import sha1
import click
from platformio import fs
from platformio.compat import (
PY2,
WINDOWS,
glob_recursive,
hashlib_encode_data,
string_types,
)
from platformio.compat import PY2, WINDOWS, hashlib_encode_data, string_types
from platformio.project import exception
from platformio.project.options import ProjectOptions
@ -122,7 +117,7 @@ class ProjectConfigBase(object):
for pattern in self.get("platformio", "extra_configs", []):
if pattern.startswith("~"):
pattern = fs.expanduser(pattern)
for item in glob_recursive(pattern):
for item in glob.glob(pattern, recursive=True):
self.read(item)
def _maintain_renaimed_options(self):
@ -402,7 +397,7 @@ class ProjectConfigDirsMixin(object):
"%s-%s"
% (
os.path.basename(project_dir),
sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
hashlib.sha1(hashlib_encode_data(project_dir)).hexdigest()[:10],
),
)

2
tox.ini
View File

@ -23,7 +23,7 @@ known_third_party=OpenSSL, SCons, jsonrpc, twisted, zope
passenv = *
usedevelop = True
deps =
black
py36,py37,py38,py39: black
isort
pylint
pytest