From 6a903886496b2f8ee5ffcab1e78bfe8e6f44ab13 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 12 Aug 2019 19:44:37 +0300 Subject: [PATCH] Move FS related helpers to fs.py module --- platformio/app.py | 8 +- platformio/builder/main.py | 4 +- platformio/builder/tools/piolib.py | 16 +-- platformio/builder/tools/piomisc.py | 6 +- platformio/builder/tools/pioplatform.py | 6 +- platformio/builder/tools/pioupload.py | 4 +- platformio/commands/boards.py | 6 +- platformio/commands/ci.py | 8 +- platformio/commands/debug/client.py | 6 +- platformio/commands/debug/command.py | 10 +- .../commands/home/rpc/handlers/piocore.py | 4 +- .../commands/home/rpc/handlers/project.py | 10 +- platformio/commands/init.py | 4 +- platformio/commands/lib.py | 4 +- platformio/commands/remote.py | 4 +- platformio/commands/run/command.py | 4 +- platformio/commands/run/helpers.py | 6 +- platformio/commands/test/command.py | 4 +- platformio/commands/test/native.py | 6 +- platformio/fs.py | 120 ++++++++++++++++++ platformio/ide/projectgenerator.py | 10 +- platformio/maintenance.py | 4 +- platformio/managers/core.py | 4 +- platformio/managers/package.py | 16 +-- platformio/managers/platform.py | 10 +- platformio/util.py | 107 ++-------------- 26 files changed, 209 insertions(+), 182 deletions(-) create mode 100644 platformio/fs.py diff --git a/platformio/app.py b/platformio/app.py index 75eea6db..efdf5ed8 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -22,7 +22,7 @@ from time import time import requests -from platformio import exception, lockfile, util +from platformio import exception, fs, lockfile from platformio.compat import (WINDOWS, dump_json_to_unicode, hashlib_encode_data) from platformio.proc import is_ci @@ -113,7 +113,7 @@ class State(object): try: self._lock_state_file() if isfile(self.path): - self._storage = util.load_json(self.path) + self._storage = fs.load_json(self.path) assert isinstance(self._storage, dict) except (AssertionError, ValueError, UnicodeDecodeError, exception.InvalidJSONFile): @@ -290,7 +290,7 @@ class ContentCache(object): try: remove(path) if not listdir(dirname(path)): - util.rmtree_(dirname(path)) + fs.rmtree(dirname(path)) except OSError: pass @@ -304,7 +304,7 @@ class ContentCache(object): def clean(self): if not self.cache_dir or not isdir(self.cache_dir): return - util.rmtree_(self.cache_dir) + fs.rmtree(self.cache_dir) def clean_cache(): diff --git a/platformio/builder/main.py b/platformio/builder/main.py index 8fa012d0..0e46f8d0 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -27,7 +27,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error -from platformio import util +from platformio import fs from platformio.compat import PY2, dump_json_to_unicode from platformio.managers.platform import PlatformBase from platformio.proc import get_pythonexe_path @@ -51,7 +51,7 @@ DEFAULT_ENV_OPTIONS = dict( "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform", "pioproject", "piowinhooks", "piolib", "pioupload", "piomisc", "pioide" ], - toolpath=[join(util.get_source_dir(), "builder", "tools")], + toolpath=[join(fs.get_source_dir(), "builder", "tools")], variables=clivars, # Propagating External Environment diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 66f9db57..73d681d9 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -31,7 +31,7 @@ from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import DefaultEnvironment # pylint: disable=import-error -from platformio import exception, util +from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool from platformio.compat import (WINDOWS, get_file_contents, hashlib_encode_data, string_types) @@ -261,7 +261,7 @@ class LibBuilderBase(object): return {} def process_extra_options(self): - with util.cd(self.path): + with fs.cd(self.path): self.env.ProcessFlags(self.build_flags) if self.extra_script: self.env.SConscriptChdir(1) @@ -533,7 +533,7 @@ class MbedLibBuilder(LibBuilderBase): def load_manifest(self): if not isfile(join(self.path, "module.json")): return {} - return util.load_json(join(self.path, "module.json")) + return fs.load_json(join(self.path, "module.json")) @property def include_dir(self): @@ -611,7 +611,7 @@ class MbedLibBuilder(LibBuilderBase): def _mbed_lib_conf_parse_macros(self, mbed_lib_path): macros = {} cppdefines = str(self.env.Flatten(self.env.subst("$CPPDEFINES"))) - manifest = util.load_json(mbed_lib_path) + manifest = fs.load_json(mbed_lib_path) # default macros for macro in manifest.get("macros", []): @@ -682,7 +682,7 @@ class PlatformIOLibBuilder(LibBuilderBase): def load_manifest(self): assert isfile(join(self.path, "library.json")) - manifest = util.load_json(join(self.path, "library.json")) + manifest = fs.load_json(join(self.path, "library.json")) assert "name" in manifest # replace "espressif" old name dev/platform with ESP8266 @@ -700,14 +700,14 @@ class PlatformIOLibBuilder(LibBuilderBase): @property def include_dir(self): if "includeDir" in self._manifest.get("build", {}): - with util.cd(self.path): + with fs.cd(self.path): return realpath(self._manifest.get("build").get("includeDir")) return LibBuilderBase.include_dir.fget(self) @property def src_dir(self): if "srcDir" in self._manifest.get("build", {}): - with util.cd(self.path): + with fs.cd(self.path): return realpath(self._manifest.get("build").get("srcDir")) return LibBuilderBase.src_dir.fget(self) @@ -1000,7 +1000,7 @@ def ConfigureProjectLibBuilder(env): def _get_vcs_info(lb): path = LibraryManager.get_src_manifest_path(lb.path) - return util.load_json(path) if path else None + return fs.load_json(path) if path else None def _correct_found_libs(lib_builders): # build full dependency graph diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 1e01d59e..d7e17b0b 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -24,7 +24,7 @@ from tempfile import mkstemp from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error -from platformio import util +from platformio import fs, util from platformio.compat import get_file_contents, glob_escape from platformio.managers.core import get_core_package_dir from platformio.proc import exec_command @@ -295,7 +295,7 @@ def PioClean(env, clean_dir): print("Removed %s" % (dst if clean_rel_path.startswith(".") else relpath(dst))) print("Done cleaning") - util.rmtree_(clean_dir) + fs.rmtree(clean_dir) env.Exit(0) @@ -333,7 +333,7 @@ def GetExtraScripts(env, scope): items.append(item[len(scope) + 1:]) if not items: return items - with util.cd(env.subst("$PROJECT_DIR")): + with fs.cd(env.subst("$PROJECT_DIR")): return [realpath(item) for item in items] diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 09d71287..8179982e 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -20,7 +20,7 @@ from os.path import isdir, isfile, join from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error -from platformio import exception, util +from platformio import exception, fs, util from platformio.compat import WINDOWS from platformio.managers.platform import PlatformFactory from platformio.project.config import ProjectOptions @@ -129,7 +129,7 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements src_manifest_path = platform.pm.get_src_manifest_path( platform.get_dir()) if src_manifest_path: - src_manifest = util.load_json(src_manifest_path) + src_manifest = fs.load_json(src_manifest_path) if "version" in src_manifest: data.append("#" + src_manifest['version']) if int(ARGUMENTS.get("PIOVERBOSE", 0)): @@ -152,7 +152,7 @@ def PrintConfiguration(env): # pylint: disable=too-many-statements ram = board_config.get("upload", {}).get("maximum_ram_size") flash = board_config.get("upload", {}).get("maximum_size") data.append("%s RAM, %s Flash" % - (util.format_filesize(ram), util.format_filesize(flash))) + (fs.format_filesize(ram), fs.format_filesize(flash))) return data def _get_debug_data(): diff --git a/platformio/builder/tools/pioupload.py b/platformio/builder/tools/pioupload.py index dd03894a..a7ea2f4e 100644 --- a/platformio/builder/tools/pioupload.py +++ b/platformio/builder/tools/pioupload.py @@ -25,7 +25,7 @@ from time import sleep from SCons.Script import ARGUMENTS # pylint: disable=import-error from serial import Serial, SerialException -from platformio import exception, util +from platformio import exception, fs, util from platformio.compat import WINDOWS from platformio.proc import exec_command @@ -156,7 +156,7 @@ def AutodetectUploadPort(*args, **kwargs): env.Replace(UPLOAD_PORT=_look_for_mbed_disk()) else: try: - util.ensure_udev_rules() + fs.ensure_udev_rules() except exception.InvalidUdevRules as e: sys.stderr.write("\n%s\n\n" % e) env.Replace(UPLOAD_PORT=_look_for_serial_port()) diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 6aff1681..8473e7f7 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -16,7 +16,7 @@ import json import click -from platformio import util +from platformio import fs from platformio.compat import dump_json_to_unicode from platformio.managers.platform import PlatformManager @@ -65,8 +65,8 @@ def print_boards(boards): BOARDLIST_TPL.format(type=click.style(board['id'], fg="cyan"), mcu=board['mcu'], frequency="%dMHz" % (board['fcpu'] / 1000000), - flash=util.format_filesize(board['rom']), - ram=util.format_filesize(board['ram']), + flash=fs.format_filesize(board['rom']), + ram=fs.format_filesize(board['ram']), name=board['name'])) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index 55ef07ad..8ad68c2b 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -20,7 +20,7 @@ from tempfile import mkdtemp import click -from platformio import app, util +from platformio import app, fs from platformio.commands.init import cli as cmd_init from platformio.commands.init import validate_boards from platformio.commands.run import cli as cmd_run @@ -89,7 +89,7 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches app.set_session_var("force_option", True) if not keep_build_dir and isdir(build_dir): - util.rmtree_(build_dir) + fs.rmtree(build_dir) if not isdir(build_dir): makedirs(build_dir) @@ -119,7 +119,7 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose) finally: if not keep_build_dir: - util.rmtree_(build_dir) + fs.rmtree(build_dir) def _copy_contents(dst_dir, contents): @@ -161,7 +161,7 @@ def _exclude_contents(dst_dir, patterns): for path in contents: path = abspath(path) if isdir(path): - util.rmtree_(path) + fs.rmtree(path) elif isfile(path): remove(path) diff --git a/platformio/commands/debug/client.py b/platformio/commands/debug/client.py index d17636c1..d40951a0 100644 --- a/platformio/commands/debug/client.py +++ b/platformio/commands/debug/client.py @@ -26,7 +26,7 @@ from twisted.internet import reactor # pylint: disable=import-error from twisted.internet import stdio # pylint: disable=import-error from twisted.internet import task # pylint: disable=import-error -from platformio import app, exception, util +from platformio import app, exception, fs, proc, util from platformio.commands.debug import helpers, initcfgs from platformio.commands.debug.process import BaseProcess from platformio.commands.debug.server import DebugServer @@ -198,7 +198,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes def processEnded(self, reason): # pylint: disable=unused-argument self._unlock_session() if self._gdbsrc_dir and isdir(self._gdbsrc_dir): - util.rmtree_(self._gdbsrc_dir) + fs.rmtree(self._gdbsrc_dir) if self._debug_server: self._debug_server.terminate() @@ -274,7 +274,7 @@ class GDBClient(BaseProcess): # pylint: disable=too-many-instance-attributes else: kill = ["kill", pid] try: - util.exec_command(kill) + proc.exec_command(kill) except: # pylint: disable=bare-except pass diff --git a/platformio/commands/debug/command.py b/platformio/commands/debug/command.py index 14c4f665..8e78e728 100644 --- a/platformio/commands/debug/command.py +++ b/platformio/commands/debug/command.py @@ -21,7 +21,7 @@ from os.path import isfile, join import click -from platformio import exception, util +from platformio import exception, fs, proc, util from platformio.commands.debug import helpers from platformio.managers.core import inject_contrib_pysite from platformio.project.config import ProjectConfig @@ -61,7 +61,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, if os.getenv(sysenv): project_dir = os.getenv(sysenv) - with util.cd(project_dir): + with fs.cd(project_dir): config = ProjectConfig.get_instance( project_conf or join(project_dir, "platformio.ini")) config.validate(envs=[environment] if environment else None) @@ -83,16 +83,14 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface, "Could not load debug configuration") if "--version" in __unprocessed: - result = util.exec_command([configuration['gdb_path'], "--version"]) + result = proc.exec_command([configuration['gdb_path'], "--version"]) if result['returncode'] == 0: return click.echo(result['out']) raise exception.PlatformioException("\n".join( [result['out'], result['err']])) try: - util.ensure_udev_rules() - except NameError: - pass + fs.ensure_udev_rules() except exception.InvalidUdevRules as e: for line in str(e).split("\n") + [""]: click.echo( diff --git a/platformio/commands/home/rpc/handlers/piocore.py b/platformio/commands/home/rpc/handlers/piocore.py index a88f495d..cb5651bf 100644 --- a/platformio/commands/home/rpc/handlers/piocore.py +++ b/platformio/commands/home/rpc/handlers/piocore.py @@ -24,7 +24,7 @@ import jsonrpc # pylint: disable=import-error from twisted.internet import threads # pylint: disable=import-error from twisted.internet import utils # pylint: disable=import-error -from platformio import __main__, __version__, util +from platformio import __main__, __version__, fs from platformio.commands.home import helpers from platformio.compat import (PY2, get_filesystem_encoding, is_bytes, string_types) @@ -88,7 +88,7 @@ class PIOCoreRPC(object): args[i] = str(arg) def _call_inline(): - with util.cd(cwd): + with fs.cd(cwd): exit_code = __main__.main(["-c"] + args) return (PIOCoreRPC.thread_stdout.get_value_and_reset(), PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code) diff --git a/platformio/commands/home/rpc/handlers/project.py b/platformio/commands/home/rpc/handlers/project.py index 4ca5af63..1f13d767 100644 --- a/platformio/commands/home/rpc/handlers/project.py +++ b/platformio/commands/home/rpc/handlers/project.py @@ -22,7 +22,7 @@ from os.path import (basename, expanduser, getmtime, isdir, isfile, join, import jsonrpc # pylint: disable=import-error -from platformio import exception, util +from platformio import exception, fs from platformio.commands.home.rpc.handlers.app import AppRPC from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC from platformio.compat import PY2, get_filesystem_encoding @@ -77,7 +77,7 @@ class ProjectRPC(object): data = {} boards = [] try: - with util.cd(project_dir): + with fs.cd(project_dir): data = _get_project_data(project_dir) except exception.PlatformIOProjectException: continue @@ -196,7 +196,7 @@ class ProjectRPC(object): ]) # yapf: disable if not main_content: return project_dir - with util.cd(project_dir): + with fs.cd(project_dir): src_dir = get_project_src_dir() main_path = join(src_dir, "main.cpp") if isfile(main_path): @@ -249,10 +249,10 @@ class ProjectRPC(object): @staticmethod def _finalize_arduino_import(_, project_dir, arduino_project_dir): - with util.cd(project_dir): + with fs.cd(project_dir): src_dir = get_project_src_dir() if isdir(src_dir): - util.rmtree_(src_dir) + fs.rmtree(src_dir) shutil.copytree(arduino_project_dir, src_dir) return project_dir diff --git a/platformio/commands/init.py b/platformio/commands/init.py index 09eddacf..7695a479 100644 --- a/platformio/commands/init.py +++ b/platformio/commands/init.py @@ -19,7 +19,7 @@ from os.path import isdir, isfile, join import click -from platformio import exception, util +from platformio import exception, fs from platformio.commands.platform import \ platform_install as cli_platform_install from platformio.ide.projectgenerator import ProjectGenerator @@ -156,7 +156,7 @@ def get_best_envname(project_dir, boards=None): def init_base_project(project_dir): ProjectConfig(join(project_dir, "platformio.ini")).save() - with util.cd(project_dir): + with fs.cd(project_dir): dir_to_readme = [ (get_project_src_dir(), None), (get_project_include_dir(), init_include_readme), diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 1d75c961..d7876abb 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -20,7 +20,7 @@ from os.path import isdir, join import click import semantic_version -from platformio import exception, util +from platformio import exception, fs, util from platformio.commands import PlatformioCLI from platformio.compat import dump_json_to_unicode from platformio.managers.lib import (LibraryManager, get_builtin_libs, @@ -99,7 +99,7 @@ def cli(ctx, **options): if not is_platformio_project(storage_dir): ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir) continue - with util.cd(storage_dir): + with fs.cd(storage_dir): libdeps_dir = get_project_libdeps_dir() config = ProjectConfig.get_instance(join(storage_dir, "platformio.ini")) diff --git a/platformio/commands/remote.py b/platformio/commands/remote.py index 8dcdf9a2..b5649979 100644 --- a/platformio/commands/remote.py +++ b/platformio/commands/remote.py @@ -21,7 +21,7 @@ from time import sleep import click -from platformio import exception, util +from platformio import exception, fs from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.compat import get_file_contents from platformio.managers.core import pioplus_call @@ -202,4 +202,4 @@ def device_monitor(ctx, **kwargs): ctx.invoke(cmd_device_monitor, **kwargs) t.join(2) finally: - util.rmtree_(sock_dir) + fs.rmtree(sock_dir) diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 543ec424..bc717fdf 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -19,7 +19,7 @@ from time import time import click -from platformio import exception, util +from platformio import exception, fs from platformio.commands.device import device_monitor as cmd_device_monitor from platformio.commands.run.helpers import (clean_build_dir, handle_legacy_libdeps, @@ -73,7 +73,7 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, if isfile(project_dir): project_dir = find_project_dir_above(project_dir) - with util.cd(project_dir): + with fs.cd(project_dir): config = ProjectConfig.get_instance( project_conf or join(project_dir, "platformio.ini")) config.validate(environment) diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index 3058acc1..2c5a0c2c 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -18,7 +18,7 @@ from time import time import click -from platformio import util +from platformio import fs from platformio.project.helpers import (compute_project_checksum, get_project_dir, get_project_libdeps_dir) @@ -47,7 +47,7 @@ def clean_build_dir(build_dir, config): # remove legacy ".pioenvs" folder legacy_build_dir = join(get_project_dir(), ".pioenvs") if isdir(legacy_build_dir) and legacy_build_dir != build_dir: - util.rmtree_(legacy_build_dir) + fs.rmtree(legacy_build_dir) checksum_file = join(build_dir, "project.checksum") checksum = compute_project_checksum(config) @@ -58,7 +58,7 @@ def clean_build_dir(build_dir, config): with open(checksum_file) as f: if f.read() == checksum: return - util.rmtree_(build_dir) + fs.rmtree(build_dir) makedirs(build_dir) with open(checksum_file, "w") as f: diff --git a/platformio/commands/test/command.py b/platformio/commands/test/command.py index d330a410..e4386ab9 100644 --- a/platformio/commands/test/command.py +++ b/platformio/commands/test/command.py @@ -21,7 +21,7 @@ from time import time import click -from platformio import exception, util +from platformio import exception, fs from platformio.commands.run.helpers import print_header from platformio.commands.test.embedded import EmbeddedTestProcessor from platformio.commands.test.native import NativeTestProcessor @@ -76,7 +76,7 @@ def cli( # pylint: disable=redefined-builtin ctx, environment, ignore, filter, upload_port, test_port, project_dir, project_conf, without_building, without_uploading, without_testing, no_reset, monitor_rts, monitor_dtr, verbose): - with util.cd(project_dir): + with fs.cd(project_dir): test_dir = get_project_test_dir() if not isdir(test_dir): raise exception.TestDirNotExists(test_dir) diff --git a/platformio/commands/test/native.py b/platformio/commands/test/native.py index 7367094f..ed46aaad 100644 --- a/platformio/commands/test/native.py +++ b/platformio/commands/test/native.py @@ -14,7 +14,7 @@ from os.path import join -from platformio import util +from platformio import fs, proc from platformio.commands.test.processor import TestProcessorBase from platformio.proc import LineBufferedAsyncPipe from platformio.project.helpers import get_project_build_dir @@ -33,9 +33,9 @@ class NativeTestProcessor(TestProcessorBase): return self.run() def run(self): - with util.cd(self.options['project_dir']): + with fs.cd(self.options['project_dir']): build_dir = get_project_build_dir() - result = util.exec_command( + result = proc.exec_command( [join(build_dir, self.env_name, "program")], stdout=LineBufferedAsyncPipe(self.on_run_out), stderr=LineBufferedAsyncPipe(self.on_run_out)) diff --git a/platformio/fs.py b/platformio/fs.py new file mode 100644 index 00000000..3d01b05d --- /dev/null +++ b/platformio/fs.py @@ -0,0 +1,120 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import shutil +import stat +import sys + +import click + +from platformio import exception +from platformio.compat import get_file_contents + + +class cd(object): + + def __init__(self, new_path): + self.new_path = new_path + self.prev_path = os.getcwd() + + def __enter__(self): + os.chdir(self.new_path) + + def __exit__(self, etype, value, traceback): + os.chdir(self.prev_path) + + +def get_source_dir(): + curpath = os.path.abspath(__file__) + if not os.path.isfile(curpath): + for p in sys.path: + if os.path.isfile(os.path.join(p, __file__)): + curpath = os.path.join(p, __file__) + break + return os.path.dirname(curpath) + + +def load_json(file_path): + try: + with open(file_path, "r") as f: + return json.load(f) + except ValueError: + raise exception.InvalidJSONFile(file_path) + + +def format_filesize(filesize): + base = 1024 + unit = 0 + suffix = "B" + filesize = float(filesize) + if filesize < base: + return "%d%s" % (filesize, suffix) + for i, suffix in enumerate("KMGTPEZY"): + unit = base**(i + 2) + if filesize >= unit: + continue + if filesize % (base**(i + 1)): + return "%.2f%sB" % ((base * filesize / unit), suffix) + break + return "%d%sB" % ((base * filesize / unit), suffix) + + +def ensure_udev_rules(): + from platformio.util import get_systype + + def _rules_to_set(rules_path): + return set(l.strip() for l in get_file_contents(rules_path).split("\n") + if l.strip() and not l.startswith("#")) + + if "linux" not in get_systype(): + return None + installed_rules = [ + "/etc/udev/rules.d/99-platformio-udev.rules", + "/lib/udev/rules.d/99-platformio-udev.rules" + ] + if not any(os.path.isfile(p) for p in installed_rules): + raise exception.MissedUdevRules + + origin_path = os.path.abspath( + os.path.join(get_source_dir(), "..", "scripts", + "99-platformio-udev.rules")) + if not os.path.isfile(origin_path): + return None + + origin_rules = _rules_to_set(origin_path) + for rules_path in installed_rules: + if not os.path.isfile(rules_path): + continue + current_rules = _rules_to_set(rules_path) + if not origin_rules <= current_rules: + raise exception.OutdatedUdevRules(rules_path) + + return True + + +def rmtree(path): + + def _onerror(_, name, __): + try: + os.chmod(name, stat.S_IWRITE) + os.remove(name) + except Exception as e: # pylint: disable=broad-except + click.secho("%s \nPlease manually remove the file `%s`" % + (str(e), name), + fg="red", + err=True) + + return shutil.rmtree(path, onerror=_onerror) diff --git a/platformio/ide/projectgenerator.py b/platformio/ide/projectgenerator.py index debdeeab..363e7522 100644 --- a/platformio/ide/projectgenerator.py +++ b/platformio/ide/projectgenerator.py @@ -20,7 +20,7 @@ from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath import bottle -from platformio import util +from platformio import fs, util from platformio.compat import WINDOWS, get_file_contents from platformio.proc import where_is_program from platformio.project.config import ProjectConfig @@ -39,7 +39,7 @@ class ProjectGenerator(object): @staticmethod def get_supported_ides(): - tpls_dir = join(util.get_source_dir(), "ide", "tpls") + tpls_dir = join(fs.get_source_dir(), "ide", "tpls") return sorted( [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))]) @@ -54,7 +54,7 @@ class ProjectGenerator(object): tpl_vars.update( load_project_ide_data(self.project_dir, self.env_name) or {}) - with util.cd(self.project_dir): + with fs.cd(self.project_dir): tpl_vars.update({ "project_name": basename(self.project_dir), "src_files": self.get_src_files(), @@ -79,7 +79,7 @@ class ProjectGenerator(object): def get_src_files(self): result = [] - with util.cd(self.project_dir): + with fs.cd(self.project_dir): for root, _, files in os.walk(get_project_src_dir()): for f in files: result.append(relpath(join(root, f))) @@ -87,7 +87,7 @@ class ProjectGenerator(object): def get_tpls(self): tpls = [] - tpls_dir = join(util.get_source_dir(), "ide", "tpls", self.ide) + tpls_dir = join(fs.get_source_dir(), "ide", "tpls", self.ide) for root, _, files in os.walk(tpls_dir): for f in files: if not f.endswith(".tpl"): diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 56712c25..012644d1 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -19,7 +19,7 @@ from time import time import click import semantic_version -from platformio import __version__, app, exception, telemetry, util +from platformio import __version__, app, exception, fs, telemetry, util from platformio.commands import PlatformioCLI from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib import lib_update as cmd_lib_update @@ -208,7 +208,7 @@ def check_platformio_upgrade(): fg="cyan", nl=False) click.secho("`.", fg="yellow") - elif join("Cellar", "platformio") in util.get_source_dir(): + elif join("Cellar", "platformio") in fs.get_source_dir(): click.secho("brew update && brew upgrade", fg="cyan", nl=False) click.secho("` command.", fg="yellow") else: diff --git a/platformio/managers/core.py b/platformio/managers/core.py index 11e4b5fc..74d8afe3 100644 --- a/platformio/managers/core.py +++ b/platformio/managers/core.py @@ -17,7 +17,7 @@ import subprocess import sys from os.path import dirname, join -from platformio import __version__, exception, util +from platformio import __version__, exception, fs from platformio.compat import PY2, WINDOWS from platformio.managers.package import PackageManager from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path @@ -120,7 +120,7 @@ def pioplus_call(args, **kwargs): pythonexe_path = get_pythonexe_path() os.environ['PYTHONEXEPATH'] = pythonexe_path os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite") - os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "") + os.environ['PIOCOREPYSITEDIR'] = dirname(fs.get_source_dir() or "") if dirname(pythonexe_path) not in os.environ['PATH'].split(os.pathsep): os.environ['PATH'] = (os.pathsep).join( [dirname(pythonexe_path), os.environ['PATH']]) diff --git a/platformio/managers/package.py b/platformio/managers/package.py index 82043114..9d99b1a3 100644 --- a/platformio/managers/package.py +++ b/platformio/managers/package.py @@ -25,7 +25,7 @@ import click import requests import semantic_version -from platformio import __version__, app, exception, telemetry, util +from platformio import __version__, app, exception, fs, telemetry, util from platformio.compat import hashlib_encode_data from platformio.downloader import FileDownloader from platformio.lockfile import LockFile @@ -359,13 +359,13 @@ class PkgInstallerMixin(object): manifest_path = self.get_manifest_path(pkg_dir) src_manifest_path = self.get_src_manifest_path(pkg_dir) if src_manifest_path: - src_manifest = util.load_json(src_manifest_path) + src_manifest = fs.load_json(src_manifest_path) if not manifest_path and not src_manifest_path: return None if manifest_path and manifest_path.endswith(".json"): - manifest = util.load_json(manifest_path) + manifest = fs.load_json(manifest_path) elif manifest_path and manifest_path.endswith(".properties"): with codecs.open(manifest_path, encoding="utf-8") as fp: for line in fp.readlines(): @@ -498,7 +498,7 @@ class PkgInstallerMixin(object): if isfile(_url): self.unpack(_url, tmp_dir) else: - util.rmtree_(tmp_dir) + fs.rmtree(tmp_dir) shutil.copytree(_url, tmp_dir) elif url.startswith(("http://", "https://")): dlpath = self.download(url, tmp_dir, sha1) @@ -523,7 +523,7 @@ class PkgInstallerMixin(object): return self._install_from_tmp_dir(_tmp_dir, requirements) finally: if isdir(tmp_dir): - util.rmtree_(tmp_dir) + fs.rmtree(tmp_dir) return None def _update_src_manifest(self, data, src_dir): @@ -532,7 +532,7 @@ class PkgInstallerMixin(object): src_manifest_path = join(src_dir, self.SRC_MANIFEST_NAME) _data = {} if isfile(src_manifest_path): - _data = util.load_json(src_manifest_path) + _data = fs.load_json(src_manifest_path) _data.update(data) with open(src_manifest_path, "w") as fp: json.dump(_data, fp) @@ -602,7 +602,7 @@ class PkgInstallerMixin(object): # remove previous/not-satisfied package if isdir(pkg_dir): - util.rmtree_(pkg_dir) + fs.rmtree(pkg_dir) shutil.move(tmp_dir, pkg_dir) assert isdir(pkg_dir) self.cache_reset() @@ -768,7 +768,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin): if islink(pkg_dir): os.unlink(pkg_dir) else: - util.rmtree_(pkg_dir) + fs.rmtree(pkg_dir) self.cache_reset() # unfix package with the same name diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index b8fbdf7c..996d7162 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -22,7 +22,7 @@ from os.path import basename, dirname, isdir, isfile, join import click import semantic_version -from platformio import __version__, app, exception, util +from platformio import __version__, app, exception, fs, util from platformio.compat import PY2, hashlib_encode_data, is_bytes from platformio.managers.core import get_core_package_dir from platformio.managers.package import BasePkgManager, PackageManager @@ -237,7 +237,7 @@ class PlatformFactory(object): name = pm.load_manifest(platform_dir)['name'] elif name.endswith("platform.json") and isfile(name): platform_dir = dirname(name) - name = util.load_json(name)['name'] + name = fs.load_json(name)['name'] else: name, requirements, url = pm.parse_pkg_uri(name, requirements) platform_dir = pm.get_package_dir(name, requirements, url) @@ -404,7 +404,7 @@ class PlatformRunMixin(object): join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q", "--warn=no-no-parallel-support", "--jobs", str(jobs), - "--sconstruct", join(util.get_source_dir(), "builder", "main.py") + "--sconstruct", join(fs.get_source_dir(), "builder", "main.py") ] # yapf: disable args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) # pylint: disable=protected-access @@ -494,7 +494,7 @@ class PlatformBase( # pylint: disable=too-many-public-methods self.verbose = False self._BOARDS_CACHE = {} - self._manifest = util.load_json(manifest_path) + self._manifest = fs.load_json(manifest_path) self._custom_packages = None self.pm = PackageManager(get_project_packages_dir(), @@ -693,7 +693,7 @@ class PlatformBoardConfig(object): assert isfile(manifest_path) self.manifest_path = manifest_path try: - self._manifest = util.load_json(manifest_path) + self._manifest = fs.load_json(manifest_path) except ValueError: raise exception.InvalidBoardManifest(manifest_path) if not set(["name", "url", "vendor"]) <= set(self._manifest): diff --git a/platformio/util.py b/platformio/util.py index f2c97c2b..b3109323 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -17,35 +17,25 @@ import os import platform import re import socket -import stat import sys import time from contextlib import contextmanager from functools import wraps from glob import glob -from os.path import abspath, basename, dirname, isfile, join -from shutil import rmtree import click import requests from platformio import __apiurl__, __version__, exception from platformio.commands import PlatformioCLI -from platformio.compat import PY2, WINDOWS, get_file_contents -from platformio.proc import exec_command, is_ci +from platformio.compat import PY2, WINDOWS +from platformio.fs import cd # pylint: disable=unused-import +from platformio.fs import load_json # pylint: disable=unused-import +from platformio.fs import rmtree as rmtree_ # pylint: disable=unused-import +from platformio.proc import exec_command # pylint: disable=unused-import +from platformio.proc import is_ci # pylint: disable=unused-import - -class cd(object): - - def __init__(self, new_path): - self.new_path = new_path - self.prev_path = os.getcwd() - - def __enter__(self): - os.chdir(self.new_path) - - def __exit__(self, etype, value, traceback): - os.chdir(self.prev_path) +# KEEP unused imports for backward compatibility with PIO Core 3.0 API class memoized(object): @@ -119,14 +109,6 @@ def capture_std_streams(stdout, stderr=None): sys.stderr = _stderr -def load_json(file_path): - try: - with open(file_path, "r") as f: - return json.load(f) - except ValueError: - raise exception.InvalidJSONFile(file_path) - - def get_systype(): type_ = platform.system().lower() arch = platform.machine().lower() @@ -141,16 +123,6 @@ def pioversion_to_intstr(): return [int(i) for i in vermatch.group(1).split(".")[:3]] -def get_source_dir(): - curpath = abspath(__file__) - if not isfile(curpath): - for p in sys.path: - if isfile(join(p, __file__)): - curpath = join(p, __file__) - break - return dirname(curpath) - - def change_filemtime(path, mtime): os.utime(path, (mtime, mtime)) @@ -221,7 +193,7 @@ def get_logical_devices(): continue items.append({ "path": match.group(1), - "name": basename(match.group(1)) + "name": os.path.basename(match.group(1)) }) return items @@ -461,23 +433,6 @@ def parse_date(datestr): return time.strptime(datestr) -def format_filesize(filesize): - base = 1024 - unit = 0 - suffix = "B" - filesize = float(filesize) - if filesize < base: - return "%d%s" % (filesize, suffix) - for i, suffix in enumerate("KMGTPEZY"): - unit = base**(i + 2) - if filesize >= unit: - continue - if filesize % (base**(i + 1)): - return "%.2f%sB" % ((base * filesize / unit), suffix) - break - return "%d%sB" % ((base * filesize / unit), suffix) - - def merge_dicts(d1, d2, path=None): if path is None: path = [] @@ -490,37 +445,6 @@ def merge_dicts(d1, d2, path=None): return d1 -def ensure_udev_rules(): - - def _rules_to_set(rules_path): - return set(l.strip() for l in get_file_contents(rules_path).split("\n") - if l.strip() and not l.startswith("#")) - - if "linux" not in get_systype(): - return None - installed_rules = [ - "/etc/udev/rules.d/99-platformio-udev.rules", - "/lib/udev/rules.d/99-platformio-udev.rules" - ] - if not any(isfile(p) for p in installed_rules): - raise exception.MissedUdevRules - - origin_path = abspath( - join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules")) - if not isfile(origin_path): - return None - - origin_rules = _rules_to_set(origin_path) - for rules_path in installed_rules: - if not isfile(rules_path): - continue - current_rules = _rules_to_set(rules_path) - if not origin_rules <= current_rules: - raise exception.OutdatedUdevRules(rules_path) - - return True - - def get_original_version(version): if version.count(".") != 2: return None @@ -530,18 +454,3 @@ def get_original_version(version): if int(raw) <= 9999: return "%s.%s" % (raw[:-2], int(raw[-2:])) return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:])) - - -def rmtree_(path): - - def _onerror(_, name, __): - try: - os.chmod(name, stat.S_IWRITE) - os.remove(name) - except Exception as e: # pylint: disable=broad-except - click.secho("%s \nPlease manually remove the file `%s`" % - (str(e), name), - fg="red", - err=True) - - return rmtree(path, onerror=_onerror)