Move FS related helpers to fs.py module

platformio/app.py

@@ -22,7 +22,7 @@ from time import time
 
 import requests
 
-from platformio import exception, lockfile, util
+from platformio import exception, fs, lockfile
 from platformio.compat import (WINDOWS, dump_json_to_unicode,
                                hashlib_encode_data)
 from platformio.proc import is_ci
@@ -113,7 +113,7 @@ class State(object):
         try:
             self._lock_state_file()
             if isfile(self.path):
-                self._storage = util.load_json(self.path)
+                self._storage = fs.load_json(self.path)
             assert isinstance(self._storage, dict)
         except (AssertionError, ValueError, UnicodeDecodeError,
                 exception.InvalidJSONFile):
@@ -290,7 +290,7 @@ class ContentCache(object):
                     try:
                         remove(path)
                         if not listdir(dirname(path)):
-                            util.rmtree_(dirname(path))
+                            fs.rmtree(dirname(path))
                     except OSError:
                         pass
 
@@ -304,7 +304,7 @@ class ContentCache(object):
     def clean(self):
         if not self.cache_dir or not isdir(self.cache_dir):
             return
-        util.rmtree_(self.cache_dir)
+        fs.rmtree(self.cache_dir)
 
 
 def clean_cache():

platformio/builder/main.py

@@ -27,7 +27,7 @@ from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 from SCons.Script import Import  # pylint: disable=import-error
 from SCons.Script import Variables  # pylint: disable=import-error
 
-from platformio import util
+from platformio import fs
 from platformio.compat import PY2, dump_json_to_unicode
 from platformio.managers.platform import PlatformBase
 from platformio.proc import get_pythonexe_path
@@ -51,7 +51,7 @@ DEFAULT_ENV_OPTIONS = dict(
         "ar", "gas", "gcc", "g++", "gnulink", "platformio", "pioplatform",
         "pioproject", "piowinhooks", "piolib", "pioupload", "piomisc", "pioide"
     ],
-    toolpath=[join(util.get_source_dir(), "builder", "tools")],
+    toolpath=[join(fs.get_source_dir(), "builder", "tools")],
     variables=clivars,
 
     # Propagating External Environment

platformio/builder/tools/piolib.py

@@ -31,7 +31,7 @@ from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 from SCons.Script import DefaultEnvironment  # pylint: disable=import-error
 
-from platformio import exception, util
+from platformio import exception, fs, util
 from platformio.builder.tools import platformio as piotool
 from platformio.compat import (WINDOWS, get_file_contents, hashlib_encode_data,
                                string_types)
@@ -261,7 +261,7 @@ class LibBuilderBase(object):
         return {}
 
     def process_extra_options(self):
-        with util.cd(self.path):
+        with fs.cd(self.path):
             self.env.ProcessFlags(self.build_flags)
             if self.extra_script:
                 self.env.SConscriptChdir(1)
@@ -533,7 +533,7 @@ class MbedLibBuilder(LibBuilderBase):
     def load_manifest(self):
         if not isfile(join(self.path, "module.json")):
             return {}
-        return util.load_json(join(self.path, "module.json"))
+        return fs.load_json(join(self.path, "module.json"))
 
     @property
     def include_dir(self):
@@ -611,7 +611,7 @@ class MbedLibBuilder(LibBuilderBase):
     def _mbed_lib_conf_parse_macros(self, mbed_lib_path):
         macros = {}
         cppdefines = str(self.env.Flatten(self.env.subst("$CPPDEFINES")))
-        manifest = util.load_json(mbed_lib_path)
+        manifest = fs.load_json(mbed_lib_path)
 
         # default macros
         for macro in manifest.get("macros", []):
@@ -682,7 +682,7 @@ class PlatformIOLibBuilder(LibBuilderBase):
 
     def load_manifest(self):
         assert isfile(join(self.path, "library.json"))
-        manifest = util.load_json(join(self.path, "library.json"))
+        manifest = fs.load_json(join(self.path, "library.json"))
         assert "name" in manifest
 
         # replace "espressif" old name dev/platform with ESP8266
@@ -700,14 +700,14 @@ class PlatformIOLibBuilder(LibBuilderBase):
     @property
     def include_dir(self):
         if "includeDir" in self._manifest.get("build", {}):
-            with util.cd(self.path):
+            with fs.cd(self.path):
                 return realpath(self._manifest.get("build").get("includeDir"))
         return LibBuilderBase.include_dir.fget(self)
 
     @property
     def src_dir(self):
         if "srcDir" in self._manifest.get("build", {}):
-            with util.cd(self.path):
+            with fs.cd(self.path):
                 return realpath(self._manifest.get("build").get("srcDir"))
         return LibBuilderBase.src_dir.fget(self)
 
@@ -1000,7 +1000,7 @@ def ConfigureProjectLibBuilder(env):
 
     def _get_vcs_info(lb):
         path = LibraryManager.get_src_manifest_path(lb.path)
-        return util.load_json(path) if path else None
+        return fs.load_json(path) if path else None
 
     def _correct_found_libs(lib_builders):
         # build full dependency graph

platformio/builder/tools/piomisc.py

@@ -24,7 +24,7 @@ from tempfile import mkstemp
 from SCons.Action import Action  # pylint: disable=import-error
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 
-from platformio import util
+from platformio import fs, util
 from platformio.compat import get_file_contents, glob_escape
 from platformio.managers.core import get_core_package_dir
 from platformio.proc import exec_command
@@ -295,7 +295,7 @@ def PioClean(env, clean_dir):
             print("Removed %s" %
                   (dst if clean_rel_path.startswith(".") else relpath(dst)))
     print("Done cleaning")
-    util.rmtree_(clean_dir)
+    fs.rmtree(clean_dir)
     env.Exit(0)
 
 
@@ -333,7 +333,7 @@ def GetExtraScripts(env, scope):
             items.append(item[len(scope) + 1:])
     if not items:
         return items
-    with util.cd(env.subst("$PROJECT_DIR")):
+    with fs.cd(env.subst("$PROJECT_DIR")):
         return [realpath(item) for item in items]
 
 

platformio/builder/tools/pioplatform.py

@@ -20,7 +20,7 @@ from os.path import isdir, isfile, join
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 
-from platformio import exception, util
+from platformio import exception, fs, util
 from platformio.compat import WINDOWS
 from platformio.managers.platform import PlatformFactory
 from platformio.project.config import ProjectOptions
@@ -129,7 +129,7 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
         src_manifest_path = platform.pm.get_src_manifest_path(
             platform.get_dir())
         if src_manifest_path:
-            src_manifest = util.load_json(src_manifest_path)
+            src_manifest = fs.load_json(src_manifest_path)
             if "version" in src_manifest:
                 data.append("#" + src_manifest['version'])
             if int(ARGUMENTS.get("PIOVERBOSE", 0)):
@@ -152,7 +152,7 @@ def PrintConfiguration(env):  # pylint: disable=too-many-statements
         ram = board_config.get("upload", {}).get("maximum_ram_size")
         flash = board_config.get("upload", {}).get("maximum_size")
         data.append("%s RAM, %s Flash" %
-                    (util.format_filesize(ram), util.format_filesize(flash)))
+                    (fs.format_filesize(ram), fs.format_filesize(flash)))
         return data
 
     def _get_debug_data():

platformio/builder/tools/pioupload.py

@@ -25,7 +25,7 @@ from time import sleep
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from serial import Serial, SerialException
 
-from platformio import exception, util
+from platformio import exception, fs, util
 from platformio.compat import WINDOWS
 from platformio.proc import exec_command
 
@@ -156,7 +156,7 @@ def AutodetectUploadPort(*args, **kwargs):
         env.Replace(UPLOAD_PORT=_look_for_mbed_disk())
     else:
         try:
-            util.ensure_udev_rules()
+            fs.ensure_udev_rules()
         except exception.InvalidUdevRules as e:
             sys.stderr.write("\n%s\n\n" % e)
         env.Replace(UPLOAD_PORT=_look_for_serial_port())

platformio/commands/boards.py

@@ -16,7 +16,7 @@ import json
 
 import click
 
-from platformio import util
+from platformio import fs
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.platform import PlatformManager
 
@@ -65,8 +65,8 @@ def print_boards(boards):
             BOARDLIST_TPL.format(type=click.style(board['id'], fg="cyan"),
                                  mcu=board['mcu'],
                                  frequency="%dMHz" % (board['fcpu'] / 1000000),
-                                 flash=util.format_filesize(board['rom']),
+                                 flash=fs.format_filesize(board['rom']),
-                                 ram=util.format_filesize(board['ram']),
+                                 ram=fs.format_filesize(board['ram']),
                                  name=board['name']))
 
 

platformio/commands/ci.py

@@ -20,7 +20,7 @@ from tempfile import mkdtemp
 
 import click
 
-from platformio import app, util
+from platformio import app, fs
 from platformio.commands.init import cli as cmd_init
 from platformio.commands.init import validate_boards
 from platformio.commands.run import cli as cmd_run
@@ -89,7 +89,7 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
         app.set_session_var("force_option", True)
 
         if not keep_build_dir and isdir(build_dir):
-            util.rmtree_(build_dir)
+            fs.rmtree(build_dir)
         if not isdir(build_dir):
             makedirs(build_dir)
 
@@ -119,7 +119,7 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
         ctx.invoke(cmd_run, project_dir=build_dir, verbose=verbose)
     finally:
         if not keep_build_dir:
-            util.rmtree_(build_dir)
+            fs.rmtree(build_dir)
 
 
 def _copy_contents(dst_dir, contents):
@@ -161,7 +161,7 @@ def _exclude_contents(dst_dir, patterns):
     for path in contents:
         path = abspath(path)
         if isdir(path):
-            util.rmtree_(path)
+            fs.rmtree(path)
         elif isfile(path):
             remove(path)
 

platformio/commands/debug/client.py

@@ -26,7 +26,7 @@ from twisted.internet import reactor  # pylint: disable=import-error
 from twisted.internet import stdio  # pylint: disable=import-error
 from twisted.internet import task  # pylint: disable=import-error
 
-from platformio import app, exception, util
+from platformio import app, exception, fs, proc, util
 from platformio.commands.debug import helpers, initcfgs
 from platformio.commands.debug.process import BaseProcess
 from platformio.commands.debug.server import DebugServer
@@ -198,7 +198,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
     def processEnded(self, reason):  # pylint: disable=unused-argument
         self._unlock_session()
         if self._gdbsrc_dir and isdir(self._gdbsrc_dir):
-            util.rmtree_(self._gdbsrc_dir)
+            fs.rmtree(self._gdbsrc_dir)
         if self._debug_server:
             self._debug_server.terminate()
 
@@ -274,7 +274,7 @@ class GDBClient(BaseProcess):  # pylint: disable=too-many-instance-attributes
         else:
             kill = ["kill", pid]
         try:
-            util.exec_command(kill)
+            proc.exec_command(kill)
         except:  # pylint: disable=bare-except
             pass
 

platformio/commands/debug/command.py

@@ -21,7 +21,7 @@ from os.path import isfile, join
 
 import click
 
-from platformio import exception, util
+from platformio import exception, fs, proc, util
 from platformio.commands.debug import helpers
 from platformio.managers.core import inject_contrib_pysite
 from platformio.project.config import ProjectConfig
@@ -61,7 +61,7 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface,
         if os.getenv(sysenv):
             project_dir = os.getenv(sysenv)
 
-    with util.cd(project_dir):
+    with fs.cd(project_dir):
         config = ProjectConfig.get_instance(
             project_conf or join(project_dir, "platformio.ini"))
         config.validate(envs=[environment] if environment else None)
@@ -83,16 +83,14 @@ def cli(ctx, project_dir, project_conf, environment, verbose, interface,
             "Could not load debug configuration")
 
     if "--version" in __unprocessed:
-        result = util.exec_command([configuration['gdb_path'], "--version"])
+        result = proc.exec_command([configuration['gdb_path'], "--version"])
         if result['returncode'] == 0:
             return click.echo(result['out'])
         raise exception.PlatformioException("\n".join(
             [result['out'], result['err']]))
 
     try:
-        util.ensure_udev_rules()
+        fs.ensure_udev_rules()
-    except NameError:
-        pass
     except exception.InvalidUdevRules as e:
         for line in str(e).split("\n") + [""]:
             click.echo(

platformio/commands/home/rpc/handlers/piocore.py

@@ -24,7 +24,7 @@ import jsonrpc  # pylint: disable=import-error
 from twisted.internet import threads  # pylint: disable=import-error
 from twisted.internet import utils  # pylint: disable=import-error
 
-from platformio import __main__, __version__, util
+from platformio import __main__, __version__, fs
 from platformio.commands.home import helpers
 from platformio.compat import (PY2, get_filesystem_encoding, is_bytes,
                                string_types)
@@ -88,7 +88,7 @@ class PIOCoreRPC(object):
                 args[i] = str(arg)
 
         def _call_inline():
-            with util.cd(cwd):
+            with fs.cd(cwd):
                 exit_code = __main__.main(["-c"] + args)
             return (PIOCoreRPC.thread_stdout.get_value_and_reset(),
                     PIOCoreRPC.thread_stderr.get_value_and_reset(), exit_code)

platformio/commands/home/rpc/handlers/project.py

@@ -22,7 +22,7 @@ from os.path import (basename, expanduser, getmtime, isdir, isfile, join,
 
 import jsonrpc  # pylint: disable=import-error
 
-from platformio import exception, util
+from platformio import exception, fs
 from platformio.commands.home.rpc.handlers.app import AppRPC
 from platformio.commands.home.rpc.handlers.piocore import PIOCoreRPC
 from platformio.compat import PY2, get_filesystem_encoding
@@ -77,7 +77,7 @@ class ProjectRPC(object):
             data = {}
             boards = []
             try:
-                with util.cd(project_dir):
+                with fs.cd(project_dir):
                     data = _get_project_data(project_dir)
             except exception.PlatformIOProjectException:
                 continue
@@ -196,7 +196,7 @@ class ProjectRPC(object):
             ])   # yapf: disable
         if not main_content:
             return project_dir
-        with util.cd(project_dir):
+        with fs.cd(project_dir):
             src_dir = get_project_src_dir()
             main_path = join(src_dir, "main.cpp")
             if isfile(main_path):
@@ -249,10 +249,10 @@ class ProjectRPC(object):
 
     @staticmethod
     def _finalize_arduino_import(_, project_dir, arduino_project_dir):
-        with util.cd(project_dir):
+        with fs.cd(project_dir):
             src_dir = get_project_src_dir()
             if isdir(src_dir):
-                util.rmtree_(src_dir)
+                fs.rmtree(src_dir)
             shutil.copytree(arduino_project_dir, src_dir)
         return project_dir
 

platformio/commands/init.py

@@ -19,7 +19,7 @@ from os.path import isdir, isfile, join
 
 import click
 
-from platformio import exception, util
+from platformio import exception, fs
 from platformio.commands.platform import \
     platform_install as cli_platform_install
 from platformio.ide.projectgenerator import ProjectGenerator
@@ -156,7 +156,7 @@ def get_best_envname(project_dir, boards=None):
 
 def init_base_project(project_dir):
     ProjectConfig(join(project_dir, "platformio.ini")).save()
-    with util.cd(project_dir):
+    with fs.cd(project_dir):
         dir_to_readme = [
             (get_project_src_dir(), None),
             (get_project_include_dir(), init_include_readme),

platformio/commands/lib.py

@@ -20,7 +20,7 @@ from os.path import isdir, join
 import click
 import semantic_version
 
-from platformio import exception, util
+from platformio import exception, fs, util
 from platformio.commands import PlatformioCLI
 from platformio.compat import dump_json_to_unicode
 from platformio.managers.lib import (LibraryManager, get_builtin_libs,
@@ -99,7 +99,7 @@ def cli(ctx, **options):
         if not is_platformio_project(storage_dir):
             ctx.meta[CTX_META_STORAGE_DIRS_KEY].append(storage_dir)
             continue
-        with util.cd(storage_dir):
+        with fs.cd(storage_dir):
             libdeps_dir = get_project_libdeps_dir()
         config = ProjectConfig.get_instance(join(storage_dir,
                                                  "platformio.ini"))

platformio/commands/remote.py

@@ -21,7 +21,7 @@ from time import sleep
 
 import click
 
-from platformio import exception, util
+from platformio import exception, fs
 from platformio.commands.device import device_monitor as cmd_device_monitor
 from platformio.compat import get_file_contents
 from platformio.managers.core import pioplus_call
@@ -202,4 +202,4 @@ def device_monitor(ctx, **kwargs):
         ctx.invoke(cmd_device_monitor, **kwargs)
         t.join(2)
     finally:
-        util.rmtree_(sock_dir)
+        fs.rmtree(sock_dir)

platformio/commands/run/command.py

@@ -19,7 +19,7 @@ from time import time
 
 import click
 
-from platformio import exception, util
+from platformio import exception, fs
 from platformio.commands.device import device_monitor as cmd_device_monitor
 from platformio.commands.run.helpers import (clean_build_dir,
                                              handle_legacy_libdeps,
@@ -73,7 +73,7 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs,
     if isfile(project_dir):
         project_dir = find_project_dir_above(project_dir)
 
-    with util.cd(project_dir):
+    with fs.cd(project_dir):
         config = ProjectConfig.get_instance(
             project_conf or join(project_dir, "platformio.ini"))
         config.validate(environment)

platformio/commands/run/helpers.py

@@ -18,7 +18,7 @@ from time import time
 
 import click
 
-from platformio import util
+from platformio import fs
 from platformio.project.helpers import (compute_project_checksum,
                                         get_project_dir,
                                         get_project_libdeps_dir)
@@ -47,7 +47,7 @@ def clean_build_dir(build_dir, config):
     # remove legacy ".pioenvs" folder
     legacy_build_dir = join(get_project_dir(), ".pioenvs")
     if isdir(legacy_build_dir) and legacy_build_dir != build_dir:
-        util.rmtree_(legacy_build_dir)
+        fs.rmtree(legacy_build_dir)
 
     checksum_file = join(build_dir, "project.checksum")
     checksum = compute_project_checksum(config)
@@ -58,7 +58,7 @@ def clean_build_dir(build_dir, config):
             with open(checksum_file) as f:
                 if f.read() == checksum:
                     return
-        util.rmtree_(build_dir)
+        fs.rmtree(build_dir)
 
     makedirs(build_dir)
     with open(checksum_file, "w") as f:

platformio/commands/test/command.py

@@ -21,7 +21,7 @@ from time import time
 
 import click
 
-from platformio import exception, util
+from platformio import exception, fs
 from platformio.commands.run.helpers import print_header
 from platformio.commands.test.embedded import EmbeddedTestProcessor
 from platformio.commands.test.native import NativeTestProcessor
@@ -76,7 +76,7 @@ def cli(  # pylint: disable=redefined-builtin
         ctx, environment, ignore, filter, upload_port, test_port, project_dir,
         project_conf, without_building, without_uploading, without_testing,
         no_reset, monitor_rts, monitor_dtr, verbose):
-    with util.cd(project_dir):
+    with fs.cd(project_dir):
         test_dir = get_project_test_dir()
         if not isdir(test_dir):
             raise exception.TestDirNotExists(test_dir)

platformio/commands/test/native.py

@@ -14,7 +14,7 @@
 
 from os.path import join
 
-from platformio import util
+from platformio import fs, proc
 from platformio.commands.test.processor import TestProcessorBase
 from platformio.proc import LineBufferedAsyncPipe
 from platformio.project.helpers import get_project_build_dir
@@ -33,9 +33,9 @@ class NativeTestProcessor(TestProcessorBase):
         return self.run()
 
     def run(self):
-        with util.cd(self.options['project_dir']):
+        with fs.cd(self.options['project_dir']):
             build_dir = get_project_build_dir()
-        result = util.exec_command(
+        result = proc.exec_command(
             [join(build_dir, self.env_name, "program")],
             stdout=LineBufferedAsyncPipe(self.on_run_out),
             stderr=LineBufferedAsyncPipe(self.on_run_out))

platformio/fs.py

@@ -0,0 +1,120 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import shutil
+import stat
+import sys
+
+import click
+
+from platformio import exception
+from platformio.compat import get_file_contents
+
+
+class cd(object):
+
+    def __init__(self, new_path):
+        self.new_path = new_path
+        self.prev_path = os.getcwd()
+
+    def __enter__(self):
+        os.chdir(self.new_path)
+
+    def __exit__(self, etype, value, traceback):
+        os.chdir(self.prev_path)
+
+
+def get_source_dir():
+    curpath = os.path.abspath(__file__)
+    if not os.path.isfile(curpath):
+        for p in sys.path:
+            if os.path.isfile(os.path.join(p, __file__)):
+                curpath = os.path.join(p, __file__)
+                break
+    return os.path.dirname(curpath)
+
+
+def load_json(file_path):
+    try:
+        with open(file_path, "r") as f:
+            return json.load(f)
+    except ValueError:
+        raise exception.InvalidJSONFile(file_path)
+
+
+def format_filesize(filesize):
+    base = 1024
+    unit = 0
+    suffix = "B"
+    filesize = float(filesize)
+    if filesize < base:
+        return "%d%s" % (filesize, suffix)
+    for i, suffix in enumerate("KMGTPEZY"):
+        unit = base**(i + 2)
+        if filesize >= unit:
+            continue
+        if filesize % (base**(i + 1)):
+            return "%.2f%sB" % ((base * filesize / unit), suffix)
+        break
+    return "%d%sB" % ((base * filesize / unit), suffix)
+
+
+def ensure_udev_rules():
+    from platformio.util import get_systype
+
+    def _rules_to_set(rules_path):
+        return set(l.strip() for l in get_file_contents(rules_path).split("\n")
+                   if l.strip() and not l.startswith("#"))
+
+    if "linux" not in get_systype():
+        return None
+    installed_rules = [
+        "/etc/udev/rules.d/99-platformio-udev.rules",
+        "/lib/udev/rules.d/99-platformio-udev.rules"
+    ]
+    if not any(os.path.isfile(p) for p in installed_rules):
+        raise exception.MissedUdevRules
+
+    origin_path = os.path.abspath(
+        os.path.join(get_source_dir(), "..", "scripts",
+                     "99-platformio-udev.rules"))
+    if not os.path.isfile(origin_path):
+        return None
+
+    origin_rules = _rules_to_set(origin_path)
+    for rules_path in installed_rules:
+        if not os.path.isfile(rules_path):
+            continue
+        current_rules = _rules_to_set(rules_path)
+        if not origin_rules <= current_rules:
+            raise exception.OutdatedUdevRules(rules_path)
+
+    return True
+
+
+def rmtree(path):
+
+    def _onerror(_, name, __):
+        try:
+            os.chmod(name, stat.S_IWRITE)
+            os.remove(name)
+        except Exception as e:  # pylint: disable=broad-except
+            click.secho("%s \nPlease manually remove the file `%s`" %
+                        (str(e), name),
+                        fg="red",
+                        err=True)
+
+    return shutil.rmtree(path, onerror=_onerror)

platformio/ide/projectgenerator.py

@@ -20,7 +20,7 @@ from os.path import abspath, basename, expanduser, isdir, isfile, join, relpath
 
 import bottle
 
-from platformio import util
+from platformio import fs, util
 from platformio.compat import WINDOWS, get_file_contents
 from platformio.proc import where_is_program
 from platformio.project.config import ProjectConfig
@@ -39,7 +39,7 @@ class ProjectGenerator(object):
 
     @staticmethod
     def get_supported_ides():
-        tpls_dir = join(util.get_source_dir(), "ide", "tpls")
+        tpls_dir = join(fs.get_source_dir(), "ide", "tpls")
         return sorted(
             [d for d in os.listdir(tpls_dir) if isdir(join(tpls_dir, d))])
 
@@ -54,7 +54,7 @@ class ProjectGenerator(object):
         tpl_vars.update(
             load_project_ide_data(self.project_dir, self.env_name) or {})
 
-        with util.cd(self.project_dir):
+        with fs.cd(self.project_dir):
             tpl_vars.update({
                 "project_name": basename(self.project_dir),
                 "src_files": self.get_src_files(),
@@ -79,7 +79,7 @@ class ProjectGenerator(object):
 
     def get_src_files(self):
         result = []
-        with util.cd(self.project_dir):
+        with fs.cd(self.project_dir):
             for root, _, files in os.walk(get_project_src_dir()):
                 for f in files:
                     result.append(relpath(join(root, f)))
@@ -87,7 +87,7 @@ class ProjectGenerator(object):
 
     def get_tpls(self):
         tpls = []
-        tpls_dir = join(util.get_source_dir(), "ide", "tpls", self.ide)
+        tpls_dir = join(fs.get_source_dir(), "ide", "tpls", self.ide)
         for root, _, files in os.walk(tpls_dir):
             for f in files:
                 if not f.endswith(".tpl"):

platformio/maintenance.py

@@ -19,7 +19,7 @@ from time import time
 import click
 import semantic_version
 
-from platformio import __version__, app, exception, telemetry, util
+from platformio import __version__, app, exception, fs, telemetry, util
 from platformio.commands import PlatformioCLI
 from platformio.commands.lib import CTX_META_STORAGE_DIRS_KEY
 from platformio.commands.lib import lib_update as cmd_lib_update
@@ -208,7 +208,7 @@ def check_platformio_upgrade():
                     fg="cyan",
                     nl=False)
         click.secho("`.", fg="yellow")
-    elif join("Cellar", "platformio") in util.get_source_dir():
+    elif join("Cellar", "platformio") in fs.get_source_dir():
         click.secho("brew update && brew upgrade", fg="cyan", nl=False)
         click.secho("` command.", fg="yellow")
     else:

platformio/managers/core.py

@@ -17,7 +17,7 @@ import subprocess
 import sys
 from os.path import dirname, join
 
-from platformio import __version__, exception, util
+from platformio import __version__, exception, fs
 from platformio.compat import PY2, WINDOWS
 from platformio.managers.package import PackageManager
 from platformio.proc import copy_pythonpath_to_osenv, get_pythonexe_path
@@ -120,7 +120,7 @@ def pioplus_call(args, **kwargs):
     pythonexe_path = get_pythonexe_path()
     os.environ['PYTHONEXEPATH'] = pythonexe_path
     os.environ['PYTHONPYSITEDIR'] = get_core_package_dir("contrib-pysite")
-    os.environ['PIOCOREPYSITEDIR'] = dirname(util.get_source_dir() or "")
+    os.environ['PIOCOREPYSITEDIR'] = dirname(fs.get_source_dir() or "")
     if dirname(pythonexe_path) not in os.environ['PATH'].split(os.pathsep):
         os.environ['PATH'] = (os.pathsep).join(
             [dirname(pythonexe_path), os.environ['PATH']])

platformio/managers/package.py

@@ -25,7 +25,7 @@ import click
 import requests
 import semantic_version
 
-from platformio import __version__, app, exception, telemetry, util
+from platformio import __version__, app, exception, fs, telemetry, util
 from platformio.compat import hashlib_encode_data
 from platformio.downloader import FileDownloader
 from platformio.lockfile import LockFile
@@ -359,13 +359,13 @@ class PkgInstallerMixin(object):
         manifest_path = self.get_manifest_path(pkg_dir)
         src_manifest_path = self.get_src_manifest_path(pkg_dir)
         if src_manifest_path:
-            src_manifest = util.load_json(src_manifest_path)
+            src_manifest = fs.load_json(src_manifest_path)
 
         if not manifest_path and not src_manifest_path:
             return None
 
         if manifest_path and manifest_path.endswith(".json"):
-            manifest = util.load_json(manifest_path)
+            manifest = fs.load_json(manifest_path)
         elif manifest_path and manifest_path.endswith(".properties"):
             with codecs.open(manifest_path, encoding="utf-8") as fp:
                 for line in fp.readlines():
@@ -498,7 +498,7 @@ class PkgInstallerMixin(object):
                 if isfile(_url):
                     self.unpack(_url, tmp_dir)
                 else:
-                    util.rmtree_(tmp_dir)
+                    fs.rmtree(tmp_dir)
                     shutil.copytree(_url, tmp_dir)
             elif url.startswith(("http://", "https://")):
                 dlpath = self.download(url, tmp_dir, sha1)
@@ -523,7 +523,7 @@ class PkgInstallerMixin(object):
             return self._install_from_tmp_dir(_tmp_dir, requirements)
         finally:
             if isdir(tmp_dir):
-                util.rmtree_(tmp_dir)
+                fs.rmtree(tmp_dir)
         return None
 
     def _update_src_manifest(self, data, src_dir):
@@ -532,7 +532,7 @@ class PkgInstallerMixin(object):
         src_manifest_path = join(src_dir, self.SRC_MANIFEST_NAME)
         _data = {}
         if isfile(src_manifest_path):
-            _data = util.load_json(src_manifest_path)
+            _data = fs.load_json(src_manifest_path)
         _data.update(data)
         with open(src_manifest_path, "w") as fp:
             json.dump(_data, fp)
@@ -602,7 +602,7 @@ class PkgInstallerMixin(object):
 
         # remove previous/not-satisfied package
         if isdir(pkg_dir):
-            util.rmtree_(pkg_dir)
+            fs.rmtree(pkg_dir)
         shutil.move(tmp_dir, pkg_dir)
         assert isdir(pkg_dir)
         self.cache_reset()
@@ -768,7 +768,7 @@ class BasePkgManager(PkgRepoMixin, PkgInstallerMixin):
             if islink(pkg_dir):
                 os.unlink(pkg_dir)
             else:
-                util.rmtree_(pkg_dir)
+                fs.rmtree(pkg_dir)
             self.cache_reset()
 
             # unfix package with the same name

platformio/managers/platform.py

@@ -22,7 +22,7 @@ from os.path import basename, dirname, isdir, isfile, join
 import click
 import semantic_version
 
-from platformio import __version__, app, exception, util
+from platformio import __version__, app, exception, fs, util
 from platformio.compat import PY2, hashlib_encode_data, is_bytes
 from platformio.managers.core import get_core_package_dir
 from platformio.managers.package import BasePkgManager, PackageManager
@@ -237,7 +237,7 @@ class PlatformFactory(object):
             name = pm.load_manifest(platform_dir)['name']
         elif name.endswith("platform.json") and isfile(name):
             platform_dir = dirname(name)
-            name = util.load_json(name)['name']
+            name = fs.load_json(name)['name']
         else:
             name, requirements, url = pm.parse_pkg_uri(name, requirements)
             platform_dir = pm.get_package_dir(name, requirements, url)
@@ -404,7 +404,7 @@ class PlatformRunMixin(object):
             join(get_core_package_dir("tool-scons"), "script", "scons"),
             "-Q", "--warn=no-no-parallel-support",
             "--jobs", str(jobs),
-            "--sconstruct", join(util.get_source_dir(), "builder", "main.py")
+            "--sconstruct", join(fs.get_source_dir(), "builder", "main.py")
         ]  # yapf: disable
         args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
         # pylint: disable=protected-access
@@ -494,7 +494,7 @@ class PlatformBase(  # pylint: disable=too-many-public-methods
         self.verbose = False
 
         self._BOARDS_CACHE = {}
-        self._manifest = util.load_json(manifest_path)
+        self._manifest = fs.load_json(manifest_path)
         self._custom_packages = None
 
         self.pm = PackageManager(get_project_packages_dir(),
@@ -693,7 +693,7 @@ class PlatformBoardConfig(object):
         assert isfile(manifest_path)
         self.manifest_path = manifest_path
         try:
-            self._manifest = util.load_json(manifest_path)
+            self._manifest = fs.load_json(manifest_path)
         except ValueError:
             raise exception.InvalidBoardManifest(manifest_path)
         if not set(["name", "url", "vendor"]) <= set(self._manifest):

platformio/util.py

@@ -17,35 +17,25 @@ import os
 import platform
 import re
 import socket
-import stat
 import sys
 import time
 from contextlib import contextmanager
 from functools import wraps
 from glob import glob
-from os.path import abspath, basename, dirname, isfile, join
-from shutil import rmtree
 
 import click
 import requests
 
 from platformio import __apiurl__, __version__, exception
 from platformio.commands import PlatformioCLI
-from platformio.compat import PY2, WINDOWS, get_file_contents
+from platformio.compat import PY2, WINDOWS
-from platformio.proc import exec_command, is_ci
+from platformio.fs import cd  # pylint: disable=unused-import
+from platformio.fs import load_json  # pylint: disable=unused-import
+from platformio.fs import rmtree as rmtree_  # pylint: disable=unused-import
+from platformio.proc import exec_command  # pylint: disable=unused-import
+from platformio.proc import is_ci  # pylint: disable=unused-import
 
-
+# KEEP unused imports for backward compatibility with PIO Core 3.0 API
-class cd(object):
-
-    def __init__(self, new_path):
-        self.new_path = new_path
-        self.prev_path = os.getcwd()
-
-    def __enter__(self):
-        os.chdir(self.new_path)
-
-    def __exit__(self, etype, value, traceback):
-        os.chdir(self.prev_path)
 
 
 class memoized(object):
@@ -119,14 +109,6 @@ def capture_std_streams(stdout, stderr=None):
     sys.stderr = _stderr
 
 
-def load_json(file_path):
-    try:
-        with open(file_path, "r") as f:
-            return json.load(f)
-    except ValueError:
-        raise exception.InvalidJSONFile(file_path)
-
-
 def get_systype():
     type_ = platform.system().lower()
     arch = platform.machine().lower()
@@ -141,16 +123,6 @@ def pioversion_to_intstr():
     return [int(i) for i in vermatch.group(1).split(".")[:3]]
 
 
-def get_source_dir():
-    curpath = abspath(__file__)
-    if not isfile(curpath):
-        for p in sys.path:
-            if isfile(join(p, __file__)):
-                curpath = join(p, __file__)
-                break
-    return dirname(curpath)
-
-
 def change_filemtime(path, mtime):
     os.utime(path, (mtime, mtime))
 
@@ -221,7 +193,7 @@ def get_logical_devices():
             continue
         items.append({
             "path": match.group(1),
-            "name": basename(match.group(1))
+            "name": os.path.basename(match.group(1))
         })
     return items
 
@@ -461,23 +433,6 @@ def parse_date(datestr):
     return time.strptime(datestr)
 
 
-def format_filesize(filesize):
-    base = 1024
-    unit = 0
-    suffix = "B"
-    filesize = float(filesize)
-    if filesize < base:
-        return "%d%s" % (filesize, suffix)
-    for i, suffix in enumerate("KMGTPEZY"):
-        unit = base**(i + 2)
-        if filesize >= unit:
-            continue
-        if filesize % (base**(i + 1)):
-            return "%.2f%sB" % ((base * filesize / unit), suffix)
-        break
-    return "%d%sB" % ((base * filesize / unit), suffix)
-
-
 def merge_dicts(d1, d2, path=None):
     if path is None:
         path = []
@@ -490,37 +445,6 @@ def merge_dicts(d1, d2, path=None):
     return d1
 
 
-def ensure_udev_rules():
-
-    def _rules_to_set(rules_path):
-        return set(l.strip() for l in get_file_contents(rules_path).split("\n")
-                   if l.strip() and not l.startswith("#"))
-
-    if "linux" not in get_systype():
-        return None
-    installed_rules = [
-        "/etc/udev/rules.d/99-platformio-udev.rules",
-        "/lib/udev/rules.d/99-platformio-udev.rules"
-    ]
-    if not any(isfile(p) for p in installed_rules):
-        raise exception.MissedUdevRules
-
-    origin_path = abspath(
-        join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules"))
-    if not isfile(origin_path):
-        return None
-
-    origin_rules = _rules_to_set(origin_path)
-    for rules_path in installed_rules:
-        if not isfile(rules_path):
-            continue
-        current_rules = _rules_to_set(rules_path)
-        if not origin_rules <= current_rules:
-            raise exception.OutdatedUdevRules(rules_path)
-
-    return True
-
-
 def get_original_version(version):
     if version.count(".") != 2:
         return None
@@ -530,18 +454,3 @@ def get_original_version(version):
     if int(raw) <= 9999:
         return "%s.%s" % (raw[:-2], int(raw[-2:]))
     return "%s.%s.%s" % (raw[:-4], int(raw[-4:-2]), int(raw[-2:]))
-
-
-def rmtree_(path):
-
-    def _onerror(_, name, __):
-        try:
-            os.chmod(name, stat.S_IWRITE)
-            os.remove(name)
-        except Exception as e:  # pylint: disable=broad-except
-            click.secho("%s \nPlease manually remove the file `%s`" %
-                        (str(e), name),
-                        fg="red",
-                        err=True)
-
-    return rmtree(path, onerror=_onerror)