Refactor dev-platform API

Makefile

@@ -27,7 +27,7 @@ clean: clean-docs
 
 profile:
 	# Usage $ > make PIOARGS="boards" profile
-	python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS}
+	python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS}
 	snakeviz .tox/.tmp/cprofile.prof
 
 publish:

platformio/builder/main.py

@@ -30,7 +30,7 @@ from SCons.Script import Variables  # pylint: disable=import-error
 
 from platformio import compat, fs
 from platformio.compat import dump_json_to_unicode
-from platformio.managers.platform import PlatformBase
+from platformio.platform.base import PlatformBase
 from platformio.proc import get_pythonexe_path
 from platformio.project.helpers import get_project_dir
 

platformio/builder/tools/pioplatform.py

@@ -14,15 +14,16 @@
 
 from __future__ import absolute_import
 
+import os
 import sys
-from os.path import isdir, isfile, join
 
 from SCons.Script import ARGUMENTS  # pylint: disable=import-error
 from SCons.Script import COMMAND_LINE_TARGETS  # pylint: disable=import-error
 
-from platformio import exception, fs, util
+from platformio import fs, util
 from platformio.compat import WINDOWS
-from platformio.managers.platform import PlatformFactory
+from platformio.platform.exception import UnknownBoard
+from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectOptions
 
 # pylint: disable=too-many-branches, too-many-locals
@@ -34,7 +35,7 @@ def PioPlatform(env):
     if "framework" in variables:
         # support PIO Core 3.0 dev/platforms
         variables["pioframework"] = variables["framework"]
-    p = PlatformFactory.newPlatform(env["PLATFORM_MANIFEST"])
+    p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"]))
     p.configure_default_packages(variables, COMMAND_LINE_TARGETS)
     return p
 
@@ -46,7 +47,7 @@ def BoardConfig(env, board=None):
             board = board or env.get("BOARD")
             assert board, "BoardConfig: Board is not defined"
             return p.board_config(board)
-        except (AssertionError, exception.UnknownBoard) as e:
+        except (AssertionError, UnknownBoard) as e:
             sys.stderr.write("Error: %s\n" % str(e))
             env.Exit(1)
 
@@ -55,8 +56,8 @@ def GetFrameworkScript(env, framework):
     p = env.PioPlatform()
     assert p.frameworks and framework in p.frameworks
     script_path = env.subst(p.frameworks[framework]["script"])
-    if not isfile(script_path):
+    if not os.path.isfile(script_path):
-        script_path = join(p.get_dir(), script_path)
+        script_path = os.path.join(p.get_dir(), script_path)
     return script_path
 
 
@@ -75,17 +76,24 @@ def LoadPioPlatform(env):
             continue
         pkg_dir = p.get_package_dir(name)
         env.PrependENVPath(
-            "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir
+            "PATH",
+            os.path.join(pkg_dir, "bin")
+            if os.path.isdir(os.path.join(pkg_dir, "bin"))
+            else pkg_dir,
         )
-        if not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain":
+        if (
+            not WINDOWS
+            and os.path.isdir(os.path.join(pkg_dir, "lib"))
+            and type_ != "toolchain"
+        ):
             env.PrependENVPath(
                 "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH",
-                join(pkg_dir, "lib"),
+                os.path.join(pkg_dir, "lib"),
             )
 
     # Platform specific LD Scripts
-    if isdir(join(p.get_dir(), "ldscripts")):
+    if os.path.isdir(os.path.join(p.get_dir(), "ldscripts")):
-        env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")])
+        env.Prepend(LIBPATH=[os.path.join(p.get_dir(), "ldscripts")])
 
     if "BOARD" not in env:
         return

platformio/commands/debug/helpers.py

@@ -20,13 +20,14 @@ from hashlib import sha1
 from io import BytesIO
 from os.path import isfile
 
-from platformio import exception, fs, util
+from platformio import fs, util
 from platformio.commands import PlatformioCLI
 from platformio.commands.debug.exception import DebugInvalidOptionsError
 from platformio.commands.platform import platform_install as cmd_platform_install
 from platformio.commands.run.command import cli as cmd_run
 from platformio.compat import is_bytes
-from platformio.managers.platform import PlatformFactory
+from platformio.platform.exception import UnknownPlatform
+from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.options import ProjectOptions
 
@@ -94,14 +95,14 @@ def validate_debug_options(cmd_ctx, env_options):
         return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items]
 
     try:
-        platform = PlatformFactory.newPlatform(env_options["platform"])
+        platform = PlatformFactory.new(env_options["platform"])
-    except exception.UnknownPlatform:
+    except UnknownPlatform:
         cmd_ctx.invoke(
             cmd_platform_install,
             platforms=[env_options["platform"]],
             skip_default_package=True,
         )
-        platform = PlatformFactory.newPlatform(env_options["platform"])
+        platform = PlatformFactory.new(env_options["platform"])
 
     board_config = platform.board_config(env_options["board"])
     tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool"))

platformio/commands/device/command.py

@@ -22,7 +22,7 @@ from serial.tools import miniterm
 from platformio import exception, fs, util
 from platformio.commands.device import helpers as device_helpers
 from platformio.compat import dump_json_to_unicode
-from platformio.managers.platform import PlatformFactory
+from platformio.platform.factory import PlatformFactory
 from platformio.project.exception import NotPlatformIOProjectError
 
 
@@ -192,7 +192,7 @@ def device_monitor(**kwargs):  # pylint: disable=too-many-branches
     platform = None
     if "platform" in project_options:
         with fs.cd(kwargs["project_dir"]):
-            platform = PlatformFactory.newPlatform(project_options["platform"])
+            platform = PlatformFactory.new(project_options["platform"])
             device_helpers.register_platform_filters(
                 platform, kwargs["project_dir"], kwargs["environment"]
             )

platformio/commands/lib/helpers.py

@@ -15,8 +15,9 @@
 import os
 
 from platformio.compat import ci_strings_are_equal
-from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.managers.platform import PlatformManager
 from platformio.package.meta import PackageSpec
+from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
 from platformio.project.exception import InvalidProjectConfError
 
@@ -29,7 +30,7 @@ def get_builtin_libs(storage_names=None):
     storage_names = storage_names or []
     pm = PlatformManager()
     for manifest in pm.get_installed():
-        p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
+        p = PlatformFactory.new(manifest["__pkg_dir"])
         for storage in p.get_lib_storages():
             if storage_names and storage["name"] not in storage_names:
                 continue

platformio/commands/platform.py

@@ -16,10 +16,12 @@ from os.path import dirname, isdir
 
 import click
 
-from platformio import app, exception, util
+from platformio import app, util
 from platformio.commands.boards import print_boards
 from platformio.compat import dump_json_to_unicode
-from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.managers.platform import PlatformManager
+from platformio.platform.exception import UnknownPlatform
+from platformio.platform.factory import PlatformFactory
 
 
 @click.group(short_help="Platform Manager")
@@ -64,12 +66,12 @@ def _get_registry_platforms():
 def _get_platform_data(*args, **kwargs):
     try:
         return _get_installed_platform_data(*args, **kwargs)
-    except exception.UnknownPlatform:
+    except UnknownPlatform:
         return _get_registry_platform_data(*args, **kwargs)
 
 
 def _get_installed_platform_data(platform, with_boards=True, expose_packages=True):
-    p = PlatformFactory.newPlatform(platform)
+    p = PlatformFactory.new(platform)
     data = dict(
         name=p.name,
         title=p.title,
@@ -232,7 +234,7 @@ def platform_list(json_output):
 def platform_show(platform, json_output):  # pylint: disable=too-many-branches
     data = _get_platform_data(platform)
     if not data:
-        raise exception.UnknownPlatform(platform)
+        raise UnknownPlatform(platform)
     if json_output:
         return click.echo(dump_json_to_unicode(data))
 
@@ -384,10 +386,7 @@ def platform_update(  # pylint: disable=too-many-locals
             if not pkg_dir:
                 continue
             latest = pm.outdated(pkg_dir, requirements)
-            if (
+            if not latest and not PlatformFactory.new(pkg_dir).are_outdated_packages():
-                not latest
-                and not PlatformFactory.newPlatform(pkg_dir).are_outdated_packages()
-            ):
                 continue
             data = _get_installed_platform_data(
                 pkg_dir, with_boards=False, expose_packages=False

platformio/commands/project.py

@@ -20,10 +20,11 @@ import os
 import click
 from tabulate import tabulate
 
-from platformio import exception, fs
+from platformio import fs
 from platformio.commands.platform import platform_install as cli_platform_install
 from platformio.ide.projectgenerator import ProjectGenerator
 from platformio.managers.platform import PlatformManager
+from platformio.platform.exception import UnknownBoard
 from platformio.project.config import ProjectConfig
 from platformio.project.exception import NotPlatformIOProjectError
 from platformio.project.helpers import is_platformio_project, load_project_ide_data
@@ -112,7 +113,7 @@ def validate_boards(ctx, param, value):  # pylint: disable=W0613
     for id_ in value:
         try:
             pm.board_config(id_)
-        except exception.UnknownBoard:
+        except UnknownBoard:
             raise click.BadParameter(
                 "`%s`. Please search for board ID using `platformio boards` "
                 "command" % id_

platformio/commands/run/processor.py

@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from platformio import exception
 from platformio.commands.platform import platform_install as cmd_platform_install
 from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME
-from platformio.managers.platform import PlatformFactory
+from platformio.platform.exception import UnknownPlatform
+from platformio.platform.factory import PlatformFactory
 from platformio.project.exception import UndefinedEnvPlatformError
 
 # pylint: disable=too-many-instance-attributes
@@ -67,14 +67,14 @@ class EnvironmentProcessor(object):
             build_targets.remove("monitor")
 
         try:
-            p = PlatformFactory.newPlatform(self.options["platform"])
+            p = PlatformFactory.new(self.options["platform"])
-        except exception.UnknownPlatform:
+        except UnknownPlatform:
             self.cmd_ctx.invoke(
                 cmd_platform_install,
                 platforms=[self.options["platform"]],
                 skip_default_package=True,
             )
-            p = PlatformFactory.newPlatform(self.options["platform"])
+            p = PlatformFactory.new(self.options["platform"])
 
         result = p.run(build_vars, build_targets, self.silent, self.verbose, self.jobs)
         return result["returncode"] == 0

platformio/commands/test/embedded.py

@@ -19,7 +19,7 @@ import serial
 
 from platformio import exception, util
 from platformio.commands.test.processor import TestProcessorBase
-from platformio.managers.platform import PlatformFactory
+from platformio.platform.factory import PlatformFactory
 
 
 class EmbeddedTestProcessor(TestProcessorBase):
@@ -108,7 +108,7 @@ class EmbeddedTestProcessor(TestProcessorBase):
             return self.env_options.get("test_port")
 
         assert set(["platform", "board"]) & set(self.env_options.keys())
-        p = PlatformFactory.newPlatform(self.env_options["platform"])
+        p = PlatformFactory.new(self.env_options["platform"])
         board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", [])
         port = None
         elapsed = 0

platformio/exception.py

@@ -47,44 +47,6 @@ class AbortedByUser(UserSideException):
     MESSAGE = "Aborted by user"
 
 
-#
-# Development Platform
-#
-
-
-class UnknownPlatform(PlatformioException):
-
-    MESSAGE = "Unknown development platform '{0}'"
-
-
-class IncompatiblePlatform(PlatformioException):
-
-    MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}"
-
-
-class PlatformNotInstalledYet(PlatformioException):
-
-    MESSAGE = (
-        "The platform '{0}' has not been installed yet. "
-        "Use `platformio platform install {0}` command"
-    )
-
-
-class UnknownBoard(PlatformioException):
-
-    MESSAGE = "Unknown board ID '{0}'"
-
-
-class InvalidBoardManifest(PlatformioException):
-
-    MESSAGE = "Invalid board JSON manifest '{0}'"
-
-
-class UnknownFramework(PlatformioException):
-
-    MESSAGE = "Unknown framework '{0}'"
-
-
 # Package Manager
 
 
@@ -195,11 +157,6 @@ class InternetIsOffline(UserSideException):
     )
 
 
-class BuildScriptNotFound(PlatformioException):
-
-    MESSAGE = "Invalid path '{0}' to build script"
-
-
 class InvalidSettingName(UserSideException):
 
     MESSAGE = "Invalid setting with the name '{0}'"

platformio/maintenance.py

@@ -25,11 +25,12 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY
 from platformio.commands.lib.command import lib_update as cmd_lib_update
 from platformio.commands.platform import platform_update as cmd_platform_update
 from platformio.commands.upgrade import get_latest_version
-from platformio.managers.platform import PlatformFactory, PlatformManager
+from platformio.managers.platform import PlatformManager
 from platformio.package.manager.core import update_core_packages
 from platformio.package.manager.library import LibraryPackageManager
 from platformio.package.manager.tool import ToolPackageManager
 from platformio.package.meta import PackageSpec
+from platformio.platform.factory import PlatformFactory
 from platformio.proc import is_container
 
 
@@ -278,9 +279,7 @@ def check_internal_updates(ctx, what):  # pylint: disable=too-many-branches
             conds = [
                 pm.outdated(manifest["__pkg_dir"]),
                 what == "platforms"
-                and PlatformFactory.newPlatform(
+                and PlatformFactory.new(manifest["__pkg_dir"]).are_outdated_packages(),
-                    manifest["__pkg_dir"]
-                ).are_outdated_packages(),
             ]
             if any(conds):
                 outdated_items.append(manifest["name"])

platformio/managers/platform.py

@@ -14,31 +14,16 @@
 
 # pylint: disable=too-many-public-methods, too-many-instance-attributes
 
-import base64
-import os
-import re
-import subprocess
-import sys
-from os.path import basename, dirname, isdir, isfile, join
 
-import click
+from os.path import isdir, isfile, join
-import semantic_version
 
-from platformio import __version__, app, exception, fs, proc, telemetry, util
+from platformio import app, exception, util
-from platformio.commands.debug.exception import (
-    DebugInvalidOptionsError,
-    DebugSupportError,
-)
-from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module
 from platformio.managers.package import BasePkgManager, PackageManager
-from platformio.package.manager.core import get_core_package_dir
+from platformio.platform.base import PlatformBase  # pylint: disable=unused-import
+from platformio.platform.exception import UnknownBoard, UnknownPlatform
+from platformio.platform.factory import PlatformFactory
 from platformio.project.config import ProjectConfig
 
-try:
-    from urllib.parse import quote
-except ImportError:
-    from urllib import quote
-
 
 class PlatformManager(BasePkgManager):
     def __init__(self, package_dir=None, repositories=None):
@@ -83,7 +68,7 @@ class PlatformManager(BasePkgManager):
         platform_dir = BasePkgManager.install(
             self, name, requirements, silent=silent, force=force
         )
-        p = PlatformFactory.newPlatform(platform_dir)
+        p = PlatformFactory.new(platform_dir)
 
         if with_all_packages:
             with_packages = list(p.packages.keys())
@@ -114,9 +99,9 @@ class PlatformManager(BasePkgManager):
             pkg_dir = self.get_package_dir(name, requirements, url)
 
         if not pkg_dir:
-            raise exception.UnknownPlatform(package)
+            raise UnknownPlatform(package)
 
-        p = PlatformFactory.newPlatform(pkg_dir)
+        p = PlatformFactory.new(pkg_dir)
         BasePkgManager.uninstall(self, pkg_dir, requirements)
         p.uninstall_python_packages()
         p.on_uninstalled()
@@ -138,15 +123,15 @@ class PlatformManager(BasePkgManager):
             pkg_dir = self.get_package_dir(name, requirements, url)
 
         if not pkg_dir:
-            raise exception.UnknownPlatform(package)
+            raise UnknownPlatform(package)
 
-        p = PlatformFactory.newPlatform(pkg_dir)
+        p = PlatformFactory.new(pkg_dir)
         pkgs_before = list(p.get_installed_packages())
 
         missed_pkgs = set()
         if not only_packages:
             BasePkgManager.update(self, pkg_dir, requirements, only_check)
-            p = PlatformFactory.newPlatform(pkg_dir)
+            p = PlatformFactory.new(pkg_dir)
             missed_pkgs = set(pkgs_before) & set(p.packages)
             missed_pkgs -= set(p.get_installed_packages())
 
@@ -164,7 +149,7 @@ class PlatformManager(BasePkgManager):
         self.cache_reset()
         deppkgs = {}
         for manifest in PlatformManager().get_installed():
-            p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
+            p = PlatformFactory.new(manifest["__pkg_dir"])
             for pkgname, pkgmanifest in p.get_installed_packages().items():
                 if pkgname not in deppkgs:
                     deppkgs[pkgname] = set()
@@ -190,7 +175,7 @@ class PlatformManager(BasePkgManager):
     def get_installed_boards(self):
         boards = []
         for manifest in self.get_installed():
-            p = PlatformFactory.newPlatform(manifest["__pkg_dir"])
+            p = PlatformFactory.new(manifest["__pkg_dir"])
             for config in p.get_boards().values():
                 board = config.get_brief_data()
                 if board not in boards:
@@ -224,705 +209,4 @@ class PlatformManager(BasePkgManager):
                 not platform or manifest["platform"] == platform
             ):
                 return manifest
-        raise exception.UnknownBoard(id_)
+        raise UnknownBoard(id_)
-
-
-class PlatformFactory(object):
-    @staticmethod
-    def get_clsname(name):
-        name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I)
-        return "%s%sPlatform" % (name.upper()[0], name.lower()[1:])
-
-    @staticmethod
-    def load_module(name, path):
-        try:
-            return load_python_module("platformio.managers.platform.%s" % name, path)
-        except ImportError:
-            raise exception.UnknownPlatform(name)
-
-    @classmethod
-    def newPlatform(cls, name, requirements=None):
-        pm = PlatformManager()
-        platform_dir = None
-        if isdir(name):
-            platform_dir = name
-            name = pm.load_manifest(platform_dir)["name"]
-        elif name.endswith("platform.json") and isfile(name):
-            platform_dir = dirname(name)
-            name = fs.load_json(name)["name"]
-        else:
-            name, requirements, url = pm.parse_pkg_uri(name, requirements)
-            platform_dir = pm.get_package_dir(name, requirements, url)
-            if platform_dir:
-                name = pm.load_manifest(platform_dir)["name"]
-
-        if not platform_dir:
-            raise exception.UnknownPlatform(
-                name if not requirements else "%s@%s" % (name, requirements)
-            )
-
-        platform_cls = None
-        if isfile(join(platform_dir, "platform.py")):
-            platform_cls = getattr(
-                cls.load_module(name, join(platform_dir, "platform.py")),
-                cls.get_clsname(name),
-            )
-        else:
-            platform_cls = type(str(cls.get_clsname(name)), (PlatformBase,), {})
-
-        _instance = platform_cls(join(platform_dir, "platform.json"))
-        assert isinstance(_instance, PlatformBase)
-        return _instance
-
-
-class PlatformPackagesMixin(object):
-    def install_packages(  # pylint: disable=too-many-arguments
-        self,
-        with_packages=None,
-        without_packages=None,
-        skip_default_package=False,
-        silent=False,
-        force=False,
-    ):
-        with_packages = set(self.find_pkg_names(with_packages or []))
-        without_packages = set(self.find_pkg_names(without_packages or []))
-
-        upkgs = with_packages | without_packages
-        ppkgs = set(self.packages)
-        if not upkgs.issubset(ppkgs):
-            raise exception.UnknownPackage(", ".join(upkgs - ppkgs))
-
-        for name, opts in self.packages.items():
-            version = opts.get("version", "")
-            if name in without_packages:
-                continue
-            if name in with_packages or not (
-                skip_default_package or opts.get("optional", False)
-            ):
-                if ":" in version:
-                    self.pm.install(
-                        "%s=%s" % (name, version), silent=silent, force=force
-                    )
-                else:
-                    self.pm.install(name, version, silent=silent, force=force)
-
-        return True
-
-    def find_pkg_names(self, candidates):
-        result = []
-        for candidate in candidates:
-            found = False
-
-            # lookup by package types
-            for _name, _opts in self.packages.items():
-                if _opts.get("type") == candidate:
-                    result.append(_name)
-                    found = True
-
-            if (
-                self.frameworks
-                and candidate.startswith("framework-")
-                and candidate[10:] in self.frameworks
-            ):
-                result.append(self.frameworks[candidate[10:]]["package"])
-                found = True
-
-            if not found:
-                result.append(candidate)
-
-        return result
-
-    def update_packages(self, only_check=False):
-        for name, manifest in self.get_installed_packages().items():
-            requirements = self.packages[name].get("version", "")
-            if ":" in requirements:
-                _, requirements, __ = self.pm.parse_pkg_uri(requirements)
-            self.pm.update(manifest["__pkg_dir"], requirements, only_check)
-
-    def get_installed_packages(self):
-        items = {}
-        for name in self.packages:
-            pkg_dir = self.get_package_dir(name)
-            if pkg_dir:
-                items[name] = self.pm.load_manifest(pkg_dir)
-        return items
-
-    def are_outdated_packages(self):
-        for name, manifest in self.get_installed_packages().items():
-            requirements = self.packages[name].get("version", "")
-            if ":" in requirements:
-                _, requirements, __ = self.pm.parse_pkg_uri(requirements)
-            if self.pm.outdated(manifest["__pkg_dir"], requirements):
-                return True
-        return False
-
-    def get_package_dir(self, name):
-        version = self.packages[name].get("version", "")
-        if ":" in version:
-            return self.pm.get_package_dir(
-                *self.pm.parse_pkg_uri("%s=%s" % (name, version))
-            )
-        return self.pm.get_package_dir(name, version)
-
-    def get_package_version(self, name):
-        pkg_dir = self.get_package_dir(name)
-        if not pkg_dir:
-            return None
-        return self.pm.load_manifest(pkg_dir).get("version")
-
-    def dump_used_packages(self):
-        result = []
-        for name, options in self.packages.items():
-            if options.get("optional"):
-                continue
-            pkg_dir = self.get_package_dir(name)
-            if not pkg_dir:
-                continue
-            manifest = self.pm.load_manifest(pkg_dir)
-            item = {"name": manifest["name"], "version": manifest["version"]}
-            if manifest.get("__src_url"):
-                item["src_url"] = manifest.get("__src_url")
-            result.append(item)
-        return result
-
-
-class PlatformRunMixin(object):
-
-    LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I)
-
-    @staticmethod
-    def encode_scons_arg(value):
-        data = base64.urlsafe_b64encode(hashlib_encode_data(value))
-        return data.decode() if is_bytes(data) else data
-
-    @staticmethod
-    def decode_scons_arg(data):
-        value = base64.urlsafe_b64decode(data)
-        return value.decode() if is_bytes(value) else value
-
-    def run(  # pylint: disable=too-many-arguments
-        self, variables, targets, silent, verbose, jobs
-    ):
-        assert isinstance(variables, dict)
-        assert isinstance(targets, list)
-
-        options = self.config.items(env=variables["pioenv"], as_dict=True)
-        if "framework" in options:
-            # support PIO Core 3.0 dev/platforms
-            options["pioframework"] = options["framework"]
-        self.configure_default_packages(options, targets)
-        self.install_packages(silent=True)
-
-        self._report_non_sensitive_data(options, targets)
-
-        self.silent = silent
-        self.verbose = verbose or app.get_setting("force_verbose")
-
-        if "clean" in targets:
-            targets = ["-c", "."]
-
-        variables["platform_manifest"] = self.manifest_path
-
-        if "build_script" not in variables:
-            variables["build_script"] = self.get_build_script()
-        if not isfile(variables["build_script"]):
-            raise exception.BuildScriptNotFound(variables["build_script"])
-
-        result = self._run_scons(variables, targets, jobs)
-        assert "returncode" in result
-
-        return result
-
-    def _report_non_sensitive_data(self, options, targets):
-        topts = options.copy()
-        topts["platform_packages"] = [
-            dict(name=item["name"], version=item["version"])
-            for item in self.dump_used_packages()
-        ]
-        topts["platform"] = {"name": self.name, "version": self.version}
-        if self.src_version:
-            topts["platform"]["src_version"] = self.src_version
-        telemetry.send_run_environment(topts, targets)
-
-    def _run_scons(self, variables, targets, jobs):
-        args = [
-            proc.get_pythonexe_path(),
-            join(get_core_package_dir("tool-scons"), "script", "scons"),
-            "-Q",
-            "--warn=no-no-parallel-support",
-            "--jobs",
-            str(jobs),
-            "--sconstruct",
-            join(fs.get_source_dir(), "builder", "main.py"),
-        ]
-        args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
-        # pylint: disable=protected-access
-        args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0))
-        args += targets
-
-        # encode and append variables
-        for key, value in variables.items():
-            args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value)))
-
-        proc.copy_pythonpath_to_osenv()
-
-        if targets and "menuconfig" in targets:
-            return proc.exec_command(
-                args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin
-            )
-
-        if click._compat.isatty(sys.stdout):
-
-            def _write_and_flush(stream, data):
-                try:
-                    stream.write(data)
-                    stream.flush()
-                except IOError:
-                    pass
-
-            return proc.exec_command(
-                args,
-                stdout=proc.BuildAsyncPipe(
-                    line_callback=self._on_stdout_line,
-                    data_callback=lambda data: _write_and_flush(sys.stdout, data),
-                ),
-                stderr=proc.BuildAsyncPipe(
-                    line_callback=self._on_stderr_line,
-                    data_callback=lambda data: _write_and_flush(sys.stderr, data),
-                ),
-            )
-
-        return proc.exec_command(
-            args,
-            stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line),
-            stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line),
-        )
-
-    def _on_stdout_line(self, line):
-        if "`buildprog' is up to date." in line:
-            return
-        self._echo_line(line, level=1)
-
-    def _on_stderr_line(self, line):
-        is_error = self.LINE_ERROR_RE.search(line) is not None
-        self._echo_line(line, level=3 if is_error else 2)
-
-        a_pos = line.find("fatal error:")
-        b_pos = line.rfind(": No such file or directory")
-        if a_pos == -1 or b_pos == -1:
-            return
-        self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip())
-
-    def _echo_line(self, line, level):
-        if line.startswith("scons: "):
-            line = line[7:]
-        assert 1 <= level <= 3
-        if self.silent and (level < 2 or not line):
-            return
-        fg = (None, "yellow", "red")[level - 1]
-        if level == 1 and "is up to date" in line:
-            fg = "green"
-        click.secho(line, fg=fg, err=level > 1, nl=False)
-
-    @staticmethod
-    def _echo_missed_dependency(filename):
-        if "/" in filename or not filename.endswith((".h", ".hpp")):
-            return
-        banner = """
-{dots}
-* Looking for {filename_styled} dependency? Check our library registry!
-*
-* CLI  > platformio lib search "header:{filename}"
-* Web  > {link}
-*
-{dots}
-""".format(
-            filename=filename,
-            filename_styled=click.style(filename, fg="cyan"),
-            link=click.style(
-                "https://platformio.org/lib/search?query=header:%s"
-                % quote(filename, safe=""),
-                fg="blue",
-            ),
-            dots="*" * (56 + len(filename)),
-        )
-        click.echo(banner, err=True)
-
-
-class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
-
-    PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__))
-    _BOARDS_CACHE = {}
-
-    def __init__(self, manifest_path):
-        self.manifest_path = manifest_path
-        self.silent = False
-        self.verbose = False
-
-        self._manifest = fs.load_json(manifest_path)
-        self._BOARDS_CACHE = {}
-        self._custom_packages = None
-
-        self.config = ProjectConfig.get_instance()
-        self.pm = PackageManager(
-            self.config.get_optional_dir("packages"), self.package_repositories
-        )
-
-        self._src_manifest = None
-        src_manifest_path = self.pm.get_src_manifest_path(self.get_dir())
-        if src_manifest_path:
-            self._src_manifest = fs.load_json(src_manifest_path)
-
-        # if self.engines and "platformio" in self.engines:
-        #     if self.PIO_VERSION not in semantic_version.SimpleSpec(
-        #             self.engines['platformio']):
-        #         raise exception.IncompatiblePlatform(self.name,
-        #                                              str(self.PIO_VERSION))
-
-    @property
-    def name(self):
-        return self._manifest["name"]
-
-    @property
-    def title(self):
-        return self._manifest["title"]
-
-    @property
-    def description(self):
-        return self._manifest["description"]
-
-    @property
-    def version(self):
-        return self._manifest["version"]
-
-    @property
-    def src_version(self):
-        return self._src_manifest.get("version") if self._src_manifest else None
-
-    @property
-    def src_url(self):
-        return self._src_manifest.get("url") if self._src_manifest else None
-
-    @property
-    def homepage(self):
-        return self._manifest.get("homepage")
-
-    @property
-    def repository_url(self):
-        return self._manifest.get("repository", {}).get("url")
-
-    @property
-    def license(self):
-        return self._manifest.get("license")
-
-    @property
-    def frameworks(self):
-        return self._manifest.get("frameworks")
-
-    @property
-    def engines(self):
-        return self._manifest.get("engines")
-
-    @property
-    def package_repositories(self):
-        return self._manifest.get("packageRepositories")
-
-    @property
-    def manifest(self):
-        return self._manifest
-
-    @property
-    def packages(self):
-        packages = self._manifest.get("packages", {})
-        for item in self._custom_packages or []:
-            name = item
-            version = "*"
-            if "@" in item:
-                name, version = item.split("@", 2)
-            name = name.strip()
-            if name not in packages:
-                packages[name] = {}
-            packages[name].update({"version": version.strip(), "optional": False})
-        return packages
-
-    @property
-    def python_packages(self):
-        return self._manifest.get("pythonPackages")
-
-    def get_dir(self):
-        return dirname(self.manifest_path)
-
-    def get_build_script(self):
-        main_script = join(self.get_dir(), "builder", "main.py")
-        if isfile(main_script):
-            return main_script
-        raise NotImplementedError()
-
-    def is_embedded(self):
-        for opts in self.packages.values():
-            if opts.get("type") == "uploader":
-                return True
-        return False
-
-    def get_boards(self, id_=None):
-        def _append_board(board_id, manifest_path):
-            config = PlatformBoardConfig(manifest_path)
-            if "platform" in config and config.get("platform") != self.name:
-                return
-            if "platforms" in config and self.name not in config.get("platforms"):
-                return
-            config.manifest["platform"] = self.name
-            self._BOARDS_CACHE[board_id] = config
-
-        bdirs = [
-            self.config.get_optional_dir("boards"),
-            join(self.config.get_optional_dir("core"), "boards"),
-            join(self.get_dir(), "boards"),
-        ]
-
-        if id_ is None:
-            for boards_dir in bdirs:
-                if not isdir(boards_dir):
-                    continue
-                for item in sorted(os.listdir(boards_dir)):
-                    _id = item[:-5]
-                    if not item.endswith(".json") or _id in self._BOARDS_CACHE:
-                        continue
-                    _append_board(_id, join(boards_dir, item))
-        else:
-            if id_ not in self._BOARDS_CACHE:
-                for boards_dir in bdirs:
-                    if not isdir(boards_dir):
-                        continue
-                    manifest_path = join(boards_dir, "%s.json" % id_)
-                    if isfile(manifest_path):
-                        _append_board(id_, manifest_path)
-                        break
-            if id_ not in self._BOARDS_CACHE:
-                raise exception.UnknownBoard(id_)
-        return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE
-
-    def board_config(self, id_):
-        return self.get_boards(id_)
-
-    def get_package_type(self, name):
-        return self.packages[name].get("type")
-
-    def configure_default_packages(self, options, targets):
-        # override user custom packages
-        self._custom_packages = options.get("platform_packages")
-
-        # enable used frameworks
-        for framework in options.get("framework", []):
-            if not self.frameworks:
-                continue
-            framework = framework.lower().strip()
-            if not framework or framework not in self.frameworks:
-                continue
-            _pkg_name = self.frameworks[framework].get("package")
-            if _pkg_name:
-                self.packages[_pkg_name]["optional"] = False
-
-        # enable upload tools for upload targets
-        if any(["upload" in t for t in targets] + ["program" in targets]):
-            for name, opts in self.packages.items():
-                if opts.get("type") == "uploader":
-                    self.packages[name]["optional"] = False
-                # skip all packages in "nobuild" mode
-                # allow only upload tools and frameworks
-                elif "nobuild" in targets and opts.get("type") != "framework":
-                    self.packages[name]["optional"] = True
-
-    def get_lib_storages(self):
-        storages = {}
-        for opts in (self.frameworks or {}).values():
-            if "package" not in opts:
-                continue
-            pkg_dir = self.get_package_dir(opts["package"])
-            if not pkg_dir or not isdir(join(pkg_dir, "libraries")):
-                continue
-            libs_dir = join(pkg_dir, "libraries")
-            storages[libs_dir] = opts["package"]
-            libcores_dir = join(libs_dir, "__cores__")
-            if not isdir(libcores_dir):
-                continue
-            for item in os.listdir(libcores_dir):
-                libcore_dir = join(libcores_dir, item)
-                if not isdir(libcore_dir):
-                    continue
-                storages[libcore_dir] = "%s-core-%s" % (opts["package"], item)
-
-        return [dict(name=name, path=path) for path, name in storages.items()]
-
-    def on_installed(self):
-        pass
-
-    def on_uninstalled(self):
-        pass
-
-    def install_python_packages(self):
-        if not self.python_packages:
-            return None
-        click.echo(
-            "Installing Python packages: %s"
-            % ", ".join(list(self.python_packages.keys())),
-        )
-        args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"]
-        for name, requirements in self.python_packages.items():
-            if any(c in requirements for c in ("<", ">", "=")):
-                args.append("%s%s" % (name, requirements))
-            else:
-                args.append("%s==%s" % (name, requirements))
-        try:
-            return subprocess.call(args) == 0
-        except Exception as e:  # pylint: disable=broad-except
-            click.secho(
-                "Could not install Python packages -> %s" % e, fg="red", err=True
-            )
-
-    def uninstall_python_packages(self):
-        if not self.python_packages:
-            return
-        click.echo("Uninstalling Python packages")
-        args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"]
-        args.extend(list(self.python_packages.keys()))
-        try:
-            subprocess.call(args) == 0
-        except Exception as e:  # pylint: disable=broad-except
-            click.secho(
-                "Could not install Python packages -> %s" % e, fg="red", err=True
-            )
-
-
-class PlatformBoardConfig(object):
-    def __init__(self, manifest_path):
-        self._id = basename(manifest_path)[:-5]
-        assert isfile(manifest_path)
-        self.manifest_path = manifest_path
-        try:
-            self._manifest = fs.load_json(manifest_path)
-        except ValueError:
-            raise exception.InvalidBoardManifest(manifest_path)
-        if not set(["name", "url", "vendor"]) <= set(self._manifest):
-            raise exception.PlatformioException(
-                "Please specify name, url and vendor fields for " + manifest_path
-            )
-
-    def get(self, path, default=None):
-        try:
-            value = self._manifest
-            for k in path.split("."):
-                value = value[k]
-            # pylint: disable=undefined-variable
-            if PY2 and isinstance(value, unicode):
-                # cast to plain string from unicode for PY2, resolves issue in
-                # dev/platform when BoardConfig.get() is used in pair with
-                # os.path.join(file_encoding, unicode_encoding)
-                try:
-                    value = value.encode("utf-8")
-                except UnicodeEncodeError:
-                    pass
-            return value
-        except KeyError:
-            if default is not None:
-                return default
-        raise KeyError("Invalid board option '%s'" % path)
-
-    def update(self, path, value):
-        newdict = None
-        for key in path.split(".")[::-1]:
-            if newdict is None:
-                newdict = {key: value}
-            else:
-                newdict = {key: newdict}
-        util.merge_dicts(self._manifest, newdict)
-
-    def __contains__(self, key):
-        try:
-            self.get(key)
-            return True
-        except KeyError:
-            return False
-
-    @property
-    def id(self):
-        return self._id
-
-    @property
-    def id_(self):
-        return self.id
-
-    @property
-    def manifest(self):
-        return self._manifest
-
-    def get_brief_data(self):
-        result = {
-            "id": self.id,
-            "name": self._manifest["name"],
-            "platform": self._manifest.get("platform"),
-            "mcu": self._manifest.get("build", {}).get("mcu", "").upper(),
-            "fcpu": int(
-                "".join(
-                    [
-                        c
-                        for c in str(self._manifest.get("build", {}).get("f_cpu", "0L"))
-                        if c.isdigit()
-                    ]
-                )
-            ),
-            "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0),
-            "rom": self._manifest.get("upload", {}).get("maximum_size", 0),
-            "frameworks": self._manifest.get("frameworks"),
-            "vendor": self._manifest["vendor"],
-            "url": self._manifest["url"],
-        }
-        if self._manifest.get("connectivity"):
-            result["connectivity"] = self._manifest.get("connectivity")
-        debug = self.get_debug_data()
-        if debug:
-            result["debug"] = debug
-        return result
-
-    def get_debug_data(self):
-        if not self._manifest.get("debug", {}).get("tools"):
-            return None
-        tools = {}
-        for name, options in self._manifest["debug"]["tools"].items():
-            tools[name] = {}
-            for key, value in options.items():
-                if key in ("default", "onboard") and value:
-                    tools[name][key] = value
-        return {"tools": tools}
-
-    def get_debug_tool_name(self, custom=None):
-        debug_tools = self._manifest.get("debug", {}).get("tools")
-        tool_name = custom
-        if tool_name == "custom":
-            return tool_name
-        if not debug_tools:
-            telemetry.send_event("Debug", "Request", self.id)
-            raise DebugSupportError(self._manifest["name"])
-        if tool_name:
-            if tool_name in debug_tools:
-                return tool_name
-            raise DebugInvalidOptionsError(
-                "Unknown debug tool `%s`. Please use one of `%s` or `custom`"
-                % (tool_name, ", ".join(sorted(list(debug_tools))))
-            )
-
-        # automatically select best tool
-        data = {"default": [], "onboard": [], "external": []}
-        for key, value in debug_tools.items():
-            if value.get("default"):
-                data["default"].append(key)
-            elif value.get("onboard"):
-                data["onboard"].append(key)
-            data["external"].append(key)
-
-        for key, value in data.items():
-            if not value:
-                continue
-            return sorted(value)[0]
-
-        assert any(item for item in data)

platformio/platform/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/platform/_packages.py

@@ -0,0 +1,126 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.package.exception import UnknownPackageError
+
+
+class PlatformPackagesMixin(object):
+    def install_packages(  # pylint: disable=too-many-arguments
+        self,
+        with_packages=None,
+        without_packages=None,
+        skip_default_package=False,
+        silent=False,
+        force=False,
+    ):
+        with_packages = set(self.find_pkg_names(with_packages or []))
+        without_packages = set(self.find_pkg_names(without_packages or []))
+
+        upkgs = with_packages | without_packages
+        ppkgs = set(self.packages)
+        if not upkgs.issubset(ppkgs):
+            raise UnknownPackageError(", ".join(upkgs - ppkgs))
+
+        for name, opts in self.packages.items():
+            version = opts.get("version", "")
+            if name in without_packages:
+                continue
+            if name in with_packages or not (
+                skip_default_package or opts.get("optional", False)
+            ):
+                if ":" in version:
+                    self.pm.install(
+                        "%s=%s" % (name, version), silent=silent, force=force
+                    )
+                else:
+                    self.pm.install(name, version, silent=silent, force=force)
+
+        return True
+
+    def find_pkg_names(self, candidates):
+        result = []
+        for candidate in candidates:
+            found = False
+
+            # lookup by package types
+            for _name, _opts in self.packages.items():
+                if _opts.get("type") == candidate:
+                    result.append(_name)
+                    found = True
+
+            if (
+                self.frameworks
+                and candidate.startswith("framework-")
+                and candidate[10:] in self.frameworks
+            ):
+                result.append(self.frameworks[candidate[10:]]["package"])
+                found = True
+
+            if not found:
+                result.append(candidate)
+
+        return result
+
+    def update_packages(self, only_check=False):
+        for name, manifest in self.get_installed_packages().items():
+            requirements = self.packages[name].get("version", "")
+            if ":" in requirements:
+                _, requirements, __ = self.pm.parse_pkg_uri(requirements)
+            self.pm.update(manifest["__pkg_dir"], requirements, only_check)
+
+    def get_installed_packages(self):
+        items = {}
+        for name in self.packages:
+            pkg_dir = self.get_package_dir(name)
+            if pkg_dir:
+                items[name] = self.pm.load_manifest(pkg_dir)
+        return items
+
+    def are_outdated_packages(self):
+        for name, manifest in self.get_installed_packages().items():
+            requirements = self.packages[name].get("version", "")
+            if ":" in requirements:
+                _, requirements, __ = self.pm.parse_pkg_uri(requirements)
+            if self.pm.outdated(manifest["__pkg_dir"], requirements):
+                return True
+        return False
+
+    def get_package_dir(self, name):
+        version = self.packages[name].get("version", "")
+        if ":" in version:
+            return self.pm.get_package_dir(
+                *self.pm.parse_pkg_uri("%s=%s" % (name, version))
+            )
+        return self.pm.get_package_dir(name, version)
+
+    def get_package_version(self, name):
+        pkg_dir = self.get_package_dir(name)
+        if not pkg_dir:
+            return None
+        return self.pm.load_manifest(pkg_dir).get("version")
+
+    def dump_used_packages(self):
+        result = []
+        for name, options in self.packages.items():
+            if options.get("optional"):
+                continue
+            pkg_dir = self.get_package_dir(name)
+            if not pkg_dir:
+                continue
+            manifest = self.pm.load_manifest(pkg_dir)
+            item = {"name": manifest["name"], "version": manifest["version"]}
+            if manifest.get("__src_url"):
+                item["src_url"] = manifest.get("__src_url")
+            result.append(item)
+        return result

platformio/platform/_run.py

@@ -0,0 +1,193 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import os
+import re
+import sys
+
+import click
+
+from platformio import app, fs, proc, telemetry
+from platformio.compat import hashlib_encode_data, is_bytes
+from platformio.package.manager.core import get_core_package_dir
+from platformio.platform.exception import BuildScriptNotFound
+
+try:
+    from urllib.parse import quote
+except ImportError:
+    from urllib import quote
+
+
+class PlatformRunMixin(object):
+
+    LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I)
+
+    @staticmethod
+    def encode_scons_arg(value):
+        data = base64.urlsafe_b64encode(hashlib_encode_data(value))
+        return data.decode() if is_bytes(data) else data
+
+    @staticmethod
+    def decode_scons_arg(data):
+        value = base64.urlsafe_b64decode(data)
+        return value.decode() if is_bytes(value) else value
+
+    def run(  # pylint: disable=too-many-arguments
+        self, variables, targets, silent, verbose, jobs
+    ):
+        assert isinstance(variables, dict)
+        assert isinstance(targets, list)
+
+        options = self.config.items(env=variables["pioenv"], as_dict=True)
+        if "framework" in options:
+            # support PIO Core 3.0 dev/platforms
+            options["pioframework"] = options["framework"]
+        self.configure_default_packages(options, targets)
+        self.install_packages(silent=True)
+
+        self._report_non_sensitive_data(options, targets)
+
+        self.silent = silent
+        self.verbose = verbose or app.get_setting("force_verbose")
+
+        if "clean" in targets:
+            targets = ["-c", "."]
+
+        variables["platform_manifest"] = self.manifest_path
+
+        if "build_script" not in variables:
+            variables["build_script"] = self.get_build_script()
+        if not os.path.isfile(variables["build_script"]):
+            raise BuildScriptNotFound(variables["build_script"])
+
+        result = self._run_scons(variables, targets, jobs)
+        assert "returncode" in result
+
+        return result
+
+    def _report_non_sensitive_data(self, options, targets):
+        topts = options.copy()
+        topts["platform_packages"] = [
+            dict(name=item["name"], version=item["version"])
+            for item in self.dump_used_packages()
+        ]
+        topts["platform"] = {"name": self.name, "version": self.version}
+        if self.src_version:
+            topts["platform"]["src_version"] = self.src_version
+        telemetry.send_run_environment(topts, targets)
+
+    def _run_scons(self, variables, targets, jobs):
+        args = [
+            proc.get_pythonexe_path(),
+            os.path.join(get_core_package_dir("tool-scons"), "script", "scons"),
+            "-Q",
+            "--warn=no-no-parallel-support",
+            "--jobs",
+            str(jobs),
+            "--sconstruct",
+            os.path.join(fs.get_source_dir(), "builder", "main.py"),
+        ]
+        args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
+        # pylint: disable=protected-access
+        args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0))
+        args += targets
+
+        # encode and append variables
+        for key, value in variables.items():
+            args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value)))
+
+        proc.copy_pythonpath_to_osenv()
+
+        if targets and "menuconfig" in targets:
+            return proc.exec_command(
+                args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin
+            )
+
+        if click._compat.isatty(sys.stdout):
+
+            def _write_and_flush(stream, data):
+                try:
+                    stream.write(data)
+                    stream.flush()
+                except IOError:
+                    pass
+
+            return proc.exec_command(
+                args,
+                stdout=proc.BuildAsyncPipe(
+                    line_callback=self._on_stdout_line,
+                    data_callback=lambda data: _write_and_flush(sys.stdout, data),
+                ),
+                stderr=proc.BuildAsyncPipe(
+                    line_callback=self._on_stderr_line,
+                    data_callback=lambda data: _write_and_flush(sys.stderr, data),
+                ),
+            )
+
+        return proc.exec_command(
+            args,
+            stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line),
+            stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line),
+        )
+
+    def _on_stdout_line(self, line):
+        if "`buildprog' is up to date." in line:
+            return
+        self._echo_line(line, level=1)
+
+    def _on_stderr_line(self, line):
+        is_error = self.LINE_ERROR_RE.search(line) is not None
+        self._echo_line(line, level=3 if is_error else 2)
+
+        a_pos = line.find("fatal error:")
+        b_pos = line.rfind(": No such file or directory")
+        if a_pos == -1 or b_pos == -1:
+            return
+        self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip())
+
+    def _echo_line(self, line, level):
+        if line.startswith("scons: "):
+            line = line[7:]
+        assert 1 <= level <= 3
+        if self.silent and (level < 2 or not line):
+            return
+        fg = (None, "yellow", "red")[level - 1]
+        if level == 1 and "is up to date" in line:
+            fg = "green"
+        click.secho(line, fg=fg, err=level > 1, nl=False)
+
+    @staticmethod
+    def _echo_missed_dependency(filename):
+        if "/" in filename or not filename.endswith((".h", ".hpp")):
+            return
+        banner = """
+{dots}
+* Looking for {filename_styled} dependency? Check our library registry!
+*
+* CLI  > platformio lib search "header:{filename}"
+* Web  > {link}
+*
+{dots}
+""".format(
+            filename=filename,
+            filename_styled=click.style(filename, fg="cyan"),
+            link=click.style(
+                "https://platformio.org/lib/search?query=header:%s"
+                % quote(filename, safe=""),
+                fg="blue",
+            ),
+            dots="*" * (56 + len(filename)),
+        )
+        click.echo(banner, err=True)

platformio/platform/base.py

@@ -0,0 +1,274 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+import click
+import semantic_version
+
+from platformio import __version__, fs, proc, util
+from platformio.managers.package import PackageManager
+from platformio.platform._packages import PlatformPackagesMixin
+from platformio.platform._run import PlatformRunMixin
+from platformio.platform.board import PlatformBoardConfig
+from platformio.platform.exception import UnknownBoard
+from platformio.project.config import ProjectConfig
+
+
+class PlatformBase(  # pylint: disable=too-many-instance-attributes,too-many-public-methods
+    PlatformPackagesMixin, PlatformRunMixin
+):
+
+    PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__))
+    _BOARDS_CACHE = {}
+
+    def __init__(self, manifest_path):
+        self.manifest_path = manifest_path
+        self.silent = False
+        self.verbose = False
+
+        self._manifest = fs.load_json(manifest_path)
+        self._BOARDS_CACHE = {}
+        self._custom_packages = None
+
+        self.config = ProjectConfig.get_instance()
+        self.pm = PackageManager(
+            self.config.get_optional_dir("packages"), self.package_repositories
+        )
+
+        self._src_manifest = None
+        src_manifest_path = self.pm.get_src_manifest_path(self.get_dir())
+        if src_manifest_path:
+            self._src_manifest = fs.load_json(src_manifest_path)
+
+        # if self.engines and "platformio" in self.engines:
+        #     if self.PIO_VERSION not in semantic_version.SimpleSpec(
+        #             self.engines['platformio']):
+        #         raise exception.IncompatiblePlatform(self.name,
+        #                                              str(self.PIO_VERSION))
+
+    @property
+    def name(self):
+        return self._manifest["name"]
+
+    @property
+    def title(self):
+        return self._manifest["title"]
+
+    @property
+    def description(self):
+        return self._manifest["description"]
+
+    @property
+    def version(self):
+        return self._manifest["version"]
+
+    @property
+    def src_version(self):
+        return self._src_manifest.get("version") if self._src_manifest else None
+
+    @property
+    def src_url(self):
+        return self._src_manifest.get("url") if self._src_manifest else None
+
+    @property
+    def homepage(self):
+        return self._manifest.get("homepage")
+
+    @property
+    def repository_url(self):
+        return self._manifest.get("repository", {}).get("url")
+
+    @property
+    def license(self):
+        return self._manifest.get("license")
+
+    @property
+    def frameworks(self):
+        return self._manifest.get("frameworks")
+
+    @property
+    def engines(self):
+        return self._manifest.get("engines")
+
+    @property
+    def package_repositories(self):
+        return self._manifest.get("packageRepositories")
+
+    @property
+    def manifest(self):
+        return self._manifest
+
+    @property
+    def packages(self):
+        packages = self._manifest.get("packages", {})
+        for item in self._custom_packages or []:
+            name = item
+            version = "*"
+            if "@" in item:
+                name, version = item.split("@", 2)
+            name = name.strip()
+            if name not in packages:
+                packages[name] = {}
+            packages[name].update({"version": version.strip(), "optional": False})
+        return packages
+
+    @property
+    def python_packages(self):
+        return self._manifest.get("pythonPackages")
+
+    def get_dir(self):
+        return os.path.dirname(self.manifest_path)
+
+    def get_build_script(self):
+        main_script = os.path.join(self.get_dir(), "builder", "main.py")
+        if os.path.isfile(main_script):
+            return main_script
+        raise NotImplementedError()
+
+    def is_embedded(self):
+        for opts in self.packages.values():
+            if opts.get("type") == "uploader":
+                return True
+        return False
+
+    def get_boards(self, id_=None):
+        def _append_board(board_id, manifest_path):
+            config = PlatformBoardConfig(manifest_path)
+            if "platform" in config and config.get("platform") != self.name:
+                return
+            if "platforms" in config and self.name not in config.get("platforms"):
+                return
+            config.manifest["platform"] = self.name
+            self._BOARDS_CACHE[board_id] = config
+
+        bdirs = [
+            self.config.get_optional_dir("boards"),
+            os.path.join(self.config.get_optional_dir("core"), "boards"),
+            os.path.join(self.get_dir(), "boards"),
+        ]
+
+        if id_ is None:
+            for boards_dir in bdirs:
+                if not os.path.isdir(boards_dir):
+                    continue
+                for item in sorted(os.listdir(boards_dir)):
+                    _id = item[:-5]
+                    if not item.endswith(".json") or _id in self._BOARDS_CACHE:
+                        continue
+                    _append_board(_id, os.path.join(boards_dir, item))
+        else:
+            if id_ not in self._BOARDS_CACHE:
+                for boards_dir in bdirs:
+                    if not os.path.isdir(boards_dir):
+                        continue
+                    manifest_path = os.path.join(boards_dir, "%s.json" % id_)
+                    if os.path.isfile(manifest_path):
+                        _append_board(id_, manifest_path)
+                        break
+            if id_ not in self._BOARDS_CACHE:
+                raise UnknownBoard(id_)
+        return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE
+
+    def board_config(self, id_):
+        return self.get_boards(id_)
+
+    def get_package_type(self, name):
+        return self.packages[name].get("type")
+
+    def configure_default_packages(self, options, targets):
+        # override user custom packages
+        self._custom_packages = options.get("platform_packages")
+
+        # enable used frameworks
+        for framework in options.get("framework", []):
+            if not self.frameworks:
+                continue
+            framework = framework.lower().strip()
+            if not framework or framework not in self.frameworks:
+                continue
+            _pkg_name = self.frameworks[framework].get("package")
+            if _pkg_name:
+                self.packages[_pkg_name]["optional"] = False
+
+        # enable upload tools for upload targets
+        if any(["upload" in t for t in targets] + ["program" in targets]):
+            for name, opts in self.packages.items():
+                if opts.get("type") == "uploader":
+                    self.packages[name]["optional"] = False
+                # skip all packages in "nobuild" mode
+                # allow only upload tools and frameworks
+                elif "nobuild" in targets and opts.get("type") != "framework":
+                    self.packages[name]["optional"] = True
+
+    def get_lib_storages(self):
+        storages = {}
+        for opts in (self.frameworks or {}).values():
+            if "package" not in opts:
+                continue
+            pkg_dir = self.get_package_dir(opts["package"])
+            if not pkg_dir or not os.path.isdir(os.path.join(pkg_dir, "libraries")):
+                continue
+            libs_dir = os.path.join(pkg_dir, "libraries")
+            storages[libs_dir] = opts["package"]
+            libcores_dir = os.path.join(libs_dir, "__cores__")
+            if not os.path.isdir(libcores_dir):
+                continue
+            for item in os.listdir(libcores_dir):
+                libcore_dir = os.path.join(libcores_dir, item)
+                if not os.path.isdir(libcore_dir):
+                    continue
+                storages[libcore_dir] = "%s-core-%s" % (opts["package"], item)
+
+        return [dict(name=name, path=path) for path, name in storages.items()]
+
+    def on_installed(self):
+        pass
+
+    def on_uninstalled(self):
+        pass
+
+    def install_python_packages(self):
+        if not self.python_packages:
+            return None
+        click.echo(
+            "Installing Python packages: %s"
+            % ", ".join(list(self.python_packages.keys())),
+        )
+        args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"]
+        for name, requirements in self.python_packages.items():
+            if any(c in requirements for c in ("<", ">", "=")):
+                args.append("%s%s" % (name, requirements))
+            else:
+                args.append("%s==%s" % (name, requirements))
+        try:
+            return subprocess.call(args) == 0
+        except Exception as e:  # pylint: disable=broad-except
+            click.secho(
+                "Could not install Python packages -> %s" % e, fg="red", err=True
+            )
+
+    def uninstall_python_packages(self):
+        if not self.python_packages:
+            return
+        click.echo("Uninstalling Python packages")
+        args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"]
+        args.extend(list(self.python_packages.keys()))
+        try:
+            subprocess.call(args) == 0
+        except Exception as e:  # pylint: disable=broad-except
+            click.secho(
+                "Could not install Python packages -> %s" % e, fg="red", err=True
+            )

platformio/platform/board.py

@@ -0,0 +1,158 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from platformio import fs, telemetry, util
+from platformio.commands.debug.exception import (
+    DebugInvalidOptionsError,
+    DebugSupportError,
+)
+from platformio.compat import PY2
+from platformio.exception import UserSideException
+from platformio.platform.exception import InvalidBoardManifest
+
+
+class PlatformBoardConfig(object):
+    def __init__(self, manifest_path):
+        self._id = os.path.basename(manifest_path)[:-5]
+        assert os.path.isfile(manifest_path)
+        self.manifest_path = manifest_path
+        try:
+            self._manifest = fs.load_json(manifest_path)
+        except ValueError:
+            raise InvalidBoardManifest(manifest_path)
+        if not set(["name", "url", "vendor"]) <= set(self._manifest):
+            raise UserSideException(
+                "Please specify name, url and vendor fields for " + manifest_path
+            )
+
+    def get(self, path, default=None):
+        try:
+            value = self._manifest
+            for k in path.split("."):
+                value = value[k]
+            # pylint: disable=undefined-variable
+            if PY2 and isinstance(value, unicode):
+                # cast to plain string from unicode for PY2, resolves issue in
+                # dev/platform when BoardConfig.get() is used in pair with
+                # os.path.join(file_encoding, unicode_encoding)
+                try:
+                    value = value.encode("utf-8")
+                except UnicodeEncodeError:
+                    pass
+            return value
+        except KeyError:
+            if default is not None:
+                return default
+        raise KeyError("Invalid board option '%s'" % path)
+
+    def update(self, path, value):
+        newdict = None
+        for key in path.split(".")[::-1]:
+            if newdict is None:
+                newdict = {key: value}
+            else:
+                newdict = {key: newdict}
+        util.merge_dicts(self._manifest, newdict)
+
+    def __contains__(self, key):
+        try:
+            self.get(key)
+            return True
+        except KeyError:
+            return False
+
+    @property
+    def id(self):
+        return self._id
+
+    @property
+    def id_(self):
+        return self.id
+
+    @property
+    def manifest(self):
+        return self._manifest
+
+    def get_brief_data(self):
+        result = {
+            "id": self.id,
+            "name": self._manifest["name"],
+            "platform": self._manifest.get("platform"),
+            "mcu": self._manifest.get("build", {}).get("mcu", "").upper(),
+            "fcpu": int(
+                "".join(
+                    [
+                        c
+                        for c in str(self._manifest.get("build", {}).get("f_cpu", "0L"))
+                        if c.isdigit()
+                    ]
+                )
+            ),
+            "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0),
+            "rom": self._manifest.get("upload", {}).get("maximum_size", 0),
+            "frameworks": self._manifest.get("frameworks"),
+            "vendor": self._manifest["vendor"],
+            "url": self._manifest["url"],
+        }
+        if self._manifest.get("connectivity"):
+            result["connectivity"] = self._manifest.get("connectivity")
+        debug = self.get_debug_data()
+        if debug:
+            result["debug"] = debug
+        return result
+
+    def get_debug_data(self):
+        if not self._manifest.get("debug", {}).get("tools"):
+            return None
+        tools = {}
+        for name, options in self._manifest["debug"]["tools"].items():
+            tools[name] = {}
+            for key, value in options.items():
+                if key in ("default", "onboard") and value:
+                    tools[name][key] = value
+        return {"tools": tools}
+
+    def get_debug_tool_name(self, custom=None):
+        debug_tools = self._manifest.get("debug", {}).get("tools")
+        tool_name = custom
+        if tool_name == "custom":
+            return tool_name
+        if not debug_tools:
+            telemetry.send_event("Debug", "Request", self.id)
+            raise DebugSupportError(self._manifest["name"])
+        if tool_name:
+            if tool_name in debug_tools:
+                return tool_name
+            raise DebugInvalidOptionsError(
+                "Unknown debug tool `%s`. Please use one of `%s` or `custom`"
+                % (tool_name, ", ".join(sorted(list(debug_tools))))
+            )
+
+        # automatically select best tool
+        data = {"default": [], "onboard": [], "external": []}
+        for key, value in debug_tools.items():
+            if value.get("default"):
+                data["default"].append(key)
+            elif value.get("onboard"):
+                data["onboard"].append(key)
+            data["external"].append(key)
+
+        for key, value in data.items():
+            if not value:
+                continue
+            return sorted(value)[0]
+
+        assert any(item for item in data)

platformio/platform/exception.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.exception import PlatformioException
+
+
+class PlatformException(PlatformioException):
+    pass
+
+
+class UnknownPlatform(PlatformException):
+
+    MESSAGE = "Unknown development platform '{0}'"
+
+
+class IncompatiblePlatform(PlatformException):
+
+    MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}"
+
+
+class UnknownBoard(PlatformException):
+
+    MESSAGE = "Unknown board ID '{0}'"
+
+
+class InvalidBoardManifest(PlatformException):
+
+    MESSAGE = "Invalid board JSON manifest '{0}'"
+
+
+class UnknownFramework(PlatformException):
+
+    MESSAGE = "Unknown framework '{0}'"
+
+
+class BuildScriptNotFound(PlatformException):
+
+    MESSAGE = "Invalid path '{0}' to build script"

platformio/platform/factory.py

@@ -0,0 +1,60 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+
+from platformio.compat import load_python_module
+from platformio.package.manager.platform import PlatformPackageManager
+from platformio.platform.base import PlatformBase
+from platformio.platform.exception import UnknownPlatform
+
+
+class PlatformFactory(object):
+    @staticmethod
+    def get_clsname(name):
+        name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I)
+        return "%s%sPlatform" % (name.upper()[0], name.lower()[1:])
+
+    @staticmethod
+    def load_module(name, path):
+        try:
+            return load_python_module("platformio.platform.%s" % name, path)
+        except ImportError:
+            raise UnknownPlatform(name)
+
+    @classmethod
+    def new(cls, pkg_or_spec):
+        pkg = PlatformPackageManager().get_package(
+            "file://%s" % pkg_or_spec if os.path.isdir(pkg_or_spec) else pkg_or_spec
+        )
+        if not pkg:
+            raise UnknownPlatform(pkg_or_spec)
+
+        platform_cls = None
+        if os.path.isfile(os.path.join(pkg.path, "platform.py")):
+            platform_cls = getattr(
+                cls.load_module(
+                    pkg.metadata.name, os.path.join(pkg.path, "platform.py")
+                ),
+                cls.get_clsname(pkg.metadata.name),
+            )
+        else:
+            platform_cls = type(
+                str(cls.get_clsname(pkg.metadata.name)), (PlatformBase,), {}
+            )
+
+        _instance = platform_cls(os.path.join(pkg.path, "platform.json"))
+        assert isinstance(_instance, PlatformBase)
+        return _instance