From 276ca61cded9a97bf8bc4f67cf143d1618919b4b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 14 Aug 2020 16:39:15 +0300 Subject: [PATCH] Refactor dev-platform API --- Makefile | 2 +- platformio/builder/main.py | 2 +- platformio/builder/tools/pioplatform.py | 32 +- platformio/commands/debug/helpers.py | 11 +- platformio/commands/device/command.py | 4 +- platformio/commands/lib/helpers.py | 5 +- platformio/commands/platform.py | 17 +- platformio/commands/project.py | 5 +- platformio/commands/run/processor.py | 10 +- platformio/commands/test/embedded.py | 4 +- platformio/exception.py | 43 -- platformio/maintenance.py | 7 +- platformio/managers/platform.py | 744 +----------------------- platformio/platform/__init__.py | 13 + platformio/platform/_packages.py | 126 ++++ platformio/platform/_run.py | 193 ++++++ platformio/platform/base.py | 274 +++++++++ platformio/platform/board.py | 158 +++++ platformio/platform/exception.py | 49 ++ platformio/platform/factory.py | 60 ++ 20 files changed, 941 insertions(+), 818 deletions(-) create mode 100644 platformio/platform/__init__.py create mode 100644 platformio/platform/_packages.py create mode 100644 platformio/platform/_run.py create mode 100644 platformio/platform/base.py create mode 100644 platformio/platform/board.py create mode 100644 platformio/platform/exception.py create mode 100644 platformio/platform/factory.py diff --git a/Makefile b/Makefile index 36b5d396..6b22d261 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ clean: clean-docs profile: # Usage $ > make PIOARGS="boards" profile - python -m cProfile -o .tox/.tmp/cprofile.prof $(shell which platformio) ${PIOARGS} + python -m cProfile -o .tox/.tmp/cprofile.prof -m platformio ${PIOARGS} snakeviz .tox/.tmp/cprofile.prof publish: diff --git a/platformio/builder/main.py b/platformio/builder/main.py index a0a8ab12..e73f6869 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -30,7 +30,7 @@ from SCons.Script import Variables # pylint: disable=import-error from platformio import compat, fs from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformBase +from platformio.platform.base import PlatformBase from platformio.proc import get_pythonexe_path from platformio.project.helpers import get_project_dir diff --git a/platformio/builder/tools/pioplatform.py b/platformio/builder/tools/pioplatform.py index 8c047365..fe7d6e28 100644 --- a/platformio/builder/tools/pioplatform.py +++ b/platformio/builder/tools/pioplatform.py @@ -14,15 +14,16 @@ from __future__ import absolute_import +import os import sys -from os.path import isdir, isfile, join from SCons.Script import ARGUMENTS # pylint: disable=import-error from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error -from platformio import exception, fs, util +from platformio import fs, util from platformio.compat import WINDOWS -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownBoard +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectOptions # pylint: disable=too-many-branches, too-many-locals @@ -34,7 +35,7 @@ def PioPlatform(env): if "framework" in variables: # support PIO Core 3.0 dev/platforms variables["pioframework"] = variables["framework"] - p = PlatformFactory.newPlatform(env["PLATFORM_MANIFEST"]) + p = PlatformFactory.new(os.path.dirname(env["PLATFORM_MANIFEST"])) p.configure_default_packages(variables, COMMAND_LINE_TARGETS) return p @@ -46,7 +47,7 @@ def BoardConfig(env, board=None): board = board or env.get("BOARD") assert board, "BoardConfig: Board is not defined" return p.board_config(board) - except (AssertionError, exception.UnknownBoard) as e: + except (AssertionError, UnknownBoard) as e: sys.stderr.write("Error: %s\n" % str(e)) env.Exit(1) @@ -55,8 +56,8 @@ def GetFrameworkScript(env, framework): p = env.PioPlatform() assert p.frameworks and framework in p.frameworks script_path = env.subst(p.frameworks[framework]["script"]) - if not isfile(script_path): - script_path = join(p.get_dir(), script_path) + if not os.path.isfile(script_path): + script_path = os.path.join(p.get_dir(), script_path) return script_path @@ -75,17 +76,24 @@ def LoadPioPlatform(env): continue pkg_dir = p.get_package_dir(name) env.PrependENVPath( - "PATH", join(pkg_dir, "bin") if isdir(join(pkg_dir, "bin")) else pkg_dir + "PATH", + os.path.join(pkg_dir, "bin") + if os.path.isdir(os.path.join(pkg_dir, "bin")) + else pkg_dir, ) - if not WINDOWS and isdir(join(pkg_dir, "lib")) and type_ != "toolchain": + if ( + not WINDOWS + and os.path.isdir(os.path.join(pkg_dir, "lib")) + and type_ != "toolchain" + ): env.PrependENVPath( "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH", - join(pkg_dir, "lib"), + os.path.join(pkg_dir, "lib"), ) # Platform specific LD Scripts - if isdir(join(p.get_dir(), "ldscripts")): - env.Prepend(LIBPATH=[join(p.get_dir(), "ldscripts")]) + if os.path.isdir(os.path.join(p.get_dir(), "ldscripts")): + env.Prepend(LIBPATH=[os.path.join(p.get_dir(), "ldscripts")]) if "BOARD" not in env: return diff --git a/platformio/commands/debug/helpers.py b/platformio/commands/debug/helpers.py index 4604a861..657e8c48 100644 --- a/platformio/commands/debug/helpers.py +++ b/platformio/commands/debug/helpers.py @@ -20,13 +20,14 @@ from hashlib import sha1 from io import BytesIO from os.path import isfile -from platformio import exception, fs, util +from platformio import fs, util from platformio.commands import PlatformioCLI from platformio.commands.debug.exception import DebugInvalidOptionsError from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.run.command import cli as cmd_run from platformio.compat import is_bytes -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.options import ProjectOptions @@ -94,14 +95,14 @@ def validate_debug_options(cmd_ctx, env_options): return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items] try: - platform = PlatformFactory.newPlatform(env_options["platform"]) - except exception.UnknownPlatform: + platform = PlatformFactory.new(env_options["platform"]) + except UnknownPlatform: cmd_ctx.invoke( cmd_platform_install, platforms=[env_options["platform"]], skip_default_package=True, ) - platform = PlatformFactory.newPlatform(env_options["platform"]) + platform = PlatformFactory.new(env_options["platform"]) board_config = platform.board_config(env_options["board"]) tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool")) diff --git a/platformio/commands/device/command.py b/platformio/commands/device/command.py index e93b1214..463116f9 100644 --- a/platformio/commands/device/command.py +++ b/platformio/commands/device/command.py @@ -22,7 +22,7 @@ from serial.tools import miniterm from platformio import exception, fs, util from platformio.commands.device import helpers as device_helpers from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformFactory +from platformio.platform.factory import PlatformFactory from platformio.project.exception import NotPlatformIOProjectError @@ -192,7 +192,7 @@ def device_monitor(**kwargs): # pylint: disable=too-many-branches platform = None if "platform" in project_options: with fs.cd(kwargs["project_dir"]): - platform = PlatformFactory.newPlatform(project_options["platform"]) + platform = PlatformFactory.new(project_options["platform"]) device_helpers.register_platform_filters( platform, kwargs["project_dir"], kwargs["environment"] ) diff --git a/platformio/commands/lib/helpers.py b/platformio/commands/lib/helpers.py index a5cc07e3..23892ac8 100644 --- a/platformio/commands/lib/helpers.py +++ b/platformio/commands/lib/helpers.py @@ -15,8 +15,9 @@ import os from platformio.compat import ci_strings_are_equal -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.platform import PlatformManager from platformio.package.meta import PackageSpec +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.exception import InvalidProjectConfError @@ -29,7 +30,7 @@ def get_builtin_libs(storage_names=None): storage_names = storage_names or [] pm = PlatformManager() for manifest in pm.get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) for storage in p.get_lib_storages(): if storage_names and storage["name"] not in storage_names: continue diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index deaeb431..14d5a1f2 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -16,10 +16,12 @@ from os.path import dirname, isdir import click -from platformio import app, exception, util +from platformio import app, util from platformio.commands.boards import print_boards from platformio.compat import dump_json_to_unicode -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.platform import PlatformManager +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory @click.group(short_help="Platform Manager") @@ -64,12 +66,12 @@ def _get_registry_platforms(): def _get_platform_data(*args, **kwargs): try: return _get_installed_platform_data(*args, **kwargs) - except exception.UnknownPlatform: + except UnknownPlatform: return _get_registry_platform_data(*args, **kwargs) def _get_installed_platform_data(platform, with_boards=True, expose_packages=True): - p = PlatformFactory.newPlatform(platform) + p = PlatformFactory.new(platform) data = dict( name=p.name, title=p.title, @@ -232,7 +234,7 @@ def platform_list(json_output): def platform_show(platform, json_output): # pylint: disable=too-many-branches data = _get_platform_data(platform) if not data: - raise exception.UnknownPlatform(platform) + raise UnknownPlatform(platform) if json_output: return click.echo(dump_json_to_unicode(data)) @@ -384,10 +386,7 @@ def platform_update( # pylint: disable=too-many-locals if not pkg_dir: continue latest = pm.outdated(pkg_dir, requirements) - if ( - not latest - and not PlatformFactory.newPlatform(pkg_dir).are_outdated_packages() - ): + if not latest and not PlatformFactory.new(pkg_dir).are_outdated_packages(): continue data = _get_installed_platform_data( pkg_dir, with_boards=False, expose_packages=False diff --git a/platformio/commands/project.py b/platformio/commands/project.py index c261a9d9..6194a915 100644 --- a/platformio/commands/project.py +++ b/platformio/commands/project.py @@ -20,10 +20,11 @@ import os import click from tabulate import tabulate -from platformio import exception, fs +from platformio import fs from platformio.commands.platform import platform_install as cli_platform_install from platformio.ide.projectgenerator import ProjectGenerator from platformio.managers.platform import PlatformManager +from platformio.platform.exception import UnknownBoard from platformio.project.config import ProjectConfig from platformio.project.exception import NotPlatformIOProjectError from platformio.project.helpers import is_platformio_project, load_project_ide_data @@ -112,7 +113,7 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 for id_ in value: try: pm.board_config(id_) - except exception.UnknownBoard: + except UnknownBoard: raise click.BadParameter( "`%s`. Please search for board ID using `platformio boards` " "command" % id_ diff --git a/platformio/commands/run/processor.py b/platformio/commands/run/processor.py index 23ccc333..d07c581c 100644 --- a/platformio/commands/run/processor.py +++ b/platformio/commands/run/processor.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from platformio import exception from platformio.commands.platform import platform_install as cmd_platform_install from platformio.commands.test.processor import CTX_META_TEST_RUNNING_NAME -from platformio.managers.platform import PlatformFactory +from platformio.platform.exception import UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.exception import UndefinedEnvPlatformError # pylint: disable=too-many-instance-attributes @@ -67,14 +67,14 @@ class EnvironmentProcessor(object): build_targets.remove("monitor") try: - p = PlatformFactory.newPlatform(self.options["platform"]) - except exception.UnknownPlatform: + p = PlatformFactory.new(self.options["platform"]) + except UnknownPlatform: self.cmd_ctx.invoke( cmd_platform_install, platforms=[self.options["platform"]], skip_default_package=True, ) - p = PlatformFactory.newPlatform(self.options["platform"]) + p = PlatformFactory.new(self.options["platform"]) result = p.run(build_vars, build_targets, self.silent, self.verbose, self.jobs) return result["returncode"] == 0 diff --git a/platformio/commands/test/embedded.py b/platformio/commands/test/embedded.py index 6f47eafc..ca658496 100644 --- a/platformio/commands/test/embedded.py +++ b/platformio/commands/test/embedded.py @@ -19,7 +19,7 @@ import serial from platformio import exception, util from platformio.commands.test.processor import TestProcessorBase -from platformio.managers.platform import PlatformFactory +from platformio.platform.factory import PlatformFactory class EmbeddedTestProcessor(TestProcessorBase): @@ -108,7 +108,7 @@ class EmbeddedTestProcessor(TestProcessorBase): return self.env_options.get("test_port") assert set(["platform", "board"]) & set(self.env_options.keys()) - p = PlatformFactory.newPlatform(self.env_options["platform"]) + p = PlatformFactory.new(self.env_options["platform"]) board_hwids = p.board_config(self.env_options["board"]).get("build.hwids", []) port = None elapsed = 0 diff --git a/platformio/exception.py b/platformio/exception.py index 9ab0e4d8..91fd67cc 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -47,44 +47,6 @@ class AbortedByUser(UserSideException): MESSAGE = "Aborted by user" -# -# Development Platform -# - - -class UnknownPlatform(PlatformioException): - - MESSAGE = "Unknown development platform '{0}'" - - -class IncompatiblePlatform(PlatformioException): - - MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}" - - -class PlatformNotInstalledYet(PlatformioException): - - MESSAGE = ( - "The platform '{0}' has not been installed yet. " - "Use `platformio platform install {0}` command" - ) - - -class UnknownBoard(PlatformioException): - - MESSAGE = "Unknown board ID '{0}'" - - -class InvalidBoardManifest(PlatformioException): - - MESSAGE = "Invalid board JSON manifest '{0}'" - - -class UnknownFramework(PlatformioException): - - MESSAGE = "Unknown framework '{0}'" - - # Package Manager @@ -195,11 +157,6 @@ class InternetIsOffline(UserSideException): ) -class BuildScriptNotFound(PlatformioException): - - MESSAGE = "Invalid path '{0}' to build script" - - class InvalidSettingName(UserSideException): MESSAGE = "Invalid setting with the name '{0}'" diff --git a/platformio/maintenance.py b/platformio/maintenance.py index cf2e0698..4b47d50a 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -25,11 +25,12 @@ from platformio.commands.lib.command import CTX_META_STORAGE_DIRS_KEY from platformio.commands.lib.command import lib_update as cmd_lib_update from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.managers.platform import PlatformFactory, PlatformManager +from platformio.managers.platform import PlatformManager from platformio.package.manager.core import update_core_packages from platformio.package.manager.library import LibraryPackageManager from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec +from platformio.platform.factory import PlatformFactory from platformio.proc import is_container @@ -278,9 +279,7 @@ def check_internal_updates(ctx, what): # pylint: disable=too-many-branches conds = [ pm.outdated(manifest["__pkg_dir"]), what == "platforms" - and PlatformFactory.newPlatform( - manifest["__pkg_dir"] - ).are_outdated_packages(), + and PlatformFactory.new(manifest["__pkg_dir"]).are_outdated_packages(), ] if any(conds): outdated_items.append(manifest["name"]) diff --git a/platformio/managers/platform.py b/platformio/managers/platform.py index 8548bba6..c0e0f98e 100644 --- a/platformio/managers/platform.py +++ b/platformio/managers/platform.py @@ -14,31 +14,16 @@ # pylint: disable=too-many-public-methods, too-many-instance-attributes -import base64 -import os -import re -import subprocess -import sys -from os.path import basename, dirname, isdir, isfile, join -import click -import semantic_version +from os.path import isdir, isfile, join -from platformio import __version__, app, exception, fs, proc, telemetry, util -from platformio.commands.debug.exception import ( - DebugInvalidOptionsError, - DebugSupportError, -) -from platformio.compat import PY2, hashlib_encode_data, is_bytes, load_python_module +from platformio import app, exception, util from platformio.managers.package import BasePkgManager, PackageManager -from platformio.package.manager.core import get_core_package_dir +from platformio.platform.base import PlatformBase # pylint: disable=unused-import +from platformio.platform.exception import UnknownBoard, UnknownPlatform +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig -try: - from urllib.parse import quote -except ImportError: - from urllib import quote - class PlatformManager(BasePkgManager): def __init__(self, package_dir=None, repositories=None): @@ -83,7 +68,7 @@ class PlatformManager(BasePkgManager): platform_dir = BasePkgManager.install( self, name, requirements, silent=silent, force=force ) - p = PlatformFactory.newPlatform(platform_dir) + p = PlatformFactory.new(platform_dir) if with_all_packages: with_packages = list(p.packages.keys()) @@ -114,9 +99,9 @@ class PlatformManager(BasePkgManager): pkg_dir = self.get_package_dir(name, requirements, url) if not pkg_dir: - raise exception.UnknownPlatform(package) + raise UnknownPlatform(package) - p = PlatformFactory.newPlatform(pkg_dir) + p = PlatformFactory.new(pkg_dir) BasePkgManager.uninstall(self, pkg_dir, requirements) p.uninstall_python_packages() p.on_uninstalled() @@ -138,15 +123,15 @@ class PlatformManager(BasePkgManager): pkg_dir = self.get_package_dir(name, requirements, url) if not pkg_dir: - raise exception.UnknownPlatform(package) + raise UnknownPlatform(package) - p = PlatformFactory.newPlatform(pkg_dir) + p = PlatformFactory.new(pkg_dir) pkgs_before = list(p.get_installed_packages()) missed_pkgs = set() if not only_packages: BasePkgManager.update(self, pkg_dir, requirements, only_check) - p = PlatformFactory.newPlatform(pkg_dir) + p = PlatformFactory.new(pkg_dir) missed_pkgs = set(pkgs_before) & set(p.packages) missed_pkgs -= set(p.get_installed_packages()) @@ -164,7 +149,7 @@ class PlatformManager(BasePkgManager): self.cache_reset() deppkgs = {} for manifest in PlatformManager().get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) for pkgname, pkgmanifest in p.get_installed_packages().items(): if pkgname not in deppkgs: deppkgs[pkgname] = set() @@ -190,7 +175,7 @@ class PlatformManager(BasePkgManager): def get_installed_boards(self): boards = [] for manifest in self.get_installed(): - p = PlatformFactory.newPlatform(manifest["__pkg_dir"]) + p = PlatformFactory.new(manifest["__pkg_dir"]) for config in p.get_boards().values(): board = config.get_brief_data() if board not in boards: @@ -224,705 +209,4 @@ class PlatformManager(BasePkgManager): not platform or manifest["platform"] == platform ): return manifest - raise exception.UnknownBoard(id_) - - -class PlatformFactory(object): - @staticmethod - def get_clsname(name): - name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) - return "%s%sPlatform" % (name.upper()[0], name.lower()[1:]) - - @staticmethod - def load_module(name, path): - try: - return load_python_module("platformio.managers.platform.%s" % name, path) - except ImportError: - raise exception.UnknownPlatform(name) - - @classmethod - def newPlatform(cls, name, requirements=None): - pm = PlatformManager() - platform_dir = None - if isdir(name): - platform_dir = name - name = pm.load_manifest(platform_dir)["name"] - elif name.endswith("platform.json") and isfile(name): - platform_dir = dirname(name) - name = fs.load_json(name)["name"] - else: - name, requirements, url = pm.parse_pkg_uri(name, requirements) - platform_dir = pm.get_package_dir(name, requirements, url) - if platform_dir: - name = pm.load_manifest(platform_dir)["name"] - - if not platform_dir: - raise exception.UnknownPlatform( - name if not requirements else "%s@%s" % (name, requirements) - ) - - platform_cls = None - if isfile(join(platform_dir, "platform.py")): - platform_cls = getattr( - cls.load_module(name, join(platform_dir, "platform.py")), - cls.get_clsname(name), - ) - else: - platform_cls = type(str(cls.get_clsname(name)), (PlatformBase,), {}) - - _instance = platform_cls(join(platform_dir, "platform.json")) - assert isinstance(_instance, PlatformBase) - return _instance - - -class PlatformPackagesMixin(object): - def install_packages( # pylint: disable=too-many-arguments - self, - with_packages=None, - without_packages=None, - skip_default_package=False, - silent=False, - force=False, - ): - with_packages = set(self.find_pkg_names(with_packages or [])) - without_packages = set(self.find_pkg_names(without_packages or [])) - - upkgs = with_packages | without_packages - ppkgs = set(self.packages) - if not upkgs.issubset(ppkgs): - raise exception.UnknownPackage(", ".join(upkgs - ppkgs)) - - for name, opts in self.packages.items(): - version = opts.get("version", "") - if name in without_packages: - continue - if name in with_packages or not ( - skip_default_package or opts.get("optional", False) - ): - if ":" in version: - self.pm.install( - "%s=%s" % (name, version), silent=silent, force=force - ) - else: - self.pm.install(name, version, silent=silent, force=force) - - return True - - def find_pkg_names(self, candidates): - result = [] - for candidate in candidates: - found = False - - # lookup by package types - for _name, _opts in self.packages.items(): - if _opts.get("type") == candidate: - result.append(_name) - found = True - - if ( - self.frameworks - and candidate.startswith("framework-") - and candidate[10:] in self.frameworks - ): - result.append(self.frameworks[candidate[10:]]["package"]) - found = True - - if not found: - result.append(candidate) - - return result - - def update_packages(self, only_check=False): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - self.pm.update(manifest["__pkg_dir"], requirements, only_check) - - def get_installed_packages(self): - items = {} - for name in self.packages: - pkg_dir = self.get_package_dir(name) - if pkg_dir: - items[name] = self.pm.load_manifest(pkg_dir) - return items - - def are_outdated_packages(self): - for name, manifest in self.get_installed_packages().items(): - requirements = self.packages[name].get("version", "") - if ":" in requirements: - _, requirements, __ = self.pm.parse_pkg_uri(requirements) - if self.pm.outdated(manifest["__pkg_dir"], requirements): - return True - return False - - def get_package_dir(self, name): - version = self.packages[name].get("version", "") - if ":" in version: - return self.pm.get_package_dir( - *self.pm.parse_pkg_uri("%s=%s" % (name, version)) - ) - return self.pm.get_package_dir(name, version) - - def get_package_version(self, name): - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - return None - return self.pm.load_manifest(pkg_dir).get("version") - - def dump_used_packages(self): - result = [] - for name, options in self.packages.items(): - if options.get("optional"): - continue - pkg_dir = self.get_package_dir(name) - if not pkg_dir: - continue - manifest = self.pm.load_manifest(pkg_dir) - item = {"name": manifest["name"], "version": manifest["version"]} - if manifest.get("__src_url"): - item["src_url"] = manifest.get("__src_url") - result.append(item) - return result - - -class PlatformRunMixin(object): - - LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) - - @staticmethod - def encode_scons_arg(value): - data = base64.urlsafe_b64encode(hashlib_encode_data(value)) - return data.decode() if is_bytes(data) else data - - @staticmethod - def decode_scons_arg(data): - value = base64.urlsafe_b64decode(data) - return value.decode() if is_bytes(value) else value - - def run( # pylint: disable=too-many-arguments - self, variables, targets, silent, verbose, jobs - ): - assert isinstance(variables, dict) - assert isinstance(targets, list) - - options = self.config.items(env=variables["pioenv"], as_dict=True) - if "framework" in options: - # support PIO Core 3.0 dev/platforms - options["pioframework"] = options["framework"] - self.configure_default_packages(options, targets) - self.install_packages(silent=True) - - self._report_non_sensitive_data(options, targets) - - self.silent = silent - self.verbose = verbose or app.get_setting("force_verbose") - - if "clean" in targets: - targets = ["-c", "."] - - variables["platform_manifest"] = self.manifest_path - - if "build_script" not in variables: - variables["build_script"] = self.get_build_script() - if not isfile(variables["build_script"]): - raise exception.BuildScriptNotFound(variables["build_script"]) - - result = self._run_scons(variables, targets, jobs) - assert "returncode" in result - - return result - - def _report_non_sensitive_data(self, options, targets): - topts = options.copy() - topts["platform_packages"] = [ - dict(name=item["name"], version=item["version"]) - for item in self.dump_used_packages() - ] - topts["platform"] = {"name": self.name, "version": self.version} - if self.src_version: - topts["platform"]["src_version"] = self.src_version - telemetry.send_run_environment(topts, targets) - - def _run_scons(self, variables, targets, jobs): - args = [ - proc.get_pythonexe_path(), - join(get_core_package_dir("tool-scons"), "script", "scons"), - "-Q", - "--warn=no-no-parallel-support", - "--jobs", - str(jobs), - "--sconstruct", - join(fs.get_source_dir(), "builder", "main.py"), - ] - args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) - # pylint: disable=protected-access - args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) - args += targets - - # encode and append variables - for key, value in variables.items(): - args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - - proc.copy_pythonpath_to_osenv() - - if targets and "menuconfig" in targets: - return proc.exec_command( - args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin - ) - - if click._compat.isatty(sys.stdout): - - def _write_and_flush(stream, data): - try: - stream.write(data) - stream.flush() - except IOError: - pass - - return proc.exec_command( - args, - stdout=proc.BuildAsyncPipe( - line_callback=self._on_stdout_line, - data_callback=lambda data: _write_and_flush(sys.stdout, data), - ), - stderr=proc.BuildAsyncPipe( - line_callback=self._on_stderr_line, - data_callback=lambda data: _write_and_flush(sys.stderr, data), - ), - ) - - return proc.exec_command( - args, - stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), - stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), - ) - - def _on_stdout_line(self, line): - if "`buildprog' is up to date." in line: - return - self._echo_line(line, level=1) - - def _on_stderr_line(self, line): - is_error = self.LINE_ERROR_RE.search(line) is not None - self._echo_line(line, level=3 if is_error else 2) - - a_pos = line.find("fatal error:") - b_pos = line.rfind(": No such file or directory") - if a_pos == -1 or b_pos == -1: - return - self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) - - def _echo_line(self, line, level): - if line.startswith("scons: "): - line = line[7:] - assert 1 <= level <= 3 - if self.silent and (level < 2 or not line): - return - fg = (None, "yellow", "red")[level - 1] - if level == 1 and "is up to date" in line: - fg = "green" - click.secho(line, fg=fg, err=level > 1, nl=False) - - @staticmethod - def _echo_missed_dependency(filename): - if "/" in filename or not filename.endswith((".h", ".hpp")): - return - banner = """ -{dots} -* Looking for {filename_styled} dependency? Check our library registry! -* -* CLI > platformio lib search "header:{filename}" -* Web > {link} -* -{dots} -""".format( - filename=filename, - filename_styled=click.style(filename, fg="cyan"), - link=click.style( - "https://platformio.org/lib/search?query=header:%s" - % quote(filename, safe=""), - fg="blue", - ), - dots="*" * (56 + len(filename)), - ) - click.echo(banner, err=True) - - -class PlatformBase(PlatformPackagesMixin, PlatformRunMixin): - - PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) - _BOARDS_CACHE = {} - - def __init__(self, manifest_path): - self.manifest_path = manifest_path - self.silent = False - self.verbose = False - - self._manifest = fs.load_json(manifest_path) - self._BOARDS_CACHE = {} - self._custom_packages = None - - self.config = ProjectConfig.get_instance() - self.pm = PackageManager( - self.config.get_optional_dir("packages"), self.package_repositories - ) - - self._src_manifest = None - src_manifest_path = self.pm.get_src_manifest_path(self.get_dir()) - if src_manifest_path: - self._src_manifest = fs.load_json(src_manifest_path) - - # if self.engines and "platformio" in self.engines: - # if self.PIO_VERSION not in semantic_version.SimpleSpec( - # self.engines['platformio']): - # raise exception.IncompatiblePlatform(self.name, - # str(self.PIO_VERSION)) - - @property - def name(self): - return self._manifest["name"] - - @property - def title(self): - return self._manifest["title"] - - @property - def description(self): - return self._manifest["description"] - - @property - def version(self): - return self._manifest["version"] - - @property - def src_version(self): - return self._src_manifest.get("version") if self._src_manifest else None - - @property - def src_url(self): - return self._src_manifest.get("url") if self._src_manifest else None - - @property - def homepage(self): - return self._manifest.get("homepage") - - @property - def repository_url(self): - return self._manifest.get("repository", {}).get("url") - - @property - def license(self): - return self._manifest.get("license") - - @property - def frameworks(self): - return self._manifest.get("frameworks") - - @property - def engines(self): - return self._manifest.get("engines") - - @property - def package_repositories(self): - return self._manifest.get("packageRepositories") - - @property - def manifest(self): - return self._manifest - - @property - def packages(self): - packages = self._manifest.get("packages", {}) - for item in self._custom_packages or []: - name = item - version = "*" - if "@" in item: - name, version = item.split("@", 2) - name = name.strip() - if name not in packages: - packages[name] = {} - packages[name].update({"version": version.strip(), "optional": False}) - return packages - - @property - def python_packages(self): - return self._manifest.get("pythonPackages") - - def get_dir(self): - return dirname(self.manifest_path) - - def get_build_script(self): - main_script = join(self.get_dir(), "builder", "main.py") - if isfile(main_script): - return main_script - raise NotImplementedError() - - def is_embedded(self): - for opts in self.packages.values(): - if opts.get("type") == "uploader": - return True - return False - - def get_boards(self, id_=None): - def _append_board(board_id, manifest_path): - config = PlatformBoardConfig(manifest_path) - if "platform" in config and config.get("platform") != self.name: - return - if "platforms" in config and self.name not in config.get("platforms"): - return - config.manifest["platform"] = self.name - self._BOARDS_CACHE[board_id] = config - - bdirs = [ - self.config.get_optional_dir("boards"), - join(self.config.get_optional_dir("core"), "boards"), - join(self.get_dir(), "boards"), - ] - - if id_ is None: - for boards_dir in bdirs: - if not isdir(boards_dir): - continue - for item in sorted(os.listdir(boards_dir)): - _id = item[:-5] - if not item.endswith(".json") or _id in self._BOARDS_CACHE: - continue - _append_board(_id, join(boards_dir, item)) - else: - if id_ not in self._BOARDS_CACHE: - for boards_dir in bdirs: - if not isdir(boards_dir): - continue - manifest_path = join(boards_dir, "%s.json" % id_) - if isfile(manifest_path): - _append_board(id_, manifest_path) - break - if id_ not in self._BOARDS_CACHE: - raise exception.UnknownBoard(id_) - return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE - - def board_config(self, id_): - return self.get_boards(id_) - - def get_package_type(self, name): - return self.packages[name].get("type") - - def configure_default_packages(self, options, targets): - # override user custom packages - self._custom_packages = options.get("platform_packages") - - # enable used frameworks - for framework in options.get("framework", []): - if not self.frameworks: - continue - framework = framework.lower().strip() - if not framework or framework not in self.frameworks: - continue - _pkg_name = self.frameworks[framework].get("package") - if _pkg_name: - self.packages[_pkg_name]["optional"] = False - - # enable upload tools for upload targets - if any(["upload" in t for t in targets] + ["program" in targets]): - for name, opts in self.packages.items(): - if opts.get("type") == "uploader": - self.packages[name]["optional"] = False - # skip all packages in "nobuild" mode - # allow only upload tools and frameworks - elif "nobuild" in targets and opts.get("type") != "framework": - self.packages[name]["optional"] = True - - def get_lib_storages(self): - storages = {} - for opts in (self.frameworks or {}).values(): - if "package" not in opts: - continue - pkg_dir = self.get_package_dir(opts["package"]) - if not pkg_dir or not isdir(join(pkg_dir, "libraries")): - continue - libs_dir = join(pkg_dir, "libraries") - storages[libs_dir] = opts["package"] - libcores_dir = join(libs_dir, "__cores__") - if not isdir(libcores_dir): - continue - for item in os.listdir(libcores_dir): - libcore_dir = join(libcores_dir, item) - if not isdir(libcore_dir): - continue - storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) - - return [dict(name=name, path=path) for path, name in storages.items()] - - def on_installed(self): - pass - - def on_uninstalled(self): - pass - - def install_python_packages(self): - if not self.python_packages: - return None - click.echo( - "Installing Python packages: %s" - % ", ".join(list(self.python_packages.keys())), - ) - args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"] - for name, requirements in self.python_packages.items(): - if any(c in requirements for c in ("<", ">", "=")): - args.append("%s%s" % (name, requirements)) - else: - args.append("%s==%s" % (name, requirements)) - try: - return subprocess.call(args) == 0 - except Exception as e: # pylint: disable=broad-except - click.secho( - "Could not install Python packages -> %s" % e, fg="red", err=True - ) - - def uninstall_python_packages(self): - if not self.python_packages: - return - click.echo("Uninstalling Python packages") - args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"] - args.extend(list(self.python_packages.keys())) - try: - subprocess.call(args) == 0 - except Exception as e: # pylint: disable=broad-except - click.secho( - "Could not install Python packages -> %s" % e, fg="red", err=True - ) - - -class PlatformBoardConfig(object): - def __init__(self, manifest_path): - self._id = basename(manifest_path)[:-5] - assert isfile(manifest_path) - self.manifest_path = manifest_path - try: - self._manifest = fs.load_json(manifest_path) - except ValueError: - raise exception.InvalidBoardManifest(manifest_path) - if not set(["name", "url", "vendor"]) <= set(self._manifest): - raise exception.PlatformioException( - "Please specify name, url and vendor fields for " + manifest_path - ) - - def get(self, path, default=None): - try: - value = self._manifest - for k in path.split("."): - value = value[k] - # pylint: disable=undefined-variable - if PY2 and isinstance(value, unicode): - # cast to plain string from unicode for PY2, resolves issue in - # dev/platform when BoardConfig.get() is used in pair with - # os.path.join(file_encoding, unicode_encoding) - try: - value = value.encode("utf-8") - except UnicodeEncodeError: - pass - return value - except KeyError: - if default is not None: - return default - raise KeyError("Invalid board option '%s'" % path) - - def update(self, path, value): - newdict = None - for key in path.split(".")[::-1]: - if newdict is None: - newdict = {key: value} - else: - newdict = {key: newdict} - util.merge_dicts(self._manifest, newdict) - - def __contains__(self, key): - try: - self.get(key) - return True - except KeyError: - return False - - @property - def id(self): - return self._id - - @property - def id_(self): - return self.id - - @property - def manifest(self): - return self._manifest - - def get_brief_data(self): - result = { - "id": self.id, - "name": self._manifest["name"], - "platform": self._manifest.get("platform"), - "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), - "fcpu": int( - "".join( - [ - c - for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) - if c.isdigit() - ] - ) - ), - "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), - "rom": self._manifest.get("upload", {}).get("maximum_size", 0), - "frameworks": self._manifest.get("frameworks"), - "vendor": self._manifest["vendor"], - "url": self._manifest["url"], - } - if self._manifest.get("connectivity"): - result["connectivity"] = self._manifest.get("connectivity") - debug = self.get_debug_data() - if debug: - result["debug"] = debug - return result - - def get_debug_data(self): - if not self._manifest.get("debug", {}).get("tools"): - return None - tools = {} - for name, options in self._manifest["debug"]["tools"].items(): - tools[name] = {} - for key, value in options.items(): - if key in ("default", "onboard") and value: - tools[name][key] = value - return {"tools": tools} - - def get_debug_tool_name(self, custom=None): - debug_tools = self._manifest.get("debug", {}).get("tools") - tool_name = custom - if tool_name == "custom": - return tool_name - if not debug_tools: - telemetry.send_event("Debug", "Request", self.id) - raise DebugSupportError(self._manifest["name"]) - if tool_name: - if tool_name in debug_tools: - return tool_name - raise DebugInvalidOptionsError( - "Unknown debug tool `%s`. Please use one of `%s` or `custom`" - % (tool_name, ", ".join(sorted(list(debug_tools)))) - ) - - # automatically select best tool - data = {"default": [], "onboard": [], "external": []} - for key, value in debug_tools.items(): - if value.get("default"): - data["default"].append(key) - elif value.get("onboard"): - data["onboard"].append(key) - data["external"].append(key) - - for key, value in data.items(): - if not value: - continue - return sorted(value)[0] - - assert any(item for item in data) + raise UnknownBoard(id_) diff --git a/platformio/platform/__init__.py b/platformio/platform/__init__.py new file mode 100644 index 00000000..b0514903 --- /dev/null +++ b/platformio/platform/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/platformio/platform/_packages.py b/platformio/platform/_packages.py new file mode 100644 index 00000000..e626eb4b --- /dev/null +++ b/platformio/platform/_packages.py @@ -0,0 +1,126 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.package.exception import UnknownPackageError + + +class PlatformPackagesMixin(object): + def install_packages( # pylint: disable=too-many-arguments + self, + with_packages=None, + without_packages=None, + skip_default_package=False, + silent=False, + force=False, + ): + with_packages = set(self.find_pkg_names(with_packages or [])) + without_packages = set(self.find_pkg_names(without_packages or [])) + + upkgs = with_packages | without_packages + ppkgs = set(self.packages) + if not upkgs.issubset(ppkgs): + raise UnknownPackageError(", ".join(upkgs - ppkgs)) + + for name, opts in self.packages.items(): + version = opts.get("version", "") + if name in without_packages: + continue + if name in with_packages or not ( + skip_default_package or opts.get("optional", False) + ): + if ":" in version: + self.pm.install( + "%s=%s" % (name, version), silent=silent, force=force + ) + else: + self.pm.install(name, version, silent=silent, force=force) + + return True + + def find_pkg_names(self, candidates): + result = [] + for candidate in candidates: + found = False + + # lookup by package types + for _name, _opts in self.packages.items(): + if _opts.get("type") == candidate: + result.append(_name) + found = True + + if ( + self.frameworks + and candidate.startswith("framework-") + and candidate[10:] in self.frameworks + ): + result.append(self.frameworks[candidate[10:]]["package"]) + found = True + + if not found: + result.append(candidate) + + return result + + def update_packages(self, only_check=False): + for name, manifest in self.get_installed_packages().items(): + requirements = self.packages[name].get("version", "") + if ":" in requirements: + _, requirements, __ = self.pm.parse_pkg_uri(requirements) + self.pm.update(manifest["__pkg_dir"], requirements, only_check) + + def get_installed_packages(self): + items = {} + for name in self.packages: + pkg_dir = self.get_package_dir(name) + if pkg_dir: + items[name] = self.pm.load_manifest(pkg_dir) + return items + + def are_outdated_packages(self): + for name, manifest in self.get_installed_packages().items(): + requirements = self.packages[name].get("version", "") + if ":" in requirements: + _, requirements, __ = self.pm.parse_pkg_uri(requirements) + if self.pm.outdated(manifest["__pkg_dir"], requirements): + return True + return False + + def get_package_dir(self, name): + version = self.packages[name].get("version", "") + if ":" in version: + return self.pm.get_package_dir( + *self.pm.parse_pkg_uri("%s=%s" % (name, version)) + ) + return self.pm.get_package_dir(name, version) + + def get_package_version(self, name): + pkg_dir = self.get_package_dir(name) + if not pkg_dir: + return None + return self.pm.load_manifest(pkg_dir).get("version") + + def dump_used_packages(self): + result = [] + for name, options in self.packages.items(): + if options.get("optional"): + continue + pkg_dir = self.get_package_dir(name) + if not pkg_dir: + continue + manifest = self.pm.load_manifest(pkg_dir) + item = {"name": manifest["name"], "version": manifest["version"]} + if manifest.get("__src_url"): + item["src_url"] = manifest.get("__src_url") + result.append(item) + return result diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py new file mode 100644 index 00000000..39e30fce --- /dev/null +++ b/platformio/platform/_run.py @@ -0,0 +1,193 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import os +import re +import sys + +import click + +from platformio import app, fs, proc, telemetry +from platformio.compat import hashlib_encode_data, is_bytes +from platformio.package.manager.core import get_core_package_dir +from platformio.platform.exception import BuildScriptNotFound + +try: + from urllib.parse import quote +except ImportError: + from urllib import quote + + +class PlatformRunMixin(object): + + LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) + + @staticmethod + def encode_scons_arg(value): + data = base64.urlsafe_b64encode(hashlib_encode_data(value)) + return data.decode() if is_bytes(data) else data + + @staticmethod + def decode_scons_arg(data): + value = base64.urlsafe_b64decode(data) + return value.decode() if is_bytes(value) else value + + def run( # pylint: disable=too-many-arguments + self, variables, targets, silent, verbose, jobs + ): + assert isinstance(variables, dict) + assert isinstance(targets, list) + + options = self.config.items(env=variables["pioenv"], as_dict=True) + if "framework" in options: + # support PIO Core 3.0 dev/platforms + options["pioframework"] = options["framework"] + self.configure_default_packages(options, targets) + self.install_packages(silent=True) + + self._report_non_sensitive_data(options, targets) + + self.silent = silent + self.verbose = verbose or app.get_setting("force_verbose") + + if "clean" in targets: + targets = ["-c", "."] + + variables["platform_manifest"] = self.manifest_path + + if "build_script" not in variables: + variables["build_script"] = self.get_build_script() + if not os.path.isfile(variables["build_script"]): + raise BuildScriptNotFound(variables["build_script"]) + + result = self._run_scons(variables, targets, jobs) + assert "returncode" in result + + return result + + def _report_non_sensitive_data(self, options, targets): + topts = options.copy() + topts["platform_packages"] = [ + dict(name=item["name"], version=item["version"]) + for item in self.dump_used_packages() + ] + topts["platform"] = {"name": self.name, "version": self.version} + if self.src_version: + topts["platform"]["src_version"] = self.src_version + telemetry.send_run_environment(topts, targets) + + def _run_scons(self, variables, targets, jobs): + args = [ + proc.get_pythonexe_path(), + os.path.join(get_core_package_dir("tool-scons"), "script", "scons"), + "-Q", + "--warn=no-no-parallel-support", + "--jobs", + str(jobs), + "--sconstruct", + os.path.join(fs.get_source_dir(), "builder", "main.py"), + ] + args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + # pylint: disable=protected-access + args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) + args += targets + + # encode and append variables + for key, value in variables.items(): + args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) + + proc.copy_pythonpath_to_osenv() + + if targets and "menuconfig" in targets: + return proc.exec_command( + args, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin + ) + + if click._compat.isatty(sys.stdout): + + def _write_and_flush(stream, data): + try: + stream.write(data) + stream.flush() + except IOError: + pass + + return proc.exec_command( + args, + stdout=proc.BuildAsyncPipe( + line_callback=self._on_stdout_line, + data_callback=lambda data: _write_and_flush(sys.stdout, data), + ), + stderr=proc.BuildAsyncPipe( + line_callback=self._on_stderr_line, + data_callback=lambda data: _write_and_flush(sys.stderr, data), + ), + ) + + return proc.exec_command( + args, + stdout=proc.LineBufferedAsyncPipe(line_callback=self._on_stdout_line), + stderr=proc.LineBufferedAsyncPipe(line_callback=self._on_stderr_line), + ) + + def _on_stdout_line(self, line): + if "`buildprog' is up to date." in line: + return + self._echo_line(line, level=1) + + def _on_stderr_line(self, line): + is_error = self.LINE_ERROR_RE.search(line) is not None + self._echo_line(line, level=3 if is_error else 2) + + a_pos = line.find("fatal error:") + b_pos = line.rfind(": No such file or directory") + if a_pos == -1 or b_pos == -1: + return + self._echo_missed_dependency(line[a_pos + 12 : b_pos].strip()) + + def _echo_line(self, line, level): + if line.startswith("scons: "): + line = line[7:] + assert 1 <= level <= 3 + if self.silent and (level < 2 or not line): + return + fg = (None, "yellow", "red")[level - 1] + if level == 1 and "is up to date" in line: + fg = "green" + click.secho(line, fg=fg, err=level > 1, nl=False) + + @staticmethod + def _echo_missed_dependency(filename): + if "/" in filename or not filename.endswith((".h", ".hpp")): + return + banner = """ +{dots} +* Looking for {filename_styled} dependency? Check our library registry! +* +* CLI > platformio lib search "header:{filename}" +* Web > {link} +* +{dots} +""".format( + filename=filename, + filename_styled=click.style(filename, fg="cyan"), + link=click.style( + "https://platformio.org/lib/search?query=header:%s" + % quote(filename, safe=""), + fg="blue", + ), + dots="*" * (56 + len(filename)), + ) + click.echo(banner, err=True) diff --git a/platformio/platform/base.py b/platformio/platform/base.py new file mode 100644 index 00000000..a2fcd158 --- /dev/null +++ b/platformio/platform/base.py @@ -0,0 +1,274 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +import click +import semantic_version + +from platformio import __version__, fs, proc, util +from platformio.managers.package import PackageManager +from platformio.platform._packages import PlatformPackagesMixin +from platformio.platform._run import PlatformRunMixin +from platformio.platform.board import PlatformBoardConfig +from platformio.platform.exception import UnknownBoard +from platformio.project.config import ProjectConfig + + +class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods + PlatformPackagesMixin, PlatformRunMixin +): + + PIO_VERSION = semantic_version.Version(util.pepver_to_semver(__version__)) + _BOARDS_CACHE = {} + + def __init__(self, manifest_path): + self.manifest_path = manifest_path + self.silent = False + self.verbose = False + + self._manifest = fs.load_json(manifest_path) + self._BOARDS_CACHE = {} + self._custom_packages = None + + self.config = ProjectConfig.get_instance() + self.pm = PackageManager( + self.config.get_optional_dir("packages"), self.package_repositories + ) + + self._src_manifest = None + src_manifest_path = self.pm.get_src_manifest_path(self.get_dir()) + if src_manifest_path: + self._src_manifest = fs.load_json(src_manifest_path) + + # if self.engines and "platformio" in self.engines: + # if self.PIO_VERSION not in semantic_version.SimpleSpec( + # self.engines['platformio']): + # raise exception.IncompatiblePlatform(self.name, + # str(self.PIO_VERSION)) + + @property + def name(self): + return self._manifest["name"] + + @property + def title(self): + return self._manifest["title"] + + @property + def description(self): + return self._manifest["description"] + + @property + def version(self): + return self._manifest["version"] + + @property + def src_version(self): + return self._src_manifest.get("version") if self._src_manifest else None + + @property + def src_url(self): + return self._src_manifest.get("url") if self._src_manifest else None + + @property + def homepage(self): + return self._manifest.get("homepage") + + @property + def repository_url(self): + return self._manifest.get("repository", {}).get("url") + + @property + def license(self): + return self._manifest.get("license") + + @property + def frameworks(self): + return self._manifest.get("frameworks") + + @property + def engines(self): + return self._manifest.get("engines") + + @property + def package_repositories(self): + return self._manifest.get("packageRepositories") + + @property + def manifest(self): + return self._manifest + + @property + def packages(self): + packages = self._manifest.get("packages", {}) + for item in self._custom_packages or []: + name = item + version = "*" + if "@" in item: + name, version = item.split("@", 2) + name = name.strip() + if name not in packages: + packages[name] = {} + packages[name].update({"version": version.strip(), "optional": False}) + return packages + + @property + def python_packages(self): + return self._manifest.get("pythonPackages") + + def get_dir(self): + return os.path.dirname(self.manifest_path) + + def get_build_script(self): + main_script = os.path.join(self.get_dir(), "builder", "main.py") + if os.path.isfile(main_script): + return main_script + raise NotImplementedError() + + def is_embedded(self): + for opts in self.packages.values(): + if opts.get("type") == "uploader": + return True + return False + + def get_boards(self, id_=None): + def _append_board(board_id, manifest_path): + config = PlatformBoardConfig(manifest_path) + if "platform" in config and config.get("platform") != self.name: + return + if "platforms" in config and self.name not in config.get("platforms"): + return + config.manifest["platform"] = self.name + self._BOARDS_CACHE[board_id] = config + + bdirs = [ + self.config.get_optional_dir("boards"), + os.path.join(self.config.get_optional_dir("core"), "boards"), + os.path.join(self.get_dir(), "boards"), + ] + + if id_ is None: + for boards_dir in bdirs: + if not os.path.isdir(boards_dir): + continue + for item in sorted(os.listdir(boards_dir)): + _id = item[:-5] + if not item.endswith(".json") or _id in self._BOARDS_CACHE: + continue + _append_board(_id, os.path.join(boards_dir, item)) + else: + if id_ not in self._BOARDS_CACHE: + for boards_dir in bdirs: + if not os.path.isdir(boards_dir): + continue + manifest_path = os.path.join(boards_dir, "%s.json" % id_) + if os.path.isfile(manifest_path): + _append_board(id_, manifest_path) + break + if id_ not in self._BOARDS_CACHE: + raise UnknownBoard(id_) + return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE + + def board_config(self, id_): + return self.get_boards(id_) + + def get_package_type(self, name): + return self.packages[name].get("type") + + def configure_default_packages(self, options, targets): + # override user custom packages + self._custom_packages = options.get("platform_packages") + + # enable used frameworks + for framework in options.get("framework", []): + if not self.frameworks: + continue + framework = framework.lower().strip() + if not framework or framework not in self.frameworks: + continue + _pkg_name = self.frameworks[framework].get("package") + if _pkg_name: + self.packages[_pkg_name]["optional"] = False + + # enable upload tools for upload targets + if any(["upload" in t for t in targets] + ["program" in targets]): + for name, opts in self.packages.items(): + if opts.get("type") == "uploader": + self.packages[name]["optional"] = False + # skip all packages in "nobuild" mode + # allow only upload tools and frameworks + elif "nobuild" in targets and opts.get("type") != "framework": + self.packages[name]["optional"] = True + + def get_lib_storages(self): + storages = {} + for opts in (self.frameworks or {}).values(): + if "package" not in opts: + continue + pkg_dir = self.get_package_dir(opts["package"]) + if not pkg_dir or not os.path.isdir(os.path.join(pkg_dir, "libraries")): + continue + libs_dir = os.path.join(pkg_dir, "libraries") + storages[libs_dir] = opts["package"] + libcores_dir = os.path.join(libs_dir, "__cores__") + if not os.path.isdir(libcores_dir): + continue + for item in os.listdir(libcores_dir): + libcore_dir = os.path.join(libcores_dir, item) + if not os.path.isdir(libcore_dir): + continue + storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) + + return [dict(name=name, path=path) for path, name in storages.items()] + + def on_installed(self): + pass + + def on_uninstalled(self): + pass + + def install_python_packages(self): + if not self.python_packages: + return None + click.echo( + "Installing Python packages: %s" + % ", ".join(list(self.python_packages.keys())), + ) + args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"] + for name, requirements in self.python_packages.items(): + if any(c in requirements for c in ("<", ">", "=")): + args.append("%s%s" % (name, requirements)) + else: + args.append("%s==%s" % (name, requirements)) + try: + return subprocess.call(args) == 0 + except Exception as e: # pylint: disable=broad-except + click.secho( + "Could not install Python packages -> %s" % e, fg="red", err=True + ) + + def uninstall_python_packages(self): + if not self.python_packages: + return + click.echo("Uninstalling Python packages") + args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"] + args.extend(list(self.python_packages.keys())) + try: + subprocess.call(args) == 0 + except Exception as e: # pylint: disable=broad-except + click.secho( + "Could not install Python packages -> %s" % e, fg="red", err=True + ) diff --git a/platformio/platform/board.py b/platformio/platform/board.py new file mode 100644 index 00000000..900892cd --- /dev/null +++ b/platformio/platform/board.py @@ -0,0 +1,158 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from platformio import fs, telemetry, util +from platformio.commands.debug.exception import ( + DebugInvalidOptionsError, + DebugSupportError, +) +from platformio.compat import PY2 +from platformio.exception import UserSideException +from platformio.platform.exception import InvalidBoardManifest + + +class PlatformBoardConfig(object): + def __init__(self, manifest_path): + self._id = os.path.basename(manifest_path)[:-5] + assert os.path.isfile(manifest_path) + self.manifest_path = manifest_path + try: + self._manifest = fs.load_json(manifest_path) + except ValueError: + raise InvalidBoardManifest(manifest_path) + if not set(["name", "url", "vendor"]) <= set(self._manifest): + raise UserSideException( + "Please specify name, url and vendor fields for " + manifest_path + ) + + def get(self, path, default=None): + try: + value = self._manifest + for k in path.split("."): + value = value[k] + # pylint: disable=undefined-variable + if PY2 and isinstance(value, unicode): + # cast to plain string from unicode for PY2, resolves issue in + # dev/platform when BoardConfig.get() is used in pair with + # os.path.join(file_encoding, unicode_encoding) + try: + value = value.encode("utf-8") + except UnicodeEncodeError: + pass + return value + except KeyError: + if default is not None: + return default + raise KeyError("Invalid board option '%s'" % path) + + def update(self, path, value): + newdict = None + for key in path.split(".")[::-1]: + if newdict is None: + newdict = {key: value} + else: + newdict = {key: newdict} + util.merge_dicts(self._manifest, newdict) + + def __contains__(self, key): + try: + self.get(key) + return True + except KeyError: + return False + + @property + def id(self): + return self._id + + @property + def id_(self): + return self.id + + @property + def manifest(self): + return self._manifest + + def get_brief_data(self): + result = { + "id": self.id, + "name": self._manifest["name"], + "platform": self._manifest.get("platform"), + "mcu": self._manifest.get("build", {}).get("mcu", "").upper(), + "fcpu": int( + "".join( + [ + c + for c in str(self._manifest.get("build", {}).get("f_cpu", "0L")) + if c.isdigit() + ] + ) + ), + "ram": self._manifest.get("upload", {}).get("maximum_ram_size", 0), + "rom": self._manifest.get("upload", {}).get("maximum_size", 0), + "frameworks": self._manifest.get("frameworks"), + "vendor": self._manifest["vendor"], + "url": self._manifest["url"], + } + if self._manifest.get("connectivity"): + result["connectivity"] = self._manifest.get("connectivity") + debug = self.get_debug_data() + if debug: + result["debug"] = debug + return result + + def get_debug_data(self): + if not self._manifest.get("debug", {}).get("tools"): + return None + tools = {} + for name, options in self._manifest["debug"]["tools"].items(): + tools[name] = {} + for key, value in options.items(): + if key in ("default", "onboard") and value: + tools[name][key] = value + return {"tools": tools} + + def get_debug_tool_name(self, custom=None): + debug_tools = self._manifest.get("debug", {}).get("tools") + tool_name = custom + if tool_name == "custom": + return tool_name + if not debug_tools: + telemetry.send_event("Debug", "Request", self.id) + raise DebugSupportError(self._manifest["name"]) + if tool_name: + if tool_name in debug_tools: + return tool_name + raise DebugInvalidOptionsError( + "Unknown debug tool `%s`. Please use one of `%s` or `custom`" + % (tool_name, ", ".join(sorted(list(debug_tools)))) + ) + + # automatically select best tool + data = {"default": [], "onboard": [], "external": []} + for key, value in debug_tools.items(): + if value.get("default"): + data["default"].append(key) + elif value.get("onboard"): + data["onboard"].append(key) + data["external"].append(key) + + for key, value in data.items(): + if not value: + continue + return sorted(value)[0] + + assert any(item for item in data) diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py new file mode 100644 index 00000000..40431d7f --- /dev/null +++ b/platformio/platform/exception.py @@ -0,0 +1,49 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.exception import PlatformioException + + +class PlatformException(PlatformioException): + pass + + +class UnknownPlatform(PlatformException): + + MESSAGE = "Unknown development platform '{0}'" + + +class IncompatiblePlatform(PlatformException): + + MESSAGE = "Development platform '{0}' is not compatible with PIO Core v{1}" + + +class UnknownBoard(PlatformException): + + MESSAGE = "Unknown board ID '{0}'" + + +class InvalidBoardManifest(PlatformException): + + MESSAGE = "Invalid board JSON manifest '{0}'" + + +class UnknownFramework(PlatformException): + + MESSAGE = "Unknown framework '{0}'" + + +class BuildScriptNotFound(PlatformException): + + MESSAGE = "Invalid path '{0}' to build script" diff --git a/platformio/platform/factory.py b/platformio/platform/factory.py new file mode 100644 index 00000000..99e5f7c4 --- /dev/null +++ b/platformio/platform/factory.py @@ -0,0 +1,60 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +from platformio.compat import load_python_module +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.base import PlatformBase +from platformio.platform.exception import UnknownPlatform + + +class PlatformFactory(object): + @staticmethod + def get_clsname(name): + name = re.sub(r"[^\da-z\_]+", "", name, flags=re.I) + return "%s%sPlatform" % (name.upper()[0], name.lower()[1:]) + + @staticmethod + def load_module(name, path): + try: + return load_python_module("platformio.platform.%s" % name, path) + except ImportError: + raise UnknownPlatform(name) + + @classmethod + def new(cls, pkg_or_spec): + pkg = PlatformPackageManager().get_package( + "file://%s" % pkg_or_spec if os.path.isdir(pkg_or_spec) else pkg_or_spec + ) + if not pkg: + raise UnknownPlatform(pkg_or_spec) + + platform_cls = None + if os.path.isfile(os.path.join(pkg.path, "platform.py")): + platform_cls = getattr( + cls.load_module( + pkg.metadata.name, os.path.join(pkg.path, "platform.py") + ), + cls.get_clsname(pkg.metadata.name), + ) + else: + platform_cls = type( + str(cls.get_clsname(pkg.metadata.name)), (PlatformBase,), {} + ) + + _instance = platform_cls(os.path.join(pkg.path, "platform.json")) + assert isinstance(_instance, PlatformBase) + return _instance