diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2765b55d..9a64d882 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -65,7 +65,7 @@ jobs: mkdir ./${{ env.LATEST_DOCS_DIR }} tar -xzf ./docs.tar.gz -C ./${{ env.LATEST_DOCS_DIR }} - name: Delete Artifact - uses: geekyeggo/delete-artifact@v1 + uses: geekyeggo/delete-artifact@v2 with: name: docs - name: Select Docs type diff --git a/.pylintrc b/.pylintrc index 0b9ff1f8..172b43a1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -8,4 +8,5 @@ disable= invalid-name, too-few-public-methods, consider-using-f-string, - cyclic-import + cyclic-import, + use-dict-literal diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..1958f6ce --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +# Code of Conduct + +See https://piolabs.com/legal/code-of-conduct.html diff --git a/HISTORY.rst b/HISTORY.rst index b963df38..3aa710ff 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,7 @@ Release Notes .. |LDF| replace:: `LDF `__ .. |INTERPOLATION| replace:: `Interpolation of Values `__ .. |UNITTESTING| replace:: `Unit Testing `__ +.. |DEBUGGING| replace:: `Debugging `__ .. _release_notes_6: @@ -14,6 +15,28 @@ PlatformIO Core 6 **A professional collaborative platform for declarative, safety-critical, and test-driven embedded development.** +6.1.7 (2023-05-08) +~~~~~~~~~~~~~~~~~~ + +* Introduced a new ``--sample-code`` option to the `pio project init `__ command, which allows users to include sample code in the newly created project +* Added validation for `project working environment names `__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen) +* Added the ability to show a detailed library dependency tree only in `verbose mode `__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 `_) +* Added the ability to run only the `device monitor `__ when using the `pio run -t monitor `__ command, saving you time and resources by skipping the build process +* Implemented a new feature to store device monitor logs in the project's ``logs`` folder, making it easier to access and review device monitor logs for your projects (`issue #4596 `_) +* Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 `_) +* Improved source file filtering functionality for the `Static Code Analysis `__ feature, making it easier to analyze only the code you need to +* Upgraded the build engine to the latest version of SCons (4.5.2) to improve build performance, reliability, and compatibility with other tools and systems (`release notes `__) +* Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 `_) +* Restored the project generator for the `NetBeans IDE `__, providing you with more flexibility and options for your development workflow +* Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 `_, `issue #4493 `_, `issue #4607 `_) +* Resolved an issue where the `build_cache_dir `__ setting was not being recognized consistently across multiple environments (`issue #4574 `_) +* Resolved an issue where organization details could not be updated using the `pio org update `__ command +* Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 `_) +* Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 `_) +* Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 `_) +* Resolved an issue where multiple targets were not executed sequentially (`issue #4604 `_) +* Resolved an issue where upgrading PlatformIO Core fails on Windows with Python 3.11 (`issue #4540 `_) + 6.1.6 (2023-01-23) ~~~~~~~~~~~~~~~~~~ diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..3d71c1ec --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,34 @@ +# Security Policy + +## Supported Versions + +We are committed to ensuring the security and protection of PlatformIO Core. +To this end, we support only the following versions: + +| Version | Supported | +| ------- | ------------------ | +| 6.1.x | :white_check_mark: | +| < 6.1 | :x: | + +Unsupported versions of the PlatformIO Core may have known vulnerabilities or security issues that could compromise the security of our organization's systems and data. +Therefore, it is important that all developers use only supported versions of the PlatformIO Core. + +## Reporting a Vulnerability + +We take the security of our systems and data very seriously. We encourage responsible disclosure of any vulnerabilities or security issues that you may find in our systems or applications. If you believe you have discovered a vulnerability, please report it to us immediately. + +To report a vulnerability, please send an email to our security team at contact@piolabs.com. Please include as much information as possible, including: + +- A description of the vulnerability and how it can be exploited +- Steps to reproduce the vulnerability +- Any additional information that can help us understand and reproduce the vulnerability + +Once we receive your report, our security team will acknowledge receipt within 24 hours and will work to validate the reported vulnerability. We will provide periodic updates on the progress of the vulnerability assessment, and will notify you once a fix has been deployed. + +If the vulnerability is accepted, we will work to remediate the issue as quickly as possible. We may also provide credit or recognition to the individual who reported the vulnerability, at our discretion. + +If the vulnerability is declined, we will provide a justification for our decision and may offer guidance on how to improve the report or how to test the system more effectively. + +Please note that we will not take any legal action against individuals who report vulnerabilities in good faith and in accordance with this policy. + +Thank you for helping us keep our systems and data secure. diff --git a/docs b/docs index 95c339a7..98609771 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 95c339a71162dc5bd28090f263e641a563229888 +Subproject commit 98609771ba8f78505adad20e66c6505c262f2650 diff --git a/examples b/examples index f98cb5a9..3e23b5ac 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit f98cb5a9be74b9c87b6e5f3cd530d8d7f0548825 +Subproject commit 3e23b5ac43ab7ec277ce7d68618458f3980f8089 diff --git a/platformio/__init__.py b/platformio/__init__.py index 08d95a79..7d3c8f8a 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -VERSION = (6, 1, 6) +VERSION = (6, 1, 7) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -46,8 +44,8 @@ __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __core_packages__ = { "contrib-piohome": "~3.4.2", - "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), - "tool-scons": "~4.40400.0", + "contrib-pioremote": "~1.0.0", + "tool-scons": "~4.40502.0", "tool-cppcheck": "~1.270.0", "tool-clangtidy": "~1.150005.0", "tool-pvs-studio": "~7.18.0", diff --git a/platformio/account/client.py b/platformio/account/client.py index 7aabb24d..a898de48 100644 --- a/platformio/account/client.py +++ b/platformio/account/client.py @@ -21,22 +21,18 @@ from platformio.http import HTTPClient, HTTPClientError class AccountError(PlatformioException): - MESSAGE = "{0}" class AccountNotAuthorized(AccountError): - MESSAGE = "You are not authorized! Please log in to PlatformIO Account." class AccountAlreadyAuthorized(AccountError): - MESSAGE = "You are already authorized with {0} account." class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods - SUMMARY_CACHE_TTL = 60 * 60 * 24 * 7 def __init__(self): @@ -298,7 +294,7 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods return self.fetch_json_data( "delete", "/v1/orgs/%s/owners" % orgname, - data={"username": username}, + params={"username": username}, x_with_authorization=True, ) @@ -351,6 +347,6 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods return self.fetch_json_data( "delete", "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), - data={"username": username}, + params={"username": username}, x_with_authorization=True, ) diff --git a/platformio/account/org/commands/update.py b/platformio/account/org/commands/update.py index 9da9564c..86e7cffa 100644 --- a/platformio/account/org/commands/update.py +++ b/platformio/account/org/commands/update.py @@ -22,29 +22,27 @@ from platformio.account.validate import validate_email, validate_orgname @click.argument("cur_orgname") @click.option( "--orgname", - callback=lambda _, __, value: validate_orgname(value), + callback=lambda _, __, value: validate_orgname(value) if value else value, help="A new orgname", ) -@click.option("--email") +@click.option( + "--email", + callback=lambda _, __, value: validate_email(value) if value else value, +) @click.option("--displayname") def org_update_cmd(cur_orgname, **kwargs): client = AccountClient() org = client.get_org(cur_orgname) - del org["owners"] - new_org = org.copy() + new_org = { + key: value if value is not None else org[key] for key, value in kwargs.items() + } if not any(kwargs.values()): - for field in org: - new_org[field] = click.prompt( - field.replace("_", " ").capitalize(), default=org[field] - ) - if field == "email": - validate_email(new_org[field]) - if field == "orgname": - validate_orgname(new_org[field]) - else: - new_org.update( - {key.replace("new_", ""): value for key, value in kwargs.items() if value} - ) + for key in kwargs: + new_org[key] = click.prompt(key.capitalize(), default=org[key]) + if key == "email": + validate_email(new_org[key]) + if key == "orgname": + validate_orgname(new_org[key]) client.update_org(cur_orgname, new_org) return click.secho( "The organization `%s` has been successfully updated." % cur_orgname, diff --git a/platformio/account/team/commands/create.py b/platformio/account/team/commands/create.py index 891d33b4..164ca6ac 100644 --- a/platformio/account/team/commands/create.py +++ b/platformio/account/team/commands/create.py @@ -22,9 +22,7 @@ from platformio.account.validate import validate_orgname_teamname @click.argument( "orgname_teamname", metavar="ORGNAME:TEAMNAME", - callback=lambda _, __, value: validate_orgname_teamname( - value, teamname_validate=True - ), + callback=lambda _, __, value: validate_orgname_teamname(value), ) @click.option( "--description", diff --git a/platformio/account/team/commands/update.py b/platformio/account/team/commands/update.py index 3ead0fed..c4c2f8b4 100644 --- a/platformio/account/team/commands/update.py +++ b/platformio/account/team/commands/update.py @@ -26,7 +26,7 @@ from platformio.account.validate import validate_orgname_teamname, validate_team ) @click.option( "--name", - callback=lambda _, __, value: validate_teamname(value), + callback=lambda _, __, value: validate_teamname(value) if value else value, help="A new team name", ) @click.option( @@ -36,18 +36,14 @@ def team_update_cmd(orgname_teamname, **kwargs): orgname, teamname = orgname_teamname.split(":", 1) client = AccountClient() team = client.get_team(orgname, teamname) - del team["id"] - del team["members"] - new_team = team.copy() + new_team = { + key: value if value is not None else team[key] for key, value in kwargs.items() + } if not any(kwargs.values()): - for field in team: - new_team[field] = click.prompt( - field.replace("_", " ").capitalize(), default=team[field] - ) - if field == "name": - validate_teamname(new_team[field]) - else: - new_team.update({key: value for key, value in kwargs.items() if value}) + for key in kwargs: + new_team[key] = click.prompt(key.capitalize(), default=team[key]) + if key == "name": + validate_teamname(new_team[key]) client.update_team(orgname, teamname, new_team) return click.secho( "The team %s has been successfully updated." % teamname, diff --git a/platformio/account/validate.py b/platformio/account/validate.py index 7fb77222..60d3bb2e 100644 --- a/platformio/account/validate.py +++ b/platformio/account/validate.py @@ -18,8 +18,10 @@ import click def validate_username(value, field="username"): - value = str(value).strip() - if not re.match(r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,37}$", value, flags=re.I + ): raise click.BadParameter( "Invalid %s format. " "%s must contain only alphanumeric characters " @@ -30,16 +32,22 @@ def validate_username(value, field="username"): return value +def validate_orgname(value): + return validate_username(value, "Organization name") + + def validate_email(value): - value = str(value).strip() - if not re.match(r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d_\.\+\-]+@[a-z\d\-]+\.[a-z\d\-\.]+$", value, flags=re.I + ): raise click.BadParameter("Invalid email address") return value def validate_password(value): - value = str(value).strip() - if not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value): + value = str(value).strip() if value else None + if not value or not re.match(r"^(?=.*[a-z])(?=.*\d).{8,}$", value): raise click.BadParameter( "Invalid password format. " "Password must contain at least 8 characters" @@ -48,27 +56,11 @@ def validate_password(value): return value -def validate_orgname(value): - return validate_username(value, "Organization name") - - -def validate_orgname_teamname(value, teamname_validate=False): - if ":" not in value: - raise click.BadParameter( - "Please specify organization and team name in the next" - " format - orgname:teamname. For example, mycompany:DreamTeam" - ) - teamname = str(value.strip().split(":", 1)[1]) - if teamname_validate: - validate_teamname(teamname) - return value - - def validate_teamname(value): - if not value: - return value - value = str(value).strip() - if not re.match(r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I): + value = str(value).strip() if value else None + if not value or not re.match( + r"^[a-z\d](?:[a-z\d]|[\-_ ](?=[a-z\d])){0,19}$", value, flags=re.I + ): raise click.BadParameter( "Invalid team name format. " "Team name must only contain alphanumeric characters, " @@ -77,3 +69,16 @@ def validate_teamname(value): " not be longer than 20 characters." ) return value + + +def validate_orgname_teamname(value): + value = str(value).strip() if value else None + if not value or ":" not in value: + raise click.BadParameter( + "Please specify organization and team name using the following" + " format - orgname:teamname. For example, mycompany:DreamTeam" + ) + orgname, teamname = value.split(":", 1) + validate_orgname(orgname) + validate_teamname(teamname) + return value diff --git a/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl b/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl index 15756575..c419d27e 100644 --- a/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl +++ b/platformio/assets/templates/ide-projects/vscode/.vscode/launch.json.tpl @@ -16,7 +16,7 @@ % "request": "launch", % "name": "PIO Debug (skip Pre-Debug)", % "executable": _escape_path(prog_path), -% "projectEnvName": env_name, +% "projectEnvName": env_name if forced_env_name else default_debug_env_name, % "toolchainBinDir": _escape_path(os.path.dirname(gdb_path)), % "internalConsoleOptions": "openOnSessionStart", % } @@ -28,7 +28,7 @@ % debug["name"] = "PIO Debug" % debug["preLaunchTask"] = { % "type": "PlatformIO", -% "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and original_env_name else "Pre-Debug", +% "task": ("Pre-Debug (%s)" % env_name) if len(config.envs()) > 1 and forced_env_name else "Pre-Debug", % } % noloading = predebug.copy() % noloading["name"] = "PIO Debug (without uploading)" diff --git a/platformio/builder/main.py b/platformio/builder/main.py index b56dc2a2..86702783 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -28,9 +28,9 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from SCons.Script import Import # pylint: disable=import-error from SCons.Script import Variables # pylint: disable=import-error -from platformio import app, compat, fs +from platformio import app, fs from platformio.platform.base import PlatformBase -from platformio.proc import get_pythonexe_path +from platformio.proc import get_pythonexe_path, where_is_program from platformio.project.helpers import get_project_dir AllowSubstExceptions(NameError) @@ -99,6 +99,7 @@ if not int(ARGUMENTS.get("PIOVERBOSE", 0)): DEFAULT_ENV_OPTIONS["%sSTR" % name] = "%s $TARGET" % (value) env = DefaultEnvironment(**DEFAULT_ENV_OPTIONS) +env.SConscriptChdir(False) # Load variables from CLI env.Replace( @@ -139,19 +140,6 @@ if int(ARGUMENTS.get("ISATTY", 0)): # pylint: disable=protected-access click._compat.isatty = lambda stream: True -if compat.IS_WINDOWS and sys.version_info >= (3, 8) and os.getcwd().startswith("\\\\"): - click.secho("!!! WARNING !!!\t\t" * 3, fg="red") - click.secho( - "Your project is located on a mapped network drive but the " - "current command-line shell does not support the UNC paths.", - fg="yellow", - ) - click.secho( - "Please move your project to a physical drive or check this workaround: " - "https://bit.ly/3kuU5mP\n", - fg="yellow", - ) - if env.subst("$BUILD_CACHE_DIR"): if not os.path.isdir(env.subst("$BUILD_CACHE_DIR")): os.makedirs(env.subst("$BUILD_CACHE_DIR")) @@ -170,18 +158,17 @@ if not os.path.isdir(env.subst("$BUILD_DIR")): env.LoadProjectOptions() env.LoadPioPlatform() -env.SConscriptChdir(0) env.SConsignFile( os.path.join( - "$BUILD_DIR", ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]) + "$BUILD_CACHE_DIR" if env.subst("$BUILD_CACHE_DIR") else "$BUILD_DIR", + ".sconsign%d%d" % (sys.version_info[0], sys.version_info[1]), ) ) -for item in env.GetExtraScripts("pre"): - env.SConscript(item, exports="env") +env.SConscript(env.GetExtraScripts("pre"), exports="env") if env.IsCleanTarget(): - env.CleanProject("cleanall" in COMMAND_LINE_TARGETS) + env.CleanProject(fullclean=int(ARGUMENTS.get("FULLCLEAN", 0))) env.Exit(0) env.SConscript("$BUILD_SCRIPT") @@ -191,8 +178,7 @@ if "UPLOAD_FLAGS" in env: if env.GetProjectOption("upload_command"): env.Replace(UPLOADCMD=env.GetProjectOption("upload_command")) -for item in env.GetExtraScripts("post"): - env.SConscript(item, exports="env") +env.SConscript(env.GetExtraScripts("post"), exports="env") ############################################################################## @@ -208,6 +194,13 @@ if env.get("SIZETOOL") and not ( Default("checkprogsize") if "compiledb" in COMMAND_LINE_TARGETS: + # Resolve absolute path of toolchain + for cmd in ("CC", "CXX", "AS"): + if cmd not in env: + continue + if os.path.isabs(env[cmd]): + continue + env[cmd] = where_is_program(env.subst("$%s" % cmd), env.subst("${ENV['PATH']}")) env.Alias("compiledb", env.CompilationDatabase("$COMPILATIONDB_PATH")) # Print configured protocols @@ -257,3 +250,9 @@ if "sizedata" in COMMAND_LINE_TARGETS: ) Default("sizedata") + +# issue #4604: process targets sequentially +for index, target in enumerate( + [t for t in COMMAND_LINE_TARGETS if not t.startswith("__")][1:] +): + env.Depends(target, COMMAND_LINE_TARGETS[index]) diff --git a/platformio/builder/tools/compilation_db.py b/platformio/builder/tools/compilation_db.py deleted file mode 100644 index 9fa11c86..00000000 --- a/platformio/builder/tools/compilation_db.py +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# Copyright 2020 MongoDB Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -# pylint: disable=unused-argument, protected-access, unused-variable, import-error -# Original: https://github.com/mongodb/mongo/blob/master/site_scons/site_tools/compilation_db.py - -import itertools -import json -import os - -import SCons - -from platformio.builder.tools.piobuild import SRC_ASM_EXT, SRC_C_EXT, SRC_CXX_EXT -from platformio.proc import where_is_program - -# Implements the ability for SCons to emit a compilation database for the MongoDB project. See -# http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation -# database is, and why you might want one. The only user visible entry point here is -# 'env.CompilationDatabase'. This method takes an optional 'target' to name the file that -# should hold the compilation database, otherwise, the file defaults to compile_commands.json, -# which is the name that most clang tools search for by default. - -# Is there a better way to do this than this global? Right now this exists so that the -# emitter we add can record all of the things it emits, so that the scanner for the top level -# compilation database can access the complete list, and also so that the writer has easy -# access to write all of the files. But it seems clunky. How can the emitter and the scanner -# communicate more gracefully? -__COMPILATION_DB_ENTRIES = [] - - -# We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even -# integrate with the cache, but there doesn't seem to be much call for it. -class __CompilationDbNode(SCons.Node.Python.Value): - def __init__(self, value): - SCons.Node.Python.Value.__init__(self, value) - self.Decider(changed_since_last_build_node) - - -def changed_since_last_build_node(*args, **kwargs): - """Dummy decider to force always building""" - return True - - -def makeEmitCompilationDbEntry(comstr): - """ - Effectively this creates a lambda function to capture: - * command line - * source - * target - :param comstr: unevaluated command line - :return: an emitter which has captured the above - """ - user_action = SCons.Action.Action(comstr) - - def EmitCompilationDbEntry(target, source, env): - """ - This emitter will be added to each c/c++ object build to capture the info needed - for clang tools - :param target: target node(s) - :param source: source node(s) - :param env: Environment for use building this node - :return: target(s), source(s) - """ - - # Resolve absolute path of toolchain - for cmd in ("CC", "CXX", "AS"): - if cmd not in env: - continue - if os.path.isabs(env[cmd]): - continue - env[cmd] = where_is_program( - env.subst("$%s" % cmd), env.subst("${ENV['PATH']}") - ) - - dbtarget = __CompilationDbNode(source) - - entry = env.__COMPILATIONDB_Entry( - target=dbtarget, - source=[], - __COMPILATIONDB_UTARGET=target, - __COMPILATIONDB_USOURCE=source, - __COMPILATIONDB_UACTION=user_action, - __COMPILATIONDB_ENV=env, - ) - - # Technically, these next two lines should not be required: it should be fine to - # cache the entries. However, they don't seem to update properly. Since they are quick - # to re-generate disable caching and sidestep this problem. - env.AlwaysBuild(entry) - env.NoCache(entry) - - __COMPILATION_DB_ENTRIES.append(dbtarget) - - return target, source - - return EmitCompilationDbEntry - - -def CompilationDbEntryAction(target, source, env, **kw): - """ - Create a dictionary with evaluated command line, target, source - and store that info as an attribute on the target - (Which has been stored in __COMPILATION_DB_ENTRIES array - :param target: target node(s) - :param source: source node(s) - :param env: Environment for use building this node - :param kw: - :return: None - """ - - command = env["__COMPILATIONDB_UACTION"].strfunction( - target=env["__COMPILATIONDB_UTARGET"], - source=env["__COMPILATIONDB_USOURCE"], - env=env["__COMPILATIONDB_ENV"], - ) - - entry = { - "directory": env.Dir("#").abspath, - "command": command, - "file": str(env["__COMPILATIONDB_USOURCE"][0]), - } - - target[0].write(entry) - - -def WriteCompilationDb(target, source, env): - entries = [] - - for s in __COMPILATION_DB_ENTRIES: - item = s.read() - item["file"] = os.path.abspath(item["file"]) - entries.append(item) - - with open(str(target[0]), mode="w", encoding="utf8") as target_file: - json.dump( - entries, target_file, sort_keys=True, indent=4, separators=(",", ": ") - ) - - -def ScanCompilationDb(node, env, path): - return __COMPILATION_DB_ENTRIES - - -def generate(env, **kwargs): - static_obj, shared_obj = SCons.Tool.createObjBuilders(env) - - env["COMPILATIONDB_COMSTR"] = kwargs.get( - "COMPILATIONDB_COMSTR", "Building compilation database $TARGET" - ) - - components_by_suffix = itertools.chain( - itertools.product( - [".%s" % ext for ext in SRC_C_EXT], - [ - (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"), - (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"), - ], - ), - itertools.product( - [".%s" % ext for ext in SRC_CXX_EXT], - [ - (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"), - (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"), - ], - ), - itertools.product( - [".%s" % ext for ext in SRC_ASM_EXT], - [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASCOM")], - ), - ) - - for entry in components_by_suffix: - suffix = entry[0] - builder, base_emitter, command = entry[1] - - # Assumes a dictionary emitter - emitter = builder.emitter[suffix] - builder.emitter[suffix] = SCons.Builder.ListEmitter( - [emitter, makeEmitCompilationDbEntry(command)] - ) - - env["BUILDERS"]["__COMPILATIONDB_Entry"] = SCons.Builder.Builder( - action=SCons.Action.Action(CompilationDbEntryAction, None), - ) - - env["BUILDERS"]["__COMPILATIONDB_Database"] = SCons.Builder.Builder( - action=SCons.Action.Action(WriteCompilationDb, "$COMPILATIONDB_COMSTR"), - target_scanner=SCons.Scanner.Scanner( - function=ScanCompilationDb, node_class=None - ), - ) - - def CompilationDatabase(env, target): - result = env.__COMPILATIONDB_Database(target=target, source=[]) - - env.AlwaysBuild(result) - env.NoCache(result) - - return result - - env.AddMethod(CompilationDatabase, "CompilationDatabase") - - -def exists(env): - return True diff --git a/platformio/builder/tools/piobuild.py b/platformio/builder/tools/piobuild.py index 32a744e3..ba194eb0 100644 --- a/platformio/builder/tools/piobuild.py +++ b/platformio/builder/tools/piobuild.py @@ -239,7 +239,7 @@ def ProcessUnFlags(env, flags): for scope in unflag_scopes: for unflags in parsed.values(): for unflag in unflags: - for current in env.get(scope, []): + for current in list(env.get(scope, [])): conditions = [ unflag == current, not isinstance(unflag, (tuple, list)) diff --git a/platformio/builder/tools/pioino.py b/platformio/builder/tools/pioino.py index de1f8cf8..c44e72b7 100644 --- a/platformio/builder/tools/pioino.py +++ b/platformio/builder/tools/pioino.py @@ -25,7 +25,6 @@ from platformio.compat import get_filesystem_encoding, get_locale_encoding class InoToCPPConverter: - PROTOTYPE_RE = re.compile( r"""^( (?:template\<.*\>\s*)? # template @@ -103,7 +102,7 @@ class InoToCPPConverter: return "\n".join(["#include "] + lines) if lines else None def process(self, contents): - out_file = self._main_ino + ".cpp" + out_file = re.sub(r"[\"\'\;]+", "", self._main_ino) + ".cpp" assert self._gcc_preprocess(contents, out_file) contents = self.read_safe_contents(out_file) contents = self._join_multiline_strings(contents) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index 8e1bec15..dee1d5bc 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -29,7 +29,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, fs from platformio.builder.tools import piobuild from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types -from platformio.http import HTTPClientError, InternetIsOffline +from platformio.http import HTTPClientError, InternetConnectionError from platformio.package.exception import ( MissingPackageManifestError, UnknownPackageError, @@ -109,7 +109,6 @@ class LibBuilderFactory: class LibBuilderBase: - CLASSIC_SCANNER = SCons.Scanner.C.CScanner() CCONDITIONAL_SCANNER = SCons.Scanner.C.CConditionalScanner() # Max depth of nested includes: @@ -298,11 +297,12 @@ class LibBuilderBase: with fs.cd(self.path): self.env.ProcessFlags(self.build_flags) if self.extra_script: - self.env.SConscriptChdir(1) + self.env.SConscriptChdir(True) self.env.SConscript( os.path.abspath(self.extra_script), exports={"env": self.env, "pio_lib_builder": self}, ) + self.env.SConscriptChdir(False) self.env.ProcessUnFlags(self.build_unflags) def process_dependencies(self): @@ -982,7 +982,11 @@ class ProjectAsLibBuilder(LibBuilderBase): try: lm.install(spec) did_install = True - except (HTTPClientError, UnknownPackageError, InternetIsOffline) as exc: + except ( + HTTPClientError, + UnknownPackageError, + InternetConnectionError, + ) as exc: click.secho("Warning! %s" % exc, fg="yellow") # reset cache @@ -1157,7 +1161,7 @@ def ConfigureProjectLibBuilder(env): click.echo("Path: %s" % lb.path, nl=False) click.echo(")", nl=False) click.echo("") - if lb.depbuilders: + if lb.verbose and lb.depbuilders: _print_deps_tree(lb, level + 1) project = ProjectAsLibBuilder(env, "$PROJECT_DIR") diff --git a/platformio/builder/tools/piotarget.py b/platformio/builder/tools/piotarget.py index f2dc774d..32bb5ffe 100644 --- a/platformio/builder/tools/piotarget.py +++ b/platformio/builder/tools/piotarget.py @@ -16,7 +16,6 @@ import os from SCons.Action import Action # pylint: disable=import-error from SCons.Script import ARGUMENTS # pylint: disable=import-error -from SCons.Script import COMMAND_LINE_TARGETS # pylint: disable=import-error from SCons.Script import AlwaysBuild # pylint: disable=import-error from platformio import compat, fs @@ -29,10 +28,10 @@ def VerboseAction(_, act, actstr): def IsCleanTarget(env): - return env.GetOption("clean") or ("cleanall" in COMMAND_LINE_TARGETS) + return env.GetOption("clean") -def CleanProject(env, clean_all=False): +def CleanProject(env, fullclean=False): def _relpath(path): if compat.IS_WINDOWS: prefix = os.getcwd()[:2].lower() @@ -56,7 +55,7 @@ def CleanProject(env, clean_all=False): else: print("Build environment is clean") - if clean_all and os.path.isdir(libdeps_dir): + if fullclean and os.path.isdir(libdeps_dir): _clean_dir(libdeps_dir) print("Done cleaning") diff --git a/platformio/check/cli.py b/platformio/check/cli.py index e293e8e0..d99da313 100644 --- a/platformio/check/cli.py +++ b/platformio/check/cli.py @@ -38,18 +38,15 @@ from platformio.project.helpers import find_project_dir_above, get_project_dir "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) -@click.option("--pattern", multiple=True) +@click.option("--pattern", multiple=True, hidden=True) +@click.option("-f", "--src-filters", multiple=True) @click.option("--flags", multiple=True) @click.option( "--severity", multiple=True, type=click.Choice(DefectItem.SEVERITY_LABELS.values()) @@ -67,6 +64,7 @@ def cli( environment, project_dir, project_conf, + src_filters, pattern, flags, severity, @@ -105,14 +103,24 @@ def cli( "%s: %s" % (k, ", ".join(v) if isinstance(v, list) else v) ) - default_patterns = [ - config.get("platformio", "src_dir"), - config.get("platformio", "include_dir"), + default_src_filters = [ + "+<%s>" % os.path.basename(config.get("platformio", "src_dir")), + "+<%s>" % os.path.basename(config.get("platformio", "include_dir")), ] + + src_filters = ( + src_filters + or pattern + or env_options.get( + "check_src_filters", + env_options.get("check_patterns", default_src_filters), + ) + ) + tool_options = dict( verbose=verbose, silent=silent, - patterns=pattern or env_options.get("check_patterns", default_patterns), + src_filters=src_filters, flags=flags or env_options.get("check_flags"), severity=[DefectItem.SEVERITY_LABELS[DefectItem.SEVERITY_HIGH]] if silent @@ -265,7 +273,7 @@ def print_defects_stats(results): tabular_data.append(total) headers = ["Component"] - headers.extend([l.upper() for l in severity_labels]) + headers.extend([label.upper() for label in severity_labels]) headers = [click.style(h, bold=True) for h in headers] click.echo(tabulate(tabular_data, headers=headers, numalign="center")) click.echo() diff --git a/platformio/check/defect.py b/platformio/check/defect.py index 15f9df70..5689a184 100644 --- a/platformio/check/defect.py +++ b/platformio/check/defect.py @@ -16,6 +16,7 @@ import os import click +from platformio.exception import PlatformioException from platformio.project.helpers import get_project_dir # pylint: disable=too-many-instance-attributes, redefined-builtin @@ -23,7 +24,6 @@ from platformio.project.helpers import get_project_dir class DefectItem: - SEVERITY_HIGH = 1 SEVERITY_MEDIUM = 2 SEVERITY_LOW = 4 @@ -79,7 +79,7 @@ class DefectItem: for key, value in DefectItem.SEVERITY_LABELS.items(): if label == value: return key - raise Exception("Unknown severity label -> %s" % label) + raise PlatformioException("Unknown severity label -> %s" % label) def as_dict(self): return { diff --git a/platformio/check/tools/base.py b/platformio/check/tools/base.py index d51b3158..77f0e55c 100644 --- a/platformio/check/tools/base.py +++ b/platformio/check/tools/base.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob import os import tempfile @@ -30,6 +29,7 @@ class CheckToolBase: # pylint: disable=too-many-instance-attributes self.config = config self.envname = envname self.options = options + self.project_dir = project_dir self.cc_flags = [] self.cxx_flags = [] self.cpp_includes = [] @@ -41,7 +41,7 @@ class CheckToolBase: # pylint: disable=too-many-instance-attributes self._defects = [] self._on_defect_callback = None self._bad_input = False - self._load_cpp_data(project_dir) + self._load_cpp_data() # detect all defects by default if not self.options.get("severity"): @@ -56,8 +56,8 @@ class CheckToolBase: # pylint: disable=too-many-instance-attributes for s in self.options["severity"] ] - def _load_cpp_data(self, project_dir): - data = load_build_metadata(project_dir, self.envname) + def _load_cpp_data(self): + data = load_build_metadata(self.project_dir, self.envname) if not data: return self.cc_flags = click.parser.split_arg_string(data.get("cc_flags", "")) @@ -99,6 +99,13 @@ class CheckToolBase: # pylint: disable=too-many-instance-attributes includes_file, ) result = proc.exec_command(cmd, shell=True) + + if result["returncode"] != 0: + click.echo("Warning: Failed to extract toolchain defines!") + if self.options.get("verbose"): + click.echo(result["out"]) + click.echo(result["err"]) + for line in result["out"].split("\n"): tokens = line.strip().split(" ", 2) if not tokens or tokens[0] != "#define": @@ -201,7 +208,7 @@ class CheckToolBase: # pylint: disable=too-many-instance-attributes return result @staticmethod - def get_project_target_files(patterns): + def get_project_target_files(project_dir, src_filters): c_extension = (".c",) cpp_extensions = (".cc", ".cpp", ".cxx", ".ino") header_extensions = (".h", ".hh", ".hpp", ".hxx") @@ -216,13 +223,9 @@ class CheckToolBase: # pylint: disable=too-many-instance-attributes elif path.endswith(cpp_extensions): result["c++"].append(os.path.abspath(path)) - for pattern in patterns: - for item in glob.glob(pattern, recursive=True): - if not os.path.isdir(item): - _add_file(item) - for root, _, files in os.walk(item, followlinks=True): - for f in files: - _add_file(os.path.join(root, f)) + src_filters = normalize_src_filters(src_filters) + for f in fs.match_src_files(project_dir, src_filters): + _add_file(f) return result @@ -243,3 +246,22 @@ class CheckToolBase: # pylint: disable=too-many-instance-attributes self.clean_up() return self._bad_input + + +# +# Helpers +# + + +def normalize_src_filters(src_filters): + def _normalize(src_filters): + return ( + src_filters + if src_filters.startswith(("+<", "-<")) + else "+<%s>" % src_filters + ) + + if isinstance(src_filters, (list, tuple)): + return " ".join([_normalize(f) for f in src_filters]) + + return _normalize(src_filters) diff --git a/platformio/check/tools/clangtidy.py b/platformio/check/tools/clangtidy.py index 1dd4165e..496acc7c 100644 --- a/platformio/check/tools/clangtidy.py +++ b/platformio/check/tools/clangtidy.py @@ -64,7 +64,9 @@ class ClangtidyCheckTool(CheckToolBase): ): cmd.append("--checks=*") - project_files = self.get_project_target_files(self.options["patterns"]) + project_files = self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ) src_files = [] for items in project_files.values(): diff --git a/platformio/check/tools/cppcheck.py b/platformio/check/tools/cppcheck.py index 0f8db402..c3c59537 100644 --- a/platformio/check/tools/cppcheck.py +++ b/platformio/check/tools/cppcheck.py @@ -96,7 +96,7 @@ class CppcheckCheckTool(CheckToolBase): ) click.echo() self._bad_input = True - self._buffer = "" + self._buffer = "" return None self._buffer = "" @@ -214,7 +214,9 @@ class CppcheckCheckTool(CheckToolBase): if not self.is_flag_set("--addon", self.get_flags("cppcheck")): return - for files in self.get_project_target_files(self.options["patterns"]).values(): + for files in self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ).values(): for f in files: dump_file = f + ".dump" if os.path.isfile(dump_file): @@ -243,7 +245,9 @@ class CppcheckCheckTool(CheckToolBase): def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback - project_files = self.get_project_target_files(self.options["patterns"]) + project_files = self.get_project_target_files( + self.project_dir, self.options["src_filters"] + ) src_files_scope = ("c", "c++") if not any(project_files[t] for t in src_files_scope): click.echo("Error: Nothing to check.") diff --git a/platformio/check/tools/pvsstudio.py b/platformio/check/tools/pvsstudio.py index ded65d1c..20979a00 100644 --- a/platformio/check/tools/pvsstudio.py +++ b/platformio/check/tools/pvsstudio.py @@ -227,7 +227,7 @@ class PvsStudioCheckTool(CheckToolBase): # pylint: disable=too-many-instance-at def check(self, on_defect_callback=None): self._on_defect_callback = on_defect_callback for scope, files in self.get_project_target_files( - self.options["patterns"] + self.project_dir, self.options["src_filters"] ).items(): if scope not in ("c", "c++"): continue diff --git a/platformio/cli.py b/platformio/cli.py index bdb46a3d..41aab522 100644 --- a/platformio/cli.py +++ b/platformio/cli.py @@ -19,7 +19,6 @@ import click class PlatformioCLI(click.MultiCommand): - leftover_args = [] def __init__(self, *args, **kwargs): diff --git a/platformio/commands/boards.py b/platformio/commands/boards.py index 8ca008f1..ac8367da 100644 --- a/platformio/commands/boards.py +++ b/platformio/commands/boards.py @@ -42,7 +42,7 @@ def cli(query, installed, json_output): # pylint: disable=R0912 grpboards[board["platform"]].append(board) terminal_width = shutil.get_terminal_size().columns - for (platform, boards) in sorted(grpboards.items()): + for platform, boards in sorted(grpboards.items()): click.echo("") click.echo("Platform: ", nl=False) click.secho(platform, bold=True) diff --git a/platformio/commands/ci.py b/platformio/commands/ci.py index aebd9e7b..5be31286 100644 --- a/platformio/commands/ci.py +++ b/platformio/commands/ci.py @@ -51,15 +51,13 @@ def validate_path(ctx, param, value): # pylint: disable=unused-argument @click.option( "--build-dir", default=tempfile.mkdtemp, - type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), + type=click.Path(file_okay=False, dir_okay=True, writable=True), ) @click.option("--keep-build-dir", is_flag=True) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("-O", "--project-option", multiple=True) @click.option("-e", "--environment", "environments", multiple=True) @@ -109,8 +107,8 @@ def cli( # pylint: disable=too-many-arguments, too-many-branches ctx.invoke( project_init_cmd, project_dir=build_dir, - board=board, - project_option=project_option, + boards=board, + project_options=project_option, ) # process project diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index ed3c9e8d..1c150c09 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -65,9 +65,7 @@ def invoke_command(ctx, cmd, **kwargs): "--storage-dir", multiple=True, default=None, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), help="Manage custom library storage", ) @click.option( diff --git a/platformio/commands/upgrade.py b/platformio/commands/upgrade.py index 2766dbd5..da8a2b3e 100644 --- a/platformio/commands/upgrade.py +++ b/platformio/commands/upgrade.py @@ -13,23 +13,28 @@ # limitations under the License. import json -import os import re -from zipfile import ZipFile +import subprocess import click from platformio import VERSION, __version__, app, exception -from platformio.compat import IS_WINDOWS from platformio.http import fetch_remote_content from platformio.package.manager.core import update_core_packages -from platformio.proc import exec_command, get_pythonexe_path -from platformio.project.helpers import get_project_cache_dir +from platformio.proc import get_pythonexe_path + +PYPI_JSON_URL = "https://pypi.org/pypi/platformio/json" +DEVELOP_ZIP_URL = "https://github.com/platformio/platformio-core/archive/develop.zip" +DEVELOP_INIT_SCRIPT_URL = ( + "https://raw.githubusercontent.com/platformio/platformio-core" + "/develop/platformio/__init__.py" +) @click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version") @click.option("--dev", is_flag=True, help="Use development branch") -def cli(dev): +@click.option("--verbose", "-v", is_flag=True) +def cli(dev, verbose): update_core_packages() if not dev and __version__ == get_latest_version(): return click.secho( @@ -38,29 +43,26 @@ def cli(dev): fg="green", ) - click.secho("Please wait while upgrading PlatformIO ...", fg="yellow") + click.secho("Please wait while upgrading PlatformIO Core ...", fg="yellow") + python_exe = get_pythonexe_path() to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".") - cmds = ( - ["pip", "install", "--upgrade", download_dist_package(to_develop)], - ["platformio", "--version"], - ) + pkg_spec = DEVELOP_ZIP_URL if to_develop else "platformio" - cmd = None - r = {} try: - for cmd in cmds: - cmd = [get_pythonexe_path(), "-m"] + cmd - r = exec_command(cmd) - - # try pip with disabled cache - if r["returncode"] != 0 and cmd[2] == "pip": - cmd.insert(3, "--no-cache-dir") - r = exec_command(cmd) - - assert r["returncode"] == 0 - assert "version" in r["out"] - actual_version = r["out"].strip().split("version", 1)[1].strip() + subprocess.run( + [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec], + check=True, + capture_output=not verbose, + ) + r = subprocess.run( + [python_exe, "-m", "platformio", "--version"], + check=True, + capture_output=True, + text=True, + ) + assert "version" in r.stdout + actual_version = r.stdout.split("version", 1)[1].strip() click.secho( "PlatformIO has been successfully upgraded to %s" % actual_version, fg="green", @@ -71,52 +73,24 @@ def cli(dev): click.secho( "Warning! Please restart IDE to affect PIO Home changes", fg="yellow" ) - except Exception as exc: - if not r: - raise exception.UpgradeError("\n".join([str(cmd), str(exc)])) from exc - permission_errors = ("permission denied", "not permitted") - if any(m in r["err"].lower() for m in permission_errors) and not IS_WINDOWS: - click.secho( - """ ------------------ -Permission denied ------------------ -You need the `sudo` permission to install Python packages. Try - -> sudo pip install -U platformio - -WARNING! Don't use `sudo` for the rest PlatformIO commands. -""", - fg="yellow", - err=True, - ) - raise exception.ReturnErrorCode(1) - raise exception.UpgradeError("\n".join([str(cmd), r["out"], r["err"]])) + except (AssertionError, subprocess.CalledProcessError) as exc: + click.secho( + "\nWarning!!! Could not automatically upgrade the PlatformIO Core.", + fg="red", + ) + click.secho( + "Please upgrade it manually using the following command:\n", + fg="red", + ) + click.secho(f'"{python_exe}" -m pip install -U {pkg_spec}\n', fg="cyan") + raise exception.ReturnErrorCode(1) from exc return True -def download_dist_package(to_develop): - if not to_develop: - return "platformio" - dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip" - cache_dir = get_project_cache_dir() - if not os.path.isdir(cache_dir): - os.makedirs(cache_dir) - pkg_name = os.path.join(cache_dir, "piocoredevelop.zip") - try: - with open(pkg_name, "wb") as fp: - r = exec_command( - ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True - ) - assert r["returncode"] == 0 - # check ZIP structure - with ZipFile(pkg_name) as zp: - assert zp.testzip() is None - return pkg_name - except: # pylint: disable=bare-except - pass - return dl_url +def get_pkg_spec(to_develop): + if to_develop: + return def get_latest_version(): @@ -133,10 +107,7 @@ def get_latest_version(): def get_develop_latest_version(): version = None - content = fetch_remote_content( - "https://raw.githubusercontent.com/platformio/platformio" - "/develop/platformio/__init__.py" - ) + content = fetch_remote_content(DEVELOP_INIT_SCRIPT_URL) for line in content.split("\n"): line = line.strip() if not line.startswith("VERSION"): @@ -153,5 +124,5 @@ def get_develop_latest_version(): def get_pypi_latest_version(): - content = fetch_remote_content("https://pypi.org/pypi/platformio/json") + content = fetch_remote_content(PYPI_JSON_URL) return json.loads(content)["info"]["version"] diff --git a/platformio/debug/cli.py b/platformio/debug/cli.py index 64bd1b43..12d6004a 100644 --- a/platformio/debug/cli.py +++ b/platformio/debug/cli.py @@ -28,9 +28,9 @@ from platformio.debug import helpers from platformio.debug.config.factory import DebugConfigFactory from platformio.debug.exception import DebugInvalidOptionsError from platformio.debug.process.gdb import GDBClientProcess +from platformio.exception import ReturnErrorCode from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig -from platformio.project.exception import ProjectEnvsNotAvailableError from platformio.project.helpers import is_platformio_project from platformio.project.options import ProjectOptions @@ -44,16 +44,12 @@ from platformio.project.options import ProjectOptions "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("--environment", "-e", metavar="") @click.option("--load-mode", type=ProjectOptions["env.debug_load_mode"].type) @@ -81,61 +77,57 @@ def cli( project_dir = os.getenv(name) with fs.cd(project_dir): - return _debug_in_project_dir( + project_config = ProjectConfig.get_instance(project_conf) + project_config.validate(envs=[environment] if environment else None) + env_name = environment or helpers.get_default_debug_env(project_config) + + if not interface: + return helpers.predebug_project( + ctx, project_dir, project_config, env_name, False, verbose + ) + + configure_args = ( ctx, - project_dir, - project_conf, - environment, + project_config, + env_name, load_mode, verbose, - interface, __unprocessed, ) + if helpers.is_gdbmi_mode(): + os.environ["PLATFORMIO_DISABLE_PROGRESSBAR"] = "true" + stream = helpers.GDBMIConsoleStream() + with proc.capture_std_streams(stream): + debug_config = _configure(*configure_args) + stream.close() + else: + debug_config = _configure(*configure_args) + + _run(project_dir, debug_config, __unprocessed) + + return None -def _debug_in_project_dir( - ctx, - project_dir, - project_conf, - environment, - load_mode, - verbose, - interface, - __unprocessed, -): - project_config = ProjectConfig.get_instance(project_conf) - project_config.validate(envs=[environment] if environment else None) - env_name = environment or helpers.get_default_debug_env(project_config) - - if not interface: - return helpers.predebug_project( - ctx, project_dir, project_config, env_name, False, verbose - ) - - env_options = project_config.items(env=env_name, as_dict=True) - if "platform" not in env_options: - raise ProjectEnvsNotAvailableError() - +def _configure(ctx, project_config, env_name, load_mode, verbose, __unprocessed): + platform = PlatformFactory.new( + project_config.get(f"env:{env_name}", "platform"), autoinstall=True + ) debug_config = DebugConfigFactory.new( - PlatformFactory.new(env_options["platform"], autoinstall=True), + platform, project_config, env_name, ) - if "--version" in __unprocessed: - return subprocess.run( - [debug_config.client_executable_path, "--version"], check=True + raise ReturnErrorCode( + subprocess.run( + [debug_config.client_executable_path, "--version"], check=True + ).returncode ) try: fs.ensure_udev_rules() except exception.InvalidUdevRules as exc: - click.echo( - helpers.escape_gdbmi_stream("~", str(exc) + "\n") - if helpers.is_gdbmi_mode() - else str(exc) + "\n", - nl=False, - ) + click.echo(str(exc)) rebuild_prog = False preload = debug_config.load_cmds == ["preload"] @@ -157,25 +149,10 @@ def _debug_in_project_dir( debug_config.load_cmds = [] if rebuild_prog: - if helpers.is_gdbmi_mode(): - click.echo( - helpers.escape_gdbmi_stream( - "~", "Preparing firmware for debugging...\n" - ), - nl=False, - ) - stream = helpers.GDBMIConsoleStream() - with proc.capture_std_streams(stream): - helpers.predebug_project( - ctx, project_dir, project_config, env_name, preload, verbose - ) - stream.close() - else: - click.echo("Preparing firmware for debugging...") - helpers.predebug_project( - ctx, project_dir, project_config, env_name, preload, verbose - ) - + click.echo("Preparing firmware for debugging...") + helpers.predebug_project( + ctx, os.getcwd(), project_config, env_name, preload, verbose + ) # save SHA sum of newly created prog if load_mode == "modified": helpers.is_prog_obsolete(debug_config.program_path) @@ -183,6 +160,10 @@ def _debug_in_project_dir( if not os.path.isfile(debug_config.program_path): raise DebugInvalidOptionsError("Program/firmware is missed") + return debug_config + + +def _run(project_dir, debug_config, __unprocessed): loop = asyncio.ProactorEventLoop() if IS_WINDOWS else asyncio.get_event_loop() asyncio.set_event_loop(loop) @@ -199,5 +180,3 @@ def _debug_in_project_dir( finally: client.close() loop.close() - - return True diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index 24e20516..cb496581 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -146,9 +146,9 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes def _load_build_data(self): data = load_build_metadata(os.getcwd(), self.env_name, cache=True, debug=True) - if data: - return data - raise DebugInvalidOptionsError("Could not load a build configuration") + if not data: + raise DebugInvalidOptionsError("Could not load a build configuration") + return data def _configure_server(self): # user disabled server in platformio.ini diff --git a/platformio/debug/config/blackmagic.py b/platformio/debug/config/blackmagic.py index 83d98d9c..370fdb7e 100644 --- a/platformio/debug/config/blackmagic.py +++ b/platformio/debug/config/blackmagic.py @@ -18,7 +18,6 @@ from platformio.device.finder import SerialPortFinder, is_pattern_port class BlackmagicDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target set language c diff --git a/platformio/debug/config/generic.py b/platformio/debug/config/generic.py index 1f155ecb..faa5a341 100644 --- a/platformio/debug/config/generic.py +++ b/platformio/debug/config/generic.py @@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase class GenericDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor reset halt diff --git a/platformio/debug/config/jlink.py b/platformio/debug/config/jlink.py index 03a1bd3a..a820e0aa 100644 --- a/platformio/debug/config/jlink.py +++ b/platformio/debug/config/jlink.py @@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase class JlinkDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor reset diff --git a/platformio/debug/config/mspdebug.py b/platformio/debug/config/mspdebug.py index 09266b3b..7f4d8e1a 100644 --- a/platformio/debug/config/mspdebug.py +++ b/platformio/debug/config/mspdebug.py @@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase class MspdebugDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target end diff --git a/platformio/debug/config/native.py b/platformio/debug/config/native.py index be15b5f4..413a9c62 100644 --- a/platformio/debug/config/native.py +++ b/platformio/debug/config/native.py @@ -17,7 +17,6 @@ from platformio.debug.config.base import DebugConfigBase class NativeDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target end diff --git a/platformio/debug/config/qemu.py b/platformio/debug/config/qemu.py index e9a57409..b998c782 100644 --- a/platformio/debug/config/qemu.py +++ b/platformio/debug/config/qemu.py @@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase class QemuDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor system_reset diff --git a/platformio/debug/config/renode.py b/platformio/debug/config/renode.py index 724be407..f2f62d66 100644 --- a/platformio/debug/config/renode.py +++ b/platformio/debug/config/renode.py @@ -16,7 +16,6 @@ from platformio.debug.config.base import DebugConfigBase class RenodeDebugConfig(DebugConfigBase): - GDB_INIT_SCRIPT = """ define pio_reset_halt_target monitor machine Reset diff --git a/platformio/debug/exception.py b/platformio/debug/exception.py index a1269b2f..7f4d0f4c 100644 --- a/platformio/debug/exception.py +++ b/platformio/debug/exception.py @@ -20,7 +20,6 @@ class DebugError(PlatformioException): class DebugSupportError(DebugError, UserSideException): - MESSAGE = ( "Currently, PlatformIO does not support debugging for `{0}`.\n" "Please request support at https://github.com/platformio/" diff --git a/platformio/debug/helpers.py b/platformio/debug/helpers.py index a2d89de3..8982c00f 100644 --- a/platformio/debug/helpers.py +++ b/platformio/debug/helpers.py @@ -31,7 +31,6 @@ from platformio.test.runners.factory import TestRunnerFactory class GDBMIConsoleStream(BytesIO): # pylint: disable=too-few-public-methods - STDOUT = sys.stdout def write(self, text): @@ -91,7 +90,7 @@ def predebug_project( TestSuite(env_name, debug_testname), project_config, TestRunnerOptions( - verbose=verbose, + verbose=3 if verbose else 0, without_building=False, without_debugging=False, without_uploading=not preload, diff --git a/platformio/debug/process/base.py b/platformio/debug/process/base.py index 2c9280c2..6ba8c70e 100644 --- a/platformio/debug/process/base.py +++ b/platformio/debug/process/base.py @@ -53,7 +53,6 @@ class DebugSubprocessProtocol(asyncio.SubprocessProtocol): class DebugBaseProcess: - STDOUT_CHUNK_SIZE = 2048 LOG_FILE = None diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index 4ce9aebe..ce7e82c8 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -24,7 +24,6 @@ from platformio.debug.process.client import DebugClientProcess class GDBClientProcess(DebugClientProcess): - PIO_SRC_NAME = ".pioinit" INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed" diff --git a/platformio/debug/process/server.py b/platformio/debug/process/server.py index 89b4095f..693dfa44 100644 --- a/platformio/debug/process/server.py +++ b/platformio/debug/process/server.py @@ -26,7 +26,6 @@ from platformio.proc import where_is_program class DebugServerProcess(DebugBaseProcess): - STD_BUFFER_SIZE = 1024 def __init__(self, debug_config): diff --git a/platformio/device/monitor/command.py b/platformio/device/monitor/command.py index 301de772..3d3190fe 100644 --- a/platformio/device/monitor/command.py +++ b/platformio/device/monitor/command.py @@ -104,7 +104,7 @@ from platformio.project.options import ProjectOptions "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option( "-e", @@ -132,24 +132,24 @@ def device_monitor_cmd(**options): ensure_ready=True, ).find(initial_port=options["port"]) - if options["menu_char"] == options["exit_char"]: - raise exception.UserSideException( - "--exit-char can not be the same as --menu-char" - ) - - # check for unknown filters - if options["filters"]: - known_filters = set(get_available_filters()) - unknown_filters = set(options["filters"]) - known_filters - if unknown_filters: - options["filters"] = list(known_filters & set(options["filters"])) - click.secho( - ("Warning! Skipping unknown filters `%s`. Known filters are `%s`") - % (", ".join(unknown_filters), ", ".join(sorted(known_filters))), - fg="yellow", + if options["menu_char"] == options["exit_char"]: + raise exception.UserSideException( + "--exit-char can not be the same as --menu-char" ) - start_terminal(options) + # check for unknown filters + if options["filters"]: + known_filters = set(get_available_filters()) + unknown_filters = set(options["filters"]) - known_filters + if unknown_filters: + options["filters"] = list(known_filters & set(options["filters"])) + click.secho( + ("Warning! Skipping unknown filters `%s`. Known filters are `%s`") + % (", ".join(unknown_filters), ", ".join(sorted(known_filters))), + fg="yellow", + ) + + start_terminal(options) def get_project_options(environment=None): diff --git a/platformio/device/monitor/filters/log2file.py b/platformio/device/monitor/filters/log2file.py index bf97b551..8e00d297 100644 --- a/platformio/device/monitor/filters/log2file.py +++ b/platformio/device/monitor/filters/log2file.py @@ -13,7 +13,7 @@ # limitations under the License. import io -import os.path +import os from datetime import datetime from platformio.device.monitor.filters.base import DeviceMonitorFilterBase @@ -27,8 +27,10 @@ class LogToFile(DeviceMonitorFilterBase): self._log_fp = None def __call__(self): - log_file_name = "platformio-device-monitor-%s.log" % datetime.now().strftime( - "%y%m%d-%H%M%S" + if not os.path.isdir("logs"): + os.makedirs("logs") + log_file_name = os.path.join( + "logs", "device-monitor-%s.log" % datetime.now().strftime("%y%m%d-%H%M%S") ) print("--- Logging an output to %s" % os.path.abspath(log_file_name)) # pylint: disable=consider-using-with diff --git a/platformio/device/monitor/terminal.py b/platformio/device/monitor/terminal.py index bec825e9..7b2a9b91 100644 --- a/platformio/device/monitor/terminal.py +++ b/platformio/device/monitor/terminal.py @@ -144,9 +144,8 @@ def new_serial_instance(options): # pylint: disable=too-many-branches except KeyboardInterrupt as exc: click.echo("", err=True) raise UserSideException("User aborted and port is not given") from exc - else: - if not port: - raise UserSideException("Port is not given") + if not port: + raise UserSideException("Port is not given") try: serial_instance = serial.serial_for_url( port, diff --git a/platformio/exception.py b/platformio/exception.py index a8287c04..bf2cec8c 100644 --- a/platformio/exception.py +++ b/platformio/exception.py @@ -14,7 +14,6 @@ class PlatformioException(Exception): - MESSAGE = None def __str__(self): # pragma: no cover @@ -26,7 +25,6 @@ class PlatformioException(Exception): class ReturnErrorCode(PlatformioException): - MESSAGE = "{0}" @@ -35,7 +33,6 @@ class UserSideException(PlatformioException): class AbortedByUser(UserSideException): - MESSAGE = "Aborted by user" @@ -49,7 +46,6 @@ class InvalidUdevRules(UserSideException): class MissedUdevRules(InvalidUdevRules): - MESSAGE = ( "Warning! Please install `99-platformio-udev.rules`. \nMore details: " "https://docs.platformio.org/en/latest/core/installation/udev-rules.html" @@ -57,7 +53,6 @@ class MissedUdevRules(InvalidUdevRules): class OutdatedUdevRules(InvalidUdevRules): - MESSAGE = ( "Warning! Your `{0}` are outdated. Please update or reinstall them." "\nMore details: " @@ -71,32 +66,26 @@ class OutdatedUdevRules(InvalidUdevRules): class GetSerialPortsError(PlatformioException): - MESSAGE = "No implementation for your platform ('{0}') available" class GetLatestVersionError(PlatformioException): - MESSAGE = "Can not retrieve the latest PlatformIO version" class InvalidSettingName(UserSideException): - MESSAGE = "Invalid setting with the name '{0}'" class InvalidSettingValue(UserSideException): - MESSAGE = "Invalid value '{0}' for the setting '{1}'" class InvalidJSONFile(PlatformioException): - MESSAGE = "Could not load broken JSON: {0}" class CIBuildEnvsEmpty(UserSideException): - MESSAGE = ( "Can't find PlatformIO build environments.\n" "Please specify `--board` or path to `platformio.ini` with " @@ -104,18 +93,7 @@ class CIBuildEnvsEmpty(UserSideException): ) -class UpgradeError(PlatformioException): - - MESSAGE = """{0} - -* Upgrade using `pip install -U platformio` -* Try different installation/upgrading steps: - https://docs.platformio.org/page/installation.html -""" - - class HomeDirPermissionsError(UserSideException): - MESSAGE = ( "The directory `{0}` or its parent directory is not owned by the " "current user and PlatformIO can not store configuration data.\n" @@ -126,7 +104,6 @@ class HomeDirPermissionsError(UserSideException): class CygwinEnvDetected(PlatformioException): - MESSAGE = ( "PlatformIO does not work within Cygwin environment. " "Use native Terminal instead." diff --git a/platformio/fs.py b/platformio/fs.py index bd17950b..1f43a7a3 100644 --- a/platformio/fs.py +++ b/platformio/fs.py @@ -24,7 +24,7 @@ import sys import click -from platformio import exception, proc +from platformio import exception from platformio.compat import IS_WINDOWS @@ -181,7 +181,7 @@ def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True): result = set() # correct fs directory separator src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep) - for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter): + for action, pattern in re.findall(r"(\+|\-)<([^>]+)>", src_filter): candidates = _find_candidates(pattern) if action == "+": result |= candidates @@ -193,26 +193,7 @@ def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True): def to_unix_path(path): if not IS_WINDOWS or not path: return path - return re.sub(r"[\\]+", "/", path) - - -def normalize_path(path): - path = os.path.abspath(path) - if not IS_WINDOWS or not path.startswith("\\\\"): - return path - try: - result = proc.exec_command(["net", "use"]) - if result["returncode"] != 0: - return path - share_re = re.compile(r"\s([A-Z]\:)\s+(\\\\[^\s]+)") - for line in result["out"].split("\n"): - share = share_re.search(line) - if not share: - continue - path = path.replace(share.group(2), share.group(1)) - except OSError: - pass - return path + return path.replace("\\", "/") def expanduser(path): diff --git a/platformio/home/cli.py b/platformio/home/cli.py index 55d09594..318c78fa 100644 --- a/platformio/home/cli.py +++ b/platformio/home/cli.py @@ -13,10 +13,11 @@ # limitations under the License. import mimetypes +import socket import click -from platformio.home.helpers import is_port_used +from platformio.compat import IS_WINDOWS from platformio.home.run import run_server from platformio.package.manager.core import get_core_package_dir @@ -95,3 +96,23 @@ def cli(port, host, no_open, shutdown_timeout, session_id): shutdown_timeout=shutdown_timeout, home_url=home_url, ) + + +def is_port_used(host, port): + socket.setdefaulttimeout(1) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if IS_WINDOWS: + try: + s.bind((host, port)) + s.close() + return False + except (OSError, socket.error): + pass + else: + try: + s.connect((host, port)) + s.close() + except socket.error: + return False + + return True diff --git a/platformio/home/helpers.py b/platformio/home/helpers.py deleted file mode 100644 index 0e88bde2..00000000 --- a/platformio/home/helpers.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import socket - -from platformio import util -from platformio.compat import IS_WINDOWS -from platformio.proc import where_is_program - - -@util.memoized(expire="60s") -def get_core_fullpath(): - return where_is_program("platformio" + (".exe" if IS_WINDOWS else "")) - - -def is_port_used(host, port): - socket.setdefaulttimeout(1) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if IS_WINDOWS: - try: - s.bind((host, port)) - s.close() - return False - except (OSError, socket.error): - pass - else: - try: - s.connect((host, port)) - s.close() - except socket.error: - return False - - return True diff --git a/platformio/home/rpc/handlers/account.py b/platformio/home/rpc/handlers/account.py index d857d587..f7e5dade 100644 --- a/platformio/home/rpc/handlers/account.py +++ b/platformio/home/rpc/handlers/account.py @@ -15,9 +15,10 @@ from ajsonrpc.core import JSONRPC20DispatchException from platformio.account.client import AccountClient +from platformio.home.rpc.handlers.base import BaseRPCHandler -class AccountRPC: +class AccountRPC(BaseRPCHandler): @staticmethod def call_client(method, *args, **kwargs): try: @@ -25,5 +26,5 @@ class AccountRPC: return getattr(client, method)(*args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="PIO Account Call Error", data=str(exc) + code=5000, message="PIO Account Call Error", data=str(exc) ) from exc diff --git a/platformio/home/rpc/handlers/app.py b/platformio/home/rpc/handlers/app.py index 9d7fc590..9c79e314 100644 --- a/platformio/home/rpc/handlers/app.py +++ b/platformio/home/rpc/handlers/app.py @@ -16,12 +16,12 @@ import os from pathlib import Path from platformio import __version__, app, fs, util +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project -class AppRPC: - +class AppRPC(BaseRPCHandler): IGNORE_STORAGE_KEYS = [ "cid", "coreVersion", diff --git a/platformio/home/rpc/handlers/base.py b/platformio/home/rpc/handlers/base.py new file mode 100644 index 00000000..7b3f5d8f --- /dev/null +++ b/platformio/home/rpc/handlers/base.py @@ -0,0 +1,17 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BaseRPCHandler: + factory = None diff --git a/platformio/home/rpc/handlers/ide.py b/platformio/home/rpc/handlers/ide.py index 5ff9fe33..643ee0c5 100644 --- a/platformio/home/rpc/handlers/ide.py +++ b/platformio/home/rpc/handlers/ide.py @@ -18,10 +18,10 @@ from pathlib import Path from ajsonrpc.core import JSONRPC20DispatchException from platformio.compat import aio_get_running_loop +from platformio.home.rpc.handlers.base import BaseRPCHandler -class IDERPC: - +class IDERPC(BaseRPCHandler): COMMAND_TIMEOUT = 1.5 # in seconds def __init__(self): @@ -51,11 +51,12 @@ class IDERPC: def on_command_result(self, cmd_id, value): if cmd_id not in self._cmd_queue: - return + return False if self._cmd_queue[cmd_id]["method"] == "get_pio_project_dirs": value = [str(Path(p).resolve()) for p in value] self._cmd_queue[cmd_id]["future"].set_result(value) del self._cmd_queue[cmd_id] + return True def _process_commands(self): for cmd_id in list(self._cmd_queue): diff --git a/platformio/home/rpc/handlers/misc.py b/platformio/home/rpc/handlers/misc.py index c384fea9..ab2508a4 100644 --- a/platformio/home/rpc/handlers/misc.py +++ b/platformio/home/rpc/handlers/misc.py @@ -17,10 +17,11 @@ import time from platformio.cache import ContentCache from platformio.compat import aio_create_task +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.home.rpc.handlers.os import OSRPC -class MiscRPC: +class MiscRPC(BaseRPCHandler): async def load_latest_tweets(self, data_url): cache_key = ContentCache.key_from_args(data_url, "tweets") cache_valid = "180d" diff --git a/platformio/home/rpc/handlers/os.py b/platformio/home/rpc/handlers/os.py index 333618b5..0d6b9f48 100644 --- a/platformio/home/rpc/handlers/os.py +++ b/platformio/home/rpc/handlers/os.py @@ -24,6 +24,7 @@ from starlette.concurrency import run_in_threadpool from platformio import fs from platformio.cache import ContentCache from platformio.device.list.util import list_logical_devices +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.http import HTTPSession, ensure_internet_on @@ -35,7 +36,7 @@ class HTTPAsyncSession(HTTPSession): return await run_in_threadpool(func, *args, **kwargs) -class OSRPC: +class OSRPC(BaseRPCHandler): @staticmethod async def fetch_content(url, data=None, headers=None, cache_valid=None): if not headers: @@ -89,6 +90,14 @@ class OSRPC: def open_file(path): return click.launch(path) + @staticmethod + def call_path_module_func(name, args, **kwargs): + return getattr(os.path, name)(*args, **kwargs) + + @staticmethod + def get_path_separator(): + return os.sep + @staticmethod def is_file(path): return os.path.isfile(path) @@ -156,9 +165,4 @@ class OSRPC: @staticmethod def get_logical_devices(): - items = [] - for item in list_logical_devices(): - if item["name"]: - item["name"] = item["name"] - items.append(item) - return items + return list_logical_devices() diff --git a/platformio/home/rpc/handlers/piocore.py b/platformio/home/rpc/handlers/piocore.py index df252114..080ba96c 100644 --- a/platformio/home/rpc/handlers/piocore.py +++ b/platformio/home/rpc/handlers/piocore.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio +import functools import io import json import os @@ -22,9 +24,45 @@ import click from ajsonrpc.core import JSONRPC20DispatchException from starlette.concurrency import run_in_threadpool -from platformio import __main__, __version__, fs, proc -from platformio.compat import get_locale_encoding, is_bytes -from platformio.home import helpers +from platformio import __main__, __version__, app, fs, proc, util +from platformio.compat import ( + IS_WINDOWS, + aio_create_task, + aio_get_running_loop, + get_locale_encoding, + is_bytes, +) +from platformio.exception import PlatformioException +from platformio.home.rpc.handlers.base import BaseRPCHandler + + +class PIOCoreProtocol(asyncio.SubprocessProtocol): + def __init__(self, exit_future, on_data_callback=None): + self.exit_future = exit_future + self.on_data_callback = on_data_callback + self.stdout = "" + self.stderr = "" + self._is_exited = False + self._encoding = get_locale_encoding() + + def pipe_data_received(self, fd, data): + data = data.decode(self._encoding, "replace") + pipe = ["stdin", "stdout", "stderr"][fd] + if pipe == "stdout": + self.stdout += data + if pipe == "stderr": + self.stderr += data + if self.on_data_callback: + self.on_data_callback(pipe=pipe, data=data) + + def connection_lost(self, exc): + self.process_exited() + + def process_exited(self): + if self._is_exited: + return + self.exit_future.set_result(True) + self._is_exited = True class MultiThreadingStdStream: @@ -58,11 +96,51 @@ class MultiThreadingStdStream: return result -class PIOCoreRPC: +@util.memoized(expire="60s") +def get_core_fullpath(): + return proc.where_is_program("platformio" + (".exe" if IS_WINDOWS else "")) + + +class PIOCoreRPC(BaseRPCHandler): @staticmethod def version(): return __version__ + async def exec(self, args, options=None): + loop = aio_get_running_loop() + exit_future = loop.create_future() + data_callback = functools.partial( + self._on_exec_data_received, exec_options=options + ) + if args[0] != "--caller" and app.get_session_var("caller_id"): + args = ["--caller", app.get_session_var("caller_id")] + args + transport, protocol = await loop.subprocess_exec( + lambda: PIOCoreProtocol(exit_future, data_callback), + get_core_fullpath(), + *args, + stdin=None, + **options.get("spawn", {}), + ) + await exit_future + transport.close() + return { + "stdout": protocol.stdout, + "stderr": protocol.stderr, + "returncode": transport.get_returncode(), + } + + def _on_exec_data_received(self, exec_options, pipe, data): + notification_method = exec_options.get(f"{pipe}NotificationMethod") + if not notification_method: + return + aio_create_task( + self.factory.notify_clients( + method=notification_method, + params=[data], + actor="frontend", + ) + ) + @staticmethod def setup_multithreading_std_streams(): if isinstance(sys.stdout, MultiThreadingStdStream): @@ -94,14 +172,14 @@ class PIOCoreRPC: return PIOCoreRPC._process_result(result, to_json) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="PIO Core Call Error", data=str(exc) + code=5000, message="PIO Core Call Error", data=str(exc) ) from exc @staticmethod async def _call_subprocess(args, options): result = await run_in_threadpool( proc.exec_command, - [helpers.get_core_fullpath()] + args, + [get_core_fullpath()] + args, cwd=options.get("cwd") or os.getcwd(), ) return (result["out"], result["err"], result["returncode"]) @@ -132,7 +210,7 @@ class PIOCoreRPC: err = err.decode(get_locale_encoding()) text = ("%s\n\n%s" % (out, err)).strip() if code != 0: - raise Exception(text) + raise PlatformioException(text) if not to_json: return text try: diff --git a/platformio/home/rpc/handlers/platform.py b/platformio/home/rpc/handlers/platform.py new file mode 100644 index 00000000..2cc959df --- /dev/null +++ b/platformio/home/rpc/handlers/platform.py @@ -0,0 +1,61 @@ +# Copyright (c) 2014-present PlatformIO +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from platformio.home.rpc.handlers.base import BaseRPCHandler +from platformio.package.manager.platform import PlatformPackageManager +from platformio.platform.factory import PlatformFactory + + +class PlatformRPC(BaseRPCHandler): + @staticmethod + def list_installed(options=None): + result = [] + options = options or {} + + def _matchSearchQuery(p): + searchQuery = options.get("searchQuery") + if not searchQuery: + return True + content_blocks = [p.name, p.title, p.description] + if p.frameworks: + content_blocks.append(" ".join(p.frameworks.keys())) + for board in p.get_boards().values(): + board_data = board.get_brief_data() + for key in ("id", "mcu", "vendor"): + content_blocks.append(board_data.get(key)) + return searchQuery.strip() in " ".join(content_blocks) + + pm = PlatformPackageManager() + for pkg in pm.get_installed(): + p = PlatformFactory.new(pkg) + if not _matchSearchQuery(p): + continue + result.append( + dict( + __pkg_path=pkg.path, + __pkg_meta=pkg.metadata.as_dict(), + name=p.name, + title=p.title, + description=p.description, + ) + ) + return result + + @staticmethod + def get_boards(spec): + p = PlatformFactory.new(spec) + return sorted( + [b.get_brief_data() for b in p.get_boards().values()], + key=lambda item: item["name"], + ) diff --git a/platformio/home/rpc/handlers/project.py b/platformio/home/rpc/handlers/project.py index 6c12bcae..904d00e4 100644 --- a/platformio/home/rpc/handlers/project.py +++ b/platformio/home/rpc/handlers/project.py @@ -16,10 +16,12 @@ import os import shutil import time +import semantic_version from ajsonrpc.core import JSONRPC20DispatchException -from platformio import exception, fs +from platformio import app, exception, fs from platformio.home.rpc.handlers.app import AppRPC +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.home.rpc.handlers.piocore import PIOCoreRPC from platformio.package.manager.platform import PlatformPackageManager from platformio.project.config import ProjectConfig @@ -29,7 +31,7 @@ from platformio.project.integration.generator import ProjectGenerator from platformio.project.options import get_config_options_schema -class ProjectRPC: +class ProjectRPC(BaseRPCHandler): @staticmethod def config_call(init_kwargs, method, *args): assert isinstance(init_kwargs, dict) @@ -184,83 +186,17 @@ class ProjectRPC: async def init(self, board, framework, project_dir): assert project_dir - state = AppRPC.load_state() if not os.path.isdir(project_dir): os.makedirs(project_dir) - args = ["init", "--board", board] + args = ["init", "--board", board, "--sample-code"] if framework: args.extend(["--project-option", "framework = %s" % framework]) - if ( - state["storage"]["coreCaller"] - and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() - ): - args.extend(["--ide", state["storage"]["coreCaller"]]) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) - return self._generate_project_main(project_dir, board, framework) - - @staticmethod - def _generate_project_main(project_dir, board, framework): - main_content = None - if framework == "arduino": - main_content = "\n".join( - [ - "#include ", - "", - "void setup() {", - " // put your setup code here, to run once:", - "}", - "", - "void loop() {", - " // put your main code here, to run repeatedly:", - "}", - "", - ] - ) - elif framework == "mbed": - main_content = "\n".join( - [ - "#include ", - "", - "int main() {", - "", - " // put your setup code here, to run once:", - "", - " while(1) {", - " // put your main code here, to run repeatedly:", - " }", - "}", - "", - ] - ) - if not main_content: - return project_dir - - is_cpp_project = True - pm = PlatformPackageManager() - try: - board = pm.board_config(board) - platforms = board.get("platforms", board.get("platform")) - if not isinstance(platforms, list): - platforms = [platforms] - c_based_platforms = ["intel_mcs51", "ststm8"] - is_cpp_project = not set(platforms) & set(c_based_platforms) - except exception.PlatformioException: - pass - - with fs.cd(project_dir): - config = ProjectConfig() - src_dir = config.get("platformio", "src_dir") - main_path = os.path.join( - src_dir, "main.%s" % ("cpp" if is_cpp_project else "c") - ) - if os.path.isfile(main_path): - return project_dir - if not os.path.isdir(src_dir): - os.makedirs(src_dir) - with open(main_path, mode="w", encoding="utf8") as fp: - fp.write(main_content.strip()) return project_dir @staticmethod @@ -296,11 +232,9 @@ class ProjectRPC: args.extend( ["--project-option", "lib_extra_dirs = ~/Documents/Arduino/libraries"] ) - if ( - state["storage"]["coreCaller"] - and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() - ): - args.extend(["--ide", state["storage"]["coreCaller"]]) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": project_dir, "force_subprocess": True} ) @@ -324,14 +258,50 @@ class ProjectRPC: ) shutil.copytree(project_dir, new_project_dir, symlinks=True) - state = AppRPC.load_state() args = ["init"] - if ( - state["storage"]["coreCaller"] - and state["storage"]["coreCaller"] in ProjectGenerator.get_supported_ides() - ): - args.extend(["--ide", state["storage"]["coreCaller"]]) + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) await PIOCoreRPC.call( args, options={"cwd": new_project_dir, "force_subprocess": True} ) return new_project_dir + + async def create_empty(self, configuration, options=None): + project_dir = os.path.join(configuration["location"], configuration["name"]) + if not os.path.isdir(project_dir): + os.makedirs(project_dir) + + project_options = [] + platform = configuration["platform"] + board = configuration.get("board", {}).get("id") + env_name = board or platform["name"] + if configuration.get("description"): + project_options.append(("description", configuration.get("description"))) + try: + v = semantic_version.Version(platform.get("version")) + assert not v.prerelease + project_options.append( + ("platform", "{name} @ ^{version}".format(**platform)) + ) + except (AssertionError, ValueError): + project_options.append( + ("platform", "{name} @ {version}".format(**platform)) + ) + if board: + project_options.append(("board", board)) + if configuration.get("framework"): + project_options.append(("framework", configuration["framework"]["name"])) + + args = ["project", "init", "-e", env_name, "--sample-code"] + ide = app.get_session_var("caller_id") + if ide in ProjectGenerator.get_supported_ides(): + args.extend(["--ide", ide]) + for name, value in project_options: + args.extend(["-O", f"{name}={value}"]) + + envclone = os.environ.copy() + envclone["PLATFORMIO_FORCE_ANSI"] = "true" + options = options or {} + options["spawn"] = {"env": envclone, "cwd": project_dir} + return await self.factory.manager.dispatcher["core.exec"](args, options=options) diff --git a/platformio/home/rpc/handlers/registry.py b/platformio/home/rpc/handlers/registry.py index a6d5b8bf..1370f328 100644 --- a/platformio/home/rpc/handlers/registry.py +++ b/platformio/home/rpc/handlers/registry.py @@ -13,17 +13,19 @@ # limitations under the License. from ajsonrpc.core import JSONRPC20DispatchException +from starlette.concurrency import run_in_threadpool +from platformio.home.rpc.handlers.base import BaseRPCHandler from platformio.registry.client import RegistryClient -class RegistryRPC: +class RegistryRPC(BaseRPCHandler): @staticmethod - def call_client(method, *args, **kwargs): + async def call_client(method, *args, **kwargs): try: client = RegistryClient() - return getattr(client, method)(*args, **kwargs) + return await run_in_threadpool(getattr(client, method), *args, **kwargs) except Exception as exc: # pylint: disable=bare-except raise JSONRPC20DispatchException( - code=4003, message="Registry Call Error", data=str(exc) + code=5000, message="Registry Call Error", data=str(exc) ) from exc diff --git a/platformio/home/rpc/server.py b/platformio/home/rpc/server.py index eb5bd6a5..2437e40e 100644 --- a/platformio/home/rpc/server.py +++ b/platformio/home/rpc/server.py @@ -12,17 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +from urllib.parse import parse_qs + import click +from ajsonrpc.core import JSONRPC20Error, JSONRPC20Request from ajsonrpc.dispatcher import Dispatcher -from ajsonrpc.manager import AsyncJSONRPCResponseManager +from ajsonrpc.manager import AsyncJSONRPCResponseManager, JSONRPC20Response from starlette.endpoints import WebSocketEndpoint from platformio.compat import aio_create_task, aio_get_running_loop +from platformio.http import InternetConnectionError from platformio.proc import force_exit class JSONRPCServerFactoryBase: - connection_nums = 0 shutdown_timer = None @@ -31,20 +34,25 @@ class JSONRPCServerFactoryBase: self.manager = AsyncJSONRPCResponseManager( Dispatcher(), is_server_error_verbose=True ) + self._clients = {} def __call__(self, *args, **kwargs): raise NotImplementedError def add_object_handler(self, handler, namespace): + handler.factory = self self.manager.dispatcher.add_object(handler, prefix="%s." % namespace) - def on_client_connect(self): + def on_client_connect(self, connection, actor=None): + self._clients[connection] = {"actor": actor} self.connection_nums += 1 if self.shutdown_timer: self.shutdown_timer.cancel() self.shutdown_timer = None - def on_client_disconnect(self): + def on_client_disconnect(self, connection): + if connection in self._clients: + del self._clients[connection] self.connection_nums -= 1 if self.connection_nums < 1: self.connection_nums = 0 @@ -67,6 +75,14 @@ class JSONRPCServerFactoryBase: self.shutdown_timeout, _auto_shutdown_server ) + async def notify_clients(self, method, params=None, actor=None): + for client, options in self._clients.items(): + if actor and options["actor"] != actor: + continue + request = JSONRPC20Request(method, params, is_notification=True) + await client.send_text(self.manager.serialize(request.body)) + return True + class WebSocketJSONRPCServerFactory(JSONRPCServerFactoryBase): def __call__(self, *args, **kwargs): @@ -81,17 +97,30 @@ class WebSocketJSONRPCServer(WebSocketEndpoint): async def on_connect(self, websocket): await websocket.accept() - self.factory.on_client_connect() # pylint: disable=no-member + qs = parse_qs(self.scope.get("query_string", b"")) + actors = qs.get(b"actor") + self.factory.on_client_connect( # pylint: disable=no-member + websocket, actor=actors[0].decode() if actors else None + ) async def on_receive(self, websocket, data): aio_create_task(self._handle_rpc(websocket, data)) async def on_disconnect(self, websocket, close_code): - self.factory.on_client_disconnect() # pylint: disable=no-member + self.factory.on_client_disconnect(websocket) # pylint: disable=no-member async def _handle_rpc(self, websocket, data): # pylint: disable=no-member response = await self.factory.manager.get_response_for_payload(data) if response.error and response.error.data: click.secho("Error: %s" % response.error.data, fg="red", err=True) + if InternetConnectionError.MESSAGE in response.error.data: + response = JSONRPC20Response( + id=response.id, + error=JSONRPC20Error( + code=4008, + message="No Internet Connection", + data=response.error.data, + ), + ) await websocket.send_text(self.factory.manager.serialize(response.body)) diff --git a/platformio/home/run.py b/platformio/home/run.py index 25f1081e..df3fa47e 100644 --- a/platformio/home/run.py +++ b/platformio/home/run.py @@ -32,6 +32,7 @@ from platformio.home.rpc.handlers.ide import IDERPC from platformio.home.rpc.handlers.misc import MiscRPC from platformio.home.rpc.handlers.os import OSRPC from platformio.home.rpc.handlers.piocore import PIOCoreRPC +from platformio.home.rpc.handlers.platform import PlatformRPC from platformio.home.rpc.handlers.project import ProjectRPC from platformio.home.rpc.handlers.registry import RegistryRPC from platformio.home.rpc.server import WebSocketJSONRPCServerFactory @@ -44,7 +45,7 @@ class ShutdownMiddleware: self.app = app async def __call__(self, scope, receive, send): - if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", {}): + if scope["type"] == "http" and b"__shutdown__" in scope.get("query_string", ""): await shutdown_server() await self.app(scope, receive, send) @@ -73,6 +74,7 @@ def run_server(host, port, no_open, shutdown_timeout, home_url): ws_rpc_factory.add_object_handler(OSRPC(), namespace="os") ws_rpc_factory.add_object_handler(PIOCoreRPC(), namespace="core") ws_rpc_factory.add_object_handler(ProjectRPC(), namespace="project") + ws_rpc_factory.add_object_handler(PlatformRPC(), namespace="platform") ws_rpc_factory.add_object_handler(RegistryRPC(), namespace="registry") path = urlparse(home_url).path diff --git a/platformio/http.py b/platformio/http.py index 9f4f44dd..48235f74 100644 --- a/platformio/http.py +++ b/platformio/http.py @@ -37,8 +37,7 @@ class HTTPClientError(PlatformioException): return self.message -class InternetIsOffline(UserSideException): - +class InternetConnectionError(UserSideException): MESSAGE = ( "You are not connected to the Internet.\n" "PlatformIO needs the Internet connection to" @@ -204,7 +203,7 @@ def _internet_on(): def ensure_internet_on(raise_exception=False): result = _internet_on() if raise_exception and not result: - raise InternetIsOffline() + raise InternetConnectionError() return result diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 7893a23d..3110b2c1 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -22,9 +22,8 @@ import semantic_version from platformio import __version__, app, exception, fs, telemetry from platformio.cache import cleanup_content_cache from platformio.cli import PlatformioCLI -from platformio.commands.platform import platform_update as cmd_platform_update from platformio.commands.upgrade import get_latest_version -from platformio.http import HTTPClientError, InternetIsOffline, ensure_internet_on +from platformio.http import HTTPClientError, InternetConnectionError, ensure_internet_on from platformio.package.manager.core import update_core_packages from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec @@ -51,7 +50,7 @@ def on_platformio_end(ctx, result): # pylint: disable=unused-argument check_prune_system() except ( HTTPClientError, - InternetIsOffline, + InternetConnectionError, exception.GetLatestVersionError, ): click.secho( @@ -67,15 +66,15 @@ def on_platformio_exception(e): def set_caller(caller=None): caller = caller or os.getenv("PLATFORMIO_CALLER") + if not caller: + if os.getenv("CODESPACES"): + caller = "codespaces" + elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): + caller = "vscode" + elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): + caller = "gitpod" if caller: - return app.set_session_var("caller_id", caller) - if os.getenv("CODESPACES"): - caller = "codespaces" - elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): - caller = "vscode" - elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): - caller = "gitpod" - return app.set_session_var("caller_id", caller) + app.set_session_var("caller_id", caller) class Upgrader: @@ -84,7 +83,6 @@ class Upgrader: self.to_version = pepver_to_semver(to_version) self._upgraders = [ - (semantic_version.Version("3.5.0-a.2"), self._update_dev_platforms), (semantic_version.Version("4.4.0-a.8"), self._update_pkg_metadata), ] @@ -100,11 +98,6 @@ class Upgrader: return all(result) - @staticmethod - def _update_dev_platforms(ctx): - ctx.invoke(cmd_platform_update) - return True - @staticmethod def _update_pkg_metadata(_): pm = ToolPackageManager() @@ -166,8 +159,6 @@ def after_upgrade(ctx): action="Upgrade", label="%s > %s" % (last_version, __version__), ) - else: - raise exception.UpgradeError("Auto upgrading...") # PlatformIO banner click.echo("*" * terminal_width) diff --git a/platformio/package/commands/install.py b/platformio/package/commands/install.py index 6041e357..e3eb05aa 100644 --- a/platformio/package/commands/install.py +++ b/platformio/package/commands/install.py @@ -39,7 +39,7 @@ from platformio.test.runners.factory import TestRunnerFactory "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -55,7 +55,7 @@ from platformio.test.runners.factory import TestRunnerFactory @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-f", "--force", is_flag=True, help="Reinstall package if it exists") diff --git a/platformio/package/commands/list.py b/platformio/package/commands/list.py index 9d538b7d..c2426c9c 100644 --- a/platformio/package/commands/list.py +++ b/platformio/package/commands/list.py @@ -31,7 +31,7 @@ from platformio.project.config import ProjectConfig "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -41,7 +41,7 @@ from platformio.project.config import ProjectConfig @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("--only-platforms", is_flag=True, help="List only platform packages") @@ -137,7 +137,7 @@ def list_global_packages(options): only_packages = any( options.get(type_) or options.get(f"only_{type_}") for (type_, _) in data ) - for (type_, pm) in data: + for type_, pm in data: skip_conds = [ only_packages and not options.get(type_) diff --git a/platformio/package/commands/outdated.py b/platformio/package/commands/outdated.py index 5a8c1ea4..18c3783a 100644 --- a/platformio/package/commands/outdated.py +++ b/platformio/package/commands/outdated.py @@ -58,7 +58,7 @@ class OutdatedCandidate: "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) def package_outdated_cmd(project_dir, environments): diff --git a/platformio/package/commands/pack.py b/platformio/package/commands/pack.py index 038ffc15..e9cd0f03 100644 --- a/platformio/package/commands/pack.py +++ b/platformio/package/commands/pack.py @@ -26,7 +26,7 @@ from platformio.package.pack import PackagePacker "package", default=os.getcwd, metavar="", - type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=True, dir_okay=True), ) @click.option( "-o", "--output", help="A destination path (folder or a full path to file)" diff --git a/platformio/package/commands/publish.py b/platformio/package/commands/publish.py index 58230eef..228c1411 100644 --- a/platformio/package/commands/publish.py +++ b/platformio/package/commands/publish.py @@ -47,7 +47,7 @@ def validate_datetime(ctx, param, value): # pylint: disable=unused-argument "package", default=os.getcwd, metavar="", - type=click.Path(exists=True, file_okay=True, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=True, dir_okay=True), ) @click.option( "--owner", diff --git a/platformio/package/commands/uninstall.py b/platformio/package/commands/uninstall.py index e393d2a8..2808d491 100644 --- a/platformio/package/commands/uninstall.py +++ b/platformio/package/commands/uninstall.py @@ -33,7 +33,7 @@ from platformio.project.savedeps import pkg_to_save_spec, save_project_dependenc "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -49,7 +49,7 @@ from platformio.project.savedeps import pkg_to_save_spec, save_project_dependenc @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") diff --git a/platformio/package/commands/update.py b/platformio/package/commands/update.py index 67fc5dd1..8788f7b5 100644 --- a/platformio/package/commands/update.py +++ b/platformio/package/commands/update.py @@ -33,7 +33,7 @@ from platformio.project.savedeps import pkg_to_save_spec, save_project_dependenc "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("-p", "--platform", "platforms", metavar="SPECIFICATION", multiple=True) @@ -49,7 +49,7 @@ from platformio.project.savedeps import pkg_to_save_spec, save_project_dependenc @click.option( "--storage-dir", default=None, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), help="Custom Package Manager storage for global packages", ) @click.option("-s", "--silent", is_flag=True, help="Suppress progress reporting") diff --git a/platformio/package/download.py b/platformio/package/download.py index 5045e856..17cc1f30 100644 --- a/platformio/package/download.py +++ b/platformio/package/download.py @@ -105,7 +105,7 @@ class FileDownloader: label=label, update_min_steps=min( 256 * 1024, file_size / 100 - ), # every 256Kb or less, + ), # every 256Kb or less ) as pb: for chunk in pb: pb.update(len(chunk)) diff --git a/platformio/package/exception.py b/platformio/package/exception.py index 580137a0..d38c0e60 100644 --- a/platformio/package/exception.py +++ b/platformio/package/exception.py @@ -48,12 +48,10 @@ class ManifestValidationError(ManifestException): class MissingPackageManifestError(ManifestException): - MESSAGE = "Could not find one of '{0}' manifest files in the package" class UnknownPackageError(UserSideException): - MESSAGE = ( "Could not find the package with '{0}' requirements for your system '%s'" % util.get_systype() @@ -61,7 +59,6 @@ class UnknownPackageError(UserSideException): class NotGlobalLibDir(UserSideException): - MESSAGE = ( "The `{0}` is not a PlatformIO project.\n\n" "To manage libraries in global storage `{1}`,\n" diff --git a/platformio/package/manager/_download.py b/platformio/package/manager/_download.py index 9d9c6118..bf844980 100644 --- a/platformio/package/manager/_download.py +++ b/platformio/package/manager/_download.py @@ -26,7 +26,6 @@ from platformio.package.lockfile import LockFile class PackageManagerDownloadMixin: - DOWNLOAD_CACHE_EXPIRE = 86400 * 30 # keep package in a local cache for 1 month def compute_download_path(self, *args): @@ -62,7 +61,7 @@ class PackageManagerDownloadMixin: self.set_download_utime(dl_path) return dl_path - with_progress = not silent and not app.is_disabled_progressbar() + with_progress = not app.is_disabled_progressbar() tmp_fd, tmp_path = tempfile.mkstemp(dir=self.get_download_dir()) try: with LockFile(dl_path): @@ -71,7 +70,7 @@ class PackageManagerDownloadMixin: fd.set_destination(tmp_path) fd.start(with_progress=with_progress, silent=silent) except IOError as exc: - raise_error = not with_progress + raise_error = not silent if with_progress: try: fd = FileDownloader(url) diff --git a/platformio/package/manager/_install.py b/platformio/package/manager/_install.py index c89d7b86..fc4d5ddb 100644 --- a/platformio/package/manager/_install.py +++ b/platformio/package/manager/_install.py @@ -27,7 +27,6 @@ from platformio.package.vcsclient import VCSClientFactory class PackageManagerInstallMixin: - _INSTALL_HISTORY = None # avoid circle dependencies @staticmethod diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index 28fab07b..ca5c5e01 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -12,19 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import os -import shutil -import subprocess -import sys -from datetime import date -from platformio import __core_packages__, exception, fs, util -from platformio.exception import UserSideException +from platformio import __core_packages__, exception from platformio.package.exception import UnknownPackageError from platformio.package.manager.tool import ToolPackageManager -from platformio.package.meta import PackageItem, PackageSpec -from platformio.proc import get_pythonexe_path +from platformio.package.meta import PackageSpec def get_installed_core_packages(): @@ -98,131 +91,3 @@ def remove_unnecessary_core_packages(dry_run=False): pm.uninstall(pkg) return candidates - - -def inject_contrib_pysite(): - # pylint: disable=import-outside-toplevel - from site import addsitedir - - try: - contrib_pysite_dir = get_core_package_dir("contrib-pysite") - except UnknownPackageError: - pm = ToolPackageManager() - contrib_pysite_dir = build_contrib_pysite_package( - os.path.join(pm.package_dir, "contrib-pysite") - ) - - if contrib_pysite_dir in sys.path: - return True - - addsitedir(contrib_pysite_dir) - sys.path.insert(0, contrib_pysite_dir) - - try: - # pylint: disable=import-error,unused-import,unused-variable - from OpenSSL import SSL - - except: # pylint: disable=bare-except - build_contrib_pysite_package(contrib_pysite_dir) - - return True - - -def build_contrib_pysite_package(target_dir, with_metadata=True): - systype = util.get_systype() - if os.path.isdir(target_dir): - fs.rmtree(target_dir) - os.makedirs(target_dir) - - # issue 3865: There is no "rustup" in "Raspbian GNU/Linux 10 (buster)" - os.environ["CRYPTOGRAPHY_DONT_BUILD_RUST"] = "1" - - # build dependencies - args = [ - get_pythonexe_path(), - "-m", - "pip", - "install", - "--no-compile", - "-t", - target_dir, - ] - if "linux" in systype: - args.extend(["--no-binary", ":all:"]) - try: - subprocess.run(args + get_contrib_pysite_deps(), check=True, env=os.environ) - except subprocess.CalledProcessError as exc: - if "linux" in systype: - raise UserSideException( - "\n\nPlease ensure that the next packages are installed:\n\n" - "sudo apt install python3-dev libffi-dev libssl-dev\n" - ) from exc - raise exc - - # build manifests - with open( - os.path.join(target_dir, "package.json"), mode="w", encoding="utf8" - ) as fp: - json.dump( - dict( - name="contrib-pysite", - version="2.%d%d.%s" - % ( - sys.version_info.major, - sys.version_info.minor, - date.today().strftime("%y%m%d"), - ), - system=list( - set([systype, "linux_armv6l", "linux_armv7l", "linux_armv8l"]) - ) - if systype.startswith("linux_arm") - else systype, - description="Extra Python package for PlatformIO Core", - keywords=["platformio", "platformio-core"], - homepage="https://docs.platformio.org/page/core/index.html", - repository={ - "type": "git", - "url": "https://github.com/platformio/platformio-core", - }, - ), - fp, - ) - - # generate package metadata - if with_metadata: - pm = ToolPackageManager() - pkg = PackageItem(target_dir) - pkg.metadata = pm.build_metadata( - target_dir, PackageSpec(owner="platformio", name="contrib-pysite") - ) - pkg.dump_meta() - - # remove unused files - for root, dirs, files in os.walk(target_dir): - for t in ("_test", "test", "tests"): - if t in dirs: - shutil.rmtree(os.path.join(root, t)) - for name in files: - if name.endswith((".chm", ".pyc")): - os.remove(os.path.join(root, name)) - - return target_dir - - -def get_contrib_pysite_deps(): - systype = util.get_systype() - twisted_version = "22.1.0" - if "linux_arm" in systype: - result = [ - # twisted[tls], see setup.py for %twisted_version% - "twisted == %s" % twisted_version, - # pyopenssl depends on it, use RUST-less version - "cryptography >= 3.3, < 35.0.0", - "pyopenssl >= 16.0.0, <= 21.0.0", - "service_identity >= 18.1.0, <= 21.1.0", - ] - else: - result = ["twisted[tls] == %s" % twisted_version] - if "windows" in systype: - result.append("pywin32 != 226") - return result diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index e5e948ba..6dbd480c 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -15,7 +15,7 @@ import os from platformio import util -from platformio.http import HTTPClientError, InternetIsOffline +from platformio.http import HTTPClientError, InternetConnectionError from platformio.package.exception import UnknownPackageError from platformio.package.manager.base import BasePackageManager from platformio.package.manager.core import get_installed_core_packages @@ -128,7 +128,7 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an key = "%s:%s" % (board["platform"], board["id"]) if key not in know_boards: boards.append(board) - except (HTTPClientError, InternetIsOffline): + except (HTTPClientError, InternetConnectionError): pass return sorted(boards, key=lambda b: b["name"]) diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 937bbf91..2075ac03 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -276,7 +276,7 @@ class ManifestSchema(BaseSchema): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.19" + version = "3.20" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" "v%s/json/licenses.json" % version diff --git a/platformio/package/meta.py b/platformio/package/meta.py index fbd2b734..4db53106 100644 --- a/platformio/package/meta.py +++ b/platformio/package/meta.py @@ -65,7 +65,6 @@ class PackageType: class PackageCompatibility: - KNOWN_QUALIFIERS = ("platforms", "frameworks", "authors") @classmethod @@ -468,7 +467,6 @@ class PackageMetaData: class PackageItem: - METAFILE_NAME = ".piopm" def __init__(self, path, metadata=None): diff --git a/platformio/package/unpack.py b/platformio/package/unpack.py index e39222c2..9d8919c3 100644 --- a/platformio/package/unpack.py +++ b/platformio/package/unpack.py @@ -20,11 +20,11 @@ from zipfile import ZipFile import click from platformio import fs +from platformio.compat import is_terminal from platformio.package.exception import PackageException class ExtractArchiveItemError(PackageException): - MESSAGE = ( "Could not extract `{0}` to `{1}`. Try to disable antivirus " "tool or check this solution -> https://bit.ly/faq-package-manager" @@ -159,18 +159,38 @@ class FileUnpacker: def unpack( self, dest_dir=None, with_progress=True, check_unpacked=True, silent=False - ): + ): # pylint: disable=too-many-branches assert self._archiver + label = "Unpacking" + items = self._archiver.get_items() if not dest_dir: dest_dir = os.getcwd() + if not with_progress or silent: if not silent: - click.echo("Unpacking...") - for item in self._archiver.get_items(): + click.echo(f"{label}...") + for item in items: self._archiver.extract_item(item, dest_dir) + elif not is_terminal(): + click.echo(f"{label} 0%", nl=False) + print_percent_step = 10 + printed_percents = 0 + unpacked_nums = 0 + for item in items: + self._archiver.extract_item(item, dest_dir) + unpacked_nums += 1 + if (unpacked_nums / len(items) * 100) >= ( + printed_percents + print_percent_step + ): + printed_percents += print_percent_step + click.echo(f" {printed_percents}%", nl=False) + click.echo("") else: - items = self._archiver.get_items() - with click.progressbar(items, label="Unpacking") as pb: + with click.progressbar( + items, + label=label, + update_min_steps=min(50, len(items) / 100), # every 50 files or less + ) as pb: for item in pb: self._archiver.extract_item(item, dest_dir) diff --git a/platformio/package/vcsclient.py b/platformio/package/vcsclient.py index b4dafc03..058a4983 100644 --- a/platformio/package/vcsclient.py +++ b/platformio/package/vcsclient.py @@ -58,7 +58,6 @@ class VCSClientFactory: class VCSClientBase: - command = None def __init__(self, src_dir, remote_url=None, tag=None, silent=False): @@ -128,7 +127,6 @@ class VCSClientBase: class GitClient(VCSClientBase): - command = "git" _configured = False @@ -232,7 +230,6 @@ class GitClient(VCSClientBase): class HgClient(VCSClientBase): - command = "hg" def export(self): @@ -256,7 +253,6 @@ class HgClient(VCSClientBase): class SvnClient(VCSClientBase): - command = "svn" def export(self): diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 79afff94..dfd7c406 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -25,10 +25,10 @@ from platformio import app, fs, proc, telemetry from platformio.compat import hashlib_encode_data from platformio.package.manager.core import get_core_package_dir from platformio.platform.exception import BuildScriptNotFound +from platformio.run.helpers import KNOWN_CLEAN_TARGETS, KNOWN_FULLCLEAN_TARGETS class PlatformRunMixin: - LINE_ERROR_RE = re.compile(r"(^|\s+)error:?\s+", re.I) @staticmethod @@ -57,9 +57,6 @@ class PlatformRunMixin: self.silent = silent self.verbose = verbose or app.get_setting("force_verbose") - if "clean" in targets: - targets = ["-c", "."] - variables["platform_manifest"] = self.manifest_path if "build_script" not in variables: @@ -93,16 +90,22 @@ class PlatformRunMixin: "--sconstruct", os.path.join(fs.get_source_dir(), "builder", "main.py"), ] - args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0)) + args.append("PIOVERBOSE=%d" % int(self.verbose)) # pylint: disable=protected-access - args.append("ISATTY=%d" % (1 if click._compat.isatty(sys.stdout) else 0)) - args += targets - + args.append("ISATTY=%d" % int(click._compat.isatty(sys.stdout))) # encode and append variables for key, value in variables.items(): args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value))) - proc.copy_pythonpath_to_osenv() + if set(KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS) & set(targets): + args.append("--clean") + args.append( + "FULLCLEAN=%d" + % (1 if set(KNOWN_FULLCLEAN_TARGETS) & set(targets) else 0) + ) + elif targets: + args.extend(targets) + # force SCons output to Unicode os.environ["PYTHONIOENCODING"] = "utf-8" diff --git a/platformio/platform/base.py b/platformio/platform/base.py index 49db0df4..3231bb44 100644 --- a/platformio/platform/base.py +++ b/platformio/platform/base.py @@ -29,7 +29,6 @@ from platformio.project.config import ProjectConfig class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods PlatformPackagesMixin, PlatformRunMixin ): - CORE_SEMVER = pepver_to_semver(__version__) _BOARDS_CACHE = {} @@ -208,6 +207,15 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub def configure_debug_session(self, debug_config): raise NotImplementedError + def generate_sample_code(self, project_config, environment): + raise NotImplementedError + + def on_installed(self): + pass + + def on_uninstalled(self): + pass + def get_lib_storages(self): storages = {} for opts in (self.frameworks or {}).values(): @@ -228,9 +236,3 @@ class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-pub storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) return [dict(name=name, path=path) for path, name in storages.items()] - - def on_installed(self): - pass - - def on_uninstalled(self): - pass diff --git a/platformio/platform/exception.py b/platformio/platform/exception.py index 604c3228..f044d3b7 100644 --- a/platformio/platform/exception.py +++ b/platformio/platform/exception.py @@ -20,12 +20,10 @@ class PlatformException(PlatformioException): class UnknownPlatform(PlatformException): - MESSAGE = "Unknown development platform '{0}'" class IncompatiblePlatform(PlatformException): - MESSAGE = ( "Development platform '{0}' is not compatible with PlatformIO Core v{1} and " "depends on PlatformIO Core {2}.\n" @@ -33,20 +31,16 @@ class IncompatiblePlatform(PlatformException): class UnknownBoard(PlatformException): - MESSAGE = "Unknown board ID '{0}'" class InvalidBoardManifest(PlatformException): - MESSAGE = "Invalid board JSON manifest '{0}'" class UnknownFramework(PlatformException): - MESSAGE = "Unknown framework '{0}'" class BuildScriptNotFound(PlatformException): - MESSAGE = "Invalid path '{0}' to build script" diff --git a/platformio/project/commands/config.py b/platformio/project/commands/config.py index b59ff005..9f218e80 100644 --- a/platformio/project/commands/config.py +++ b/platformio/project/commands/config.py @@ -28,7 +28,7 @@ from platformio.project.helpers import is_platformio_project "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("--json-output", is_flag=True) def project_config_cmd(project_dir, json_output): diff --git a/platformio/project/commands/init.py b/platformio/project/commands/init.py index ca57de34..098c31a5 100644 --- a/platformio/project/commands/init.py +++ b/platformio/project/commands/init.py @@ -24,12 +24,14 @@ from platformio import fs from platformio.package.commands.install import install_project_dependencies from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.exception import UnknownBoard +from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig from platformio.project.helpers import is_platformio_project from platformio.project.integration.generator import ProjectGenerator +from platformio.project.options import ProjectOptions -def validate_boards(ctx, param, value): # pylint: disable=W0613 +def validate_boards(ctx, param, value): # pylint: disable=unused-argument pm = PlatformPackageManager() for id_ in value: try: @@ -47,25 +49,33 @@ def validate_boards(ctx, param, value): # pylint: disable=W0613 "--project-dir", "-d", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), +) +@click.option( + "-b", "--board", "boards", multiple=True, metavar="ID", callback=validate_boards ) -@click.option("-b", "--board", multiple=True, metavar="ID", callback=validate_boards) @click.option("--ide", type=click.Choice(ProjectGenerator.get_supported_ides())) @click.option("-e", "--environment", help="Update existing environment") -@click.option("-O", "--project-option", multiple=True) -@click.option("--env-prefix", default="") +@click.option( + "-O", + "--project-option", + "project_options", + multiple=True, + help="A `name=value` pair", +) +@click.option("--sample-code", is_flag=True) @click.option("--no-install-dependencies", is_flag=True) +@click.option("--env-prefix", default="") @click.option("-s", "--silent", is_flag=True) def project_init_cmd( project_dir, - board, + boards, ide, environment, - project_option, - env_prefix, + project_options, + sample_code, no_install_dependencies, + env_prefix, silent, ): is_new_project = not is_platformio_project(project_dir) @@ -74,23 +84,23 @@ def project_init_cmd( print_header(project_dir) init_base_project(project_dir) - if environment: - update_project_env(project_dir, environment, project_option) - elif board: - update_board_envs(project_dir, board, project_option, env_prefix) - with fs.cd(project_dir): + if environment: + update_project_env(environment, project_options) + elif boards: + update_board_envs(project_dir, boards, project_options, env_prefix) + generator = None config = ProjectConfig.get_instance(os.path.join(project_dir, "platformio.ini")) if ide: config.validate() # init generator and pick the best env if user didn't specify - generator = ProjectGenerator(config, environment, ide, board) + generator = ProjectGenerator(config, environment, ide, boards) if not environment: environment = generator.env_name # resolve project dependencies - if not no_install_dependencies and (environment or board): + if not no_install_dependencies and (environment or boards): install_project_dependencies( options=dict( project_dir=project_dir, @@ -99,6 +109,9 @@ def project_init_cmd( ) ) + if environment and sample_code: + init_sample_code(config, environment) + if generator: if not silent: click.echo( @@ -106,31 +119,22 @@ def project_init_cmd( ) generator.generate() - if is_new_project: - init_cvs_ignore(project_dir) + if is_new_project: + init_cvs_ignore() if not silent: print_footer(is_new_project) def print_header(project_dir): - if project_dir == os.getcwd(): - click.secho("\nThe current working directory ", fg="yellow", nl=False) - try: - click.secho(project_dir, fg="cyan", nl=False) - except UnicodeEncodeError: - click.secho(json.dumps(project_dir), fg="cyan", nl=False) - click.secho(" will be used for the project.", fg="yellow") - click.echo("") - - click.echo("The next files/directories have been created in ", nl=False) + click.echo("The following files/directories have been created in ", nl=False) try: click.secho(project_dir, fg="cyan") except UnicodeEncodeError: click.secho(json.dumps(project_dir), fg="cyan") click.echo("%s - Put project header files here" % click.style("include", fg="cyan")) click.echo( - "%s - Put here project specific (private) libraries" + "%s - Put project specific (private) libraries here" % click.style("lib", fg="cyan") ) click.echo("%s - Put project source files here" % click.style("src", fg="cyan")) @@ -140,18 +144,9 @@ def print_header(project_dir): def print_footer(is_new_project): - if is_new_project: - return click.secho( - "\nProject has been successfully initialized! Useful commands:\n" - "`pio run` - process/build project from the current directory\n" - "`pio run --target upload` or `pio run -t upload` " - "- upload firmware to a target\n" - "`pio run --target clean` - clean project (remove compiled files)" - "\n`pio run --help` - additional information", - fg="green", - ) + action = "initialized" if is_new_project else "updated" return click.secho( - "Project has been successfully updated!", + f"Project has been successfully {action}!", fg="green", ) @@ -166,7 +161,7 @@ def init_base_project(project_dir): (config.get("platformio", "lib_dir"), init_lib_readme), (config.get("platformio", "test_dir"), init_test_readme), ] - for (path, cb) in dir_to_readme: + for path, cb in dir_to_readme: if os.path.isdir(path): continue os.makedirs(path) @@ -291,15 +286,15 @@ More information about PlatformIO Unit Testing: ) -def init_cvs_ignore(project_dir): - conf_path = os.path.join(project_dir, ".gitignore") +def init_cvs_ignore(): + conf_path = ".gitignore" if os.path.isfile(conf_path): return with open(conf_path, mode="w", encoding="utf8") as fp: fp.write(".pio\n") -def update_board_envs(project_dir, board_ids, project_option, env_prefix): +def update_board_envs(project_dir, boards, extra_project_options, env_prefix): config = ProjectConfig( os.path.join(project_dir, "platformio.ini"), parse_extra=False ) @@ -311,7 +306,7 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix): pm = PlatformPackageManager() modified = False - for id_ in board_ids: + for id_ in boards: board_config = pm.board_config(id_) if id_ in used_boards: continue @@ -324,7 +319,7 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix): if frameworks: envopts["framework"] = frameworks[0] - for item in project_option: + for item in extra_project_options: if "=" not in item: continue _name, _value = item.split("=", 1) @@ -340,21 +335,76 @@ def update_board_envs(project_dir, board_ids, project_option, env_prefix): config.save() -def update_project_env(project_dir, environment, project_option): - if not project_option: +def update_project_env(environment, extra_project_options=None): + if not extra_project_options: return + env_section = "env:%s" % environment + option_to_sections = {"platformio": [], env_section: []} + for item in extra_project_options: + assert "=" in item + name, value = item.split("=", 1) + name = name.strip() + destination = env_section + for option in ProjectOptions.values(): + if option.scope in option_to_sections and option.name == name: + destination = option.scope + break + option_to_sections[destination].append((name, value.strip())) + config = ProjectConfig( - os.path.join(project_dir, "platformio.ini"), parse_extra=False + "platformio.ini", parse_extra=False, expand_interpolations=False ) - - section = "env:%s" % environment - if not config.has_section(section): - config.add_section(section) - - for item in project_option: - if "=" not in item: + for section, options in option_to_sections.items(): + if not options: continue - _name, _value = item.split("=", 1) - config.set(section, _name.strip(), _value.strip()) + if not config.has_section(section): + config.add_section(section) + for name, value in options: + config.set(section, name, value) config.save() + + +def init_sample_code(config, environment): + platform_spec = config.get(f"env:{environment}", "platform", None) + if not platform_spec: + return None + p = PlatformFactory.new(platform_spec) + try: + return p.generate_sample_code(config, environment) + except NotImplementedError: + pass + + framework = config.get(f"env:{environment}", "framework", None) + if framework != ["arduino"]: + return None + main_content = """ +#include + +// put function declarations here: +int myFunction(int, int); + +void setup() { + // put your setup code here, to run once: + int result = myFunction(2, 3); +} + +void loop() { + // put your main code here, to run repeatedly: +} + +// put function definitions here: +int myFunction(int x, int y) { + return x + y; +} +""" + is_cpp_project = p.name not in ["intel_mcs51", "ststm8"] + src_dir = config.get("platformio", "src_dir") + main_path = os.path.join(src_dir, "main.%s" % ("cpp" if is_cpp_project else "c")) + if os.path.isfile(main_path): + return None + if not os.path.isdir(src_dir): + os.makedirs(src_dir) + with open(main_path, mode="w", encoding="utf8") as fp: + fp.write(main_content.strip()) + return True diff --git a/platformio/project/commands/metadata.py b/platformio/project/commands/metadata.py index 06457574..fbf71b8b 100644 --- a/platformio/project/commands/metadata.py +++ b/platformio/project/commands/metadata.py @@ -31,11 +31,11 @@ from platformio.project.helpers import load_build_metadata "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option("-e", "--environment", "environments", multiple=True) @click.option("--json-output", is_flag=True) -@click.option("--json-output-path", type=click.Path(resolve_path=True)) +@click.option("--json-output-path", type=click.Path()) def project_metadata_cmd(project_dir, environments, json_output, json_output_path): with fs.cd(project_dir): config = ProjectConfig.get_instance() diff --git a/platformio/project/config.py b/platformio/project/config.py index ee380bb9..c2b072ef 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -39,7 +39,7 @@ CONFIG_HEADER = """ class ProjectConfigBase: - + ENVNAME_RE = re.compile(r"^[a-z\d\_\-]+$", flags=re.I) INLINE_COMMENT_RE = re.compile(r"\s+;.*$") VARTPL_RE = re.compile(r"\$\{([^\.\}\()]+)\.([^\}]+)\}") @@ -389,16 +389,38 @@ class ProjectConfigBase: def validate(self, envs=None, silent=False): if not os.path.isfile(self.path): raise exception.NotPlatformIOProjectError(os.path.dirname(self.path)) + + known_envs = set(self.envs()) + # check envs - known = set(self.envs()) - if not known: + if not known_envs: raise exception.ProjectEnvsNotAvailableError() - unknown = set(list(envs or []) + self.default_envs()) - known - if unknown: - raise exception.UnknownEnvNamesError(", ".join(unknown), ", ".join(known)) + unknown_envs = set(list(envs or []) + self.default_envs()) - known_envs + if unknown_envs: + raise exception.UnknownEnvNamesError( + ", ".join(unknown_envs), ", ".join(known_envs) + ) + + for env in known_envs: + # check envs names + if not self.ENVNAME_RE.match(env): + raise exception.InvalidEnvNameError(env) + + # check simultaneous use of `monitor_raw` and `monitor_filters` + if self.get(f"env:{env}", "monitor_raw", False) and self.get( + f"env:{env}", "monitor_filters", None + ): + self.warnings.append( + "The `monitor_raw` and `monitor_filters` options cannot be " + f"used simultaneously for the `{env}` environment in the " + "`platformio.ini` file. The `monitor_filters` option will " + "be disabled to avoid conflicts." + ) + if not silent: for warning in self.warnings: click.secho("Warning! %s" % warning, fg="yellow") + return True @@ -412,7 +434,6 @@ class ProjectConfigDirsMixin: class ProjectConfig(ProjectConfigBase, ProjectConfigDirsMixin): - _instances = {} @staticmethod diff --git a/platformio/project/exception.py b/platformio/project/exception.py index aa45eb07..95681bc0 100644 --- a/platformio/project/exception.py +++ b/platformio/project/exception.py @@ -20,7 +20,6 @@ class ProjectError(PlatformioException): class NotPlatformIOProjectError(ProjectError, UserSideException): - MESSAGE = ( "Not a PlatformIO project. `platformio.ini` file has not been " "found in current working directory ({0}). To initialize new project " @@ -29,25 +28,27 @@ class NotPlatformIOProjectError(ProjectError, UserSideException): class InvalidProjectConfError(ProjectError, UserSideException): - MESSAGE = "Invalid '{0}' (project configuration file): '{1}'" class UndefinedEnvPlatformError(ProjectError, UserSideException): - MESSAGE = "Please specify platform for '{0}' environment" class ProjectEnvsNotAvailableError(ProjectError, UserSideException): - MESSAGE = "Please setup environments in `platformio.ini` file" class UnknownEnvNamesError(ProjectError, UserSideException): - MESSAGE = "Unknown environment names '{0}'. Valid names are '{1}'" -class ProjectOptionValueError(ProjectError, UserSideException): +class InvalidEnvNameError(ProjectError, UserSideException): + MESSAGE = ( + "Invalid environment name '{0}'. The name can contain " + "alphanumeric, underscore, and hyphen characters (a-z, 0-9, -, _)" + ) + +class ProjectOptionValueError(ProjectError, UserSideException): MESSAGE = "{0} for option `{1}` in section [{2}]" diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 200321f2..aec12dd9 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import re import subprocess from hashlib import sha1 @@ -24,7 +25,7 @@ from platformio.project.config import ProjectConfig def get_project_dir(): - return fs.normalize_path(os.getcwd()) + return os.getcwd() def is_platformio_project(project_dir=None): @@ -94,7 +95,16 @@ def compute_project_checksum(config): checksum = sha1(hashlib_encode_data(__version__)) # configuration file state - checksum.update(hashlib_encode_data(config.to_json())) + config_data = config.to_json() + if IS_WINDOWS: + # issue #4600: fix drive letter + config_data = re.sub( + r"([A-Z]):\\", + lambda match: "%s:\\" % match.group(1).lower(), + config_data, + flags=re.I, + ) + checksum.update(hashlib_encode_data(config_data)) # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") diff --git a/platformio/project/integration/generator.py b/platformio/project/integration/generator.py index 4ea3bbf3..654e5ac3 100644 --- a/platformio/project/integration/generator.py +++ b/platformio/project/integration/generator.py @@ -19,33 +19,34 @@ import sys import bottle from platformio import fs, util +from platformio.debug.helpers import get_default_debug_env from platformio.proc import where_is_program from platformio.project.helpers import load_build_metadata class ProjectGenerator: - def __init__(self, config, env_name, ide, board_ids=None): + def __init__(self, config, env_name, ide, boards=None): self.config = config self.project_dir = os.path.dirname(config.path) - self.original_env_name = env_name - self.env_name = str(env_name or self.get_best_envname(board_ids)) + self.forced_env_name = env_name + self.env_name = str(env_name or self.get_best_envname(boards)) self.ide = str(ide) - def get_best_envname(self, board_ids=None): + def get_best_envname(self, boards=None): envname = None default_envs = self.config.default_envs() if default_envs: envname = default_envs[0] - if not board_ids: + if not boards: return envname for env in self.config.envs(): - if not board_ids: + if not boards: return env if not envname: envname = env items = self.config.items(env=env, as_dict=True) - if "board" in items and items.get("board") in board_ids: + if "board" in items and items.get("board") in boards: return env return envname @@ -86,7 +87,8 @@ class ProjectGenerator: "platformio", "name", os.path.basename(self.project_dir) ), "project_dir": self.project_dir, - "original_env_name": self.original_env_name, + "forced_env_name": self.forced_env_name, + "default_debug_env_name": get_default_debug_env(self.config), "env_name": self.env_name, "user_home_dir": os.path.abspath(fs.expanduser("~")), "platformio_path": sys.argv[0] @@ -132,7 +134,7 @@ class ProjectGenerator: for root, _, files in os.walk(self.config.get("platformio", "src_dir")): for f in files: result.append( - os.path.relpath(os.path.join(os.path.realpath(root), f)) + os.path.relpath(os.path.join(os.path.abspath(root), f)) ) return result diff --git a/platformio/project/options.py b/platformio/project/options.py index 5e200629..2e13815a 100644 --- a/platformio/project/options.py +++ b/platformio/project/options.py @@ -114,7 +114,7 @@ def validate_dir(path): path = fs.expanduser(path) if "$" in path: path = expand_dir_templates(path) - return fs.normalize_path(path) + return os.path.abspath(path) def get_default_core_dir(): @@ -649,10 +649,10 @@ ProjectOptions = OrderedDict( ), ConfigEnvOption( group="check", - name="check_patterns", + name="check_src_filters", + oldnames=["check_patterns"], description=( - "Configure a list of target files or directories for checking " - "(Unix shell-style wildcards)" + "Configure a list of target files or directories for checking" ), multiple=True, ), diff --git a/platformio/registry/client.py b/platformio/registry/client.py index a682db57..1a89e345 100644 --- a/platformio/registry/client.py +++ b/platformio/registry/client.py @@ -142,12 +142,15 @@ class RegistryClient(HTTPClient): x_with_authorization=self.allowed_private_packages(), ) - def get_package(self, type_, owner, name, version=None): + def get_package(self, type_, owner, name, version=None, extra_path=None): try: return self.fetch_json_data( "get", - "/v3/packages/{owner}/{type}/{name}".format( - type=type_, owner=owner.lower(), name=name.lower() + "/v3/packages/{owner}/{type}/{name}{extra_path}".format( + type=type_, + owner=owner.lower(), + name=name.lower(), + extra_path=extra_path or "", ), params=dict(version=version) if version else None, x_cache_valid="1h", diff --git a/platformio/registry/mirror.py b/platformio/registry/mirror.py index 4235e88d..4b4508f6 100644 --- a/platformio/registry/mirror.py +++ b/platformio/registry/mirror.py @@ -22,7 +22,6 @@ from platformio.registry.client import RegistryClient class RegistryFileMirrorIterator: - HTTP_CLIENT_INSTANCES = {} def __init__(self, download_url): diff --git a/platformio/remote/ac/base.py b/platformio/remote/ac/base.py index 8105062c..0c631c64 100644 --- a/platformio/remote/ac/base.py +++ b/platformio/remote/ac/base.py @@ -17,7 +17,6 @@ from twisted.spread import pb # pylint: disable=import-error class AsyncCommandBase: - MAX_BUFFER_SIZE = 1024 * 1024 # 1Mb def __init__(self, options=None, on_end_callback=None): diff --git a/platformio/remote/cli.py b/platformio/remote/cli.py index bede41d1..f1775d49 100644 --- a/platformio/remote/cli.py +++ b/platformio/remote/cli.py @@ -17,7 +17,9 @@ import os import subprocess +import sys import threading +from site import addsitedir from tempfile import mkdtemp from time import sleep @@ -29,7 +31,7 @@ from platformio.device.monitor.command import ( device_monitor_cmd, get_project_options, ) -from platformio.package.manager.core import inject_contrib_pysite +from platformio.package.manager.core import get_core_package_dir from platformio.project.exception import NotPlatformIOProjectError from platformio.project.options import ProjectOptions from platformio.run.cli import cli as cmd_run @@ -41,7 +43,11 @@ from platformio.test.cli import cli as test_cmd @click.pass_context def cli(ctx, agent): ctx.obj = agent - inject_contrib_pysite() + # inject twisted dependencies + contrib_dir = get_core_package_dir("contrib-pioremote") + if contrib_dir not in sys.path: + addsitedir(contrib_dir) + sys.path.insert(0, contrib_dir) @cli.group("agent", short_help="Start a new agent or list active") @@ -56,7 +62,7 @@ def remote_agent(): "-d", "--working-dir", envvar="PLATFORMIO_REMOTE_AGENT_DIR", - type=click.Path(file_okay=False, dir_okay=True, writable=True, resolve_path=True), + type=click.Path(file_okay=False, dir_okay=True, writable=True), ) def remote_agent_start(name, share, working_dir): from platformio.remote.client.agent_service import RemoteAgentService @@ -96,9 +102,7 @@ def remote_update(agents, only_check, dry_run): "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option("--disable-auto-clean", is_flag=True) @click.option("-r", "--force-remote", is_flag=True) @@ -118,7 +122,6 @@ def remote_run( silent, verbose, ): - from platformio.remote.client.run_or_test import RunOrTestClient cr = RunOrTestClient( @@ -187,9 +190,7 @@ def remote_run( "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option("-r", "--force-remote", is_flag=True) @click.option("--without-building", is_flag=True) @@ -211,7 +212,6 @@ def remote_test( # pylint: disable=redefined-builtin without_uploading, verbose, ): - from platformio.remote.client.run_or_test import RunOrTestClient cr = RunOrTestClient( @@ -336,7 +336,7 @@ def device_list(agents, json_output): "-d", "--project-dir", default=os.getcwd, - type=click.Path(exists=True, file_okay=False, dir_okay=True, resolve_path=True), + type=click.Path(exists=True, file_okay=False, dir_okay=True), ) @click.option( "-e", @@ -345,9 +345,7 @@ def device_list(agents, json_output): ) @click.option( "--sock", - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.pass_obj @click.pass_context diff --git a/platformio/remote/client/async_base.py b/platformio/remote/client/async_base.py index 488802a5..2a672641 100644 --- a/platformio/remote/client/async_base.py +++ b/platformio/remote/client/async_base.py @@ -34,7 +34,7 @@ class AsyncClientBase(RemoteClientBase): def cb_async_result(self, result): if self._acs_total == 0: self._acs_total = len(result) - for (success, value) in result: + for success, value in result: if not success: raise pb.Error(value) self.acread_data(*value) diff --git a/platformio/remote/client/base.py b/platformio/remote/client/base.py index fe2c4fb4..cf5c2405 100644 --- a/platformio/remote/client/base.py +++ b/platformio/remote/client/base.py @@ -33,7 +33,6 @@ from platformio.remote.factory.ssl import SSLContextFactory class RemoteClientBase( # pylint: disable=too-many-instance-attributes pb.Referenceable ): - PING_DELAY = 60 PING_MAX_FAILURES = 3 DEBUG = False diff --git a/platformio/remote/client/device_list.py b/platformio/remote/client/device_list.py index a22911b4..f05bbfd2 100644 --- a/platformio/remote/client/device_list.py +++ b/platformio/remote/client/device_list.py @@ -32,7 +32,7 @@ class DeviceListClient(RemoteClientBase): def _cbResult(self, result): data = {} - for (success, value) in result: + for success, value in result: if not success: click.secho(value, fg="red", err=True) continue diff --git a/platformio/remote/client/device_monitor.py b/platformio/remote/client/device_monitor.py index 46ca19e4..1499dd58 100644 --- a/platformio/remote/client/device_monitor.py +++ b/platformio/remote/client/device_monitor.py @@ -71,7 +71,6 @@ class SMBridgeFactory(protocol.ServerFactory): class DeviceMonitorClient( # pylint: disable=too-many-instance-attributes RemoteClientBase ): - MAX_BUFFER_SIZE = 1024 * 1024 def __init__(self, agents, **kwargs): @@ -96,7 +95,7 @@ class DeviceMonitorClient( # pylint: disable=too-many-instance-attributes def _cb_device_list(self, result): devices = [] hwid_devindexes = [] - for (success, value) in result: + for success, value in result: if not success: click.secho(value, fg="red", err=True) continue diff --git a/platformio/remote/client/run_or_test.py b/platformio/remote/client/run_or_test.py index 71065640..d23d92c7 100644 --- a/platformio/remote/client/run_or_test.py +++ b/platformio/remote/client/run_or_test.py @@ -28,7 +28,6 @@ from platformio.remote.projectsync import PROJECT_SYNC_STAGE, ProjectSync class RunOrTestClient(AsyncClientBase): - MAX_ARCHIVE_SIZE = 50 * 1024 * 1024 # 50Mb UPLOAD_CHUNK_SIZE = 256 * 1024 # 256Kb @@ -147,7 +146,7 @@ class RunOrTestClient(AsyncClientBase): def cb_psync_init_result(self, result): self._acs_total = len(result) - for (success, value) in result: + for success, value in result: if not success: raise pb.Error(value) agent_id, ac_id = value diff --git a/platformio/remote/projectsync.py b/platformio/remote/projectsync.py index 820034ae..1715145f 100644 --- a/platformio/remote/projectsync.py +++ b/platformio/remote/projectsync.py @@ -47,13 +47,13 @@ class ProjectSync: def rebuild_dbindex(self): self._db = {} - for (path, relpath, cb_filter) in self.items: + for path, relpath, cb_filter in self.items: if cb_filter and not cb_filter(path): continue self._insert_to_db(path, relpath) if not isdir(path): continue - for (root, _, files) in os.walk(path, followlinks=True): + for root, _, files in os.walk(path, followlinks=True): for name in files: self._insert_to_db( join(root, name), join(relpath, root[len(path) + 1 :], name) diff --git a/platformio/run/cli.py b/platformio/run/cli.py index 6c647431..dc3eb2ee 100644 --- a/platformio/run/cli.py +++ b/platformio/run/cli.py @@ -26,7 +26,7 @@ from platformio.device.monitor.command import device_monitor_cmd from platformio.project.config import ProjectConfig from platformio.project.exception import ProjectError from platformio.project.helpers import find_project_dir_above, load_build_metadata -from platformio.run.helpers import clean_build_dir, handle_legacy_libdeps +from platformio.run.helpers import clean_build_dir from platformio.run.processor import EnvironmentProcessor from platformio.test.runners.base import CTX_META_TEST_IS_RUNNING @@ -47,16 +47,12 @@ except NotImplementedError: "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option( "-j", @@ -101,9 +97,12 @@ def cli( if os.path.isfile(project_dir): project_dir = find_project_dir_above(project_dir) + targets = list(target) if target else [] + del target + only_monitor = targets == ["monitor"] is_test_running = CTX_META_TEST_IS_RUNNING in ctx.meta + command_failed = False - results = [] with fs.cd(project_dir): config = ProjectConfig.get_instance(project_conf) config.validate(environment) @@ -112,7 +111,7 @@ def cli( return print_target_list(list(environment) or config.envs()) # clean obsolete build dir - if not disable_auto_clean: + if not only_monitor and not disable_auto_clean: build_dir = config.get("platformio", "build_dir") try: clean_build_dir(build_dir, config) @@ -125,9 +124,8 @@ def cli( fg="yellow", ) - handle_legacy_libdeps(project_dir, config) - default_envs = config.default_envs() + results = [] for env in config.envs(): skipenv = any( [ @@ -148,8 +146,7 @@ def cli( ctx, env, config, - environment, - target, + targets, upload_port, monitor_port, jobs, @@ -159,17 +156,21 @@ def cli( verbose, ) ) - - command_failed = any(r.get("succeeded") is False for r in results) - - if not is_test_running and (command_failed or not silent) and len(results) > 1: - print_processing_summary(results, verbose) + command_failed = any(r.get("succeeded") is False for r in results) + if ( + not is_test_running + and not only_monitor + and (command_failed or not silent) + and len(results) > 1 + ): + print_processing_summary(results, verbose) # Reset custom project config app.set_session_var("custom_project_conf", None) if command_failed: raise exception.ReturnErrorCode(1) + return True @@ -177,7 +178,6 @@ def process_env( ctx, name, config, - environments, targets, upload_port, monitor_port, @@ -190,35 +190,40 @@ def process_env( if not is_test_running and not silent: print_processing_header(name, config, verbose) - ep = EnvironmentProcessor( - ctx, - name, - config, - targets, - upload_port, - jobs, - program_args, - silent, - verbose, - ) - result = {"env": name, "duration": time(), "succeeded": ep.process()} - result["duration"] = time() - result["duration"] + targets = targets or config.get(f"env:{name}", "targets", []) + only_monitor = targets == ["monitor"] + result = {"env": name, "duration": time(), "succeeded": True} - # print footer on error or when is not unit testing - if not is_test_running and (not silent or not result["succeeded"]): - print_processing_footer(result) + if not only_monitor: + result["succeeded"] = EnvironmentProcessor( + ctx, + name, + config, + [t for t in targets if t != "monitor"], + upload_port, + jobs, + program_args, + silent, + verbose, + ).process() - if ( - result["succeeded"] - and "monitor" in ep.get_build_targets() - and "nobuild" not in ep.get_build_targets() - ): + if "monitor" in targets and "nobuild" not in targets: ctx.invoke( device_monitor_cmd, port=monitor_port, - environment=environments[0] if environments else None, + environment=name, ) + result["duration"] = time() - result["duration"] + + # print footer on error or when is not unit testing + if ( + not is_test_running + and not only_monitor + and (not silent or not result["succeeded"]) + ): + print_processing_footer(result) + return result diff --git a/platformio/run/helpers.py b/platformio/run/helpers.py index 925b5584..f8bfdcba 100644 --- a/platformio/run/helpers.py +++ b/platformio/run/helpers.py @@ -15,31 +15,12 @@ from os import makedirs from os.path import isdir, isfile, join -import click - from platformio import fs from platformio.project.helpers import compute_project_checksum, get_project_dir - -def handle_legacy_libdeps(project_dir, config): - legacy_libdeps_dir = join(project_dir, ".piolibdeps") - if not isdir(legacy_libdeps_dir) or legacy_libdeps_dir == config.get( - "platformio", "libdeps_dir" - ): - return - if not config.has_section("env"): - config.add_section("env") - lib_extra_dirs = config.get("env", "lib_extra_dirs", []) - lib_extra_dirs.append(legacy_libdeps_dir) - config.set("env", "lib_extra_dirs", lib_extra_dirs) - click.secho( - "DEPRECATED! A legacy library storage `{0}` has been found in a " - "project. \nPlease declare project dependencies in `platformio.ini`" - " file using `lib_deps` option and remove `{0}` folder." - "\nMore details -> https://docs.platformio.org/page/projectconf/" - "section_env_library.html#lib-deps".format(legacy_libdeps_dir), - fg="yellow", - ) +KNOWN_CLEAN_TARGETS = ("clean",) +KNOWN_FULLCLEAN_TARGETS = ("cleanall", "fullclean") +KNOWN_ALLCLEAN_TARGETS = KNOWN_CLEAN_TARGETS + KNOWN_FULLCLEAN_TARGETS def clean_build_dir(build_dir, config): diff --git a/platformio/run/processor.py b/platformio/run/processor.py index 053478ac..329c4bf6 100644 --- a/platformio/run/processor.py +++ b/platformio/run/processor.py @@ -15,6 +15,7 @@ from platformio.package.commands.install import install_project_env_dependencies from platformio.platform.factory import PlatformFactory from platformio.project.exception import UndefinedEnvPlatformError +from platformio.run.helpers import KNOWN_ALLCLEAN_TARGETS from platformio.test.runners.base import CTX_META_TEST_RUNNING_NAME # pylint: disable=too-many-instance-attributes @@ -36,7 +37,7 @@ class EnvironmentProcessor: self.cmd_ctx = cmd_ctx self.name = name self.config = config - self.targets = [str(t) for t in targets] + self.targets = targets self.upload_port = upload_port self.jobs = jobs self.program_args = program_args @@ -61,33 +62,29 @@ class EnvironmentProcessor: variables["upload_port"] = self.upload_port return variables - def get_build_targets(self): - return ( - self.targets - if self.targets - else self.config.get("env:" + self.name, "targets", []) - ) - def process(self): if "platform" not in self.options: raise UndefinedEnvPlatformError(self.name) build_vars = self.get_build_variables() - build_targets = list(self.get_build_targets()) + is_clean = set(KNOWN_ALLCLEAN_TARGETS) & set(self.targets) + build_targets = [t for t in self.targets if t not in KNOWN_ALLCLEAN_TARGETS] - # skip monitor target, we call it above - if "monitor" in build_targets: - build_targets.remove("monitor") - - if not set(["clean", "cleanall"]) & set(build_targets): - install_project_env_dependencies( - self.name, - { - "project_targets": build_targets, - "piotest_running_name": build_vars.get("piotest_running_name"), - }, - ) + # pre-clean + if is_clean: + result = PlatformFactory.new( + self.options["platform"], autoinstall=True + ).run(build_vars, self.targets, self.silent, self.verbose, self.jobs) + if not build_targets: + return result["returncode"] == 0 + install_project_env_dependencies( + self.name, + { + "project_targets": self.targets, + "piotest_running_name": build_vars.get("piotest_running_name"), + }, + ) result = PlatformFactory.new(self.options["platform"], autoinstall=True).run( build_vars, build_targets, self.silent, self.verbose, self.jobs ) diff --git a/platformio/system/commands/completion.py b/platformio/system/commands/completion.py index aac68592..5ff73bb1 100644 --- a/platformio/system/commands/completion.py +++ b/platformio/system/commands/completion.py @@ -33,7 +33,7 @@ def system_completion_cmd(): @click.argument("shell", type=click.Choice([t.value for t in ShellType])) @click.option( "--path", - type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True), + type=click.Path(file_okay=True, dir_okay=False, readable=True), help="Custom installation path of the code to be evaluated by the shell. " "The standard installation path is used by default.", ) @@ -54,7 +54,7 @@ def system_completion_install(shell, path): @click.argument("shell", type=click.Choice([t.value for t in ShellType])) @click.option( "--path", - type=click.Path(file_okay=True, dir_okay=False, readable=True, resolve_path=True), + type=click.Path(file_okay=True, dir_okay=False, readable=True), help="Custom installation path of the code to be evaluated by the shell. " "The standard installation path is used by default.", ) diff --git a/platformio/telemetry.py b/platformio/telemetry.py index 211c0bea..e9cbcd58 100644 --- a/platformio/telemetry.py +++ b/platformio/telemetry.py @@ -54,7 +54,6 @@ class TelemetryBase: class MeasurementProtocol(TelemetryBase): - TID = "UA-1768265-9" PARAMS_MAP = { "screen_name": "cd", @@ -106,18 +105,6 @@ class MeasurementProtocol(TelemetryBase): self["cd3"] = " ".join(args) def _prefill_custom_data(self): - def _filter_args(items): - result = [] - stop = False - for item in items: - item = str(item).lower() - result.append(item) - if stop: - break - if item == "account": - stop = True - return result - caller_id = str(app.get_session_var("caller_id")) self["cd1"] = util.get_systype() self["cd4"] = 1 if (not is_ci() and (caller_id or not is_container())) else 0 @@ -201,7 +188,6 @@ class MeasurementProtocol(TelemetryBase): @util.singleton class MPDataPusher: - MAX_WORKERS = 5 def __init__(self): diff --git a/platformio/test/cli.py b/platformio/test/cli.py index d0a6e53a..748580c8 100644 --- a/platformio/test/cli.py +++ b/platformio/test/cli.py @@ -48,16 +48,12 @@ from platformio.test.runners.factory import TestRunnerFactory "-d", "--project-dir", default=os.getcwd, - type=click.Path( - exists=True, file_okay=False, dir_okay=True, writable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True), ) @click.option( "-c", "--project-conf", - type=click.Path( - exists=True, file_okay=True, dir_okay=False, readable=True, resolve_path=True - ), + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), ) @click.option("--without-building", is_flag=True) @click.option("--without-uploading", is_flag=True) @@ -83,8 +79,8 @@ from platformio.test.runners.factory import TestRunnerFactory help="A program argument (multiple are allowed)", ) @click.option("--list-tests", is_flag=True) -@click.option("--json-output-path", type=click.Path(resolve_path=True)) -@click.option("--junit-output-path", type=click.Path(resolve_path=True)) +@click.option("--json-output-path", type=click.Path()) +@click.option("--junit-output-path", type=click.Path()) @click.option( "--verbose", "-v", diff --git a/platformio/test/exception.py b/platformio/test/exception.py index 12b76ea4..14cbe5ac 100644 --- a/platformio/test/exception.py +++ b/platformio/test/exception.py @@ -20,7 +20,6 @@ class UnitTestError(PlatformioException): class TestDirNotExistsError(UnitTestError, UserSideException): - MESSAGE = ( "A test folder '{0}' does not exist.\nPlease create 'test' " "directory in the project root and put a test suite.\n" diff --git a/platformio/test/helpers.py b/platformio/test/helpers.py index 01650a0b..25551ef6 100644 --- a/platformio/test/helpers.py +++ b/platformio/test/helpers.py @@ -40,7 +40,6 @@ def list_test_suites(project_config, environments, filters, ignores): test_names = list_test_names(project_config) for env_name in project_config.envs(): for test_name in test_names: - # filter and ignore patterns patterns = dict(filter=list(filters), ignore=list(ignores)) for key, value in patterns.items(): diff --git a/platformio/test/runners/base.py b/platformio/test/runners/base.py index 8f9136f1..845a8c37 100644 --- a/platformio/test/runners/base.py +++ b/platformio/test/runners/base.py @@ -18,7 +18,7 @@ from platformio.exception import ReturnErrorCode from platformio.platform.factory import PlatformFactory from platformio.test.exception import UnitTestSuiteError from platformio.test.result import TestCase, TestStatus -from platformio.test.runners.readers.program import ProgramTestOutputReader +from platformio.test.runners.readers.native import NativeTestOutputReader from platformio.test.runners.readers.serial import SerialTestOutputReader CTX_META_TEST_IS_RUNNING = __name__ + ".test_running" @@ -54,7 +54,6 @@ class TestRunnerOptions: # pylint: disable=too-many-instance-attributes class TestRunnerBase: - NAME = None EXTRA_LIB_DEPS = None TESTCASE_PARSE_RE = None @@ -161,7 +160,7 @@ class TestRunnerBase: return None click.secho("Testing...", bold=True) test_port = self.get_test_port() - program_conds = [ + native_conds = [ not self.platform.is_embedded() and (not test_port or "://" not in test_port), self.project_config.get( @@ -169,8 +168,8 @@ class TestRunnerBase: ), ] reader = ( - ProgramTestOutputReader(self) - if any(program_conds) + NativeTestOutputReader(self) + if any(native_conds) else SerialTestOutputReader(self) ) return reader.begin() diff --git a/platformio/test/runners/doctest.py b/platformio/test/runners/doctest.py index 52a0916f..dd30bc92 100644 --- a/platformio/test/runners/doctest.py +++ b/platformio/test/runners/doctest.py @@ -101,7 +101,6 @@ class DoctestTestCaseParser: class DoctestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["doctest/doctest@^2.4.9"] def __init__(self, *args, **kwargs): diff --git a/platformio/test/runners/googletest.py b/platformio/test/runners/googletest.py index b299002e..8a4d1b15 100644 --- a/platformio/test/runners/googletest.py +++ b/platformio/test/runners/googletest.py @@ -22,7 +22,6 @@ from platformio.test.runners.base import TestRunnerBase class GoogletestTestCaseParser: - # Examples: # [ RUN ] FooTest.Bar # ... @@ -89,7 +88,6 @@ class GoogletestTestCaseParser: class GoogletestTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["google/googletest@^1.12.1"] def __init__(self, *args, **kwargs): diff --git a/platformio/test/runners/readers/program.py b/platformio/test/runners/readers/native.py similarity index 91% rename from platformio/test/runners/readers/program.py rename to platformio/test/runners/readers/native.py index d80d170d..981febc3 100644 --- a/platformio/test/runners/readers/program.py +++ b/platformio/test/runners/readers/native.py @@ -24,6 +24,7 @@ from platformio.compat import ( get_filesystem_encoding, get_locale_encoding, ) +from platformio.project.helpers import load_build_metadata from platformio.test.exception import UnitTestError EXITING_TIMEOUT = 5 # seconds @@ -56,7 +57,7 @@ class ProgramProcessProtocol(asyncio.SubprocessProtocol): self._exit_timer.cancel() -class ProgramTestOutputReader: +class NativeTestOutputReader: def __init__(self, test_runner): self.test_runner = test_runner self.aio_loop = ( @@ -78,6 +79,13 @@ class ProgramTestOutputReader: "program.exe" if IS_WINDOWS else "program", ) ] + # if user changed PROGNAME + if not os.path.exists(cmd[0]): + build_data = load_build_metadata( + os.getcwd(), self.test_runner.test_suite.env_name, cache=True + ) + if build_data: + cmd[0] = build_data["prog_path"] if self.test_runner.options.program_args: cmd.extend(self.test_runner.options.program_args) return cmd diff --git a/platformio/test/runners/readers/serial.py b/platformio/test/runners/readers/serial.py index ab35775b..a9732d7f 100644 --- a/platformio/test/runners/readers/serial.py +++ b/platformio/test/runners/readers/serial.py @@ -22,7 +22,6 @@ from platformio.exception import UserSideException class SerialTestOutputReader: - SERIAL_TIMEOUT = 600 def __init__(self, test_runner): @@ -47,7 +46,7 @@ class SerialTestOutputReader: ser.open() except serial.SerialException as exc: click.secho(str(exc), fg="red", err=True) - return None + return if not self.test_runner.options.no_reset: ser.flushInput() diff --git a/platformio/test/runners/unity.py b/platformio/test/runners/unity.py index b7d8026b..25fea0ab 100644 --- a/platformio/test/runners/unity.py +++ b/platformio/test/runners/unity.py @@ -26,7 +26,6 @@ from platformio.util import strip_ansi_codes class UnityTestRunner(TestRunnerBase): - EXTRA_LIB_DEPS = ["throwtheswitch/Unity@^2.5.2"] # Examples: @@ -115,7 +114,7 @@ $framework_config_code native=dict( code=""" #include -void unityOutputStart(unsigned long baudrate) { } +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { putchar(c); } void unityOutputFlush(void) { fflush(stdout); } void unityOutputComplete(void) { } @@ -156,7 +155,7 @@ void unityOutputComplete(void) { } espidf=dict( code=""" #include -void unityOutputStart(unsigned long baudrate) { } +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { putchar(c); } void unityOutputFlush(void) { fflush(stdout); } void unityOutputComplete(void) { } @@ -166,7 +165,7 @@ void unityOutputComplete(void) { } zephyr=dict( code=""" #include -void unityOutputStart(unsigned long baudrate) { } +void unityOutputStart(unsigned long baudrate) { (void) baudrate; } void unityOutputChar(unsigned int c) { printk("%c", c); } void unityOutputFlush(void) { } void unityOutputComplete(void) { } diff --git a/platformio/util.py b/platformio/util.py index 63a16d6e..c981384a 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -96,7 +96,6 @@ class RetryStopException(RetryException): class retry: - RetryNextException = RetryNextException RetryStopException = RetryStopException diff --git a/scripts/docspregen.py b/scripts/docspregen.py index f895c25d..09dd6777 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -22,7 +22,7 @@ sys.path.append("..") import click # noqa: E402 -from platformio import fs, util # noqa: E402 +from platformio import fs # noqa: E402 from platformio.package.manager.platform import PlatformPackageManager # noqa: E402 from platformio.platform.factory import PlatformFactory # noqa: E402 @@ -41,6 +41,25 @@ RST_COPYRIGHT = """.. Copyright (c) 2014-present PlatformIO =0.27,<1", "pyserial==3.5.*", # keep in sync "device/monitor/terminal.py" "requests==2.*", + "urllib3<2", # issue 4614: urllib3 v2.0 only supports OpenSSL 1.1.1+ "requests==%s" % ("2.27.1" if PY36 else "2.*"), "semantic_version==2.10.*", "tabulate==%s" % ("0.8.10" if PY36 else "0.9.*"), ] home_requirements = [ - "aiofiles==%s" % ("0.8.0" if PY36 else "22.1.*"), + "aiofiles==%s" % ("0.8.0" if PY36 else "23.1.*"), "ajsonrpc==1.*", - "starlette==%s" % ("0.19.1" if PY36 else "0.23.*"), - "uvicorn==%s" % ("0.16.0" if PY36 else "0.20.*"), + "starlette==%s" % ("0.19.1" if PY36 else "0.26.*"), + "uvicorn==%s" % ("0.16.0" if PY36 else "0.22.*"), "wsproto==%s" % ("1.0.0" if PY36 else "1.2.*"), ] @@ -67,6 +68,8 @@ setup( "assets/templates/ide-projects/*/*.tpl", "assets/templates/ide-projects/*/.*.tpl", # include hidden files "assets/templates/ide-projects/*/.*/*.tpl", # include hidden folders + "assets/templates/ide-projects/*/*/*.tpl", # NetBeans + "assets/templates/ide-projects/*/*/*/*.tpl", # NetBeans ] }, entry_points={ diff --git a/tests/commands/test_check.py b/tests/commands/test_check.py index be6042ca..cb6f7777 100644 --- a/tests/commands/test_check.py +++ b/tests/commands/test_check.py @@ -15,8 +15,8 @@ # pylint: disable=redefined-outer-name import json +import os import sys -from os.path import isfile, join import pytest @@ -83,12 +83,12 @@ def check_dir(tmpdir_factory): def count_defects(output): error, warning, style = 0, 0, 0 - for l in output.split("\n"): - if "[high:error]" in l: + for line in output.split("\n"): + if "[high:error]" in line: error += 1 - elif "[medium:warning]" in l: + elif "[medium:warning]" in line: warning += 1 - elif "[low:style]" in l: + elif "[low:style]" in line: style += 1 return error, warning, style @@ -240,9 +240,9 @@ def test_check_includes_passed(clirunner, check_dir): result = clirunner.invoke(cmd_check, ["--project-dir", str(check_dir), "--verbose"]) inc_count = 0 - for l in result.output.split("\n"): - if l.startswith("Includes:"): - inc_count = l.count("-I") + for line in result.output.split("\n"): + if line.startswith("Includes:"): + inc_count = line.count("-I") # at least 1 include path for default mode assert inc_count > 0 @@ -259,46 +259,6 @@ def test_check_silent_mode(clirunner, validate_cliresult, check_dir): assert style == 0 -def test_check_custom_pattern_absolute_path( - clirunner, validate_cliresult, tmpdir_factory -): - project_dir = tmpdir_factory.mktemp("project") - project_dir.join("platformio.ini").write(DEFAULT_CONFIG) - - check_dir = tmpdir_factory.mktemp("custom_src_dir") - check_dir.join("main.cpp").write(TEST_CODE) - - result = clirunner.invoke( - cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] - ) - validate_cliresult(result) - - errors, warnings, style = count_defects(result.output) - - assert errors == EXPECTED_ERRORS - assert warnings == EXPECTED_WARNINGS - assert style == EXPECTED_STYLE - - -def test_check_custom_pattern_relative_path( - clirunner, validate_cliresult, tmpdir_factory -): - tmpdir = tmpdir_factory.mktemp("project") - tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) - - tmpdir.mkdir("app").join("main.cpp").write(TEST_CODE) - tmpdir.mkdir("prj").join("test.cpp").write(TEST_CODE) - - result = clirunner.invoke( - cmd_check, ["--project-dir", str(tmpdir), "--pattern=app", "--pattern=prj"] - ) - validate_cliresult(result) - - errors, warnings, style = count_defects(result.output) - - assert errors + warnings + style == EXPECTED_DEFECTS * 2 - - def test_check_no_source_files(clirunner, tmpdir): tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) tmpdir.mkdir("src") @@ -427,7 +387,7 @@ R21.4 text. validate_cliresult(result) assert "R21.3 Found MISRA defect" in result.output - assert not isfile(join(str(check_dir), "src", "main.cpp.dump")) + assert not os.path.isfile(os.path.join(str(check_dir), "src", "main.cpp.dump")) def test_check_fails_on_defects_only_with_flag(clirunner, validate_cliresult, tmpdir): @@ -607,10 +567,10 @@ framework = arduino validate_cliresult(result) project_path = fs.to_unix_path(str(tmpdir)) - for l in result.output.split("\n"): - if not l.startswith("Includes:"): + for line in result.output.split("\n"): + if not line.startswith("Includes:"): continue - for inc in l.split(" "): + for inc in line.split(" "): if inc.startswith("-I") and project_path not in inc: pytest.fail("Detected an include path from packages: " + inc) @@ -656,3 +616,147 @@ def test_check_handles_spaces_in_paths(clirunner, validate_cliresult, tmpdir_fac default_result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) validate_cliresult(default_result) + + +# +# Files filtering functionality +# + + +@pytest.mark.parametrize( + "src_filter,number_of_checked_files", + [ + (["+"], 1), + (["+"], 1), + (["+", "-"], 2), + (["-<*> + + +"], 3), + ], + ids=["Single file", "Glob pattern", "Exclude pattern", "Filter as string"], +) +def test_check_src_filter( + clirunner, + validate_cliresult, + tmpdir_factory, + src_filter, + number_of_checked_files, +): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.join("app.cpp").write(TEST_CODE) + src_dir.mkdir("uart").join("uart.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + cmd_args = ["--project-dir", str(tmpdir)] + [ + "--src-filters=%s" % f for f in src_filter + ] + + result = clirunner.invoke(cmd_check, cmd_args) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * number_of_checked_files + + +def test_check_src_filter_from_config(clirunner, validate_cliresult, tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("project") + + config = ( + DEFAULT_CONFIG + + """ +check_src_filters = + + + + + """ + ) + tmpdir.join("platformio.ini").write(config) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + assert "main.cpp" not in result.output + + +def test_check_custom_pattern_absolute_path_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + project_dir = tmpdir_factory.mktemp("project") + project_dir.join("platformio.ini").write(DEFAULT_CONFIG) + + check_dir = tmpdir_factory.mktemp("custom_src_dir") + check_dir.join("main.cpp").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, ["--project-dir", str(project_dir), "--pattern=" + str(check_dir)] + ) + + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors == EXPECTED_ERRORS + assert warnings == EXPECTED_WARNINGS + assert style == EXPECTED_STYLE + + +def test_check_custom_pattern_relative_path_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + tmpdir = tmpdir_factory.mktemp("project") + tmpdir.join("platformio.ini").write(DEFAULT_CONFIG) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("uart").join("uart.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + + result = clirunner.invoke( + cmd_check, + ["--project-dir", str(tmpdir), "--pattern=src/uart", "--pattern=src/spi"], + ) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + + +def test_check_src_filter_from_config_legacy( + clirunner, validate_cliresult, tmpdir_factory +): + tmpdir = tmpdir_factory.mktemp("project") + + config = ( + DEFAULT_CONFIG + + """ +check_patterns = + src/spi/*.c* + tests/test.cpp + """ + ) + tmpdir.join("platformio.ini").write(config) + + src_dir = tmpdir.mkdir("src") + src_dir.join("main.cpp").write(TEST_CODE) + src_dir.mkdir("spi").join("spi.cpp").write(TEST_CODE) + tmpdir.mkdir("tests").join("test.cpp").write(TEST_CODE) + + result = clirunner.invoke(cmd_check, ["--project-dir", str(tmpdir)]) + validate_cliresult(result) + + errors, warnings, style = count_defects(result.output) + + assert errors + warnings + style == EXPECTED_DEFECTS * 2 + assert "main.cpp" not in result.output diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index ae632280..f67bada4 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -112,7 +112,6 @@ def test_ci_keep_build_dir_single_src_dir( def test_ci_keep_build_dir_nested_src_dirs( clirunner, tmpdir_factory, validate_cliresult ): - build_dir = str(tmpdir_factory.mktemp("ci_build_dir")) # Split default Arduino project in two parts diff --git a/tests/commands/test_run.py b/tests/commands/test_run.py index 009f33c9..cb9e3d85 100644 --- a/tests/commands/test_run.py +++ b/tests/commands/test_run.py @@ -22,6 +22,7 @@ def test_generic_build(clirunner, validate_cliresult, tmpdir): ("-D TEST_INT=13", "-DTEST_INT=13"), ("-DTEST_SINGLE_MACRO", "-DTEST_SINGLE_MACRO"), ('-DTEST_STR_SPACE="Andrew Smith"', '"-DTEST_STR_SPACE=Andrew Smith"'), + ("-Iextra_inc", "-Iextra_inc"), ] tmpdir.join("platformio.ini").write( @@ -58,8 +59,20 @@ projenv.Append(CPPDEFINES="POST_SCRIPT_MACRO") """ ) + tmpdir.mkdir("extra_inc").join("foo.h").write( + """ +#define FOO + """ + ) + tmpdir.mkdir("src").join("main.cpp").write( """ +#include "foo.h" + +#ifndef FOO +#error "FOO" +#endif + #ifdef I_AM_ONLY_SRC_FLAG #include #else diff --git a/tests/commands/test_test.py b/tests/commands/test_test.py index 26089500..29ff20ab 100644 --- a/tests/commands/test_test.py +++ b/tests/commands/test_test.py @@ -642,7 +642,7 @@ def test_googletest_framework(clirunner, tmp_path: Path): pio_test_cmd, [ "-d", - project_dir, + str(project_dir), "-e", "native", "--json-output-path", diff --git a/tests/misc/test_misc.py b/tests/misc/test_misc.py index 52a6cac4..349e9c79 100644 --- a/tests/misc/test_misc.py +++ b/tests/misc/test_misc.py @@ -35,7 +35,7 @@ def test_ping_internet_ips(): def test_api_internet_offline(without_internet, isolated_pio_core): regclient = RegistryClient() - with pytest.raises(http.InternetIsOffline): + with pytest.raises(http.InternetConnectionError): regclient.fetch_json_data("get", "/v2/stats") diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 828412f4..a3279e05 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -901,5 +901,5 @@ def test_broken_schemas(): ) # invalid package name - with pytest.raises(ManifestValidationError, match=("are not allowed")): + with pytest.raises(ManifestValidationError, match="are not allowed"): ManifestSchema().load_manifest(dict(name="C/C++ :library", version="1.2.3")) diff --git a/tests/project/test_config.py b/tests/project/test_config.py index 6076a7d6..f559502e 100644 --- a/tests/project/test_config.py +++ b/tests/project/test_config.py @@ -23,7 +23,11 @@ import pytest from platformio import fs from platformio.project.config import ProjectConfig -from platformio.project.exception import InvalidProjectConfError, UnknownEnvNamesError +from platformio.project.exception import ( + InvalidEnvNameError, + InvalidProjectConfError, + UnknownEnvNamesError, +) BASE_CONFIG = """ [platformio] @@ -662,9 +666,21 @@ upload_tool = two [c] upload_tool = three -[env:native] +[env:na_ti-ve13] extends = a, b, c """ ) config = ProjectConfig(str(project_conf)) - assert config.get("env:native", "upload_tool") == "three" + assert config.get("env:na_ti-ve13", "upload_tool") == "three" + + +def test_invalid_env_names(tmp_path: Path): + project_conf = tmp_path / "platformio.ini" + project_conf.write_text( + """ +[env:app:1] + """ + ) + config = ProjectConfig(str(project_conf)) + with pytest.raises(InvalidEnvNameError, match=r".*Invalid environment name 'app:1"): + config.validate() diff --git a/tests/test_examples.py b/tests/test_examples.py index 27164fd7..b49aa90d 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -22,6 +22,7 @@ from platformio import fs, proc from platformio.package.manager.platform import PlatformPackageManager from platformio.platform.factory import PlatformFactory from platformio.project.config import ProjectConfig +from platformio.project.exception import ProjectError def pytest_generate_tests(metafunc): @@ -47,6 +48,8 @@ def pytest_generate_tests(metafunc): for root, _, files in os.walk(examples_dir): if "platformio.ini" not in files or ".skiptest" in files: continue + if "mbed-legacy-examples" in root: + continue group = os.path.basename(root) if "-" in group: group = group.split("-", 1)[0] @@ -64,6 +67,13 @@ def pytest_generate_tests(metafunc): def test_run(pioproject_dir): with fs.cd(pioproject_dir): config = ProjectConfig() + + # temporary fix for unreleased dev-platforms with broken env name + try: + config.validate() + except ProjectError as exc: + pytest.skip(str(exc)) + build_dir = config.get("platformio", "build_dir") if os.path.isdir(build_dir): fs.rmtree(build_dir) diff --git a/tox.ini b/tox.ini index 0a51741a..60832806 100644 --- a/tox.ini +++ b/tox.ini @@ -54,7 +54,7 @@ commands = [testenv:docs] deps = sphinx - sphinx-rtd-theme==1.1.1 + sphinx-rtd-theme==1.2.0 sphinx-notfound-page sphinx-copybutton restructuredtext-lint