diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec53bebb..5f2f4188 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ Contributing To get started, sign the Contributor License Agreement. -1. Fork the repository on GitHub. +1. Fork the repository on GitHub 2. Clone repository `git clone --recursive https://github.com/YourGithubUsername/platformio-core.git` 3. Run `pip install tox` 4. Go to the root of project where is located `tox.ini` and run `tox -e py37` @@ -18,4 +18,4 @@ To get started, si 8. Run the tests `make test` 9. Build documentation `tox -e docs` (creates a directory _build under docs where you can find the html) 10. Commit changes to your forked repository -11. Submit a Pull Request on GitHub. +11. Submit a Pull Request on GitHub diff --git a/HISTORY.rst b/HISTORY.rst index 9274b86b..c6cd902b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,15 @@ PlatformIO Core 5 **A professional collaborative platform for embedded development** +5.2.5 (2022-02-10) +~~~~~~~~~~~~~~~~~~ + +- Improved support for private packages in `PlatformIO Registry `__ +- Improved checking of available Internet connection for IPv6-only workstations (`pull #4151 `_) +- Better detecting of default PlatformIO project directory on Linux OS (`pull #4158 `_) +- Respect disabling debugging server from "platformio.ini" passing an empty value to the `debug_server `__ option +- Fixed a "module 'asyncio' has no attribute 'run'" error when launching PIO Home using Python 3.6 (`issue #4169 `_) + 5.2.4 (2021-12-15) ~~~~~~~~~~~~~~~~~~ @@ -286,7 +295,7 @@ Please check `Migration guide from 4.x to 5.0 `__ command (`issue #3522 `_) - Show ignored project environments only in the verbose mode (`issue #3641 `_) - Do not escape compiler arguments in VSCode template on Windows - - Drop support for Python 2 and 3.5. + - Drop support for Python 2 and 3.5 .. _release_notes_4: diff --git a/README.rst b/README.rst index ae40bcdf..061c0142 100644 --- a/README.rst +++ b/README.rst @@ -17,11 +17,12 @@ PlatformIO Core :target: https://pypi.python.org/pypi/platformio/ :alt: License .. image:: https://img.shields.io/badge/PlatformIO-Labs-orange.svg - :alt: Community Labs + :alt: PlatformIO Labs :target: https://piolabs.com/?utm_source=github&utm_medium=core -**Quick Links:** `Web `_ | +**Quick Links:** `Homepage `_ | `PlatformIO IDE `_ | +`Registry `_ | `Project Examples `__ | `Docs `_ | `Donate `_ | @@ -43,7 +44,7 @@ PlatformIO Core * Cross-platform IDE and Unified Debugger * Static Code Analyzer and Remote Unit Testing * Multi-platform and Multi-architecture Build System -* Firmware File Explorer and Memory Inspection. +* Firmware File Explorer and Memory Inspection Get Started ----------- @@ -70,66 +71,9 @@ Solutions Registry -------- -* `Libraries `_ -* `Development Platforms `_ -* `Frameworks `_ -* `Embedded Boards `_ - -Development Platforms ---------------------- - -* `Aceinna IMU `_ -* `ASR Microelectronics ASR605x `_ -* `Atmel AVR `_ -* `Atmel SAM `_ -* `Espressif 32 `_ -* `Espressif 8266 `_ -* `Freescale Kinetis `_ -* `Infineon XMC `_ -* `Intel ARC32 `_ -* `Intel MCS-51 (8051) `_ -* `Kendryte K210 `_ -* `Lattice iCE40 `_ -* `Maxim 32 `_ -* `Microchip PIC32 `_ -* `Nordic nRF51 `_ -* `Nordic nRF52 `_ -* `Nuclei `_ -* `NXP LPC `_ -* `RISC-V `_ -* `RISC-V GAP `_ -* `Shakti `_ -* `Silicon Labs EFM32 `_ -* `ST STM32 `_ -* `ST STM8 `_ -* `Teensy `_ -* `TI MSP430 `_ -* `TI Tiva `_ -* `WIZNet W7500 `_ - -Frameworks ----------- - -* `Arduino `_ -* `CMSIS `_ -* `ESP-IDF `_ -* `ESP8266 Non-OS SDK `_ -* `ESP8266 RTOS SDK `_ -* `Freedom E SDK `_ -* `GigaDevice GD32V SDK `_ -* `Kendryte Standalone SDK `_ -* `Kendryte FreeRTOS SDK `_ -* `libOpenCM3 `_ -* `Mbed `_ -* `Nuclei SDK `_ -* `PULP OS `_ -* `Pumbaa `_ -* `Shakti SDK `_ -* `Simba `_ -* `SPL `_ -* `STM32Cube `_ -* `WiringPi `_ -* `Zephyr `_ +* `Libraries `_ +* `Development Platforms `_ +* `Development Tools `_ Contributing ------------ diff --git a/docs b/docs index ba3fca21..bbf4d275 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit ba3fca21eab09226c194f0a23463b3c253d2e069 +Subproject commit bbf4d27508f4fe600dfb5706641bdccf6b2a762e diff --git a/examples b/examples index d722c23d..dcafbd19 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit d722c23da47639dcc92e3c9b997df61b9ee1bfe3 +Subproject commit dcafbd192ee19fdb310136fa62335a3ce13ec517 diff --git a/platformio/__init__.py b/platformio/__init__.py index 075823ca..0c9e9bd5 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (5, 2, 4) +VERSION = (5, 2, 5) __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" @@ -47,7 +47,7 @@ __pioremote_endpoint__ = "ssl:host=remote.platformio.org:port=4413" __default_requests_timeout__ = (10, None) # (connect, read) __core_packages__ = { - "contrib-piohome": "~3.4.0", + "contrib-piohome": "~3.4.1", "contrib-pysite": "~2.%d%d.0" % (sys.version_info.major, sys.version_info.minor), "tool-unity": "~1.20500.0", "tool-scons": "~4.40300.0", diff --git a/platformio/app.py b/platformio/app.py index 374f4014..6c152c17 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -253,29 +253,14 @@ def is_disabled_progressbar(): def get_cid(): - # pylint: disable=import-outside-toplevel - from platformio.clients.http import fetch_remote_content - cid = get_state_item("cid") if cid: return cid uid = None - if os.getenv("C9_UID"): - uid = os.getenv("C9_UID") + if os.getenv("GITHUB_USER"): + uid = os.getenv("GITHUB_USER") elif os.getenv("GITPOD_GIT_USER_NAME"): uid = os.getenv("GITPOD_GIT_USER_NAME") - elif os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")): - try: - uid = json.loads( - fetch_remote_content( - "{api}/user?token={token}".format( - api=os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")), - token=os.getenv("USER_TOKEN"), - ) - ) - ).get("id") - except: # pylint: disable=bare-except - pass if not uid: uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py index bd6f35f4..3b57c1e7 100644 --- a/platformio/builder/tools/piolib.py +++ b/platformio/builder/tools/piolib.py @@ -32,7 +32,7 @@ from SCons.Script import DefaultEnvironment # pylint: disable=import-error from platformio import exception, fs, util from platformio.builder.tools import platformio as piotool -from platformio.clients.http import InternetIsOffline +from platformio.clients.http import HTTPClientError, InternetIsOffline from platformio.compat import IS_WINDOWS, hashlib_encode_data, string_types from platformio.package.exception import UnknownPackageError from platformio.package.manager.library import LibraryPackageManager @@ -939,7 +939,7 @@ class ProjectAsLibBuilder(LibBuilderBase): try: lm.install(spec) did_install = True - except (UnknownPackageError, InternetIsOffline) as e: + except (HTTPClientError, UnknownPackageError, InternetIsOffline) as e: click.secho("Warning! %s" % e, fg="yellow") # reset cache diff --git a/platformio/clients/account.py b/platformio/clients/account.py index c7c2c6fa..60349934 100644 --- a/platformio/clients/account.py +++ b/platformio/clients/account.py @@ -16,7 +16,7 @@ import os import time from platformio import __accounts_api__, app -from platformio.clients.http import HTTPClient +from platformio.clients.http import HTTPClient, HTTPClientError from platformio.exception import PlatformioException @@ -61,13 +61,33 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods del account[key] app.set_state_item("account", account) - def send_auth_request(self, *args, **kwargs): - headers = kwargs.get("headers", {}) - if "Authorization" not in headers: - token = self.fetch_authentication_token() - headers["Authorization"] = "Bearer %s" % token - kwargs["headers"] = headers - return self.fetch_json_data(*args, **kwargs) + def fetch_json_data(self, *args, **kwargs): + try: + return super(AccountClient, self).fetch_json_data(*args, **kwargs) + except HTTPClientError as exc: + raise AccountError(exc) from exc + + def fetch_authentication_token(self): + if os.environ.get("PLATFORMIO_AUTH_TOKEN"): + return os.environ.get("PLATFORMIO_AUTH_TOKEN") + auth = app.get_state_item("account", {}).get("auth", {}) + if auth.get("access_token") and auth.get("access_token_expire"): + if auth.get("access_token_expire") > time.time(): + return auth.get("access_token") + if auth.get("refresh_token"): + try: + data = self.fetch_json_data( + "post", + "/v1/login", + headers={ + "Authorization": "Bearer %s" % auth.get("refresh_token") + }, + ) + app.set_state_item("account", data) + return data.get("auth").get("access_token") + except AccountError: + self.delete_local_session() + raise AccountNotAuthorized() def login(self, username, password): try: @@ -119,10 +139,11 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods return True def change_password(self, old_password, new_password): - return self.send_auth_request( + return self.fetch_json_data( "post", "/v1/password", data={"old_password": old_password, "new_password": new_password}, + x_with_authorization=True, ) def registration( @@ -150,10 +171,11 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods ) def auth_token(self, password, regenerate): - return self.send_auth_request( + return self.fetch_json_data( "post", "/v1/token", data={"password": password, "regenerate": 1 if regenerate else 0}, + x_with_authorization=True, ).get("auth_token") def forgot_password(self, username): @@ -164,18 +186,20 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods ) def get_profile(self): - return self.send_auth_request( + return self.fetch_json_data( "get", "/v1/profile", + x_with_authorization=True, ) def update_profile(self, profile, current_password): profile["current_password"] = current_password self.delete_local_state("summary") - response = self.send_auth_request( + response = self.fetch_json_data( "put", "/v1/profile", data=profile, + x_with_authorization=True, ) return response @@ -193,9 +217,10 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods "username": account.get("username"), } } - result = self.send_auth_request( + result = self.fetch_json_data( "get", "/v1/summary", + x_with_authorization=True, ) account["summary"] = dict( profile=result.get("profile"), @@ -211,119 +236,121 @@ class AccountClient(HTTPClient): # pylint:disable=too-many-public-methods return self.get_account_info(offline=True).get("profile").get("username") def destroy_account(self): - return self.send_auth_request("delete", "/v1/account") + return self.fetch_json_data( + "delete", + "/v1/account", + x_with_authorization=True, + ) def create_org(self, orgname, email, displayname): - return self.send_auth_request( + return self.fetch_json_data( "post", "/v1/orgs", data={"orgname": orgname, "email": email, "displayname": displayname}, + x_with_authorization=True, ) def get_org(self, orgname): - return self.send_auth_request("get", "/v1/orgs/%s" % orgname) + return self.fetch_json_data( + "get", + "/v1/orgs/%s" % orgname, + x_with_authorization=True, + ) def list_orgs(self): - return self.send_auth_request( + return self.fetch_json_data( "get", "/v1/orgs", + x_with_authorization=True, ) def update_org(self, orgname, data): - return self.send_auth_request( - "put", "/v1/orgs/%s" % orgname, data={k: v for k, v in data.items() if v} + return self.fetch_json_data( + "put", + "/v1/orgs/%s" % orgname, + data={k: v for k, v in data.items() if v}, + x_with_authorization=True, ) def destroy_org(self, orgname): - return self.send_auth_request( + return self.fetch_json_data( "delete", "/v1/orgs/%s" % orgname, + x_with_authorization=True, ) def add_org_owner(self, orgname, username): - return self.send_auth_request( + return self.fetch_json_data( "post", "/v1/orgs/%s/owners" % orgname, data={"username": username}, + x_with_authorization=True, ) def list_org_owners(self, orgname): - return self.send_auth_request( + return self.fetch_json_data( "get", "/v1/orgs/%s/owners" % orgname, + x_with_authorization=True, ) def remove_org_owner(self, orgname, username): - return self.send_auth_request( + return self.fetch_json_data( "delete", "/v1/orgs/%s/owners" % orgname, data={"username": username}, + x_with_authorization=True, ) def create_team(self, orgname, teamname, description): - return self.send_auth_request( + return self.fetch_json_data( "post", "/v1/orgs/%s/teams" % orgname, data={"name": teamname, "description": description}, + x_with_authorization=True, ) def destroy_team(self, orgname, teamname): - return self.send_auth_request( + return self.fetch_json_data( "delete", "/v1/orgs/%s/teams/%s" % (orgname, teamname), + x_with_authorization=True, ) def get_team(self, orgname, teamname): - return self.send_auth_request( + return self.fetch_json_data( "get", "/v1/orgs/%s/teams/%s" % (orgname, teamname), + x_with_authorization=True, ) def list_teams(self, orgname): - return self.send_auth_request( + return self.fetch_json_data( "get", "/v1/orgs/%s/teams" % orgname, + x_with_authorization=True, ) def update_team(self, orgname, teamname, data): - return self.send_auth_request( + return self.fetch_json_data( "put", "/v1/orgs/%s/teams/%s" % (orgname, teamname), data={k: v for k, v in data.items() if v}, + x_with_authorization=True, ) def add_team_member(self, orgname, teamname, username): - return self.send_auth_request( + return self.fetch_json_data( "post", "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), data={"username": username}, + x_with_authorization=True, ) def remove_team_member(self, orgname, teamname, username): - return self.send_auth_request( + return self.fetch_json_data( "delete", "/v1/orgs/%s/teams/%s/members" % (orgname, teamname), data={"username": username}, + x_with_authorization=True, ) - - def fetch_authentication_token(self): - if os.environ.get("PLATFORMIO_AUTH_TOKEN"): - return os.environ.get("PLATFORMIO_AUTH_TOKEN") - auth = app.get_state_item("account", {}).get("auth", {}) - if auth.get("access_token") and auth.get("access_token_expire"): - if auth.get("access_token_expire") > time.time(): - return auth.get("access_token") - if auth.get("refresh_token"): - try: - data = self.fetch_json_data( - "post", - "/v1/login", - headers={ - "Authorization": "Bearer %s" % auth.get("refresh_token") - }, - ) - app.set_state_item("account", data) - return data.get("auth").get("access_token") - except AccountError: - self.delete_local_session() - raise AccountNotAuthorized() diff --git a/platformio/clients/http.py b/platformio/clients/http.py index 0f3e3860..3cf247b4 100644 --- a/platformio/clients/http.py +++ b/platformio/clients/http.py @@ -21,7 +21,7 @@ import requests.adapters from requests.packages.urllib3.util.retry import Retry # pylint:disable=import-error from platformio import __check_internet_hosts__, __default_requests_timeout__, app, util -from platformio.cache import ContentCache +from platformio.cache import ContentCache, cleanup_content_cache from platformio.exception import PlatformioException, UserSideException try: @@ -117,6 +117,21 @@ class HTTPClient(object): # check Internet before and resolve issue with 60 seconds timeout ensure_internet_on(raise_exception=True) + headers = kwargs.get("headers", {}) + with_authorization = ( + kwargs.pop("x_with_authorization") + if "x_with_authorization" in kwargs + else False + ) + if with_authorization and "Authorization" not in headers: + # pylint: disable=import-outside-toplevel + from platformio.clients.account import AccountClient + + headers["Authorization"] = ( + "Bearer %s" % AccountClient().fetch_authentication_token() + ) + kwargs["headers"] = headers + # set default timeout if "timeout" not in kwargs: kwargs["timeout"] = __default_requests_timeout__ @@ -134,7 +149,9 @@ class HTTPClient(object): raise HTTPClientError(str(e)) def fetch_json_data(self, method, path, **kwargs): - cache_valid = kwargs.pop("cache_valid") if "cache_valid" in kwargs else None + if method != "get": + cleanup_content_cache("http") + cache_valid = kwargs.pop("x_cache_valid") if "x_cache_valid" in kwargs else None if not cache_valid: return self._parse_json_response(self.send_request(method, path, **kwargs)) cache_key = ContentCache.key_from_args( @@ -179,8 +196,9 @@ def _internet_on(): continue requests.get("http://%s" % host, allow_redirects=False, timeout=timeout) return True - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((host, 80)) + # try to resolve `host` for both AF_INET and AF_INET6, and then try to connect + # to all possible addresses (IPv4 and IPv6) in turn until a connection succeeds: + s = socket.create_connection((host, 80)) s.close() return True except: # pylint: disable=bare-except diff --git a/platformio/clients/registry.py b/platformio/clients/registry.py index 3a45a48e..aba734ff 100644 --- a/platformio/clients/registry.py +++ b/platformio/clients/registry.py @@ -13,7 +13,7 @@ # limitations under the License. from platformio import __registry_api__, fs -from platformio.clients.account import AccountClient +from platformio.clients.account import AccountClient, AccountError from platformio.clients.http import HTTPClient, HTTPClientError # pylint: disable=too-many-arguments @@ -23,19 +23,29 @@ class RegistryClient(HTTPClient): def __init__(self): super(RegistryClient, self).__init__(__registry_api__) - def send_auth_request(self, *args, **kwargs): - headers = kwargs.get("headers", {}) - if "Authorization" not in headers: - token = AccountClient().fetch_authentication_token() - headers["Authorization"] = "Bearer %s" % token - kwargs["headers"] = headers - return self.fetch_json_data(*args, **kwargs) + @staticmethod + def allowed_private_packages(): + private_permissions = set( + [ + "service.registry.publish-private-tool", + "service.registry.publish-private-platform", + "service.registry.publish-private-library", + ] + ) + try: + info = AccountClient().get_account_info() or {} + for item in info.get("packages", []): + if set(item.keys()) & private_permissions: + return True + except AccountError: + pass + return False def publish_package( # pylint: disable=redefined-builtin self, owner, type, archive_path, released_at=None, private=False, notify=True ): with open(archive_path, "rb") as fp: - return self.send_auth_request( + return self.fetch_json_data( "post", "/v3/packages/%s/%s" % (owner, type), params={ @@ -50,6 +60,7 @@ class RegistryClient(HTTPClient): ), }, data=fp, + x_with_authorization=True, ) def unpublish_package( # pylint: disable=redefined-builtin @@ -58,36 +69,40 @@ class RegistryClient(HTTPClient): path = "/v3/packages/%s/%s/%s" % (owner, type, name) if version: path += "/" + version - return self.send_auth_request( - "delete", - path, - params={"undo": 1 if undo else 0}, + return self.fetch_json_data( + "delete", path, params={"undo": 1 if undo else 0}, x_with_authorization=True ) def update_resource(self, urn, private): - return self.send_auth_request( + return self.fetch_json_data( "put", "/v3/resources/%s" % urn, data={"private": int(private)}, + x_with_authorization=True, ) def grant_access_for_resource(self, urn, client, level): - return self.send_auth_request( + return self.fetch_json_data( "put", "/v3/resources/%s/access" % urn, data={"client": client, "level": level}, + x_with_authorization=True, ) def revoke_access_from_resource(self, urn, client): - return self.send_auth_request( + return self.fetch_json_data( "delete", "/v3/resources/%s/access" % urn, data={"client": client}, + x_with_authorization=True, ) def list_resources(self, owner): - return self.send_auth_request( - "get", "/v3/resources", params={"owner": owner} if owner else None + return self.fetch_json_data( + "get", + "/v3/resources", + params={"owner": owner} if owner else None, + x_with_authorization=True, ) def list_packages(self, query=None, filters=None, page=None): @@ -117,7 +132,11 @@ class RegistryClient(HTTPClient): if page: params["page"] = int(page) return self.fetch_json_data( - "get", "/v3/search", params=params, cache_valid="1h" + "get", + "/v3/search", + params=params, + x_cache_valid="1h", + x_with_authorization=self.allowed_private_packages(), ) def get_package(self, type_, owner, name, version=None): @@ -128,7 +147,8 @@ class RegistryClient(HTTPClient): type=type_, owner=owner.lower(), name=name.lower() ), params=dict(version=version) if version else None, - cache_valid="1h", + x_cache_valid="1h", + x_with_authorization=self.allowed_private_packages(), ) except HTTPClientError as e: if e.response is not None and e.response.status_code == 404: diff --git a/platformio/commands/access.py b/platformio/commands/access.py index d9fb3970..74f48e2b 100644 --- a/platformio/commands/access.py +++ b/platformio/commands/access.py @@ -134,6 +134,14 @@ def access_list(owner, urn_type, json_output): table_data = [] table_data.append(("URN:", resource.get("urn"))) table_data.append(("Owner:", resource.get("owner"))) + table_data.append( + ( + "Access:", + click.style("Private", fg="red") + if resource.get("private", False) + else "Public", + ) + ) table_data.append( ( "Access level(s):", diff --git a/platformio/commands/lib/command.py b/platformio/commands/lib/command.py index 7b6c7bb7..d29b7388 100644 --- a/platformio/commands/lib/command.py +++ b/platformio/commands/lib/command.py @@ -355,7 +355,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): "get", "/v2/lib/search", params=dict(query=" ".join(query), page=page), - cache_valid="1d", + x_cache_valid="1d", ) if json_output: @@ -408,7 +408,7 @@ def lib_search(query, json_output, page, noninteractive, **filters): "get", "/v2/lib/search", params=dict(query=" ".join(query), page=int(result["page"]) + 1), - cache_valid="1d", + x_cache_valid="1d", ) @@ -440,7 +440,9 @@ def lib_show(library, json_output): lm = LibraryPackageManager() lib_id = lm.reveal_registry_package_id(library, silent=json_output) regclient = lm.get_registry_client_instance() - lib = regclient.fetch_json_data("get", "/v2/lib/info/%d" % lib_id, cache_valid="1h") + lib = regclient.fetch_json_data( + "get", "/v2/lib/info/%d" % lib_id, x_cache_valid="1h" + ) if json_output: return click.echo(json.dumps(lib)) @@ -535,7 +537,7 @@ def lib_register(config_url): # pylint: disable=unused-argument @click.option("--json-output", is_flag=True) def lib_stats(json_output): regclient = LibraryPackageManager().get_registry_client_instance() - result = regclient.fetch_json_data("get", "/v2/lib/stats", cache_valid="1h") + result = regclient.fetch_json_data("get", "/v2/lib/stats", x_cache_valid="1h") if json_output: return click.echo(json.dumps(result)) diff --git a/platformio/commands/package.py b/platformio/commands/package.py index 8ce79dd5..3e3bdaa6 100644 --- a/platformio/commands/package.py +++ b/platformio/commands/package.py @@ -191,6 +191,12 @@ def package_publish( # pylint: disable=too-many-arguments, too-many-locals abort=True, ) + click.secho( + "The package publishing may take some time depending " + "on your Internet connection and the package size.", + fg="yellow", + ) + click.echo("Publishing...") response = RegistryClient().publish_package( owner, type_, archive_path, released_at, private, notify ) diff --git a/platformio/commands/platform.py b/platformio/commands/platform.py index 84945e39..287f5760 100644 --- a/platformio/commands/platform.py +++ b/platformio/commands/platform.py @@ -61,7 +61,7 @@ def platform_frameworks(query, json_output): regclient = PlatformPackageManager().get_registry_client_instance() frameworks = [] for framework in regclient.fetch_json_data( - "get", "/v2/frameworks", cache_valid="1d" + "get", "/v2/frameworks", x_cache_valid="1d" ): if query == "all": query = "" @@ -354,7 +354,7 @@ def _print_platforms(platforms): def _get_registry_platforms(): regclient = PlatformPackageManager().get_registry_client_instance() - return regclient.fetch_json_data("get", "/v2/platforms", cache_valid="1d") + return regclient.fetch_json_data("get", "/v2/platforms", x_cache_valid="1d") def _get_platform_data(*args, **kwargs): diff --git a/platformio/debug/config/base.py b/platformio/debug/config/base.py index bcafa8fd..db7ddfba 100644 --- a/platformio/debug/config/base.py +++ b/platformio/debug/config/base.py @@ -153,7 +153,14 @@ class DebugConfigBase: # pylint: disable=too-many-instance-attributes raise DebugInvalidOptionsError("Could not load a build configuration") def _configure_server(self): + # user disabled server in platformio.ini + if "debug_server" in self.env_options and not self.env_options.get( + "debug_server" + ): + return None + result = None + # specific server per a system if isinstance(self.tool_settings.get("server", {}), list): for item in self.tool_settings["server"][:]: diff --git a/platformio/maintenance.py b/platformio/maintenance.py index ba370032..82aa4a0f 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -35,7 +35,6 @@ from platformio.package.manager.tool import ToolPackageManager from platformio.package.meta import PackageSpec from platformio.package.version import pepver_to_semver from platformio.platform.factory import PlatformFactory -from platformio.proc import is_container def on_platformio_start(ctx, force, caller): @@ -78,17 +77,12 @@ def set_caller(caller=None): caller = caller or os.getenv("PLATFORMIO_CALLER") if caller: return app.set_session_var("caller_id", caller) - if os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): + if os.getenv("CODESPACES"): + caller = "codespaces" + elif os.getenv("VSCODE_PID") or os.getenv("VSCODE_NLS_CONFIG"): caller = "vscode" - elif os.getenv("GITPOD_INSTANCE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): + elif os.getenv("GITPOD_WORKSPACE_ID") or os.getenv("GITPOD_WORKSPACE_URL"): caller = "gitpod" - elif is_container(): - if os.getenv("C9_UID"): - caller = "C9" - elif os.getenv("USER") == "cabox": - caller = "CA" - elif os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")): - caller = "Che" return app.set_session_var("caller_id", caller) diff --git a/platformio/package/manager/_registry.py b/platformio/package/manager/_registry.py index 80d42889..e488b5b3 100644 --- a/platformio/package/manager/_registry.py +++ b/platformio/package/manager/_registry.py @@ -54,6 +54,7 @@ class RegistryFileMirrorIterator(object): params=dict(bypass=",".join(self._visited_mirrors)) if self._visited_mirrors else None, + x_with_authorization=RegistryClient.allowed_private_packages(), ) stop_conditions = [ response.status_code not in (302, 307), diff --git a/platformio/package/manager/core.py b/platformio/package/manager/core.py index fe479c74..50b7e34e 100644 --- a/platformio/package/manager/core.py +++ b/platformio/package/manager/core.py @@ -212,7 +212,7 @@ def build_contrib_pysite_package(target_dir, with_metadata=True): def get_contrib_pysite_deps(): - twisted_version = "20.3.0" + twisted_version = "21.7.0" result = [ # twisted[tls], see setup.py for %twisted_version% "twisted == %s" % twisted_version, @@ -221,21 +221,6 @@ def get_contrib_pysite_deps(): "pyopenssl >= 16.0.0, <= 21.0.0", "service_identity >= 18.1.0, <= 21.1.0", ] - - sys_type = util.get_systype() - py_version = "%d%d" % (sys.version_info.major, sys.version_info.minor) - if "windows" in sys_type: - result.append("pypiwin32 == 223") - # workaround for twisted wheels - twisted_wheel = ( - "https://download.lfd.uci.edu/pythonlibs/x2tqcw5k/Twisted-" - "%s-cp%s-cp%s-win%s.whl" - % ( - twisted_version, - py_version, - py_version, - "_amd64" if "amd64" in sys_type else "32", - ) - ) - result[0] = twisted_wheel + if "windows" in util.get_systype(): + result.append("pywin32 != 226") return result diff --git a/platformio/package/manager/platform.py b/platformio/package/manager/platform.py index ecce6a22..0b438018 100644 --- a/platformio/package/manager/platform.py +++ b/platformio/package/manager/platform.py @@ -140,7 +140,7 @@ class PlatformPackageManager(BasePackageManager): # pylint: disable=too-many-an def get_registered_boards(self): return self.get_registry_client_instance().fetch_json_data( - "get", "/v2/boards", cache_valid="1d" + "get", "/v2/boards", x_cache_valid="1d" ) def get_all_boards(self): diff --git a/platformio/package/manifest/parser.py b/platformio/package/manifest/parser.py index 54417c0c..556339a7 100644 --- a/platformio/package/manifest/parser.py +++ b/platformio/package/manifest/parser.py @@ -608,7 +608,6 @@ class LibraryPropertiesManifestParser(BaseManifestParser): return None def _parse_export(self): - result = {"exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"]} include = None if self.remote_url: url_attrs = urlparse(self.remote_url) @@ -621,8 +620,8 @@ class LibraryPropertiesManifestParser(BaseManifestParser): or None ) if include: - result["include"] = [include] - return result + return dict(include=[include]) + return None @staticmethod def _parse_dependencies(raw): diff --git a/platformio/package/manifest/schema.py b/platformio/package/manifest/schema.py index 60d0d60c..78bf43a7 100644 --- a/platformio/package/manifest/schema.py +++ b/platformio/package/manifest/schema.py @@ -259,7 +259,7 @@ class ManifestSchema(BaseSchema): @staticmethod @memoized(expire="1h") def load_spdx_licenses(): - version = "3.15" + version = "3.16" spdx_data_url = ( "https://raw.githubusercontent.com/spdx/license-list-data/" "v%s/json/licenses.json" % version diff --git a/platformio/package/pack.py b/platformio/package/pack.py index 2d1eea46..84d835ed 100644 --- a/platformio/package/pack.py +++ b/platformio/package/pack.py @@ -22,7 +22,11 @@ import tempfile from platformio import fs from platformio.compat import IS_WINDOWS from platformio.package.exception import PackageException, UserSideException -from platformio.package.manifest.parser import ManifestFileType, ManifestParserFactory +from platformio.package.manifest.parser import ( + LibraryPropertiesManifestParser, + ManifestFileType, + ManifestParserFactory, +) from platformio.package.manifest.schema import ManifestSchema from platformio.package.meta import PackageItem from platformio.package.unpack import FileUnpacker @@ -43,6 +47,7 @@ class PackagePacker(object): ".cache", "**/.cache", "**/__pycache__", + "**/*.pyc", # VCS ".git/", ".hg/", @@ -50,16 +55,26 @@ class PackagePacker(object): ] EXCLUDE_EXTRA = [ # Tests - "tests?", + "test", + "tests", # Docs "doc", "docs", "mkdocs", + "doxygen", + "*.doxyfile", + "html", + "media", "**/*.[pP][dD][fF]", - "**/*.[dD][oO][cC]?", - "**/*.[pP][pP][tT]?", + "**/*.[dD][oO][cC]", + "**/*.[dD][oO][cC][xX]", + "**/*.[pP][pP][tT]", + "**/*.[pP][pP][tT][xX]", + "**/*.[xX][lL][sS]", + "**/*.[xX][lL][sS][xX]", "**/*.[dD][oO][xX]", - "**/*.[hH][tT][mM]?", + "**/*.[hH][tT][mM]", + "**/*.[hH][tT][mM][lL]", "**/*.[tT][eE][xX]", "**/*.[jJ][sS]", "**/*.[cC][sS][sS]", @@ -75,14 +90,13 @@ class PackagePacker(object): "**/*.[mM][pP][34]", "**/*.[pP][sS][dD]", "**/*.[wW][aA][wW]", + "**/*.sqlite", ] EXCLUDE_LIBRARY_EXTRA = [ "assets", "extra", + "extras", "resources", - "html", - "media", - "doxygen", "**/build/", "**/*.flat", "**/*.[jJ][aA][rR]", @@ -97,6 +111,7 @@ class PackagePacker(object): def __init__(self, package, manifest_uri=None): self.package = package self.manifest_uri = manifest_uri + self.manifest_parser = None @staticmethod def get_archive_name(name, version, system=None): @@ -128,7 +143,8 @@ class PackagePacker(object): src = tmp_dir src = self.find_source_root(src) - manifest = self.load_manifest(src) + self.manifest_parser = ManifestParserFactory.new_from_dir(src) + manifest = ManifestSchema().load_manifest(self.manifest_parser.as_dict()) filename = self.get_archive_name( manifest["name"], manifest["version"], @@ -144,11 +160,6 @@ class PackagePacker(object): finally: shutil.rmtree(tmp_dir) - @staticmethod - def load_manifest(src): - mp = ManifestParserFactory.new_from_dir(src) - return ManifestSchema().load_manifest(mp.as_dict()) - def find_source_root(self, src): if self.manifest_uri: mp = ( @@ -214,7 +225,9 @@ class PackagePacker(object): # exclude items declared in manifest result += ["-<%s>" % p for p in exclude or []] # apply extra excludes if no custom "export" field in manifest - if not include and not exclude: + if (not include and not exclude) or isinstance( + self.manifest_parser, LibraryPropertiesManifestParser + ): result += ["-<%s>" % p for p in exclude_extra] # automatically include manifests result += ["+<%s>" % p for p in self.INCLUDE_DEFAULT] diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index bd9b687f..b82475b7 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -189,7 +189,7 @@ class PlatformRunMixin(object): filename=filename, filename_styled=click.style(filename, fg="cyan"), link=click.style( - "https://platformio.org/lib/search?query=header:%s" + "https://registry.platformio.org/search?q=header:%s" % quote(filename, safe=""), fg="blue", ), diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 2736cae8..2aab20b7 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -14,12 +14,13 @@ import json import os +import subprocess from hashlib import sha1 from click.testing import CliRunner from platformio import __version__, exception, fs -from platformio.compat import IS_WINDOWS, hashlib_encode_data +from platformio.compat import IS_MACOS, IS_WINDOWS, hashlib_encode_data from platformio.project.config import ProjectConfig @@ -75,7 +76,15 @@ def get_default_projects_dir(): ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, buf) docs_dir = buf.value except: # pylint: disable=bare-except - pass + if not IS_MACOS: + try: + docs_dir = ( + subprocess.check_output(["xdg-user-dir", "DOCUMENTS"]) + .decode("utf-8") + .strip() + ) + except FileNotFoundError: # command not found + pass return os.path.join(docs_dir, "PlatformIO", "Projects") diff --git a/platformio/project/tpls/vscode/.vscode/extensions.json.tpl b/platformio/project/tpls/vscode/.vscode/extensions.json.tpl index 1b2dfdd9..26412ae7 100644 --- a/platformio/project/tpls/vscode/.vscode/extensions.json.tpl +++ b/platformio/project/tpls/vscode/.vscode/extensions.json.tpl @@ -1,23 +1,26 @@ -% import json -% import os -% import re -% -% recommendations = set(["platformio.platformio-ide"]) -% previous_json = os.path.join(project_dir, ".vscode", "extensions.json") -% if os.path.isfile(previous_json): -% fp = open(previous_json) -% contents = re.sub(r"^\s*//.*$", "", fp.read(), flags=re.M).strip() -% fp.close() -% if contents: -% recommendations |= set(json.loads(contents).get("recommendations", [])) -% end -% end -{ - // See http://go.microsoft.com/fwlink/?LinkId=827846 - // for the documentation about the extensions.json format - "recommendations": [ -% for i, item in enumerate(sorted(recommendations)): - "{{ item }}"{{ ("," if (i + 1) < len(recommendations) else "") }} -% end - ] -} +% import json +% import os +% import re +% +% recommendations = set(["platformio.platformio-ide"]) +% previous_json = os.path.join(project_dir, ".vscode", "extensions.json") +% if os.path.isfile(previous_json): +% fp = open(previous_json) +% contents = re.sub(r"^\s*//.*$", "", fp.read(), flags=re.M).strip() +% fp.close() +% if contents: +% recommendations |= set(json.loads(contents).get("recommendations", [])) +% end +% end +{ + // See http://go.microsoft.com/fwlink/?LinkId=827846 + // for the documentation about the extensions.json format + "recommendations": [ +% for i, item in enumerate(sorted(recommendations)): + "{{ item }}"{{ ("," if (i + 1) < len(recommendations) else "") }} +% end + ], + "unwantedRecommendations": [ + "ms-vscode.cpptools-extension-pack" + ] +} diff --git a/pytest.ini b/pytest.ini index 03c86580..b3dbaa57 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,5 @@ filterwarnings = error # Marshmallow - ignore:The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives:DeprecationWarning \ No newline at end of file + ignore:distutils Version classes are deprecated. Use packaging.version instead. + ignore:The distutils package is deprecated and slated for removal in Python \ No newline at end of file diff --git a/scripts/99-platformio-udev.rules b/scripts/99-platformio-udev.rules index 043ce11a..6fd87d00 100644 --- a/scripts/99-platformio-udev.rules +++ b/scripts/99-platformio-udev.rules @@ -70,6 +70,9 @@ ATTRS{idVendor}=="0451", ATTRS{idProduct}=="f432", MODE="0666", ENV{ID_MM_DEVICE #GD32V DFU Bootloader ATTRS{idVendor}=="28e9", ATTRS{idProduct}=="0189", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" +# FireBeetle-ESP32 +ATTRS{idVendor}=="1a86", ATTRS{idProduct}=="7522", MODE="0666", ENV{ID_MM_DEVICE_IGNORE}="1", ENV{ID_MM_PORT_IGNORE}="1" + # # Debuggers # diff --git a/scripts/docspregen.py b/scripts/docspregen.py index c4291cc0..d05cef91 100644 --- a/scripts/docspregen.py +++ b/scripts/docspregen.py @@ -12,18 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools +import json import os -from os.path import dirname, isdir, isfile, join, realpath -from sys import exit as sys_exit -from sys import path +import sys +import tempfile -path.append("..") +sys.path.append("..") -import click +import click # noqa: E402 -from platformio import fs, util -from platformio.package.manager.platform import PlatformPackageManager -from platformio.platform.factory import PlatformFactory +from platformio import fs, util # noqa: E402 +from platformio.package.manager.platform import PlatformPackageManager # noqa: E402 +from platformio.platform.factory import PlatformFactory # noqa: E402 try: from urlparse import ParseResult, urlparse, urlunparse @@ -42,17 +43,18 @@ RST_COPYRIGHT = """.. Copyright (c) 2014-present PlatformIO = result["total"]: + break + + +@functools.cache +def get_frameworks(): + items = {} + for pkg in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(pkg) + for name, options in (p.frameworks or {}).items(): + if name in items or not set(options.keys()).issuperset( + set(["title", "description"]) + ): + continue + items[name] = dict( + name=name, title=options["title"], description=options["description"] + ) + return sorted(items.values(), key=lambda item: item["name"]) + + +def is_compat_platform_and_framework(platform, framework): + p = PlatformFactory.new(platform) + return framework in (p.frameworks or {}).keys() + + def generate_boards_table(boards, skip_columns=None): columns = [ ("Name", ":ref:`board_{platform}_{id}`"), @@ -141,7 +185,7 @@ Frameworks - Description""" ) known = set() - for framework in API_FRAMEWORKS: + for framework in get_frameworks(): known.add(framework["name"]) if framework["name"] not in frameworks: continue @@ -259,8 +303,8 @@ Please click on board name for the further details. return lines -def generate_packages(platform, packagenames, is_embedded): - if not packagenames: +def generate_packages(platform, packages, is_embedded): + if not packages: return lines = [] lines.append( @@ -276,27 +320,21 @@ Packages * - Name - Description""" ) - for name in sorted(packagenames): - if name not in API_PACKAGES: - click.secho("Unknown package `%s`" % name, fg="red") - lines.append( - """ - * - {name} - - - """.format( - name=name - ) - ) - else: - lines.append( - """ + for name, options in dict(sorted(packages.items())).items(): + package = REGCLIENT.get_package( + "tool", options.get("owner", "platformio"), name + ) + lines.append( + """ * - `{name} <{url}>`__ - {description}""".format( - name=name, - url=campaign_url(API_PACKAGES[name]["url"]), - description=API_PACKAGES[name]["description"], - ) + name=package["name"], + url=reg_package_url( + "tool", package["owner"]["username"], package["name"] + ), + description=package["description"], ) + ) if is_embedded: lines.append( @@ -336,17 +374,23 @@ Packages return "\n".join(lines) -def generate_platform(name, rst_dir): +def generate_platform(pkg, rst_dir): + name = pkg.metadata.name print("Processing platform: %s" % name) - compatible_boards = [board for board in BOARDS if name == board["platform"]] + compatible_boards = [ + board + for board in PlatformPackageManager().get_installed_boards() + if name == board["platform"] + ] lines = [] - lines.append(RST_COPYRIGHT) + p = PlatformFactory.new(name) assert p.repository_url.endswith(".git") github_url = p.repository_url[:-4] + registry_url = reg_package_url("platform", pkg.metadata.spec.owner, name) lines.append(".. _platform_%s:" % p.name) lines.append("") @@ -354,6 +398,8 @@ def generate_platform(name, rst_dir): lines.append(p.title) lines.append("=" * len(p.title)) lines.append("") + lines.append(":Registry:") + lines.append(" `%s <%s>`__" % (registry_url, registry_url)) lines.append(":Configuration:") lines.append(" :ref:`projectconf_env_platform` = ``%s``" % p.name) lines.append("") @@ -374,7 +420,7 @@ For more detailed information please visit `vendor site <%s>`_.""" # # Extra # - if isfile(join(rst_dir, "%s_extra.rst" % name)): + if os.path.isfile(os.path.join(rst_dir, "%s_extra.rst" % name)): lines.append(".. include:: %s_extra.rst" % p.name) # @@ -389,11 +435,11 @@ Examples are listed from `%s development platform repository <%s>`_: """ % (p.title, campaign_url("%s/tree/master/examples" % github_url)) ) - examples_dir = join(p.get_dir(), "examples") - if isdir(examples_dir): + examples_dir = os.path.join(p.get_dir(), "examples") + if os.path.isdir(examples_dir): for eitem in os.listdir(examples_dir): - example_dir = join(examples_dir, eitem) - if not isdir(example_dir) or not os.listdir(example_dir): + example_dir = os.path.join(examples_dir, eitem) + if not os.path.isdir(example_dir) or not os.listdir(example_dir): continue url = "%s/tree/master/examples/%s" % (github_url, eitem) lines.append("* `%s <%s>`_" % (eitem, campaign_url(url))) @@ -407,7 +453,7 @@ Examples are listed from `%s development platform repository <%s>`_: compatible_boards, skip_board_columns=["Platform"], extra_rst="%s_debug.rst" % name - if isfile(join(rst_dir, "%s_debug.rst" % name)) + if os.path.isfile(os.path.join(rst_dir, "%s_debug.rst" % name)) else None, ) ) @@ -455,7 +501,7 @@ Upstream # # Packages # - _packages_content = generate_packages(name, p.packages.keys(), p.is_embedded()) + _packages_content = generate_packages(name, p.packages, p.is_embedded()) if _packages_content: lines.append(_packages_content) @@ -463,7 +509,7 @@ Upstream # Frameworks # compatible_frameworks = [] - for framework in API_FRAMEWORKS: + for framework in get_frameworks(): if is_compat_platform_and_framework(name, framework["name"]): compatible_frameworks.append(framework["name"]) lines.extend(generate_frameworks_contents(compatible_frameworks)) @@ -484,8 +530,7 @@ Boards ------ .. note:: - * You can list pre-configured boards by :ref:`cmd_boards` command or - `PlatformIO Boards Explorer `_ + * You can list pre-configured boards by :ref:`cmd_boards` command * For more detailed ``board`` information please scroll the tables below by horizontally. """ @@ -500,23 +545,26 @@ Boards def update_platform_docs(): - for manifest in PLATFORM_MANIFESTS: - name = manifest["name"] - platforms_dir = join(DOCS_ROOT_DIR, "platforms") - rst_path = join(platforms_dir, "%s.rst" % name) + platforms_dir = os.path.join(DOCS_ROOT_DIR, "platforms") + for pkg in PlatformPackageManager().get_installed(): + rst_path = os.path.join(platforms_dir, "%s.rst" % pkg.metadata.name) with open(rst_path, "w") as f: - f.write(generate_platform(name, platforms_dir)) + f.write(generate_platform(pkg, platforms_dir)) -def generate_framework(type_, data, rst_dir=None): +def generate_framework(type_, framework, rst_dir=None): print("Processing framework: %s" % type_) compatible_platforms = [ - m - for m in PLATFORM_MANIFESTS - if is_compat_platform_and_framework(m["name"], type_) + pkg + for pkg in PlatformPackageManager().get_installed() + if is_compat_platform_and_framework(pkg.metadata.name, type_) + ] + compatible_boards = [ + board + for board in PlatformPackageManager().get_installed_boards() + if type_ in board["frameworks"] ] - compatible_boards = [board for board in BOARDS if type_ in board["frameworks"]] lines = [] @@ -524,20 +572,13 @@ def generate_framework(type_, data, rst_dir=None): lines.append(".. _framework_%s:" % type_) lines.append("") - lines.append(data["title"]) - lines.append("=" * len(data["title"])) + lines.append(framework["title"]) + lines.append("=" * len(framework["title"])) lines.append("") lines.append(":Configuration:") lines.append(" :ref:`projectconf_env_framework` = ``%s``" % type_) lines.append("") - lines.append(data["description"]) - lines.append( - """ -For more detailed information please visit `vendor site <%s>`_. -""" - % campaign_url(data["url"]) - ) - + lines.append(framework["description"]) lines.append( """ .. contents:: Contents @@ -546,9 +587,35 @@ For more detailed information please visit `vendor site <%s>`_. ) # Extra - if isfile(join(rst_dir, "%s_extra.rst" % type_)): + if os.path.isfile(os.path.join(rst_dir, "%s_extra.rst" % type_)): lines.append(".. include:: %s_extra.rst" % type_) + if compatible_platforms: + # Platforms + lines.extend( + generate_platforms_contents( + [pkg.metadata.name for pkg in compatible_platforms] + ) + ) + + # examples + lines.append( + """ +Examples +-------- +""" + ) + for pkg in compatible_platforms: + p = PlatformFactory.new(pkg) + lines.append( + "* `%s for %s <%s>`_" + % ( + framework["title"], + p.title, + campaign_url("%s/tree/master/examples" % p.repository_url[:-4]), + ) + ) + # # Debugging # @@ -557,37 +624,11 @@ For more detailed information please visit `vendor site <%s>`_. generate_debug_contents( compatible_boards, extra_rst="%s_debug.rst" % type_ - if isfile(join(rst_dir, "%s_debug.rst" % type_)) + if os.path.isfile(os.path.join(rst_dir, "%s_debug.rst" % type_)) else None, ) ) - if compatible_platforms: - # examples - lines.append( - """ -Examples --------- -""" - ) - for manifest in compatible_platforms: - p = PlatformFactory.new(manifest["name"]) - lines.append( - "* `%s for %s <%s>`_" - % ( - data["title"], - manifest["title"], - campaign_url("%s/tree/master/examples" % p.repository_url[:-4]), - ) - ) - - # Platforms - lines.extend( - generate_platforms_contents( - [manifest["name"] for manifest in compatible_platforms] - ) - ) - # # Boards # @@ -603,8 +644,7 @@ Boards ------ .. note:: - * You can list pre-configured boards by :ref:`cmd_boards` command or - `PlatformIO Boards Explorer `_ + * You can list pre-configured boards by :ref:`cmd_boards` command * For more detailed ``board`` information please scroll the tables below by horizontally. """ ) @@ -616,10 +656,10 @@ Boards def update_framework_docs(): - for framework in API_FRAMEWORKS: + frameworks_dir = os.path.join(DOCS_ROOT_DIR, "frameworks") + for framework in get_frameworks(): name = framework["name"] - frameworks_dir = join(DOCS_ROOT_DIR, "frameworks") - rst_path = join(frameworks_dir, "%s.rst" % name) + rst_path = os.path.join(frameworks_dir, "%s.rst" % name) with open(rst_path, "w") as f: f.write(generate_framework(name, framework, frameworks_dir)) @@ -639,17 +679,17 @@ def update_boards(): """ Rapid Embedded Development, Continuous and IDE integration in a few steps with PlatformIO thanks to built-in project generator for the most -popular embedded boards and IDE. +popular embedded boards and IDEs. .. note:: - * You can list pre-configured boards by :ref:`cmd_boards` command or - `PlatformIO Boards Explorer `_ + * You can list pre-configured boards by :ref:`cmd_boards` command * For more detailed ``board`` information please scroll tables below by horizontal. """ ) platforms = {} - for data in BOARDS: + installed_boards = PlatformPackageManager().get_installed_boards() + for data in installed_boards: platform = data["platform"] if platform in platforms: platforms[platform].append(data) @@ -670,19 +710,17 @@ popular embedded boards and IDE. lines.append(" %s/%s" % (platform, board["id"])) lines.append("") - emboards_rst = join(DOCS_ROOT_DIR, "boards", "index.rst") + emboards_rst = os.path.join(DOCS_ROOT_DIR, "boards", "index.rst") with open(emboards_rst, "w") as f: f.write("\n".join(lines)) # individual board page - for data in BOARDS: - # if data['id'] != "m5stack-core-esp32": - # continue - rst_path = join( + for data in installed_boards: + rst_path = os.path.join( DOCS_ROOT_DIR, "boards", data["platform"], "%s.rst" % data["id"] ) - if not isdir(dirname(rst_path)): - os.makedirs(dirname(rst_path)) + if not os.path.isdir(os.path.dirname(rst_path)): + os.makedirs(os.path.dirname(rst_path)) update_embedded_board(rst_path, data) @@ -892,7 +930,7 @@ def update_debugging(): vendors = {} platforms = [] frameworks = [] - for data in BOARDS: + for data in PlatformPackageManager().get_installed_boards(): if not data.get("debug"): continue @@ -937,7 +975,7 @@ Boards # save with open( - join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), "r+" + os.path.join(fs.get_source_dir(), "..", "docs", "plus", "debugging.rst"), "r+" ) as fp: content = fp.read() fp.seek(0) @@ -948,8 +986,8 @@ Boards # Debug tools for tool, platforms in tool_to_platforms.items(): - tool_path = join(DOCS_ROOT_DIR, "plus", "debug-tools", "%s.rst" % tool) - if not isfile(tool_path): + tool_path = os.path.join(DOCS_ROOT_DIR, "plus", "debug-tools", "%s.rst" % tool) + if not os.path.isfile(tool_path): click.secho("Unknown debug tool `%s`" % tool, fg="red") continue platforms = sorted(set(platforms)) @@ -974,7 +1012,11 @@ Boards ) lines.extend( generate_boards_table( - [b for b in BOARDS if b["id"] in tool_to_boards[tool]], + [ + b + for b in PlatformPackageManager().get_installed_boards() + if b["id"] in tool_to_boards[tool] + ], skip_columns=None, ) ) @@ -1012,30 +1054,30 @@ def update_project_examples(): {examples} """ - project_examples_dir = join(fs.get_source_dir(), "..", "examples") + project_examples_dir = os.path.join(fs.get_source_dir(), "..", "examples") framework_examples_md_lines = {} embedded = [] desktop = [] - for manifest in PLATFORM_MANIFESTS: - p = PlatformFactory.new(manifest["name"]) + for pkg in PlatformPackageManager().get_installed(): + p = PlatformFactory.new(pkg) github_url = p.repository_url[:-4] # Platform README - platform_examples_dir = join(p.get_dir(), "examples") + platform_examples_dir = os.path.join(p.get_dir(), "examples") examples_md_lines = [] - if isdir(platform_examples_dir): + if os.path.isdir(platform_examples_dir): for item in sorted(os.listdir(platform_examples_dir)): - example_dir = join(platform_examples_dir, item) - if not isdir(example_dir) or not os.listdir(example_dir): + example_dir = os.path.join(platform_examples_dir, item) + if not os.path.isdir(example_dir) or not os.listdir(example_dir): continue url = "%s/tree/master/examples/%s" % (github_url, item) examples_md_lines.append("* [%s](%s)" % (item, url)) - readme_dir = join(project_examples_dir, "platforms", p.name) - if not isdir(readme_dir): + readme_dir = os.path.join(project_examples_dir, "platforms", p.name) + if not os.path.isdir(readme_dir): os.makedirs(readme_dir) - with open(join(readme_dir, "README.md"), "w") as fp: + with open(os.path.join(readme_dir, "README.md"), "w") as fp: fp.write( platform_readme_tpl.format( name=p.name, @@ -1046,14 +1088,14 @@ def update_project_examples(): ) # Framework README - for framework in API_FRAMEWORKS: + for framework in get_frameworks(): if not is_compat_platform_and_framework(p.name, framework["name"]): continue if framework["name"] not in framework_examples_md_lines: framework_examples_md_lines[framework["name"]] = [] lines = [] lines.append("- [%s](%s)" % (p.title, github_url)) - lines.extend(" %s" % l for l in examples_md_lines) + lines.extend(" %s" % line for line in examples_md_lines) lines.append("") framework_examples_md_lines[framework["name"]].extend(lines) @@ -1066,13 +1108,13 @@ def update_project_examples(): # Frameworks frameworks = [] - for framework in API_FRAMEWORKS: + for framework in get_frameworks(): if framework["name"] not in framework_examples_md_lines: continue - readme_dir = join(project_examples_dir, "frameworks", framework["name"]) - if not isdir(readme_dir): + readme_dir = os.path.join(project_examples_dir, "frameworks", framework["name"]) + if not os.path.isdir(readme_dir): os.makedirs(readme_dir) - with open(join(readme_dir, "README.md"), "w") as fp: + with open(os.path.join(readme_dir, "README.md"), "w") as fp: fp.write( framework_readme_tpl.format( name=framework["name"], @@ -1089,7 +1131,7 @@ def update_project_examples(): ) frameworks.append("* [%s](%s)" % (framework["title"], url)) - with open(join(project_examples_dir, "README.md"), "w") as fp: + with open(os.path.join(project_examples_dir, "README.md"), "w") as fp: fp.write( """# PlatformIO Project Examples @@ -1117,12 +1159,16 @@ def update_project_examples(): def main(): - update_platform_docs() - update_framework_docs() - update_boards() - update_debugging() - update_project_examples() + with tempfile.TemporaryDirectory() as tmp_dir: + print("Core directory: %s" % tmp_dir) + os.environ["PLATFORMIO_CORE_DIR"] = tmp_dir + install_platforms() + update_platform_docs() + update_framework_docs() + update_boards() + update_debugging() + update_project_examples() if __name__ == "__main__": - sys_exit(main()) + sys.exit(main()) diff --git a/setup.py b/setup.py index 766a1b87..02b947e0 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys from setuptools import find_packages, setup from platformio import ( @@ -28,24 +29,24 @@ from platformio.compat import PY2 minimal_requirements = [ "bottle==0.12.*", - "click>=8,<9,!=8.0.2", + "click>=8.0.3,<9", "colorama", "marshmallow%s" % (">=2,<3" if PY2 else ">=2,<4"), "pyelftools>=0.27,<1", "pyserial==3.*", "requests==2.*", - "semantic_version==2.8.*", + "semantic_version==2.9.*", "tabulate==0.8.*", ] if not PY2: - minimal_requirements.append("zeroconf==0.37.*") + minimal_requirements.append("zeroconf==0.38.*") home_requirements = [ "aiofiles==0.8.*", "ajsonrpc==1.*", - "starlette==0.17.*", - "uvicorn==0.16.*", + "starlette==0.18.*", + "uvicorn==%s" % ("0.17.*" if sys.version_info >= (3, 7) else "0.16.0"), "wsproto==1.0.*", ] diff --git a/tests/commands/test_ci.py b/tests/commands/test_ci.py index 95b31a8f..01ac9c37 100644 --- a/tests/commands/test_ci.py +++ b/tests/commands/test_ci.py @@ -119,6 +119,7 @@ def test_ci_keep_build_dir_nested_src_dirs( src_dir1 = tmpdir_factory.mktemp("src_1") src_dir1.join("src1.cpp").write( """ +#include void setup() {} """ ) @@ -126,6 +127,7 @@ void setup() {} src_dir2 = tmpdir_factory.mktemp("src_2") src_dir2.join("src2.cpp").write( """ +#include void loop() {} """ ) diff --git a/tests/package/test_manager.py b/tests/package/test_manager.py index 01bad3df..c82d3c68 100644 --- a/tests/package/test_manager.py +++ b/tests/package/test_manager.py @@ -437,10 +437,10 @@ def test_update_with_metadata(isolated_pio_core, tmpdir_factory): lm = LibraryPackageManager(str(storage_dir)) # test non SemVer in registry - pkg = lm.install("RadioHead @ <1.90", silent=True) + pkg = lm.install("adafruit/Adafruit NeoPixel @ <1.9", silent=True) outdated = lm.outdated(pkg) - assert str(outdated.current) == "1.89.0" - assert outdated.latest > semantic_version.Version("1.100.0") + assert str(outdated.current) == "1.8.7" + assert outdated.latest > semantic_version.Version("1.10.0") pkg = lm.install("ArduinoJson @ 5.10.1", silent=True) # tesy latest diff --git a/tests/package/test_manifest.py b/tests/package/test_manifest.py index 216adf5d..a07a035e 100644 --- a/tests/package/test_manifest.py +++ b/tests/package/test_manifest.py @@ -220,9 +220,6 @@ includes=Arduino.h, Arduino Space.hpp "description": "This is Arduino library", "sentence": "This is Arduino library", "frameworks": ["arduino"], - "export": { - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] - }, "authors": [ {"name": "SomeAuthor", "email": "info@author.com"}, {"name": "Maintainer Author", "maintainer": True}, @@ -270,7 +267,6 @@ includes=Arduino.h, Arduino Space.hpp ), ).as_dict() assert data["export"] == { - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"], "include": ["libraries/TestPackage"], } assert data["repository"] == { @@ -465,9 +461,6 @@ depends=First Library (=2.0.0), Second Library (>=1.2.0), Third "frameworks": ["arduino"], "platforms": ["atmelavr", "atmelsam"], "version": "1.19.1", - "export": { - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] - }, "authors": [ {"maintainer": True, "email": "olikraus@gmail.com", "name": "oliver"} ], @@ -538,9 +531,6 @@ includes=MozziGuts.h "platforms": ["*"], "frameworks": ["arduino"], "headers": ["MozziGuts.h"], - "export": { - "exclude": ["extras", "docs", "tests", "test", "*.doxyfile", "*.pdf"] - }, "authors": [ { "maintainer": True, diff --git a/tests/test_misc.py b/tests/test_misc.py index 36574ee4..9b8fc5d4 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -42,7 +42,7 @@ def test_api_internet_offline(without_internet, isolated_pio_core): def test_api_cache(monkeypatch, isolated_pio_core): regclient = RegistryClient() - api_kwargs = {"method": "get", "path": "/v2/stats", "cache_valid": "10s"} + api_kwargs = {"method": "get", "path": "/v2/stats", "x_cache_valid": "10s"} result = regclient.fetch_json_data(**api_kwargs) assert result and "boards" in result monkeypatch.setattr(http, "_internet_on", lambda: False) diff --git a/tox.ini b/tox.ini index 5d38aa33..0ee9c0ec 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ envlist = py36,py37,py38,py39 [isort] -line_length = 88 +profile = black known_third_party=OpenSSL, SCons, jsonrpc, twisted, zope [testenv]