Resolved an issue where upgrading PlatformIO Core fails on Windows with Python 3.11 // Resolve #4540

HISTORY.rst

@@ -18,22 +18,23 @@ PlatformIO Core 6
 6.1.7 (2023-??-??)
 ~~~~~~~~~~~~~~~~~~
 
-* Implemented a new feature to store device monitor logs in the project's "logs" folder, making it easier to access and review device monitor logs for your projects (`issue #4596 <https://github.com/platformio/platformio-core/issues/4596>`_)
+* Implemented a new feature to store device monitor logs in the project's ``logs`` folder, making it easier to access and review device monitor logs for your projects (`issue #4596 <https://github.com/platformio/platformio-core/issues/4596>`_)
-* Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
-* Improved source file filtering functionality for the `Static Code Analysis <https://docs.platformio.org/en/latest/advanced/static-code-analysis/index.html>`__ feature, making it easier to analyze only the code you need to
 * Added the ability to show a detailed library dependency tree only in `verbose mode <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html#cmdoption-pio-run-v>`__, which can help you understand the relationship between libraries and troubleshoot issues more effectively (`issue #4517 <https://github.com/platformio/platformio-core/issues/4517>`_)
 * Added the ability to run only the `device monitor <https://docs.platformio.org/en/latest/core/userguide/device/cmd_monitor.html>`__ when using the `pio run -t monitor <https://docs.platformio.org/en/latest/core/userguide/cmd_run.html>`__ command, saving you time and resources by skipping the build process
 * Added validation for `project working environment names <https://docs.platformio.org/en/latest/projectconf/sections/env/index.html#working-env-name>`__ to ensure that they only contain lowercase letters ``a-z``, numbers ``0-9``, and special characters ``_`` (underscore) and ``-`` (hyphen)
+* Improved support for projects located on Windows network drives, including Network Shared Folder, Dropbox, OneDrive, Google Drive, and other similar services (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
+* Improved source file filtering functionality for the `Static Code Analysis <https://docs.platformio.org/en/latest/advanced/static-code-analysis/index.html>`__ feature, making it easier to analyze only the code you need to
 * Upgraded the build engine to the latest version of SCons (4.5.2) to improve build performance, reliability, and compatibility with other tools and systems (`release notes <https://github.com/SCons/scons/releases/tag/4.5.2>`__)
 * Implemented a fix for shell injection vulnerabilities when converting INO files to CPP, ensuring your code is safe and secure (`issue #4532 <https://github.com/platformio/platformio-core/issues/4532>`_)
 * Restored the project generator for the `NetBeans IDE <https://docs.platformio.org/en/latest/integration/ide/netbeans.html>`__, providing you with more flexibility and options for your development workflow
+* Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 <https://github.com/platformio/platformio-core/issues/4425>`_, `issue #4493 <https://github.com/platformio/platformio-core/issues/4493>`_, `issue #4607 <https://github.com/platformio/platformio-core/issues/4607>`_)
 * Resolved an issue where the `build_cache_dir <https://docs.platformio.org/en/latest/projectconf/sections/platformio/options/directory/build_cache_dir.html>`__ setting was not being recognized consistently across multiple environments (`issue #4574 <https://github.com/platformio/platformio-core/issues/4574>`_)
 * Resolved an issue where organization details could not be updated using the `pio org update <https://docs.platformio.org/en/latest/core/userguide/org/cmd_update.html>`__ command
 * Resolved an issue where the incorrect debugging environment was generated for VSCode in "Auto" mode (`issue #4597 <https://github.com/platformio/platformio-core/issues/4597>`_)
 * Resolved an issue where native tests would fail if a custom program name was specified (`issue #4546 <https://github.com/platformio/platformio-core/issues/4546>`_)
 * Resolved an issue where the PlatformIO |DEBUGGING| solution was not escaping the tool installation process into MI2 correctly (`issue #4565 <https://github.com/platformio/platformio-core/issues/4565>`_)
 * Resolved an issue where multiple targets were not executed sequentially (`issue #4604 <https://github.com/platformio/platformio-core/issues/4604>`_)
-* Resolved installation issues with PIO Remote on Raspberry Pi and other small form-factor PCs (`issue #4425 <https://github.com/platformio/platformio-core/issues/4425>`_, `issue #4493 <https://github.com/platformio/platformio-core/issues/4493>`_, `issue #4607 <https://github.com/platformio/platformio-core/issues/4607>`_)
+* Resolved an issue where upgrading PlatformIO Core fails on Windows with Python 3.11 (`issue #4540 <https://github.com/platformio/platformio-core/issues/4540>`_)
 
 6.1.6 (2023-01-23)
 ~~~~~~~~~~~~~~~~~~

platformio/commands/upgrade.py

@@ -13,23 +13,28 @@
 # limitations under the License.
 
 import json
-import os
 import re
-from zipfile import ZipFile
+import subprocess
 
 import click
 
 from platformio import VERSION, __version__, app, exception
-from platformio.compat import IS_WINDOWS
 from platformio.http import fetch_remote_content
 from platformio.package.manager.core import update_core_packages
-from platformio.proc import exec_command, get_pythonexe_path
+from platformio.proc import get_pythonexe_path
-from platformio.project.helpers import get_project_cache_dir
+
+PYPI_JSON_URL = "https://pypi.org/pypi/platformio/json"
+DEVELOP_ZIP_URL = "https://github.com/platformio/platformio-core/archive/develop.zip"
+DEVELOP_INIT_SCRIPT_URL = (
+    "https://raw.githubusercontent.com/platformio/platformio-core"
+    "/develop/platformio/__init__.py"
+)
 
 
 @click.command("upgrade", short_help="Upgrade PlatformIO Core to the latest version")
 @click.option("--dev", is_flag=True, help="Use development branch")
-def cli(dev):
+@click.option("--verbose", "-v", is_flag=True)
+def cli(dev, verbose):
     update_core_packages()
     if not dev and __version__ == get_latest_version():
         return click.secho(
@@ -38,29 +43,26 @@ def cli(dev):
             fg="green",
         )
 
-    click.secho("Please wait while upgrading PlatformIO ...", fg="yellow")
+    click.secho("Please wait while upgrading PlatformIO Core ...", fg="yellow")
 
+    python_exe = get_pythonexe_path()
     to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".")
-    cmds = (
+    pkg_spec = DEVELOP_ZIP_URL if to_develop else "platformio"
-        ["pip", "install", "--upgrade", download_dist_package(to_develop)],
-        ["platformio", "--version"],
-    )
 
-    cmd = None
-    r = {}
     try:
-        for cmd in cmds:
+        subprocess.run(
-            cmd = [get_pythonexe_path(), "-m"] + cmd
+            [python_exe, "-m", "pip", "install", "--upgrade", pkg_spec],
-            r = exec_command(cmd)
+            check=True,
-
+            capture_output=not verbose,
-            # try pip with disabled cache
+        )
-            if r["returncode"] != 0 and cmd[2] == "pip":
+        r = subprocess.run(
-                cmd.insert(3, "--no-cache-dir")
+            [python_exe, "-m", "platformio", "--version"],
-                r = exec_command(cmd)
+            check=True,
-
+            capture_output=True,
-            assert r["returncode"] == 0
+            text=True,
-        assert "version" in r["out"]
+        )
-        actual_version = r["out"].strip().split("version", 1)[1].strip()
+        assert "version" in r.stdout
+        actual_version = r.stdout.split("version", 1)[1].strip()
         click.secho(
             "PlatformIO has been successfully upgraded to %s" % actual_version,
             fg="green",
@@ -71,52 +73,24 @@ def cli(dev):
             click.secho(
                 "Warning! Please restart IDE to affect PIO Home changes", fg="yellow"
             )
-    except Exception as exc:
+    except (AssertionError, subprocess.CalledProcessError) as exc:
-        if not r:
+        click.secho(
-            raise exception.UpgradeError("\n".join([str(cmd), str(exc)])) from exc
+            "\nWarning!!! Could not automatically upgrade the PlatformIO Core.",
-        permission_errors = ("permission denied", "not permitted")
+            fg="red",
-        if any(m in r["err"].lower() for m in permission_errors) and not IS_WINDOWS:
+        )
-            click.secho(
+        click.secho(
-                """
+            "Please upgrade it manually using the following command:\n",
------------------
+            fg="red",
-Permission denied
+        )
------------------
+        click.secho(f'"{python_exe}" -m pip install -U {pkg_spec}\n', fg="cyan")
-You need the `sudo` permission to install Python packages. Try
+        raise exception.ReturnErrorCode(1) from exc
-
-> sudo pip install -U platformio
-
-WARNING! Don't use `sudo` for the rest PlatformIO commands.
-""",
-                fg="yellow",
-                err=True,
-            )
-            raise exception.ReturnErrorCode(1)
-        raise exception.UpgradeError("\n".join([str(cmd), r["out"], r["err"]]))
 
     return True
 
 
-def download_dist_package(to_develop):
+def get_pkg_spec(to_develop):
-    if not to_develop:
+    if to_develop:
-        return "platformio"
+        return
-    dl_url = "https://github.com/platformio/platformio-core/archive/develop.zip"
-    cache_dir = get_project_cache_dir()
-    if not os.path.isdir(cache_dir):
-        os.makedirs(cache_dir)
-    pkg_name = os.path.join(cache_dir, "piocoredevelop.zip")
-    try:
-        with open(pkg_name, "wb") as fp:
-            r = exec_command(
-                ["curl", "-fsSL", dl_url], stdout=fp, universal_newlines=True
-            )
-            assert r["returncode"] == 0
-        # check ZIP structure
-        with ZipFile(pkg_name) as zp:
-            assert zp.testzip() is None
-        return pkg_name
-    except:  # pylint: disable=bare-except
-        pass
-    return dl_url
 
 
 def get_latest_version():
@@ -133,10 +107,7 @@ def get_latest_version():
 
 def get_develop_latest_version():
     version = None
-    content = fetch_remote_content(
+    content = fetch_remote_content(DEVELOP_INIT_SCRIPT_URL)
-        "https://raw.githubusercontent.com/platformio/platformio"
-        "/develop/platformio/__init__.py"
-    )
     for line in content.split("\n"):
         line = line.strip()
         if not line.startswith("VERSION"):
@@ -153,5 +124,5 @@ def get_develop_latest_version():
 
 
 def get_pypi_latest_version():
-    content = fetch_remote_content("https://pypi.org/pypi/platformio/json")
+    content = fetch_remote_content(PYPI_JSON_URL)
     return json.loads(content)["info"]["version"]

platformio/exception.py

@@ -93,15 +93,6 @@ class CIBuildEnvsEmpty(UserSideException):
     )
 
 
-class UpgradeError(PlatformioException):
-    MESSAGE = """{0}
-
-* Upgrade using `pip install -U platformio`
-* Try different installation/upgrading steps:
-  https://docs.platformio.org/page/installation.html
-"""
-
-
 class HomeDirPermissionsError(UserSideException):
     MESSAGE = (
         "The directory `{0}` or its parent directory is not owned by the "

platformio/maintenance.py

@@ -166,8 +166,6 @@ def after_upgrade(ctx):
                 action="Upgrade",
                 label="%s > %s" % (last_version, __version__),
             )
-        else:
-            raise exception.UpgradeError("Auto upgrading...")
 
     # PlatformIO banner
     click.echo("*" * terminal_width)