Merge branch 'feature/python2-deprecation-warn_v4.1' into 'release/v4.1'

Tools: Add Python 2 deprecation warning (v4.1)

See merge request espressif/esp-idf!11527

components/app_update/otatool.py

@@ -249,6 +249,10 @@ def _erase_ota_partition(target, ota_id):
 
 
 def main():
+    if sys.version_info[0] < 3:
+        print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.")
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print("WARNING: Python 3 versions older than 3.6 are not supported.")
     global quiet
 
     parser = argparse.ArgumentParser("ESP-IDF OTA Partitions Tool")

components/efuse/efuse_table_gen.py

@@ -458,6 +458,10 @@ def create_output_files(name, output_table, debug):
 
 
 def main():
+    if sys.version_info[0] < 3:
+        print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.", file=sys.stderr)
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print("WARNING: Python 3 versions older than 3.6 are not supported.", file=sys.stderr)
     global quiet
     global max_blk_len
     global idf_target

components/spiffs/spiffsgen.py

@@ -16,8 +16,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import division
+from __future__ import division, print_function
 import os
+import sys
 import io
 import math
 import struct
@@ -451,6 +452,10 @@ class SpiffsFS():
 
 
 def main():
+    if sys.version_info[0] < 3:
+        print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.")
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print("WARNING: Python 3 versions older than 3.6 are not supported.")
     parser = argparse.ArgumentParser(description="SPIFFS Image Generator",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 

components/ulp/esp32ulp_mapgen.py

@@ -5,7 +5,9 @@
 # Copyright (c) 2016-2017 Espressif Systems (Shanghai) PTE LTD.
 # Distributed under the terms of Apache License v2.0 found in the top-level LICENSE file.
 
+from __future__ import print_function
 from optparse import OptionParser
+import sys
 
 BASE_ADDR = 0x50000000
 
@@ -26,6 +28,10 @@ def gen_ld_h_from_sym(f_sym, f_ld, f_h):
 
 
 def main():
+    if sys.version_info[0] < 3:
+        print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.", file=sys.stderr)
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print("WARNING: Python 3 versions older than 3.6 are not supported.", file=sys.stderr)
     description = ("This application generates .h and .ld files for symbols defined in input file. "
                    "The input symbols file can be generated using nm utility like this: "
                    "esp32-ulp-nm -g -f posix <elf_file> > <symbols_file>")

docs/en/get-started/linux-setup-scratch.rst

@@ -15,15 +15,17 @@ To compile with ESP-IDF you need to get the following packages:
 
 - CentOS 7::
 
-    sudo yum install git wget ncurses-devel flex bison gperf python pyserial python-pyelftools cmake ninja-build ccache
+    sudo yum -y update && sudo yum install git wget ncurses-devel flex bison gperf python3 python3-pip cmake ninja-build ccache
+
+CentOS 7 is still supported but CentOS version 8 is recommended for a better user experience.
 
 - Ubuntu and Debian::
 
-    sudo apt-get install git wget libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache libffi-dev libssl-dev
+    sudo apt-get install git wget libncurses-dev flex bison gperf python3 python3-pip python3-setuptools python3-serial python3-cryptography python3-future python3-pyparsing python3-pyelftools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
 
 - Arch::
 
-    sudo pacman -S --needed gcc git make ncurses flex bison gperf python-pyserial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja ccache
+    sudo pacman -Sy --needed gcc git make ncurses flex bison gperf python-pyserial python-cryptography python-future python-pyparsing python-pyelftools cmake ninja ccache dfu-util
 
 .. note::
     CMake version 3.5 or newer is required for use with ESP-IDF. Older Linux distributions may require updating, enabling of a "backports" repository, or installing of a "cmake3" package rather than "cmake".
@@ -35,7 +37,7 @@ Compile the Toolchain from Source
 
   - CentOS 7::
 
-        sudo yum install gawk gperf grep gettext ncurses-devel python python-devel automake bison flex texinfo help2man libtool make
+        sudo yum install gawk gperf grep gettext ncurses-devel python3 python3-devel automake bison flex texinfo help2man libtool make
 
   - Ubuntu pre-16.04::
 
@@ -51,7 +53,7 @@ Compile the Toolchain from Source
 
   - Arch::
 
-        TODO
+        sudo pacman -Sy --needed python-pip
 
 Create the working directory and go into it::
 
@@ -72,6 +74,11 @@ Build the toolchain::
 
 Toolchain will be built in ``~/esp/crosstool-NG/builds/xtensa-esp32-elf``. Follow :ref:`instructions for standard setup <setup-linux-toolchain-add-it-to-path-legacy>` to add the toolchain to your ``PATH``.
 
+Python 2 deprecation
+====================
+
+Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for popular Linux distributions are listed above.
+
 
 Next Steps
 ==========

docs/en/get-started/linux-setup.rst

@@ -11,11 +11,13 @@ To compile with ESP-IDF you need to get the following packages:
 
 - CentOS 7::
 
-    sudo yum install git wget flex bison gperf python cmake ninja-build ccache
+    sudo yum -y update && sudo yum install git wget flex bison gperf python3 cmake ninja-build ccache
+
+CentOS 7 is still supported but CentOS version 8 is recommended for a better user experience.
 
 - Ubuntu and Debian::
 
-    sudo apt-get install git wget flex bison gperf python python-pip python-setuptools cmake ninja-build ccache libffi-dev libssl-dev
+    sudo apt-get install git wget flex bison gperf python3 python3-pip python3-setuptools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
 
 - Arch::
 
@@ -32,20 +34,47 @@ Permission issues /dev/ttyUSB0
 
 With some Linux distributions you may get the ``Failed to open port /dev/ttyUSB0`` error message when flashing the ESP32. :ref:`This can be solved by adding the current user to the dialout group<linux-dialout-group>`.
 
+Setting up Python 3 as default for CentOS
+-----------------------------------------
+
+CentOS 7 and older is providing Python 2.7 as the default interpreter.
+Python 3 is recommended instead and can be installed in old distributions as follows, or please consult the documentation of your operating system for other recommended ways to achieve this::
+
+    sudo yum -y update && sudo yum install python3 python3-pip python3-setuptools
+
+Making Python 3 the default interpreter is possible by running::
+
+    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3
+
 Setting up Python 3 as default for Ubuntu and Debian
 ----------------------------------------------------
 
-Ubuntu and Debian are still providing Python 2.7 as the default interpreter. Python 3 can be installed as follows::
+Ubuntu (version 18.04 and older) and Debian (version 9 and older) are still providing Python 2.7 as the default interpreter.
+Python 3 is recommended instead and can be installed in old distributions as follows, or please consult the documentation of your operating system for other recommended ways to achieve this::
 
     sudo apt-get install python3 python3-pip python3-setuptools
 
 Making Python 3 the default interpreter is possible by running::
 
-    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3
 
 .. note::
     This is system-wide change which may affect all of the applications.
 
+Fixing broken pip on Ubuntu 16.04
+=================================
+
+Package ``python3-pip`` could be broken without possibility to upgrade it. 
+Package has to be removed and installed manually using script `get-pip.py <https://bootstrap.pypa.io/get-pip.py>`_.::
+
+    apt remove python3-pip python3-virtualenv; rm -r ~/.local
+    rm -r ~/.espressif/python_env && python get-pip.py
+
+Python 2 deprecation
+====================
+
+Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for popular Linux distributions are listed above.
+
 Next Steps
 ==========
 

docs/en/get-started/macos-setup-scratch.rst

@@ -79,6 +79,10 @@ Build the toolchain::
 
 Toolchain will be built in ``~/esp/ctng-volume/crosstool-NG/builds/xtensa-esp32-elf``. To use it, you need to add ``~/esp/ctng-volume/crosstool-NG/builds/xtensa-esp32-elf/bin`` to ``PATH`` environment variable.
 
+Python 2 deprecation
+====================
+
+Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for macOS are listed above.
 
 Next Steps
 ==========

docs/en/get-started/macos-setup.rst

@@ -37,7 +37,7 @@ ESP-IDF will use the version of Python installed by default on macOS.
 Installing and setting up Python 3 as default
 ---------------------------------------------
 
-Basing on macOS `Catalina 10.15 release notes`_, use of Python 2.7 is not recommended and Python will not be included by default in future versions of macOS. Check what Python you currently have::
+Basing on macOS `Catalina 10.15 release notes`_, use of Python 2.7 is not recommended and Python 2.7 will not be included by default in future versions of macOS. Check what Python you currently have::
 
   python --version
 
@@ -54,7 +54,7 @@ Below is an overview of steps to install Python 3 and making it default interpre
       brew install python3
       ln -s /usr/local/bin/python3.8 /usr/local/bin/python
 
-    Adjust above directory name ``/usr/local/bin/python3.8`` to point where Python 3 has been installed. To check this directory you can run  ``which -a python``.
+    Adjust above directory name ``/usr/local/bin/python3.8`` to point where Python 3 has been installed. To check this directory you can run  ``which -a python3``.
 
   - If you have MacPorts_, you can run::
 
@@ -72,6 +72,12 @@ If the output is similar to ``Python 3.8.5``, your installation has been done su
 
     This is system-wide change which may affect all of the applications.
 
+Python 2 deprecation
+====================
+
+Python 2 reached its `end of life <https://www.python.org/doc/sunset-python-2/>`_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for macOS are listed above.
+
+
 Next Steps
 ==========
 

docs/zh_CN/get-started/linux-setup-scratch.rst

@@ -13,18 +13,21 @@
 
 - CentOS 7::
 
-    sudo yum install git wget ncurses-devel flex bison gperf python pyserial python-pyelftools cmake ninja-build ccache
+    sudo yum -y update && sudo yum install git wget ncurses-devel flex bison gperf python3 python3-pip cmake ninja-build ccache
+
+目前仍然支持 CentOS 7，但为了更好的用户体验，建议使用 CentOS 8。
 
 - Ubuntu 和 Debian::
 
-    sudo apt-get install git wget libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache libffi-dev libssl-dev
+    sudo apt-get install git wget libncurses-dev flex bison gperf python3 python3-pip python3-setuptools python3-serial python3-cryptography python3-future python3-pyparsing python3-pyelftools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
 
 - Arch::
 
-    sudo pacman -S --needed gcc git make ncurses flex bison gperf python-pyserial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja ccache
+    sudo pacman -Sy --needed gcc git make ncurses flex bison gperf python-pyserial python-cryptography python-future python-pyparsing python-pyelftools cmake ninja ccache dfu-util
 
-.. note::
+.. 注解::
-    使用 ESP-IDF 需要 CMake 3.5 或以上版本。较早版本的 Linux 可能需要升级才能向后移植仓库，或安装 "cmake3" 软件包，而不是安装 "cmake"。
+
+    使用 ESP-IDF 需要 CMake 3.5 或以上版本。较早的 Linux 发行版可能需要升级自身的软件源仓库，或开启 backports 套件库，或安装 "cmake3" 软件包（不是安装 "cmake")。
 
 从源代码编译工具链
 =================================
@@ -33,7 +36,7 @@
 
 - CentOS 7::
 
-    sudo yum install gawk gperf grep gettext ncurses-devel python python-devel automake bison flex texinfo help2man libtool make
+    sudo yum install gawk gperf grep gettext ncurses-devel python3 python3-devel automake bison flex texinfo help2man libtool make
 
 - Ubuntu pre-16.04::
 
@@ -49,7 +52,7 @@
 
 - Arch::
 
-    TODO
+    sudo pacman -Sy --needed python-pip
 
 创建工作目录，并进入该目录::
 
@@ -69,8 +72,14 @@
 编译得到的工具链会被保存到 ``~/esp/crosstool-NG/builds/xtensa-esp32-elf``。请按照 :ref:`标准设置指南 <setup-linux-toolchain-add-it-to-path-legacy>` 的介绍，将工具链添加到 ``PATH``。
 
 
+
+停用 Python 2 
+====================
+
+Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的目前主流 Linux 发行版的安装说明。
+
+
 后续步骤
 ==========
 
-继续设置开发环境，请前往 :ref:`get-started-get-esp-idf` 章节。
+请前往 :ref:`get-started-get-esp-idf` 章节继续设置开发环境。
-

docs/zh_CN/get-started/linux-setup.rst

@@ -11,11 +11,13 @@ Linux 平台工具链的标准设置
 
 - CentOS 7::
 
-    sudo yum install git wget flex bison gperf python cmake ninja-build ccache
+    sudo yum -y update && sudo yum install git wget flex bison gperf python3 cmake ninja-build ccache
+
+目前仍然支持 CentOS 7，但为了更好的用户体验，建议使用 CentOS 8。
 
 - Ubuntu 和 Debian::
 
-    sudo apt-get install git wget flex bison gperf python python-pip python-setuptools cmake ninja-build ccache libffi-dev libssl-dev
+    sudo apt-get install git wget flex bison gperf python3 python3-pip python3-setuptools cmake ninja-build ccache libffi-dev libssl-dev dfu-util
 
 - Arch::
 
@@ -30,7 +32,47 @@ Linux 平台工具链的标准设置
 权限问题 /dev/ttyUSB0
 ------------------------------------------------------------
 
-使用某些 Linux 版本向 ESP32 烧写固件时，可能会出现 ``Failed to open port /dev/ttyUSB0`` 错误消息。此时，可以将当前用户增加至 :ref:` Linux Dialout 组 <linux-dialout-group>`。
+使用某些 Linux 版本向 {IDF_TARGET_NAME} 烧录固件时，可能会出现 ``Failed to open port /dev/ttyUSB0`` 错误消息。此时可以将用户添加至 :ref:`Linux Dialout 组<linux-dialout-group>`。
+
+设置 Python 3 为 CentOS 默认 Python 版本
+----------------------------------------------------
+
+CentOS 7 及更早的版本提供 Python 2.7 作为默认解释器。但这里推荐使用 Python 3，您可以运行下方命令安装 Python 3。或者查看当前所用系统的相关文档，按照文档推荐的其它方法安装 Python 3::
+
+    sudo yum -y update && sudo yum install python3 python3-pip python3-setuptools
+
+设置 Python 3 为默认 Python 版本::
+
+    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3
+
+
+设置 Python 3 为 Ubuntu 和 Debian 默认 Python 版本
+----------------------------------------------------
+
+Ubuntu（v18.04 及之前的版本）和 Debian（v9 及之前的版本）的默认解释器为 Python 2.7，但这里推荐使用 Python 3，您可以运行下方命令安装 Python 3。或者查看当前所用系统的相关文档，按照文档推荐的其它方法安装 Python 3::
+
+    sudo apt-get install python3 python3-pip python3-setuptools
+
+设置 Python 3 为默认 Python 版本::
+
+    sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3
+
+.. 注解::
+    上述设置为全局设置，同时会影响到其它应用。
+
+修复 Ubuntu 16.04 损坏的 pip 
+=================================
+
+``python3-pip`` 包可能已损坏无法升级。需使用脚本 `get-pip.py <https://bootstrap.pypa.io/get-pip.py>`_ 手动删除并安装该包::
+
+    apt remove python3-pip python3-virtualenv; rm -r ~/.local
+    rm -r ~/.espressif/python_env && python get-pip.py
+
+停用 Python 2 
+====================
+
+Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的目前主流 Linux 发行版的安装说明。
+
 
 后续步骤
 ==========

docs/zh_CN/get-started/macos-setup-scratch.rst

@@ -1,5 +1,5 @@
 ***********************************************
-从零开始设置 MacOS 环境下的工具链
+从零开始设置 macOS 环境下的工具链
 ***********************************************
 
 :link_to_translation:`en:[English]`
@@ -80,6 +80,12 @@ MacPorts 需要完整的 XCode 软件，而 homebrew 只需要安装 XCode 命
 编译得到的工具链会被保存到 ``~/esp/ctng-volume/crosstool-NG/builds/xtensa-esp32-elf``。使用工具链前，请将 ``~/esp/ctng-volume/crosstool-NG/builds/xtensa-esp32-elf/bin`` 添加至 ``PATH`` 环境变量。
 
 
+停用 Python 2 
+====================
+
+Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的 macOS 安装说明。
+
+
 后续步骤
 ==========
 

docs/zh_CN/get-started/macos-setup.rst

@@ -35,6 +35,50 @@ ESP-IDF 将使用 Mac OS 上默认安装的 Python 版本。
 
     则必须安装 XCode 命令行工具，具体可运行 ``xcode-select --install``。
 
+安装并设置 Python 3 为默认版本
+---------------------------------------------
+
+`Catalina 10.15 发布说明`_ 中表示不推荐使用 Python 2.7 版本，在未来的 macOS 版本中也不会默认包含 Python 2.7。执行以下命令来检查您当前使用的 Python 版本::
+
+  python --version
+
+如果输出结果是 ``Python 2.7.17``，则代表您的默认解析器是 Python 2.7。这时需要您运行以下命令检查电脑上是否已经安装过 Python 3::
+
+  python3 --version
+
+如果运行上述命令出现错误，则代表电脑上没有安装 Python 3。
+
+请根据以下步骤安装 Python 3 并使其成为默认解释器：
+
+  - 使用 HomeBrew_ 进行安装的方法如下::
+
+      brew install python3
+      ln -s /usr/local/bin/python3.8 /usr/local/bin/python
+
+    将上述的目录名 ``/usr/local/bin/python3.8`` 修改为 Python 3 所在的目录。您可以运行 ``which -a python3`` 来查看 Python 3 所在的目录。
+
+  - 使用 MacPorts_ 进行安装的方法如下::
+
+      sudo port install python38
+      sudo port select --set python python38
+
+现在您可以打开终端窗口验证默认运行的 Python 版本::
+
+  python --version
+
+如果输出结果类似于 ``Python 3.8.5`` 则代表安装成功。
+
+.. 注解::
+
+    上述设置为全局设置，同时会影响到其它应用。
+
+
+
+停用 Python 2 
+====================
+
+Python 2 已经 `结束生命周期 <https://www.python.org/doc/sunset-python-2/>`_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的 macOS 安装说明。
+
 后续步骤
 ==========
 
@@ -46,4 +90,5 @@ ESP-IDF 将使用 Mac OS 上默认安装的 Python 版本。
 .. _ccache: https://ccache.samba.org/
 .. _homebrew: https://brew.sh/
 .. _MacPorts: https://www.macports.org/install.php
+.. _Catalina 10.15 发布说明: https://developer.apple.com/documentation/macos-release-notes/macos-catalina-10_15-release-notes
 

tools/ci/check_build_warnings.py

@@ -29,6 +29,9 @@ IGNORE_WARNS = [
         r"reassigning to symbol",
         r"changes choice state",
         r"crosstool_version_check\.cmake",
+        r"CryptographyDeprecationWarning",
+        r"Python 3 versions older than 3.6 are not supported.",
+        r"Python 2 is deprecated and will be removed in future versions."
     ]
 ]
 

tools/ci/config/build.yml

@@ -56,6 +56,8 @@ build_ssc_esp32s2beta:
 
 build_esp_idf_tests_make:
   extends: .build_esp_idf_unit_test_template
+  variables:
+      PYTHON_VER: 3
   script:
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
@@ -72,6 +74,8 @@ build_esp_idf_tests_make:
 
 build_esp_idf_tests_cmake:
   extends: .build_esp_idf_unit_test_template
+  variables:
+      PYTHON_VER: 3
   script:
     - export PATH="$IDF_PATH/tools:$PATH"
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}

tools/idf.py

@@ -82,6 +82,12 @@ def check_environment():
         print("Setting IDF_PATH environment variable: %s" % detected_idf_path)
         os.environ["IDF_PATH"] = detected_idf_path
 
+    # check Python version
+    if sys.version_info[0] < 3:
+        print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.")
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
+        print("WARNING: Python 3 versions older than 3.6 are not supported.")
+
     # check Python dependencies
     checks_output.append("Checking Python dependencies...")
     try: