Merge branch 'feature/tools_fix_arm64_arm32_detection_v5.1' into 'release/v5.1'

fix(idf_tools): Fix platform detection - arm64 hardware but it's running in arm32 environment (v5.1)

See merge request espressif/esp-idf!33396

tools/ci/check_type_comments.py

@@ -29,7 +29,7 @@ def types_valid_ignored_rules(file_name):  # type: (str) -> bool
     """
     Run Mypy check with rules for ignore list on the given file, return TRUE if Mypy check passes
     """
-    mypy_exit_code = subprocess.call('mypy {} --python-version 3.7 --allow-untyped-defs'.format(file_name), shell=True)
+    mypy_exit_code = subprocess.call('mypy {} --python-version 3.8 --allow-untyped-defs'.format(file_name), shell=True)
     return not bool(mypy_exit_code)
 
 

tools/idf_tools.py

@@ -164,6 +164,42 @@ class Platforms:
         'Linux-arm': PLATFORM_LINUX_ARM32,
     }
 
+    @staticmethod
+    def detect_linux_arm_platform(supposed_platform):  # type: (Optional[str]) -> Optional[str]
+        """
+        We probe the python binary to check exactly what environment the script is running in.
+
+        ARM platform may run on armhf hardware but having armel installed packages.
+        To avoid possible armel/armhf libraries mixing need to define user's
+        packages architecture to use the same
+        See note section in https://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html#index-mfloat-abi
+
+        ARM platform may run on aarch64 hardware but having armhf installed packages
+        (it happens if a docker container is running on arm64 hardware, but using an armhf image).
+
+        """
+        if supposed_platform not in (PLATFORM_LINUX_ARM32, PLATFORM_LINUX_ARMHF, PLATFORM_LINUX_ARM64):
+            return supposed_platform
+
+        # suppose that installed python was built with the right ABI
+        with open(sys.executable, 'rb') as f:
+            # see ELF header description in https://man7.org/linux/man-pages/man5/elf.5.html, offsets depend on ElfN size
+            if int.from_bytes(f.read(4), sys.byteorder) != int.from_bytes(b'\x7fELF', sys.byteorder):
+                return supposed_platform  # ELF magic not found. Use the default platform name from PLATFORM_FROM_NAME
+            f.seek(18)  # seek to e_machine
+            e_machine = int.from_bytes(f.read(2), sys.byteorder)
+            if e_machine == 183:  # EM_AARCH64, https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst
+                supposed_platform = PLATFORM_LINUX_ARM64
+            elif e_machine == 40:  # EM_ARM, https://github.com/ARM-software/abi-aa/blob/main/aaelf32/aaelf32.rst
+                f.seek(36)  # seek to e_flags
+                e_flags = int.from_bytes(f.read(4), sys.byteorder)
+                if e_flags & 0x400:
+                    supposed_platform = PLATFORM_LINUX_ARMHF
+                else:
+                    supposed_platform = PLATFORM_LINUX_ARM32
+
+        return supposed_platform
+
     @staticmethod
     def get(platform_alias):  # type: (Optional[str]) -> Optional[str]
         if platform_alias is None:
@@ -171,21 +207,8 @@ class Platforms:
 
         if platform_alias == 'any' and CURRENT_PLATFORM:
             platform_alias = CURRENT_PLATFORM
-
         platform_name = Platforms.PLATFORM_FROM_NAME.get(platform_alias, None)
-
+        platform_name = Platforms.detect_linux_arm_platform(platform_name)
-        # ARM platform may run on armhf hardware but having armel installed packages.
-        # To avoid possible armel/armhf libraries mixing need to define user's
-        # packages architecture to use the same
-        # See note section in https://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html#index-mfloat-abi
-        if platform_name in (PLATFORM_LINUX_ARM32, PLATFORM_LINUX_ARMHF) and 'arm' in platform.machine():
-            # suppose that installed python was built with a right ABI
-            with open(sys.executable, 'rb') as f:
-                if int.from_bytes(f.read(4), sys.byteorder) != int.from_bytes(b'\x7fELF', sys.byteorder):
-                    return platform_name  # ELF magic not found. Use default platform name from PLATFORM_FROM_NAME
-                f.seek(36)  # seek to e_flags (https://man7.org/linux/man-pages/man5/elf.5.html)
-                e_flags = int.from_bytes(f.read(4), sys.byteorder)
-                platform_name = PLATFORM_LINUX_ARMHF if e_flags & 0x400 else PLATFORM_LINUX_ARM32
         return platform_name
 
     @staticmethod
@@ -2349,7 +2372,7 @@ class ChecksumFileParser():
         try:
             for bytes_str, hash_str in zip(self.checksum[0::2], self.checksum[1::2]):
                 bytes_filename = self.parseLine(r'^# (\S*):', bytes_str)
-                hash_filename = self.parseLine(r'^\S* \*(\S*)', hash_str)
+                hash_filename = self.parseLine(r'^\S* [\* ](\S*)', hash_str)
                 if hash_filename != bytes_filename:
                     fatal('filename in hash-line and in bytes-line are not the same')
                     raise SystemExit(1)

tools/test_idf_tools/test_idf_tools.py

@@ -2,7 +2,6 @@
 #
 # SPDX-FileCopyrightText: 2019-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import json
 import os
 import re
@@ -10,6 +9,7 @@ import shutil
 import sys
 import tempfile
 import unittest
+from unittest.mock import patch
 
 try:
     from contextlib import redirect_stdout
@@ -650,5 +650,30 @@ class TestMaintainer(unittest.TestCase):
             self.assertEqual(json.load(f1), expected_json, "Please check 'tools/tools.new.json' to find a cause!")
 
 
+class TestArmDetection(unittest.TestCase):
+
+    ELF_HEADERS = {
+        idf_tools.PLATFORM_LINUX_ARM64: 'platform_detection/arm64_header.elf',
+        idf_tools.PLATFORM_LINUX_ARMHF: 'platform_detection/armhf_header.elf',
+        idf_tools.PLATFORM_LINUX_ARM32: 'platform_detection/arm32_header.elf',
+    }
+
+    ARM_PLATFORMS = {
+        idf_tools.PLATFORM_LINUX_ARM64,
+        idf_tools.PLATFORM_LINUX_ARMHF,
+        idf_tools.PLATFORM_LINUX_ARM32,
+    }
+
+    def test_arm_detection(self):
+        for platform in idf_tools.Platforms.PLATFORM_FROM_NAME.values():
+            with patch('sys.executable', __file__):  # use invalid ELF as executable. In this case passed parameter must return
+                self.assertEqual(idf_tools.Platforms.detect_linux_arm_platform(platform), platform)
+        # detect_linux_arm_platform() intended to return arch that detected in sys.executable ELF
+        for exec_platform in (idf_tools.PLATFORM_LINUX_ARM64, idf_tools.PLATFORM_LINUX_ARMHF, idf_tools.PLATFORM_LINUX_ARM32):
+            with patch('sys.executable', TestArmDetection.ELF_HEADERS[exec_platform]):
+                for platform in TestArmDetection.ARM_PLATFORMS:
+                    self.assertEqual(idf_tools.Platforms.detect_linux_arm_platform(platform), exec_platform)
+
+
 if __name__ == '__main__':
     unittest.main()