Merge branch 'bugfix/add_dummy_to_total_size_v4.4' into 'release/v4.4'

Tools: fix bug with total size calculation issue (v4.4)

See merge request espressif/esp-idf!16859

tools/idf_size.py

@@ -6,7 +6,7 @@
 # Includes information which is not shown in "xtensa-esp32-elf-size",
 # or easy to parse from "xtensa-esp32-elf-objdump" or raw map files.
 #
-# SPDX-FileCopyrightText: 2017-2021 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2017-2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 #
 from __future__ import division, print_function, unicode_literals
@@ -228,11 +228,10 @@ def load_map_data(map_file: TextIO) -> Tuple[str, Dict, Dict]:
     detected_chip = detect_target_chip(map_file)
     sections = load_sections(map_file)
 
-    # Exclude the .dummy section, which usually means shared region among I/D buses
+    # Exclude the dummy and .text_end section, which usually means shared region among I/D buses
-    dummy_keys = [key for key in sections if key.endswith(('.dummy'))]
+    for key in list(sections.keys()):
-    if dummy_keys:
+        if key.endswith(('dummy', '.text_end')):
-        sections.pop(*dummy_keys)
+            sections.pop(key)
-
     return detected_chip, segments, sections
 
 
@@ -569,8 +568,6 @@ class StructureForSummary(object):
         r.dram_total = get_size(dram_filter)
         iram_filter = filter(in_iram, segments)
         r.iram_total = get_size(iram_filter)
-        if r.diram_total == 0:
-            r.diram_total = r.dram_total + r.iram_total
 
         def filter_in_section(sections: Iterable[MemRegions.Region], section_to_check: str) -> List[MemRegions.Region]:
             return list(filter(lambda x: LinkingSections.in_section(x.section, section_to_check), sections))  # type: ignore
@@ -578,8 +575,6 @@ class StructureForSummary(object):
         dram_sections = list(filter(in_dram, sections))
         iram_sections = list(filter(in_iram, sections))
         diram_sections = list(filter(in_diram, sections))
-        if not diram_sections:
-            diram_sections = dram_sections + iram_sections
         flash_sections = filter_in_section(sections, 'flash')
 
         dram_data_list = filter_in_section(dram_sections, 'data')

tools/test_idf_size/expected_output.json

@@ -14,19 +14,19 @@
     "iram_total": 131072,
     "used_iram_ratio": 0.297027587890625,
     "iram_remain": 92140,
-    "diram_data": 9324,
+    "diram_data": 0,
-    "diram_bss": 8296,
+    "diram_bss": 0,
-    "diram_text": 37908,
+    "diram_text": 0,
-    "diram_vectors": 1024,
+    "diram_vectors": 0,
     "diram_rodata": 0,
     "diram_other": 0,
-    "diram_total": 311808,
+    "diram_total": 0,
-    "used_diram": 56552,
+    "used_diram": 0,
-    "used_diram_ratio": 0.18136802134646962,
+    "used_diram_ratio": 0,
-    "diram_remain": 255256,
+    "diram_remain": 0,
     "flash_code": 146944,
     "flash_rodata": 39580,
     "flash_other": 0,
     "used_flash_non_ram": 186524,
-    "total_size": 283036
+    "total_size": 234780
 }

tools/test_idf_size/expected_output.txt

@@ -5,12 +5,7 @@ Used static DRAM:   17620 bytes ( 163116 remain, 9.7% used)
 Used static IRAM:   38932 bytes (  92140 remain, 29.7% used)
       .text size:   37908 bytes
    .vectors size:    1024 bytes
-Used stat D/IRAM:   56552 bytes ( 255256 remain, 18.1% used)
-      .data size:    9324 bytes
-      .bss  size:    8296 bytes
-      .text size:   37908 bytes
-   .vectors size:    1024 bytes
 Used Flash size :  186524 bytes
       .text     :  146944 bytes
       .rodata   :   39580 bytes
-Total image size:  283036 bytes (.bin may be padded larger)
+Total image size:  234780 bytes (.bin may be padded larger)

tools/test_idf_size/mem_test.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+#
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+#
+import argparse
+import json
+import os
+import re
+from typing import Dict
+
+IDF_PATH = os.environ['IDF_PATH']
+MAX_SIZE_DIFF = 50
+
+
+def mem_test(size_json: dict, esptool_output: list) -> None:
+    seg_len = {}  # type: Dict[str, int]
+    for i in esptool_output:
+        tmp = i.split(' ')
+        if tmp[0] == 'Segment':
+            # tmp look like ['Segment', '2:', 'len', '0x02780', 'load', '0x3fc90610', 'file_offs', '0x00007ab0', '[BYTE_ACCESSIBLE,MEM_INTERNAL,DRAM]']
+            # tmp[3] contains the size of the segment and tmp[8] contains the name of the memory segment
+            esptool_mem = {'mem_type':tmp[8], 'size':tmp[3]}
+            seg = re.sub(r'MEM_INTERNAL|,|BYTE_ACCESSIBLE|\n|\[|\]', '', esptool_mem['mem_type'])
+            # If there are two IRAMs in esptool output it will compute these two IRAM lengths in a seg_len['IRAM']
+            seg_len[seg] = int(esptool_mem['size'], 16) if seg not in seg_len else seg_len[seg] + int(esptool_mem['size'], 16)
+    # including flash_other to DROM because flash_other contain .flash.appdesc that includes in DROM that produced by esptool
+    size_from_map = [('IROM', size_json['flash_code']), ('IRAM', size_json['iram_text'] + size_json['iram_vectors'] + size_json['diram_text']
+                     + size_json['diram_vectors']), ('DROM', size_json['flash_rodata'] + size_json['flash_other']), ('DRAM', size_json
+                     ['dram_data'] + size_json['diram_data'])]
+    for mem_type, size in size_from_map:
+        if abs(size - seg_len[mem_type]) > MAX_SIZE_DIFF:
+            raise RuntimeError(mem_type + " segment in idf_size isn't correct regarding esptool")
+    print('Test complete without errors')
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description='mem_test.py - a tool to test accuracy of the sizes of the memory segments regarding idf.py size by esptool')
+
+    parser.add_argument(
+        'size_json', help='JSON file with the output of the idf.py size',
+        type=argparse.FileType('r'))
+    parser.add_argument(
+        'esptool_output', help='File with the output of the esptool',
+        type=argparse.FileType('r'))
+
+    args = parser.parse_args()
+    mem_test(json.loads(args.size_json.read()), args.esptool_output.read().split('\n'))
+
+
+if __name__ == '__main__':
+    main()

tools/test_idf_size/size_schema.json

@@ -77,10 +77,7 @@
                 "(^\\.flash\\.(rodata|text|appdesc|rodata_noload)$)": {
                     "$ref": "#/$defs/archive_details"
                 },
-                "(^\\.flash_rodata_dummy$)": {
+                "(^\\.iram0\\.(text|vectors|bss|data)$)": {
-                    "$ref": "#/$defs/archive_details"
-                },
-                "(^\\.iram0\\.(text|vectors|text_end|bss|data)$)": {
                     "$ref": "#/$defs/archive_details"
                 },
                 "(^\\.rtc\\.(bss|data|text)$)": {

tools/test_idf_size/test.sh

@@ -1,8 +1,32 @@
 #!/usr/bin/env bash
 
 
+memory_test () {
+    pushd $IDF_PATH/examples/get-started/hello_world \
+    && echo -e "\n***\nBuilding project for $1..." &>> $IDF_PATH/tools/test_idf_size/output \
+    && idf.py set-target $1 \
+    && idf.py build \
+    && echo -e "\n***\nRunning mem_test.py for $1..." &>> $IDF_PATH/tools/test_idf_size/output \
+    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json build/hello_world.map > size_output.json \
+    && python $IDF_PATH/components/esptool_py/esptool/esptool.py --chip $1 image_info build/hello_world.bin > esptool_output \
+    && python -m coverage run -a $IDF_PATH/tools/test_idf_size/mem_test.py size_output.json esptool_output &>> $IDF_PATH/tools/test_idf_size/output \
+    && popd
+}
+
+json_test() {
+    echo -e "\n***\nProducing JSON output for $1..." &>> output \
+    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json app_$1.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
+    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archives app_$1.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
+    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --files app_$1.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
+    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archive_details libdriver.a app_$1.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output
+}
+
 { python -m coverage debug sys \
     && python -m coverage erase &> output \
+    && memory_test esp32 \
+    && memory_test esp32s2 \
+    && memory_test esp32s3 \
+    && memory_test esp32c3 \
     && echo -e "\n***\nRunning idf_size.py..." &>> output \
     && python -m coverage run -a $IDF_PATH/tools/idf_size.py app.map &>> output \
     && echo -e "\n***\nRunning idf_size.py on bootloader..." &>> output \
@@ -112,26 +136,10 @@
     && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archives app.map --diff app2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
     && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --files app.map --diff app2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
     && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archive_details libdriver.a app.map --diff app2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && echo -e "\n***\nProducing JSON output for esp32s2..." &>> output \
+    && json_test esp32s2 \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json app_esp32s2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
+    && json_test esp32c3 \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archives app_esp32s2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
+    && json_test esp32h2 \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --files app_esp32s2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
+    && json_test esp32s3 \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archive_details libdriver.a app_esp32s2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && echo -e "\n***\nProducing JSON output for esp32c3..." &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json app_esp32c3.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archives app_esp32c3.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --files app_esp32c3.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archive_details libdriver.a app_esp32c3.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && echo -e "\n***\nProducing JSON output for esp32h2..." &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json app_esp32h2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archives app_esp32h2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --files app_esp32h2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archive_details libdriver.a app_esp32h2.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && echo -e "\n***\nProducing JSON output for esp32s3..." &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json app_esp32s3.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archives app_esp32s3.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --files app_esp32s3.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
-    && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --archive_details libdriver.a app_esp32s3.map | python $IDF_PATH/tools/test_idf_size/json_validate_test.py &>> output \
     && echo -e "\n***\nProducing JSON file output..." &>> output \
     && python -m coverage run -a $IDF_PATH/tools/idf_size.py --json --output-file output.json app.map &>> output \
     && echo -e "\n***\nProducing text file output..." &>> output \