diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bca981e555..dd4974b908 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,11 +56,12 @@ variables: # Docker images BOT_DOCKER_IMAGE_TAG: ":latest" - ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env:v4.4-1-v5" - ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env:v4.4-1" - AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test:v4.4-1-1" - CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis:v4.4-1-2" + ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.0:1" + AFL_FUZZER_TEST_IMAGE: "${CI_DOCKER_REGISTRY}/afl-fuzzer-test-v5.0:1-1" + CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.0:1-1" + ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.0:1-1" SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3" + LINUX_SHELL_IMAGE: "${CI_DOCKER_REGISTRY}/linux-shells:2" # target test config file, used by assign test job CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml" diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 2eca5b01e8..d545664c9e 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -57,10 +57,8 @@ test_ldgen_on_host: extends: .host_test_template script: - cd tools/ldgen/test - - ./test_fragments.py - - ./test_generation.py - - ./test_entity.py - - ./test_output_commands.py + - export PYTHONPATH=$PYTHONPATH:.. + - python -m unittest variables: LC_ALL: C.UTF-8 @@ -317,7 +315,7 @@ test_mkuf2: test_autocomplete: extends: .host_test_template - image: $CI_DOCKER_REGISTRY/linux-shells:1 + image: $LINUX_SHELL_IMAGE artifacts: when: on_failure paths: @@ -328,7 +326,7 @@ test_autocomplete: test_detect_python: extends: .host_test_template - image: $CI_DOCKER_REGISTRY/linux-shells:1 + image: $LINUX_SHELL_IMAGE script: - cd ${IDF_PATH} - shellcheck -s sh tools/detect_python.sh diff --git a/.pylintrc b/.pylintrc index 019a561e44..8b57c7b550 100644 --- a/.pylintrc +++ b/.pylintrc @@ -151,6 +151,7 @@ disable=print-statement, too-many-branches, too-many-statements, ungrouped-imports, # since we have isort in pre-commit + no-name-in-module, # since we have flake8 to check this # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/components/espcoredump/test/test_espcoredump.sh b/components/espcoredump/test/test_espcoredump.sh index c67cf1ebe9..5732ae40b2 100755 --- a/components/espcoredump/test/test_espcoredump.sh +++ b/components/espcoredump/test/test_espcoredump.sh @@ -11,26 +11,21 @@ else elf_dir=$1 fi -if ! command -v coverage &> /dev/null; then - echo "coverage could not be found, please install it ('pip install coverage')" - exit 1 -fi - SUPPORTED_TARGETS=("esp32" "esp32s2" "esp32c3" "esp32s3" ) res=0 -coverage erase +python -m coverage erase for chip in "${SUPPORTED_TARGETS[@]}"; do { echo "run b64 decoding tests on $chip" - coverage run -a --source=corefile ../espcoredump.py --chip="$chip" --gdb-timeout-sec 5 info_corefile -m -t b64 -c "${chip}/coredump.b64" -s "${chip}/core.elf" "${elf_dir}/${chip}.elf" &>"${chip}/output" && + python -m coverage run -a --source=corefile ../espcoredump.py --chip="$chip" --gdb-timeout-sec 5 info_corefile -m -t b64 -c "${chip}/coredump.b64" -s "${chip}/core.elf" "${elf_dir}/${chip}.elf" &>"${chip}/output" && diff "${chip}/expected_output" "${chip}/output" && - coverage run -a --source=corefile ../espcoredump.py --chip="$chip" --gdb-timeout-sec 5 info_corefile -m -t elf -c "${chip}/core.elf" "${elf_dir}/${chip}.elf" &>"${chip}/output2" && + python -m coverage run -a --source=corefile ../espcoredump.py --chip="$chip" --gdb-timeout-sec 5 info_corefile -m -t elf -c "${chip}/core.elf" "${elf_dir}/${chip}.elf" &>"${chip}/output2" && diff "${chip}/expected_output" "${chip}/output2" } || { echo 'The test for espcoredump has failed!' res=1 } done -coverage run -a --source=corefile ./test_espcoredump.py -coverage report ../corefile/*.py ../espcoredump.py +python -m coverage run -a --source=corefile ./test_espcoredump.py +python -m coverage report ../corefile/*.py ../espcoredump.py exit $res diff --git a/components/lwip/test_afl_host/esp_netif_loopback_mock.c b/components/lwip/test_afl_host/esp_netif_loopback_mock.c index ba7bbccb3c..a86b64fd55 100644 --- a/components/lwip/test_afl_host/esp_netif_loopback_mock.c +++ b/components/lwip/test_afl_host/esp_netif_loopback_mock.c @@ -1,17 +1,10 @@ -// Copyright 2020 Espressif Systems (Shanghai) CO LTD -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* + * SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#include #include "esp_netif_lwip_internal.h" esp_err_t esp_netif_get_ip_info(esp_netif_t *esp_netif, esp_netif_ip_info_t *ip_info) diff --git a/components/lwip/test_afl_host/no_warn_host.h b/components/lwip/test_afl_host/no_warn_host.h index 37ab01ebec..8d75837192 100644 --- a/components/lwip/test_afl_host/no_warn_host.h +++ b/components/lwip/test_afl_host/no_warn_host.h @@ -3,3 +3,4 @@ #define __warning__ deprecated #define IRAM_ATTR #define __ESP_ATTR_H__ +#include diff --git a/components/mdns/test_afl_fuzz_host/esp32_mock.h b/components/mdns/test_afl_fuzz_host/esp32_mock.h index 5acf53842e..c1163e9b6b 100644 --- a/components/mdns/test_afl_fuzz_host/esp32_mock.h +++ b/components/mdns/test_afl_fuzz_host/esp32_mock.h @@ -1,16 +1,8 @@ -// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* + * SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ #ifndef _ESP32_COMPAT_H_ #define _ESP32_COMPAT_H_ @@ -20,6 +12,7 @@ #define _ESP_TASK_H_ #ifdef USE_BSD_STRING +#include #include #endif #include diff --git a/docs/en/api-guides/linker-script-generation.rst b/docs/en/api-guides/linker-script-generation.rst index ba1e99a433..c56d1744b4 100644 --- a/docs/en/api-guides/linker-script-generation.rst +++ b/docs/en/api-guides/linker-script-generation.rst @@ -24,16 +24,16 @@ This section presents a guide for quickly placing code/data to RAM and RTC memor For this guide, suppose we have the following:: - - components/ - - my_component/ - - CMakeLists.txt - - component.mk - - Kconfig - - src/ - - my_src1.c - - my_src2.c - - my_src3.c - - my_linker_fragment_file.lf + component + └── my_component + └── CMakeLists.txt + ├── component.mk + ├── Kconfig + ├── src/ + │ ├── my_src1.c + │ ├── my_src2.c + │ └── my_src3.c + └── my_linker_fragment_file.lf - a component named ``my_component`` that is archived as library ``libmy_component.a`` during build - three source files archived under the library, ``my_src1.c``, ``my_src2.c`` and ``my_src3.c`` which are compiled as ``my_src1.o``, ``my_src2.o`` and ``my_src3.o``, respectively @@ -71,7 +71,7 @@ Placing object files """""""""""""""""""" Suppose the entirety of ``my_src1.o`` is performance-critical, so it is desirable to place it in RAM. On the other hand, the entirety of ``my_src2.o`` contains symbols needed coming out of deep sleep, so it needs to be put under RTC memory. -In the the linker fragment file, we can write: +In the linker fragment file, we can write: .. code-block:: none @@ -125,6 +125,9 @@ Similarly, this places the entire component in RTC memory: entries: * (rtc) + +.. _ldgen-conditional-placements: + Configuration-dependent placements """""""""""""""""""""""""""""""""" @@ -224,6 +227,9 @@ The three fragment types share a common grammar: - type: Corresponds to the fragment type, can either be ``sections``, ``scheme`` or ``mapping``. - name: The name of the fragment, should be unique for the specified fragment type. - key, value: Contents of the fragment; each fragment type may support different keys and different grammars for the key values. + + - For :ref:`sections` and :ref:`scheme`, the only supported key is ``entries`` + - For :ref:`mappings`, both ``archive`` and ``entries`` are supported. .. note:: @@ -286,24 +292,10 @@ Condition checking behaves as you would expect an ``if...elseif/elif...else`` bl key_2: value_b - **Comments** Comment in linker fragment files begin with ``#``. Like in other languages, comment are used to provide helpful descriptions and documentation and are ignored during processing. -Compatibility with ESP-IDF v3.x Linker Script Fragment Files -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -ESP-IDF v4.0 brings some changes to the linker script fragment file grammar: - -- indentation is enforced and improperly indented fragment files generate a parse exception; this was not enforced in the old version but previous documentation and examples demonstrates properly indented grammar -- move to ``if...elif...else`` structure for conditionals, with the ability to nest checks and place entire fragments themselves inside conditionals -- mapping fragments now requires a name like other fragment types - -Linker script generator should be able to parse ESP-IDF v3.x linker fragment files that are indented properly (as demonstrated by the ESP-IDF v3.x version of this document). Backward compatibility with the previous mapping fragment grammar (optional name and the old grammar for conditionals) has also been retained but with a deprecation warning. Users should switch to the newer grammar discussed in this document as support for the old grammar is planned to be removed in the future. - -Note that linker fragment files using the new ESP-IDF v4.0 grammar is not supported on ESP-IDF v3.x, however. - Types """"" @@ -608,3 +600,14 @@ Then the corresponding excerpt from the generated linker script will be as follo Rule generated from the default scheme entry ``iram -> iram0_text``. Since the default scheme specifies an ``iram -> iram0_text`` entry, it too is placed wherever ``iram0_text`` is referenced by a marker. Since it is a rule generated from the default scheme, it comes first among all other rules collected under the same target name. The linker script template currently used is :component_file:`esp_system/ld/{IDF_TARGET_PATH_NAME}/sections.ld.in`; the generated output script ``sections.ld`` is put under its build directory. + +.. _ldgen-migrate-lf-grammar : + +Migrate to ESP-IDF v5.0 Linker Script Fragment Files Grammar +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The old grammar supported in ESP-IDF v3.x would be dropped in ESP-IDF v5.0. Here are a few notes on how to migrate properly: + +1. Now indentation is enforced and improperly indented fragment files would generate a runtime parse exception. This was not enforced in the old version but previous documentation and examples demonstrate properly indented grammar. +2. Migrate the old condition entry to the ``if...elif...else`` structure for conditionals. You can refer to the :ref:`earlier chapter` for detailed grammar. +3. mapping fragments now requires a name like other fragment types. diff --git a/docs/en/migration-guides/build-system.rst b/docs/en/migration-guides/build-system.rst index abb2f1bcb4..0a3e790361 100644 --- a/docs/en/migration-guides/build-system.rst +++ b/docs/en/migration-guides/build-system.rst @@ -1,4 +1,12 @@ Migrate Build System to ESP-IDF 5.0 =================================== +Migrating from make to cmake +---------------------------- + Please follow the :ref:`build system ` guide for migrating make-based projects no longer supported in ESP-IDF v5.0. + +Update fragment file grammar +---------------------------- + +Please follow the :ref:`migrate linker script fragment files grammar` chapter for migrating v3.x grammar to the new one. diff --git a/docs/zh_CN/api-guides/linker-script-generation.rst b/docs/zh_CN/api-guides/linker-script-generation.rst index 6ee1d3f2d0..31258be653 100644 --- a/docs/zh_CN/api-guides/linker-script-generation.rst +++ b/docs/zh_CN/api-guides/linker-script-generation.rst @@ -24,16 +24,16 @@ 假设用户有:: - - components/ - - my_component/ - - CMakeLists.txt - - component.mk - - Kconfig - - src/ - - my_src1.c - - my_src2.c - - my_src3.c - - my_linker_fragment_file.lf + component + └── my_component + └── CMakeLists.txt + ├── component.mk + ├── Kconfig + ├── src/ + │ ├── my_src1.c + │ ├── my_src2.c + │ └── my_src3.c + └── my_linker_fragment_file.lf - 名为 ``my_component`` 的组件，在构建过程中存储为 ``libmy_component.a`` 库文件 - 库文件包含的三个源文件：``my_src1.c``、``my_src2.c`` 和 ``my_src3.c``，编译后分别为 ``my_src1.o``、``my_src2.o`` 和 ``my_src3.o`` @@ -125,6 +125,8 @@ entries: * (rtc) +.. _ldgen-conditional-placements : + 根据具体配置存放 """""""""""""""""""" @@ -225,6 +227,9 @@ - 名称：片段名称，指定片段类型的片段名称应唯一。 - 键值：片段内容。每个片段类型可支持不同的键值和不同的键值语法。 + - 在 ``段`` 和 ``协议`` 中，仅支持 ``entries`` 键。 + - 在 ``映射`` 中，键支持 ``archive`` 和 ``entries``。 + .. note:: 多个片段的类型和名称相同时会引发异常。 @@ -286,24 +291,10 @@ key_2: value_b - **注释** 链接器片段文件中的注释以 ``#`` 开头。和在其他语言中一样，注释提供了有用的描述和资料，在处理过程中会被忽略。 -与 ESP-IDF v3.x 链接器脚本片段文件兼容 -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" - -ESP-IDF v4.0 变更了链接器脚本片段文件使用的一些语法： - -- 必须缩进，缩进不当的文件会产生解析异常；旧版本不强制缩进，但之前的文档和示例均遵循了正确的缩进语法 -- 条件改用 ``if...elif...else`` 结构，可以嵌套检查，将完整片段置于条件内 -- 映射片段和其他片段类型一样，需有名称 - -链接器脚本生成器可解析 ESP-IDF v3.x 版本中缩进正确的链接器片段文件（如 ESP-IDF v3.x 版本中的本文件所示），依然可以向后兼容此前的映射片段语法（可选名称和条件的旧语法），但是会有弃用警告。用户应换成本文档介绍的新语法，因为旧语法将在未来停用。 - -请注意，ESP-IDF v3.x 不支持使用 ESP-IDF v4.0 新语法的链接器片段文件。 - 类型 """"""" @@ -608,3 +599,14 @@ ESP-IDF v4.0 变更了链接器脚本片段文件使用的一些语法： 这是根据默认协议条目 ``iram -> iram0_text`` 生成的规则。默认协议指定了 ``iram -> iram0_text`` 条目，因此生成的规则同样也放在被 ``iram0_text`` 标记的地方。由于该规则是根据默认协议生成的，因此在同一目标下收集的所有规则下排在第一位。 目前使用的链接器脚本模板是 :component_file:`esp_system/ld/{IDF_TARGET_PATH_NAME}/sections.ld.in`，生成的脚本存放在构建目录下。 + +.. _ldgen-migrate-lf-grammar : + +将链接器脚本片段文件语法迁移至 ESP-IDF v5.0 适应版本 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ESP-IDF v5.0 中将不再支持 ESP-IDF v3.x 中链接器脚本片段文件的旧式语法。在迁移的过程中需注意以下几点： + +- 必须缩进，缩进不当的文件会产生解析异常；旧版本不强制缩进，但之前的文档和示例均遵循了正确的缩进语法 +- 条件改用 ``if...elif...else`` 结构，可以参照 :ref:`之前的章节` +- 映射片段和其他片段类型一样，需有名称 diff --git a/requirements.txt b/requirements.txt index 45e43877b7..5141f4481b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ cryptography>=2.1.4 # We do have cryptography binary on https://dl.espressif.com/pypi for ARM # On https://pypi.org/ are no ARM binaries as standard now -pyparsing>=2.0.3,<2.4.0 +pyparsing>=3.0.3 # https://github.com/pyparsing/pyparsing/issues/319 is fixed in 3.0.3 pyelftools>=0.22 idf-component-manager>=0.2.99-beta diff --git a/tools/ci/check_build_warnings.py b/tools/ci/check_build_warnings.py index c1e564e727..73538ebda9 100755 --- a/tools/ci/check_build_warnings.py +++ b/tools/ci/check_build_warnings.py @@ -35,7 +35,8 @@ IGNORE_WARNS = [ r'changes choice state', r'crosstool_version_check\.cmake', r'CryptographyDeprecationWarning', - r'Warning: \d+/\d+ app partitions are too small for binary' + r'Warning: \d+/\d+ app partitions are too small for binary', + r'CMake Deprecation Warning at main/lib/tinyxml2/CMakeLists\.txt:11 \(cmake_policy\)', ] ] diff --git a/tools/ci/check_copyright_ignore.txt b/tools/ci/check_copyright_ignore.txt index 05fe0c46aa..fa81931654 100644 --- a/tools/ci/check_copyright_ignore.txt +++ b/tools/ci/check_copyright_ignore.txt @@ -1243,7 +1243,6 @@ components/lwip/test_afl_host/dhcp_di.h components/lwip/test_afl_host/dhcpserver_di.h components/lwip/test_afl_host/dns_di.h components/lwip/test_afl_host/esp_attr.h -components/lwip/test_afl_host/esp_netif_loopback_mock.c components/lwip/test_afl_host/network_mock.c components/lwip/test_afl_host/no_warn_host.h components/lwip/test_afl_host/test_dhcp_client.c @@ -1350,7 +1349,6 @@ components/mdns/mdns_networking_lwip.c components/mdns/private_include/mdns_networking.h components/mdns/test/test_mdns.c components/mdns/test_afl_fuzz_host/esp32_mock.c -components/mdns/test_afl_fuzz_host/esp32_mock.h components/mdns/test_afl_fuzz_host/esp_attr.h components/mdns/test_afl_fuzz_host/esp_netif_mock.c components/mdns/test_afl_fuzz_host/mdns_di.h @@ -3028,8 +3026,6 @@ tools/ldgen/output_commands.py tools/ldgen/samples/template.ld tools/ldgen/sdkconfig.py tools/ldgen/test/data/linker_script.ld -tools/ldgen/test/test_entity.py -tools/ldgen/test/test_output_commands.py tools/mass_mfg/mfg_gen.py tools/mkdfu.py tools/mkuf2.py diff --git a/tools/ci/mypy_ignore_list.txt b/tools/ci/mypy_ignore_list.txt index 24e3307b3b..bb3e0404c0 100644 --- a/tools/ci/mypy_ignore_list.txt +++ b/tools/ci/mypy_ignore_list.txt @@ -233,13 +233,13 @@ tools/kconfig_new/test/confgen/test_confgen.py tools/kconfig_new/test/confserver/test_confserver.py tools/kconfig_new/test/gen_kconfig_doc/test_kconfig_out.py tools/kconfig_new/test/gen_kconfig_doc/test_target_visibility.py -tools/ldgen/fragments.py -tools/ldgen/generation.py tools/ldgen/ldgen.py -tools/ldgen/ldgen_common.py -tools/ldgen/linker_script.py -tools/ldgen/output_commands.py -tools/ldgen/sdkconfig.py +tools/ldgen/ldgen/entity.py +tools/ldgen/ldgen/fragments.py +tools/ldgen/ldgen/generation.py +tools/ldgen/ldgen/linker_script.py +tools/ldgen/ldgen/output_commands.py +tools/ldgen/ldgen/sdkconfig.py tools/ldgen/test/test_entity.py tools/ldgen/test/test_fragments.py tools/ldgen/test/test_generation.py diff --git a/tools/esp_app_trace/test/logtrace/test.sh b/tools/esp_app_trace/test/logtrace/test.sh index 1e24c0e300..8b49169404 100755 --- a/tools/esp_app_trace/test/logtrace/test.sh +++ b/tools/esp_app_trace/test/logtrace/test.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -{ coverage debug sys \ - && coverage erase &> output \ - && coverage run -a $IDF_PATH/tools/esp_app_trace/logtrace_proc.py adc_log.trc test.elf &>> output \ +{ python -m coverage debug sys \ + && python -m coverage erase &> output \ + && python -m coverage run -a $IDF_PATH/tools/esp_app_trace/logtrace_proc.py adc_log.trc test.elf &>> output \ && diff output expected_output \ - && coverage report \ + && python -m coverage report \ ; } || { echo 'The test for logtrace_proc has failed. Please examine the artifacts.' ; exit 1; } diff --git a/tools/esp_app_trace/test/sysview/test.sh b/tools/esp_app_trace/test/sysview/test.sh index b52649f754..2d9d4054ab 100755 --- a/tools/esp_app_trace/test/sysview/test.sh +++ b/tools/esp_app_trace/test/sysview/test.sh @@ -1,29 +1,29 @@ #!/usr/bin/env bash -{ coverage debug sys \ - && coverage erase &> output \ - && coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -d -p -b test.elf cpu0.svdat cpu1.svdat &>> output \ +{ python -m coverage debug sys \ + && python -m coverage erase &> output \ + && python -m coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -d -p -b test.elf cpu0.svdat cpu1.svdat &>> output \ && diff output expected_output \ - && coverage report \ + && python -m coverage report \ ; } || { echo 'The test for sysviewtrace_proc has failed. Please examine the artifacts.' ; exit 1; } -{ coverage debug sys \ - && coverage erase &> output.json \ - && coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -j -b test.elf cpu0.svdat cpu1.svdat &>> output.json \ +{ python -m coverage debug sys \ + && python -m coverage erase &> output.json \ + && python -m coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -j -b test.elf cpu0.svdat cpu1.svdat &>> output.json \ && diff output.json expected_output.json \ - && coverage report \ + && python -m coverage report \ ; } || { echo 'The test for sysviewtrace_proc JSON functionality has failed. Please examine the artifacts.' ; exit 1; } -{ coverage debug sys \ - && coverage erase &> output \ - && coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -d -p -b sysview_tracing_heap_log.elf heap_log_mcore.svdat &>> output \ +{ python -m coverage debug sys \ + && python -m coverage erase &> output \ + && python -m coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -d -p -b sysview_tracing_heap_log.elf heap_log_mcore.svdat &>> output \ && diff output expected_output_mcore \ - && coverage report \ + && python -m coverage report \ ; } || { echo 'The test for mcore sysviewtrace_proc functionality has failed. Please examine the artifacts.' ; exit 1; } -{ coverage debug sys \ - && coverage erase &> output.json \ - && coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -j -b sysview_tracing_heap_log.elf heap_log_mcore.svdat &>> output.json \ +{ python -m coverage debug sys \ + && python -m coverage erase &> output.json \ + && python -m coverage run -a $IDF_PATH/tools/esp_app_trace/sysviewtrace_proc.py -j -b sysview_tracing_heap_log.elf heap_log_mcore.svdat &>> output.json \ && diff output.json expected_output_mcore.json \ - && coverage report \ + && python -m coverage report \ ; } || { echo 'The test for mcore sysviewtrace_proc JSON functionality has failed. Please examine the artifacts.' ; exit 1; } diff --git a/tools/ldgen/fragments.py b/tools/ldgen/fragments.py deleted file mode 100644 index 238f198f18..0000000000 --- a/tools/ldgen/fragments.py +++ /dev/null @@ -1,607 +0,0 @@ -# -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -# -import abc -import os -import re -from collections import namedtuple -from enum import Enum - -from entity import Entity -from pyparsing import (Combine, Forward, Group, Keyword, Literal, OneOrMore, Optional, Or, ParseFatalException, - Suppress, Word, ZeroOrMore, alphanums, alphas, delimitedList, indentedBlock, nums, - originalTextFor, restOfLine) -from sdkconfig import SDKConfig - - -class FragmentFile(): - """ - Processes a fragment file and stores all parsed fragments. For - more information on how this class interacts with classes for the different fragment types, - see description of Fragment. - """ - - def __init__(self, fragment_file, sdkconfig): - try: - fragment_file = open(fragment_file, 'r') - except TypeError: - pass - - path = os.path.realpath(fragment_file.name) - - indent_stack = [1] - - class parse_ctx: - fragment = None # current fragment - key = '' # current key - keys = list() # list of keys parsed - key_grammar = None # current key grammar - - @staticmethod - def reset(): - parse_ctx.fragment_instance = None - parse_ctx.key = '' - parse_ctx.keys = list() - parse_ctx.key_grammar = None - - def fragment_type_parse_action(toks): - parse_ctx.reset() - parse_ctx.fragment = FRAGMENT_TYPES[toks[0]]() # create instance of the fragment - return None - - def expand_conditionals(toks, stmts): - try: - stmt = toks['value'] - stmts.append(stmt) - except KeyError: - try: - conditions = toks['conditional'] - for condition in conditions: - try: - _toks = condition[1] - _cond = condition[0] - if sdkconfig.evaluate_expression(_cond): - expand_conditionals(_toks, stmts) - break - except IndexError: - expand_conditionals(condition[0], stmts) - except KeyError: - for tok in toks: - expand_conditionals(tok, stmts) - - def key_body_parsed(pstr, loc, toks): - stmts = list() - expand_conditionals(toks, stmts) - - if parse_ctx.key_grammar.min and len(stmts) < parse_ctx.key_grammar.min: - raise ParseFatalException(pstr, loc, "fragment requires at least %d values for key '%s'" % - (parse_ctx.key_grammar.min, parse_ctx.key)) - - if parse_ctx.key_grammar.max and len(stmts) > parse_ctx.key_grammar.max: - raise ParseFatalException(pstr, loc, "fragment requires at most %d values for key '%s'" % - (parse_ctx.key_grammar.max, parse_ctx.key)) - - try: - parse_ctx.fragment.set_key_value(parse_ctx.key, stmts) - except Exception as e: - raise ParseFatalException(pstr, loc, "unable to add key '%s'; %s" % (parse_ctx.key, str(e))) - return None - - key = Word(alphanums + '_') + Suppress(':') - key_stmt = Forward() - - condition_block = indentedBlock(key_stmt, indent_stack) - key_stmts = OneOrMore(condition_block) - key_body = Suppress(key) + key_stmts - key_body.setParseAction(key_body_parsed) - - condition = originalTextFor(SDKConfig.get_expression_grammar()).setResultsName('condition') - if_condition = Group(Suppress('if') + condition + Suppress(':') + condition_block) - elif_condition = Group(Suppress('elif') + condition + Suppress(':') + condition_block) - else_condition = Group(Suppress('else') + Suppress(':') + condition_block) - conditional = (if_condition + Optional(OneOrMore(elif_condition)) + Optional(else_condition)).setResultsName('conditional') - - def key_parse_action(pstr, loc, toks): - key = toks[0] - - if key in parse_ctx.keys: - raise ParseFatalException(pstr, loc, "duplicate key '%s' value definition" % parse_ctx.key) - - parse_ctx.key = key - parse_ctx.keys.append(key) - - try: - parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars()[key] - key_grammar = parse_ctx.key_grammar.grammar - except KeyError: - raise ParseFatalException(pstr, loc, "key '%s' is not supported by fragment" % key) - except Exception as e: - raise ParseFatalException(pstr, loc, "unable to parse key '%s'; %s" % (key, str(e))) - - key_stmt << (conditional | Group(key_grammar).setResultsName('value')) - - return None - - def name_parse_action(pstr, loc, toks): - parse_ctx.fragment.name = toks[0] - - key.setParseAction(key_parse_action) - - ftype = Word(alphas).setParseAction(fragment_type_parse_action) - fid = Suppress(':') + Word(alphanums + '_.').setResultsName('name') - fid.setParseAction(name_parse_action) - header = Suppress('[') + ftype + fid + Suppress(']') - - def fragment_parse_action(pstr, loc, toks): - key_grammars = parse_ctx.fragment.get_key_grammars() - required_keys = set([k for (k,v) in key_grammars.items() if v.required]) - present_keys = required_keys.intersection(set(parse_ctx.keys)) - if present_keys != required_keys: - raise ParseFatalException(pstr, loc, 'required keys %s for fragment not found' % - list(required_keys - present_keys)) - return parse_ctx.fragment - - fragment_stmt = Forward() - fragment_block = indentedBlock(fragment_stmt, indent_stack) - - fragment_if_condition = Group(Suppress('if') + condition + Suppress(':') + fragment_block) - fragment_elif_condition = Group(Suppress('elif') + condition + Suppress(':') + fragment_block) - fragment_else_condition = Group(Suppress('else') + Suppress(':') + fragment_block) - fragment_conditional = (fragment_if_condition + Optional(OneOrMore(fragment_elif_condition)) + - Optional(fragment_else_condition)).setResultsName('conditional') - - fragment = (header + OneOrMore(indentedBlock(key_body, indent_stack, False))).setResultsName('value') - fragment.setParseAction(fragment_parse_action) - fragment.ignore('#' + restOfLine) - - deprecated_mapping = DeprecatedMapping.get_fragment_grammar(sdkconfig, fragment_file.name).setResultsName('value') - - fragment_stmt << (Group(deprecated_mapping) | Group(fragment) | Group(fragment_conditional)) - - def fragment_stmt_parsed(pstr, loc, toks): - stmts = list() - expand_conditionals(toks, stmts) - return stmts - - parser = ZeroOrMore(fragment_stmt) - parser.setParseAction(fragment_stmt_parsed) - - self.fragments = parser.parseFile(fragment_file, parseAll=True) - - for fragment in self.fragments: - fragment.path = path - - -class Fragment(): - """ - Base class for a fragment that can be parsed from a fragment file. All fragments - share the common grammar: - - [type:name] - key1:value1 - key2:value2 - ... - - Supporting a new fragment type means deriving a concrete class which specifies - key-value pairs that the fragment supports and what to do with the parsed key-value pairs. - - The new fragment must also be appended to FRAGMENT_TYPES, specifying the - keyword for the type and the derived class. - - The key of the key-value pair is a simple keyword string. Other parameters - that describe the key-value pair is specified in Fragment.KeyValue: - 1. grammar - pyparsing grammar to parse the value of key-value pair - 2. min - the minimum number of value in the key entry, None means no minimum - 3. max - the maximum number of value in the key entry, None means no maximum - 4. required - if the key-value pair is required in the fragment - - Setting min=max=1 means that the key has a single value. - - FragmentFile provides conditional expression evaluation, enforcing - the parameters for Fragment.Keyvalue. - """ - __metaclass__ = abc.ABCMeta - - KeyValue = namedtuple('KeyValue', 'grammar min max required') - - IDENTIFIER = Word(alphas + '_', alphanums + '_') - ENTITY = Word(alphanums + '.-_$+') - - @abc.abstractmethod - def set_key_value(self, key, parse_results): - pass - - @abc.abstractmethod - def get_key_grammars(self): - pass - - -class Sections(Fragment): - """ - Fragment which contains list of input sections. - - [sections:] - entries: - .section1 - .section2 - ... - """ - - # Unless quoted, symbol names start with a letter, underscore, or point - # and may include any letters, underscores, digits, points, and hyphens. - GNU_LD_SYMBOLS = Word(alphas + '_.', alphanums + '._-') - - entries_grammar = Combine(GNU_LD_SYMBOLS + Optional('+')) - - grammars = { - 'entries': Fragment.KeyValue(entries_grammar.setResultsName('section'), 1, None, True) - } - - """ - Utility function that returns a list of sections given a sections fragment entry, - with the '+' notation and symbol concatenation handled automatically. - """ - @staticmethod - def get_section_data_from_entry(sections_entry, symbol=None): - if not symbol: - sections = list() - sections.append(sections_entry.replace('+', '')) - sections.append(sections_entry.replace('+', '.*')) - return sections - else: - if sections_entry.endswith('+'): - section = sections_entry.replace('+', '.*') - expansion = section.replace('.*', '.' + symbol) - return (section, expansion) - else: - return (sections_entry, None) - - def set_key_value(self, key, parse_results): - if key == 'entries': - self.entries = set() - for result in parse_results: - self.entries.add(result['section']) - - def get_key_grammars(self): - return self.__class__.grammars - - -class Scheme(Fragment): - """ - Fragment which defines where the input sections defined in a Sections fragment - is going to end up, the target. The targets are markers in a linker script template - (see LinkerScript in linker_script.py). - - [scheme:] - entries: - sections1 -> target1 - ... - """ - - grammars = { - 'entries': Fragment.KeyValue(Fragment.IDENTIFIER.setResultsName('sections') + Suppress('->') + - Fragment.IDENTIFIER.setResultsName('target'), 1, None, True) - } - - def set_key_value(self, key, parse_results): - if key == 'entries': - self.entries = set() - for result in parse_results: - self.entries.add((result['sections'], result['target'])) - - def get_key_grammars(self): - return self.__class__.grammars - - -class Mapping(Fragment): - """ - Fragment which attaches a scheme to entities (see Entity in entity.py), specifying where the input - sections of the entity will end up. - - [mapping:] - archive: lib1.a - entries: - obj1:symbol1 (scheme1); section1 -> target1 KEEP SURROUND(sym1) ... - obj2 (scheme2) - ... - - Ultimately, an `entity (scheme)` entry generates an - input section description (see https://sourceware.org/binutils/docs/ld/Input-Section.html) - in the output linker script. It is possible to attach 'flags' to the - `entity (scheme)` to generate different output commands or to - emit additional keywords in the generated input section description. The - input section description, as well as other output commands, is defined in - output_commands.py. - """ - - class Flag(): - PRE_POST = (Optional(Suppress(',') + Suppress('pre').setParseAction(lambda: True).setResultsName('pre')) + - Optional(Suppress(',') + Suppress('post').setParseAction(lambda: True).setResultsName('post'))) - - class Surround(Flag): - def __init__(self, symbol): - self.symbol = symbol - self.pre = True - self.post = True - - @staticmethod - def get_grammar(): - # SURROUND(symbol) - # - # '__symbol_start', '__symbol_end' is generated before and after - # the corresponding input section description, respectively. - grammar = (Keyword('SURROUND').suppress() + - Suppress('(') + - Fragment.IDENTIFIER.setResultsName('symbol') + - Suppress(')')) - - grammar.setParseAction(lambda tok: Mapping.Surround(tok.symbol)) - return grammar - - def __eq__(self, other): - return (isinstance(other, Mapping.Surround) and - self.symbol == other.symbol) - - class Align(Flag): - - def __init__(self, alignment, pre=True, post=False): - self.alignment = alignment - self.pre = pre - self.post = post - - @staticmethod - def get_grammar(): - # ALIGN(alignment, [, pre, post]). - # - # Generates alignment command before and/or after the corresponding - # input section description, depending whether pre, post or - # both are specified. - grammar = (Keyword('ALIGN').suppress() + - Suppress('(') + - Word(nums).setResultsName('alignment') + - Mapping.Flag.PRE_POST + - Suppress(')')) - - def on_parse(tok): - alignment = int(tok.alignment) - if tok.pre == '' and tok.post == '': - res = Mapping.Align(alignment) - elif tok.pre != '' and tok.post == '': - res = Mapping.Align(alignment, tok.pre) - elif tok.pre == '' and tok.post != '': - res = Mapping.Align(alignment, False, tok.post) - else: - res = Mapping.Align(alignment, tok.pre, tok.post) - return res - - grammar.setParseAction(on_parse) - return grammar - - def __eq__(self, other): - return (isinstance(other, Mapping.Align) and - self.alignment == other.alignment and - self.pre == other.pre and - self.post == other.post) - - class Keep(Flag): - - def __init__(self): - pass - - @staticmethod - def get_grammar(): - # KEEP() - # - # Surrounds input section description with KEEP command. - grammar = Keyword('KEEP()').setParseAction(Mapping.Keep) - return grammar - - def __eq__(self, other): - return isinstance(other, Mapping.Keep) - - class Sort(Flag): - class Type(Enum): - NAME = 0 - ALIGNMENT = 1 - INIT_PRIORITY = 2 - - def __init__(self, first, second=None): - self.first = first - self.second = second - - @staticmethod - def get_grammar(): - # SORT([sort_by_first, sort_by_second]) - # - # where sort_by_first, sort_by_second = {name, alignment, init_priority} - # - # Emits SORT_BY_NAME, SORT_BY_ALIGNMENT or SORT_BY_INIT_PRIORITY - # depending on arguments. Nested sort follows linker script rules. - keywords = Keyword('name') | Keyword('alignment') | Keyword('init_priority') - grammar = (Keyword('SORT').suppress() + Suppress('(') + - keywords.setResultsName('first') + - Optional(Suppress(',') + keywords.setResultsName('second')) + Suppress(')')) - - grammar.setParseAction(lambda tok: Mapping.Sort(tok.first, tok.second if tok.second != '' else None)) - return grammar - - def __eq__(self, other): - return (isinstance(other, Mapping.Sort) and - self.first == other.first and - self.second == other.second) - - def __init__(self): - Fragment.__init__(self) - self.entries = set() - # k = (obj, symbol, scheme) - # v = list((section, target), Mapping.Flag)) - self.flags = dict() - self.deprecated = False - - def set_key_value(self, key, parse_results): - if key == 'archive': - self.archive = parse_results[0]['archive'] - elif key == 'entries': - for result in parse_results: - obj = None - symbol = None - scheme = None - - obj = result['object'] - - try: - symbol = result['symbol'] - except KeyError: - pass - - scheme = result['scheme'] - - mapping = (obj, symbol, scheme) - self.entries.add(mapping) - - try: - parsed_flags = result['sections_target_flags'] - except KeyError: - parsed_flags = [] - - if parsed_flags: - entry_flags = [] - for pf in parsed_flags: - entry_flags.append((pf.sections, pf.target, list(pf.flags))) - - try: - existing_flags = self.flags[mapping] - except KeyError: - existing_flags = list() - self.flags[mapping] = existing_flags - - existing_flags.extend(entry_flags) - - def get_key_grammars(self): - # There are three possible patterns for mapping entries: - # obj:symbol (scheme) - # obj (scheme) - # * (scheme) - # Flags can be specified for section->target in the scheme specified, ex: - # obj (scheme); section->target SURROUND(symbol), section2->target2 ALIGN(4) - obj = Fragment.ENTITY.setResultsName('object') - symbol = Suppress(':') + Fragment.IDENTIFIER.setResultsName('symbol') - scheme = Suppress('(') + Fragment.IDENTIFIER.setResultsName('scheme') + Suppress(')') - - # The flags are specified for section->target in the scheme specified - sections_target = Scheme.grammars['entries'].grammar - - flag = Or([f.get_grammar() for f in [Mapping.Keep, Mapping.Align, Mapping.Surround, Mapping.Sort]]) - - section_target_flags = Group(sections_target + Group(OneOrMore(flag)).setResultsName('flags')) - - pattern1 = obj + symbol - pattern2 = obj - pattern3 = Literal(Entity.ALL).setResultsName('object') - - entry = ((pattern1 | pattern2 | pattern3) + scheme + - Optional(Suppress(';') + delimitedList(section_target_flags).setResultsName('sections_target_flags'))) - - grammars = { - 'archive': Fragment.KeyValue(Or([Fragment.ENTITY, Word(Entity.ALL)]).setResultsName('archive'), 1, 1, True), - 'entries': Fragment.KeyValue(entry, 0, None, True) - } - - return grammars - - -class DeprecatedMapping(): - """ - Mapping fragment with old grammar in versions older than ESP-IDF v4.0. Does not conform to - requirements of the Fragment class and thus is limited when it comes to conditional expression - evaluation. - """ - - # Name of the default condition entry - DEFAULT_CONDITION = 'default' - - @staticmethod - def get_fragment_grammar(sdkconfig, fragment_file): - - # Match header [mapping] - header = Suppress('[') + Suppress('mapping') + Suppress(']') - - # There are three possible patterns for mapping entries: - # obj:symbol (scheme) - # obj (scheme) - # * (scheme) - obj = Fragment.ENTITY.setResultsName('object') - symbol = Suppress(':') + Fragment.IDENTIFIER.setResultsName('symbol') - scheme = Suppress('(') + Fragment.IDENTIFIER.setResultsName('scheme') + Suppress(')') - - pattern1 = Group(obj + symbol + scheme) - pattern2 = Group(obj + scheme) - pattern3 = Group(Literal(Entity.ALL).setResultsName('object') + scheme) - - mapping_entry = pattern1 | pattern2 | pattern3 - - # To simplify parsing, classify groups of condition-mapping entry into two types: normal and default - # A normal grouping is one with a non-default condition. The default grouping is one which contains the - # default condition - mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName('mappings') - - normal_condition = Suppress(':') + originalTextFor(SDKConfig.get_expression_grammar()) - default_condition = Optional(Suppress(':') + Literal(DeprecatedMapping.DEFAULT_CONDITION)) - - normal_group = Group(normal_condition.setResultsName('condition') + mapping_entries) - default_group = Group(default_condition + mapping_entries).setResultsName('default_group') - - normal_groups = Group(ZeroOrMore(normal_group)).setResultsName('normal_groups') - - # Any mapping fragment definition can have zero or more normal group and only one default group as a last entry. - archive = Suppress('archive') + Suppress(':') + Fragment.ENTITY.setResultsName('archive') - entries = Suppress('entries') + Suppress(':') + (normal_groups + default_group).setResultsName('entries') - - mapping = Group(header + archive + entries) - mapping.ignore('#' + restOfLine) - - def parsed_deprecated_mapping(pstr, loc, toks): - fragment = Mapping() - fragment.archive = toks[0].archive - fragment.name = re.sub(r'[^0-9a-zA-Z]+', '_', fragment.archive) - fragment.deprecated = True - - fragment.entries = set() - condition_true = False - for entries in toks[0].entries[0]: - condition = next(iter(entries.condition.asList())).strip() - condition_val = sdkconfig.evaluate_expression(condition) - - if condition_val: - for entry in entries[1]: - fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme)) - condition_true = True - break - - if not fragment.entries and not condition_true: - try: - entries = toks[0].entries[1][1] - except IndexError: - entries = toks[0].entries[1][0] - for entry in entries: - fragment.entries.add((entry.object, None if entry.symbol == '' else entry.symbol, entry.scheme)) - - if not fragment.entries: - fragment.entries.add(('*', None, 'default')) - - dep_warning = str(ParseFatalException(pstr, loc, - 'Warning: Deprecated old-style mapping fragment parsed in file %s.' % fragment_file)) - - print(dep_warning) - return fragment - - mapping.setParseAction(parsed_deprecated_mapping) - return mapping - - -FRAGMENT_TYPES = { - 'sections': Sections, - 'scheme': Scheme, - 'mapping': Mapping -} diff --git a/tools/ldgen/ldgen.py b/tools/ldgen/ldgen.py index a621c12423..767b34bbc7 100755 --- a/tools/ldgen/ldgen.py +++ b/tools/ldgen/ldgen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -13,13 +13,13 @@ import sys import tempfile from io import StringIO -from entity import EntityDB -from fragments import FragmentFile -from generation import Generation -from ldgen_common import LdGenFailure -from linker_script import LinkerScript +from ldgen.entity import EntityDB +from ldgen.fragments import parse_fragment_file +from ldgen.generation import Generation +from ldgen.ldgen_common import LdGenFailure +from ldgen.linker_script import LinkerScript +from ldgen.sdkconfig import SDKConfig from pyparsing import ParseException, ParseFatalException -from sdkconfig import SDKConfig try: import confgen @@ -148,12 +148,12 @@ def main(): for fragment_file in fragment_files: try: - fragment_file = FragmentFile(fragment_file, sdkconfig) + fragment_file = parse_fragment_file(fragment_file, sdkconfig) except (ParseException, ParseFatalException) as e: # ParseException is raised on incorrect grammar # ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate # keys, key unsupported by fragment, unexpected number of values, etc.) - raise LdGenFailure('failed to parse %s\n%s' % (fragment_file.name, str(e))) + raise LdGenFailure('failed to parse %s\n%s' % (fragment_file, str(e))) generation_model.add_fragments_from_file(fragment_file) mapping_rules = generation_model.generate(sections_infos) diff --git a/tools/ldgen/__init__.py b/tools/ldgen/ldgen/__init__.py similarity index 100% rename from tools/ldgen/__init__.py rename to tools/ldgen/ldgen/__init__.py diff --git a/tools/ldgen/entity.py b/tools/ldgen/ldgen/entity.py similarity index 81% rename from tools/ldgen/entity.py rename to tools/ldgen/ldgen/entity.py index 065f5828be..35021a51b5 100644 --- a/tools/ldgen/entity.py +++ b/tools/ldgen/ldgen/entity.py @@ -1,17 +1,6 @@ # -# Copyright 2021 Espressif Systems (Shanghai) CO LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 # import collections @@ -21,11 +10,11 @@ from enum import Enum from functools import total_ordering from pyparsing import (Group, Literal, OneOrMore, ParseException, SkipTo, Suppress, White, Word, ZeroOrMore, alphas, - nums, restOfLine) + nums, rest_of_line) @total_ordering -class Entity(): +class Entity: """ An entity refers to a library, object, symbol whose input sections can be placed or excluded from placement. @@ -60,7 +49,7 @@ class Entity(): else: raise ValueError("Invalid arguments '(%s, %s, %s)'" % (archive, obj, symbol)) - self.archive = archive + self.archive = archive self.obj = obj self.symbol = symbol @@ -93,7 +82,7 @@ class Entity(): return '%s:%s %s' % self.__repr__() def __repr__(self): - return (self.archive, self.obj, self.symbol) + return self.archive, self.obj, self.symbol def __getitem__(self, spec): res = None @@ -108,7 +97,7 @@ class Entity(): return res -class EntityDB(): +class EntityDB: """ Collection of entities extracted from libraries known in the build. Allows retrieving a list of archives, a list of object files in an archive @@ -127,11 +116,10 @@ class EntityDB(): archive_path = (Literal('In archive').suppress() + White().suppress() + # trim the colon and line ending characters from archive_path - restOfLine.setResultsName('archive_path').setParseAction(lambda s, loc, toks: s.rstrip(':\n\r '))) + rest_of_line.set_results_name('archive_path').set_parse_action( + lambda s, loc, toks: s.rstrip(':\n\r '))) parser = archive_path - results = None - try: results = parser.parseString(first_line, parseAll=True) except ParseException as p: @@ -142,7 +130,7 @@ class EntityDB(): def _get_infos_from_file(self, info): # {object}: file format elf32-xtensa-le - object_line = SkipTo(':').setResultsName('object') + Suppress(restOfLine) + object_line = SkipTo(':').set_results_name('object') + Suppress(rest_of_line) # Sections: # Idx Name ... @@ -151,13 +139,14 @@ class EntityDB(): # 00 {section} 0000000 ... # CONTENTS, ALLOC, .... - section_entry = Suppress(Word(nums)) + SkipTo(' ') + Suppress(restOfLine) + \ - Suppress(ZeroOrMore(Word(alphas) + Literal(',')) + Word(alphas)) + section_entry = (Suppress(Word(nums)) + SkipTo(' ') + Suppress(rest_of_line) + + Suppress(ZeroOrMore(Word(alphas) + Literal(',')) + Word(alphas))) - content = Group(object_line + section_start + section_header + Group(OneOrMore(section_entry)).setResultsName('sections')) - parser = Group(ZeroOrMore(content)).setResultsName('contents') - - results = None + content = Group(object_line + + section_start + + section_header + + Group(OneOrMore(section_entry)).set_results_name('sections')) + parser = Group(ZeroOrMore(content)).set_results_name('contents') try: results = parser.parseString(info.content, parseAll=True) @@ -192,7 +181,9 @@ class EntityDB(): def _match_obj(self, archive, obj): objs = self.get_objects(archive) - match_objs = fnmatch.filter(objs, obj + '.o') + fnmatch.filter(objs, obj + '.*.obj') + fnmatch.filter(objs, obj + '.obj') + match_objs = (fnmatch.filter(objs, obj + '.o') + + fnmatch.filter(objs, obj + '.*.obj') + + fnmatch.filter(objs, obj + '.obj')) if len(match_objs) > 1: raise ValueError("Multiple matches for object: '%s: %s': %s" % (archive, obj, str(match_objs))) diff --git a/tools/ldgen/ldgen/fragments.py b/tools/ldgen/ldgen/fragments.py new file mode 100644 index 0000000000..ea17f65b06 --- /dev/null +++ b/tools/ldgen/ldgen/fragments.py @@ -0,0 +1,473 @@ +# +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +# + +from typing import Any, Dict, List, Optional, Set, Tuple, Union + +from pyparsing import (Combine, Forward, Group, IndentedBlock, Keyword, LineEnd, Literal, OneOrMore, Opt, + ParseFatalException, SkipTo, Suppress, Word, ZeroOrMore, alphanums, alphas, delimited_list, + nums, rest_of_line) + + +class Empty: + """ + Return `Empty()` when the sdkconfig does not meet the conditional statements. + """ + + def __repr__(self): + return '' + + def __bool__(self): + return False + + +class Fragment: + """ + Base class for a fragment that can be parsed from a fragment file. + """ + IDENTIFIER = Word(alphas + '_', alphanums + '_') + ENTITY = Word(alphanums + '.-_$+') + + def __init__(self, name: str, entries: Set[Union[str, Tuple[str]]]): + self.name = name + self.entries = entries + + def __repr__(self): + return str(self.__dict__) + + +class Sections(Fragment): + """ + Fragment which contains list of input sections. + + [sections:] + entries: + .section1 + .section2 + ... + """ + # Unless quoted, symbol names start with a letter, underscore, or point + # and may include any letters, underscores, digits, points, and hyphens. + ENTRY = Combine(Word(alphas + '_.', alphanums + '._-') + Opt('+')) + LineEnd().suppress() + + @staticmethod + def parse_entry(toks): + # section + return toks[0] + + @staticmethod + def parse(s, loc, toks): + this = toks[0] + + name = this[0] + entries = {entry for entry in this[1] if entry} + + if not entries: + raise ParseFatalException(s, loc, 'Sections entries shouldn\'t be empty') + + return Sections(name, entries) + + @staticmethod + def get_section_data_from_entry(sections_entry, symbol=None): + """ + Returns a list of sections given a sections fragment entry, + with the '+' notation and symbol concatenation handled automatically. + """ + if not symbol: + sections = list() + sections.append(sections_entry.replace('+', '')) + sections.append(sections_entry.replace('+', '.*')) + return sections + else: + if sections_entry.endswith('+'): + section = sections_entry.replace('+', '.*') + expansion = section.replace('.*', '.' + symbol) + return section, expansion + else: + return sections_entry, None + + +class Scheme(Fragment): + """ + Fragment which defines where the input sections defined in a Sections fragment + is going to end up, the target. The targets are markers in a linker script template + (see LinkerScript in linker_script.py). + + [scheme:] + entries: + sections1 -> target1 + ... + """ + ENTRY = Fragment.IDENTIFIER + Suppress('->') + Fragment.IDENTIFIER + LineEnd().suppress() + + @staticmethod + def parse_entry(toks): + # section, target + return toks[0], toks[1] + + @staticmethod + def parse(s, loc, toks): + this = toks[0] + + name = this[0] + entries = {entry for entry in this[1] if entry} + + if not entries: + raise ParseFatalException(s, loc, 'Scheme entries shouldn\'t be empty') + + return Scheme(name, entries) + + +class EntryFlag: + def __repr__(self): + return str(self.__dict__) + + +class Surround(EntryFlag): + """ + SURROUND(symbol) + + '__symbol_start', '__symbol_end' is generated before and after + the corresponding input section description, respectively. + """ + SURROUND = (Keyword('SURROUND').suppress() + + Suppress('(') + + Fragment.IDENTIFIER + + Suppress(')')) + + def __init__(self, symbol: str): + self.symbol = symbol + self.pre = True + self.post = True + + def __eq__(self, other): + if isinstance(other, Surround): + if self.symbol == other.symbol and self.pre == other.pre and self.post == other.post: + return True + + return False + + @staticmethod + def parse(toks): + return Surround(toks[0]) + + +class Align(EntryFlag): + """ + ALIGN(alignment, [, pre, post]). + + Generates alignment command before and/or after the corresponding + input section description, depending on whether pre, post or + both are specified. + """ + PRE = Opt(Suppress(',') + Suppress('pre')).set_results_name('pre') + POST = Opt(Suppress(',') + Suppress('post')).set_results_name('post') + + ALIGN = (Keyword('ALIGN').suppress() + + Suppress('(') + + Word(nums) + + PRE + + POST + + Suppress(')')) + + def __init__(self, alignment, pre=True, post=False): + self.alignment = alignment + self.pre = pre + self.post = post + + def __eq__(self, other): + if isinstance(other, Align): + if self.alignment == other.alignment and self.pre == other.pre and self.post == other.post: + return True + + return False + + @staticmethod + def parse(toks): + alignment = int(toks[0]) + if toks.post == '': + return Align(alignment) + + if toks.pre == '' and toks.post != '': + return Align(alignment, False, True) + + return Align(alignment, True, True) + + +class Keep(EntryFlag): + """ + KEEP() + + Surrounds input section description with KEEP command. + """ + KEEP = Keyword('KEEP()') + + def __eq__(self, other): + if isinstance(other, Keep): + return True + + return False + + @staticmethod + def parse(): + return Keep() + + +class Sort(EntryFlag): + """ + SORT([sort_by_first, sort_by_second]) + + where sort_by_first, sort_by_second = {name, alignment, init_priority} + + Emits SORT_BY_NAME, SORT_BY_ALIGNMENT or SORT_BY_INIT_PRIORITY + depending on arguments. Nested sort follows linker script rules. + """ + _keywords = Keyword('name') | Keyword('alignment') | Keyword('init_priority') + SORT = (Keyword('SORT').suppress() + + Suppress('(') + + _keywords.set_results_name('first') + + Opt(Suppress(',') + _keywords.set_results_name('second')) + + Suppress(')')) + + def __init__(self, first: str, second: Optional[str] = None): + self.first = first + self.second = second + + def __eq__(self, other): + if isinstance(other, Sort): + if self.first == other.first and self.second == other.second: + return True + + return False + + @staticmethod + def parse(toks): + return Sort(toks.first, toks.second or None) + + +class Flag: + _section_target = Fragment.IDENTIFIER + Suppress('->') + Fragment.IDENTIFIER + _flag = (Surround.SURROUND.set_parse_action(Surround.parse) + | Align.ALIGN.set_parse_action(Align.parse) + | Keep.KEEP.set_parse_action(Keep.parse) + | Sort.SORT.set_parse_action(Sort.parse)) + + FLAG = _section_target + OneOrMore(_flag) + + def __init__(self, section: str, target: str, flags: List[EntryFlag]): + self.section = section + self.target = target + self.flags = flags + + def __eq__(self, other): + if isinstance(other, Flag): + if self.section == other.section and self.target == other.target and len(self.flags) == len(other.flags): + for i, j in zip(self.flags, other.flags): + if i != j: + break + else: + return True + + return False + + @staticmethod + def parse(toks): + return Flag(toks[0], toks[1], toks[2:]) + + def __repr__(self): + return str(self.__dict__) + + +class Mapping(Fragment): + """ + Fragment which attaches a scheme to entities (see Entity in entity.py), specifying where the input + sections of the entity will end up. + + [mapping:] + archive: lib1.a + entries: + obj1:symbol1 (scheme1); section1 -> target1 KEEP SURROUND(sym1) ... + obj2 (scheme2) + ... + + Ultimately, an `entity (scheme)` entry generates an + input section description (see https://sourceware.org/binutils/docs/ld/Input-Section.html) + in the output linker script. It is possible to attach 'flags' to the + `entity (scheme)` to generate different output commands or to + emit additional keywords in the generated input section description. The + input section description, as well as other output commands, is defined in + output_commands.py. + """ + + _any = Literal('*') + _obj = Word(alphas + '_', alphanums + '-_').set_results_name('object') + _sym = Fragment.IDENTIFIER.set_results_name('symbol') + + # There are three possible patterns for mapping entries: + # obj:symbol (scheme) + # obj (scheme) + # * (scheme) + _entry = (((_obj + Opt(Suppress(':') + _sym)) | _any.set_results_name('object')) + + Suppress('(') + + Fragment.IDENTIFIER.set_results_name('section') + + Suppress(')')) + + ENTRY = _entry + LineEnd().suppress() + ARCHIVE = (Word(alphanums + '.-_$+') | Literal('*')) + LineEnd().suppress() + + # Flags can be specified for section->target in the scheme specified, ex: + # obj (scheme); + # section->target SURROUND(symbol), + # section2->target2 ALIGN(4) + ENTRY_WITH_FLAG = (_entry + Suppress(';') + + delimited_list(Flag.FLAG.set_parse_action(Flag.parse))) + + def __init__(self, archive: str, flags: Dict[Any, Flag], *args, **kwargs): + super().__init__(*args, **kwargs) + self.archive = archive + self.flags = flags + + @staticmethod + def parse_archive(s, loc, toks): + this = toks[0][0] + if len(this) != 1: + raise ParseFatalException(s, loc, 'Could only specify one archive file in one mapping fragment') + + return this[0] + + @staticmethod + def parse_entry(toks): + return toks.object, toks.symbol or None, toks.section + + @staticmethod + def parse_entry_with_flag(toks): + entry = toks.object, toks.symbol or None, toks.section + return { + entry: [tok for tok in toks if isinstance(tok, Flag)] + } + + @staticmethod + def parse_entries(toks): + return toks[0] + + @staticmethod + def parse(toks): + this = toks[0] + + name = this[0] + archive = this[1] + entries_or_dict_with_flags = this[2] + + entries = set() + flags = dict() + for item in entries_or_dict_with_flags: + if isinstance(item, Empty): + continue + elif isinstance(item, dict): # entry with flags + for k, v in item.items(): + entries.add(k) + if k in flags: + flags[k].extend(v) + else: + flags[k] = v + else: + entries.add(item) + + return Mapping(archive=archive, name=name, entries=entries, flags=flags) + + +class FragmentFile: + """ + Processes a fragment file and stores all parsed fragments. For + more information on how this class interacts with classes for the different fragment types, + see description of Fragment. + """ + + def __init__(self, fragments: List[Fragment]): + self.path = None # assign later, couldn't pass extra argument while parsing + self.fragments: List[Fragment] = fragments + + def __repr__(self): + return str(self.__dict__) + + +def parse_fragment_file(path, sdkconfig): + def parse_conditional(toks): + this = toks[0] + for stmt in this: + if stmt[0] in ['if', 'elif']: # if/elif + if sdkconfig.evaluate_expression(stmt.condition): + return stmt[-1] + else: # else + return stmt[-1] + + return Empty() + + def get_conditional_stmt(_stmt): + condition = SkipTo(':').set_results_name('condition') + Suppress(':') + _suite = IndentedBlock(_stmt) + + if_decl = Literal('if') + condition + elif_decl = Literal('elif') + condition + else_decl = Literal('else:') + if_ = Group(if_decl + _suite) + elif_ = Group(elif_decl + _suite) + else_ = Group(else_decl + _suite) + return Group(if_ + Opt(OneOrMore(elif_)) + Opt(else_)).set_parse_action(parse_conditional) + + def get_suite(_stmt): + __stmt = Forward() + __conditional = get_conditional_stmt(__stmt) + __stmt <<= (comment + | _stmt + | __conditional) + return IndentedBlock(__stmt) + + def parse(toks): + return FragmentFile([tok for tok in toks if not isinstance(tok, Empty)]) + + # comment + comment = (Literal('#') + rest_of_line).set_parse_action(lambda s, l, t: Empty()) + + # section + section_entry = Sections.ENTRY.set_parse_action(Sections.parse_entry) + section_entries_suite = get_suite(section_entry) + section_header = Suppress('[sections:') + Fragment.IDENTIFIER + Suppress(']') + LineEnd().suppress() + section = Group(section_header + + Suppress('entries:') + + section_entries_suite).set_parse_action(Sections.parse) + + # scheme + scheme_entry = Scheme.ENTRY.set_parse_action(Scheme.parse_entry) + scheme_entries_suite = get_suite(scheme_entry) + scheme_header = Suppress('[scheme:') + Fragment.IDENTIFIER + Suppress(']') + LineEnd().suppress() + scheme = Group(scheme_header + + Suppress('entries:') + + scheme_entries_suite).set_parse_action(Scheme.parse) + # mapping + mapping_archive = Mapping.ARCHIVE + mapping_archive_suite = get_suite(mapping_archive) + + mapping_entry = Mapping.ENTRY.set_parse_action(Mapping.parse_entry) + mapping_entry_with_flag = Mapping.ENTRY_WITH_FLAG.set_parse_action(Mapping.parse_entry_with_flag) + mapping_entries_suite = get_suite(mapping_entry | mapping_entry_with_flag) + + mapping_header = Suppress('[mapping:') + Fragment.IDENTIFIER + Suppress(']') + mapping = Group(mapping_header + + Group(Suppress('archive:') + + mapping_archive_suite).set_parse_action(Mapping.parse_archive) + + Group(Suppress('entries:') + + mapping_entries_suite).set_parse_action(Mapping.parse_entries) + ).set_parse_action(Mapping.parse) + + # highest level + fragment = (section + | scheme + | mapping + | get_conditional_stmt(section | scheme | mapping)) + parser = ZeroOrMore(fragment).ignore(comment).set_parse_action(parse) + fragment_file = parser.parse_file(path, parse_all=True)[0] + fragment_file.path = path + + return fragment_file diff --git a/tools/ldgen/generation.py b/tools/ldgen/ldgen/generation.py similarity index 87% rename from tools/ldgen/generation.py rename to tools/ldgen/ldgen/generation.py index 74a6d0ba60..f4e87d849e 100644 --- a/tools/ldgen/generation.py +++ b/tools/ldgen/ldgen/generation.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -8,13 +8,13 @@ import fnmatch import itertools from collections import namedtuple -from entity import Entity -from fragments import Mapping, Scheme, Sections -from ldgen_common import LdGenFailure -from output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress +from .entity import Entity +from .fragments import Keep, Scheme, Sections, Sort, Surround +from .ldgen_common import LdGenFailure +from .output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress -class Placement(): +class Placement: """ A Placement is an assignment of an entity's input sections to a target in the output linker script - a precursor to the input section description. @@ -97,7 +97,7 @@ class Placement(): self.subplacements.add(subplacement) -class EntityNode(): +class EntityNode: """ Node in entity tree. An EntityNode is created from an Entity (see entity.py). @@ -134,12 +134,12 @@ class EntityNode(): def add_child(self, entity): child_specificity = self.entity.specificity.value + 1 - assert(child_specificity <= Entity.Specificity.SYMBOL.value) + assert (child_specificity <= Entity.Specificity.SYMBOL.value) name = entity[Entity.Specificity(child_specificity)] - assert(name and name != Entity.ALL) + assert (name and name != Entity.ALL) child = [c for c in self.children if c.name == name] - assert(len(child) <= 1) + assert (len(child) <= 1) if not child: child = self.child_t(self, name) @@ -174,7 +174,7 @@ class EntityNode(): for sections in self.get_output_sections(): placement = self.placements[sections] if placement.is_significant(): - assert(placement.node == self) + assert (placement.node == self) keep = False sort = None @@ -183,16 +183,16 @@ class EntityNode(): placement_flags = placement.flags if placement.flags is not None else [] for flag in placement_flags: - if isinstance(flag, Mapping.Keep): + if isinstance(flag, Keep): keep = True - elif isinstance(flag, Mapping.Sort): + elif isinstance(flag, Sort): sort = (flag.first, flag.second) else: # SURROUND or ALIGN surround_type.append(flag) for flag in surround_type: if flag.pre: - if isinstance(flag, Mapping.Surround): + if isinstance(flag, Surround): commands[placement.target].append(SymbolAtAddress('_%s_start' % flag.symbol)) else: # ALIGN commands[placement.target].append(AlignAtAddress(flag.alignment)) @@ -202,11 +202,12 @@ class EntityNode(): placement_sections = frozenset(placement.sections) command_sections = sections if sections == placement_sections else placement_sections - command = InputSectionDesc(placement.node.entity, command_sections, [e.node.entity for e in placement.exclusions], keep, sort) + command = InputSectionDesc(placement.node.entity, command_sections, + [e.node.entity for e in placement.exclusions], keep, sort) commands[placement.target].append(command) - # Generate commands for intermediate, non-explicit exclusion placements here, so that they can be enclosed by - # flags that affect the parent placement. + # Generate commands for intermediate, non-explicit exclusion placements here, + # so that they can be enclosed by flags that affect the parent placement. for subplacement in placement.subplacements: if not subplacement.flags and not subplacement.explicit: command = InputSectionDesc(subplacement.node.entity, subplacement.sections, @@ -215,7 +216,7 @@ class EntityNode(): for flag in surround_type: if flag.post: - if isinstance(flag, Mapping.Surround): + if isinstance(flag, Surround): commands[placement.target].append(SymbolAtAddress('_%s_end' % flag.symbol)) else: # ALIGN commands[placement.target].append(AlignAtAddress(flag.alignment)) @@ -248,6 +249,7 @@ class SymbolNode(EntityNode): Entities at depth=3. Represents entities with archive, object and symbol specified. """ + def __init__(self, parent, name): EntityNode.__init__(self, parent, name) self.entity = Entity(self.parent.parent.name, self.parent.name) @@ -270,6 +272,7 @@ class ObjectNode(EntityNode): An intermediate placement on this node is created, if one does not exist, and is the one excluded from its basis placement. """ + def __init__(self, parent, name): EntityNode.__init__(self, parent, name) self.child_t = SymbolNode @@ -334,6 +337,7 @@ class ArchiveNode(EntityNode): """ Entities at depth=1. Represents entities with archive specified. """ + def __init__(self, parent, name): EntityNode.__init__(self, parent, name) self.child_t = ObjectNode @@ -345,6 +349,7 @@ class RootNode(EntityNode): Single entity at depth=0. Represents entities with no specific members specified. """ + def __init__(self): EntityNode.__init__(self, None, Entity.ALL) self.child_t = ArchiveNode @@ -382,7 +387,7 @@ class Generation: for (sections_name, target_name) in scheme.entries: # Get the sections under the bucket 'target_name'. If this bucket does not exist - # is is created automatically + # is created automatically sections_in_bucket = sections_bucket[target_name] try: @@ -433,9 +438,9 @@ class Generation: entity = Entity(archive, obj, symbol) # Check the entity exists - if (self.check_mappings and - entity.specificity.value > Entity.Specificity.ARCHIVE.value and - mapping.name not in self.check_mapping_exceptions): + if (self.check_mappings + and entity.specificity.value > Entity.Specificity.ARCHIVE.value + and mapping.name not in self.check_mapping_exceptions): if not entities.check_exists(entity): message = "'%s' not found" % str(entity) raise GenerationException(message, mapping) @@ -444,11 +449,11 @@ class Generation: flags = mapping.flags[(obj, symbol, scheme_name)] # Check if all section->target defined in the current # scheme. - for (s, t, f) in flags: - if (t not in scheme_dictionary[scheme_name].keys() or - s not in [_s.name for _s in scheme_dictionary[scheme_name][t]]): - - message = "%s->%s not defined in scheme '%s'" % (s, t, scheme_name) + for flag in flags: + if (flag.target not in scheme_dictionary[scheme_name].keys() + or flag.section not in + [_s.name for _s in scheme_dictionary[scheme_name][flag.target]]): + message = "%s->%s not defined in scheme '%s'" % (flag.section, flag.target, scheme_name) raise GenerationException(message, mapping) else: flags = None @@ -460,9 +465,9 @@ class Generation: _flags = [] if flags: - for (s, t, f) in flags: - if (s, t) == (section.name, target): - _flags.extend(f) + for flag in flags: + if (flag.section, flag.target) == (section.name, target): + _flags.extend(flag.flags) sections_str = get_section_strs(section) @@ -477,18 +482,18 @@ class Generation: entity_mappings[key] = Generation.EntityMapping(entity, sections_str, target, _flags) else: # Check for conflicts. - if (target != existing.target): + if target != existing.target: raise GenerationException('Sections mapped to multiple targets.', mapping) # Combine flags here if applicable, to simplify # insertion logic. - if (_flags or existing.flags): - if ((_flags and not existing.flags) or (not _flags and existing.flags)): + if _flags or existing.flags: + if (_flags and not existing.flags) or (not _flags and existing.flags): _flags.extend(existing.flags) entity_mappings[key] = Generation.EntityMapping(entity, sections_str, target, _flags) - elif (_flags == existing.flags): + elif _flags == existing.flags: pass else: raise GenerationException('Conflicting flags specified.', mapping) @@ -517,26 +522,22 @@ class Generation: def add_fragments_from_file(self, fragment_file): for fragment in fragment_file.fragments: - dict_to_append_to = None - - if isinstance(fragment, Mapping) and fragment.deprecated and fragment.name in self.mappings.keys(): - self.mappings[fragment.name].entries |= fragment.entries + if isinstance(fragment, Scheme): + dict_to_append_to = self.schemes + elif isinstance(fragment, Sections): + dict_to_append_to = self.placements else: - if isinstance(fragment, Scheme): - dict_to_append_to = self.schemes - elif isinstance(fragment, Sections): - dict_to_append_to = self.placements - else: - dict_to_append_to = self.mappings + dict_to_append_to = self.mappings - # Raise exception when the fragment of the same type is already in the stored fragments - if fragment.name in dict_to_append_to.keys(): - stored = dict_to_append_to[fragment.name].path - new = fragment.path - message = "Duplicate definition of fragment '%s' found in %s and %s." % (fragment.name, stored, new) - raise GenerationException(message) + # Raise exception when the fragment of the same type is already in the stored fragments + if fragment.name in dict_to_append_to: + stored = dict_to_append_to[fragment.name].path + new = fragment.path + message = "Duplicate definition of fragment '%s' found in %s and %s." % ( + fragment.name, stored, new) + raise GenerationException(message) - dict_to_append_to[fragment.name] = fragment + dict_to_append_to[fragment.name] = fragment class GenerationException(LdGenFailure): diff --git a/tools/ldgen/ldgen/ldgen_common.py b/tools/ldgen/ldgen/ldgen_common.py new file mode 100644 index 0000000000..ba4346cb14 --- /dev/null +++ b/tools/ldgen/ldgen/ldgen_common.py @@ -0,0 +1,9 @@ +# +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +# + +class LdGenFailure(RuntimeError): + """ + Parent class for any ldgen runtime failure which is due to input data + """ diff --git a/tools/ldgen/linker_script.py b/tools/ldgen/ldgen/linker_script.py similarity index 86% rename from tools/ldgen/linker_script.py rename to tools/ldgen/ldgen/linker_script.py index 968ac776a5..59405e2fc9 100644 --- a/tools/ldgen/linker_script.py +++ b/tools/ldgen/ldgen/linker_script.py @@ -1,15 +1,16 @@ # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # import collections import os -from fragments import Fragment -from generation import GenerationException from pyparsing import ParseException, Suppress, White +from .fragments import Fragment +from .generation import GenerationException + class LinkerScript: """ @@ -32,24 +33,21 @@ class LinkerScript: lines = template_file.readlines() target = Fragment.IDENTIFIER - reference = Suppress('mapping') + Suppress('[') + target.setResultsName('target') + Suppress(']') - pattern = White(' \t').setResultsName('indent') + reference + reference = Suppress('mapping') + Suppress('[') + target + Suppress(']') + pattern = White(' \t') + reference # Find the markers in the template file line by line. If line does not match marker grammar, # set it as a literal to be copied as is to the output file. for line in lines: try: - parsed = pattern.parseString(line) - - indent = parsed.indent - target = parsed.target - - marker = LinkerScript.Marker(target, indent, []) - - self.members.append(marker) + parsed = pattern.parse_string(line) except ParseException: # Does not match marker syntax self.members.append(line) + else: + indent, target = parsed + marker = LinkerScript.Marker(target, indent, []) + self.members.append(marker) def fill(self, mapping_rules): for member in self.members: diff --git a/tools/ldgen/output_commands.py b/tools/ldgen/ldgen/output_commands.py similarity index 83% rename from tools/ldgen/output_commands.py rename to tools/ldgen/ldgen/output_commands.py index 05726b8d5d..7df4a85210 100644 --- a/tools/ldgen/output_commands.py +++ b/tools/ldgen/ldgen/output_commands.py @@ -1,26 +1,15 @@ # -# Copyright 2021 Espressif Systems (Shanghai) CO LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 # -from entity import Entity +from .entity import Entity # Contains classes for output section commands referred to in # https://www.acrc.bris.ac.uk/acrc/RedHat/rhel-ld-en-4/sections.html#OUTPUT-SECTION-DESCRIPTION. -class AlignAtAddress(): +class AlignAtAddress: """ Outputs assignment of builtin function ALIGN to current position: @@ -42,7 +31,7 @@ class AlignAtAddress(): self.alignment == other.alignment) -class SymbolAtAddress(): +class SymbolAtAddress: """ Outputs assignment of builtin function ABSOLUTE to a symbol for current position: @@ -65,7 +54,7 @@ class SymbolAtAddress(): self.symbol == other.symbol) -class InputSectionDesc(): +class InputSectionDesc: """ Outputs an input section description as described in https://www.acrc.bris.ac.uk/acrc/RedHat/rhel-ld-en-4/sections.html#INPUT-SECTION. @@ -76,7 +65,7 @@ class InputSectionDesc(): """ def __init__(self, entity, sections, exclusions=None, keep=False, sort=None): - assert(entity.specificity != Entity.Specificity.SYMBOL) + assert (entity.specificity != Entity.Specificity.SYMBOL) self.entity = entity self.sections = set(sections) @@ -84,8 +73,8 @@ class InputSectionDesc(): self.exclusions = set() if exclusions: - assert(not [e for e in exclusions if e.specificity == Entity.Specificity.SYMBOL or - e.specificity == Entity.Specificity.NONE]) + assert (not [e for e in exclusions if e.specificity == Entity.Specificity.SYMBOL or + e.specificity == Entity.Specificity.NONE]) self.exclusions = set(exclusions) else: self.exclusions = set() diff --git a/tools/ldgen/ldgen/sdkconfig.py b/tools/ldgen/ldgen/sdkconfig.py new file mode 100644 index 0000000000..7899ae5f91 --- /dev/null +++ b/tools/ldgen/ldgen/sdkconfig.py @@ -0,0 +1,26 @@ +# +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +# + +import kconfiglib + + +class SDKConfig: + """ + Evaluates conditional expressions based on the build's sdkconfig and Kconfig files. + """ + def __init__(self, kconfig_file, sdkconfig_file): + self.config = kconfiglib.Kconfig(kconfig_file) + self.config.load_config(sdkconfig_file) + self.config.warn = False # eval_string may contain un-declared symbol + + def evaluate_expression(self, expression): + result = self.config.eval_string(expression) + + if result == 0: # n + return False + elif result == 2: # y + return True + else: # m + raise Exception('unsupported config expression result') diff --git a/tools/ldgen/ldgen_common.py b/tools/ldgen/ldgen_common.py deleted file mode 100644 index ebeb404b36..0000000000 --- a/tools/ldgen/ldgen_common.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright 2021 Espressif Systems (Shanghai) CO LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -class LdGenFailure(RuntimeError): - """ - Parent class for any ldgen runtime failure which is due to input data - """ - def __init__(self, message): - super(LdGenFailure, self).__init__(message) diff --git a/tools/ldgen/sdkconfig.py b/tools/ldgen/sdkconfig.py deleted file mode 100644 index f6e8fa520a..0000000000 --- a/tools/ldgen/sdkconfig.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright 2021 Espressif Systems (Shanghai) CO LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import kconfiglib -from pyparsing import (Combine, Group, Literal, Optional, Word, alphanums, hexnums, infixNotation, nums, oneOf, - opAssoc, printables, quotedString, removeQuotes) - - -class SDKConfig: - """ - Evaluates conditional expressions based on the build's sdkconfig and Kconfig files. - This also defines the grammar of conditional expressions. - """ - - # A configuration entry is in the form CONFIG=VALUE. Definitions of components of that grammar - IDENTIFIER = Word(alphanums.upper() + '_') - - HEX = Combine('0x' + Word(hexnums)).setParseAction(lambda t:int(t[0], 16)) - DECIMAL = Combine(Optional(Literal('+') | Literal('-')) + Word(nums)).setParseAction(lambda t:int(t[0])) - LITERAL = Word(printables.replace(':', '')) - QUOTED_LITERAL = quotedString.setParseAction(removeQuotes) - - VALUE = HEX | DECIMAL | LITERAL | QUOTED_LITERAL - - # Operators supported by the expression evaluation - OPERATOR = oneOf(['=', '!=', '>', '<', '<=', '>=']) - - def __init__(self, kconfig_file, sdkconfig_file): - self.config = kconfiglib.Kconfig(kconfig_file) - self.config.load_config(sdkconfig_file) - - def evaluate_expression(self, expression): - result = self.config.eval_string(expression) - - if result == 0: # n - return False - elif result == 2: # y - return True - else: # m - raise Exception('unsupported config expression result') - - @staticmethod - def get_expression_grammar(): - identifier = SDKConfig.IDENTIFIER.setResultsName('identifier') - operator = SDKConfig.OPERATOR.setResultsName('operator') - value = SDKConfig.VALUE.setResultsName('value') - - test_binary = identifier + operator + value - test_single = identifier - - test = test_binary | test_single - - condition = Group(Optional('(').suppress() + test + Optional(')').suppress()) - - grammar = infixNotation(condition, [ - ('!', 1, opAssoc.RIGHT), - ('&&', 2, opAssoc.LEFT), - ('||', 2, opAssoc.LEFT)]) - - return grammar diff --git a/tools/ldgen/test/test_entity.py b/tools/ldgen/test/test_entity.py index 97c1a449d3..0f297157a8 100755 --- a/tools/ldgen/test/test_entity.py +++ b/tools/ldgen/test/test_entity.py @@ -1,29 +1,19 @@ #!/usr/bin/env python # coding=utf-8 # -# Copyright 2018-2020 Espressif Systems (Shanghai) CO LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 # +import os import sys import unittest try: - from entity import Entity, EntityDB + from ldgen.entity import Entity, EntityDB except ImportError: - sys.path.append('../') - from entity import Entity, EntityDB + sys.path.append(os.path.dirname(os.path.dirname(__file__))) + from ldgen.entity import Entity, EntityDB class EntityTest(unittest.TestCase): diff --git a/tools/ldgen/test/test_fragments.py b/tools/ldgen/test/test_fragments.py index f92d91abef..df1665cd72 100755 --- a/tools/ldgen/test/test_fragments.py +++ b/tools/ldgen/test/test_fragments.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -10,44 +10,18 @@ import tempfile import unittest from io import StringIO -from pyparsing import ParseException, ParseFatalException, Word, alphanums - try: - from fragments import FRAGMENT_TYPES, Fragment, FragmentFile, Mapping - from sdkconfig import SDKConfig + from ldgen.fragments import Align, Flag, Keep, Sort, Surround, parse_fragment_file + from ldgen.sdkconfig import SDKConfig except ImportError: - sys.path.append('../') - from fragments import FRAGMENT_TYPES, Fragment, FragmentFile, Mapping - from sdkconfig import SDKConfig + sys.path.append(os.path.dirname(os.path.dirname(__file__))) + from ldgen.fragments import Align, Flag, Keep, Sort, Surround, parse_fragment_file + from ldgen.sdkconfig import SDKConfig - -class SampleFragment(Fragment): - - grammars = { - 'key_1': Fragment.KeyValue(Word(alphanums + '_').setResultsName('value'), 0, None, True), - 'key_2': Fragment.KeyValue(Word(alphanums + '_').setResultsName('value'), 0, None, False), - 'key_3': Fragment.KeyValue(Word(alphanums + '_').setResultsName('value'), 3, 5, False) - } - - def set_key_value(self, key, parse_results): - if key == 'key_1': - self.key_1 = list() - for result in parse_results: - self.key_1.append(result['value']) - elif key == 'key_2': - self.key_2 = list() - for result in parse_results: - self.key_2.append(result['value']) - - def get_key_grammars(self): - return self.__class__.grammars - - -FRAGMENT_TYPES['test'] = SampleFragment +from pyparsing import ParseException, ParseFatalException class FragmentTest(unittest.TestCase): - def setUp(self): with tempfile.NamedTemporaryFile(delete=False) as f: self.kconfigs_source_file = os.path.join(tempfile.gettempdir(), f.name) @@ -79,46 +53,24 @@ class FragmentTest(unittest.TestCase): def test_basic(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 value_2 # comments should be ignored value_3 # this is a comment as well -key_2: value_a - + value_a # this is the last comment """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - self.assertEqual(len(fragment_file.fragments[0].key_1), 3) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[0].key_1[1], 'value_2') - self.assertEqual(fragment_file.fragments[0].key_1[2], 'value_3') - self.assertEqual(len(fragment_file.fragments[0].key_2), 1) - self.assertEqual(fragment_file.fragments[0].key_2[0], 'value_a') - - def test_duplicate_keys(self): - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: value_1 -key_1: value_a -""") - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_empty_key(self): - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: -""") - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + self.assertEqual(fragment_file.fragments[0].name, 'test') + self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_2', 'value_3', 'value_a'}) def test_conditional(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 if A = y: value_2 @@ -128,15 +80,14 @@ key_1: if B = n: value_5 """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[0].key_1[1], 'value_2') - self.assertEqual(fragment_file.fragments[0].key_1[2], 'value_3') - self.assertEqual(fragment_file.fragments[0].key_1[3], 'value_5') + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) + + self.assertEqual(fragment_file.fragments[0].name, 'test') + self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_2', 'value_3', 'value_5'}) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 if B = y: value_2 @@ -148,15 +99,14 @@ key_1: value_5 value_6 """) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[0].key_1[1], 'value_3') - self.assertEqual(fragment_file.fragments[0].key_1[2], 'value_6') + self.assertEqual(fragment_file.fragments[0].name, 'test') + self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_3', 'value_6'}) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 if A = y: value_2 @@ -168,304 +118,206 @@ key_1: value_5 value_6 value_7 -key_2: - value_a - if B != y: - value_b """) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[0].key_1[1], 'value_2') - self.assertEqual(fragment_file.fragments[0].key_1[2], 'value_4') - self.assertEqual(fragment_file.fragments[0].key_1[3], 'value_5') - self.assertEqual(fragment_file.fragments[0].key_1[4], 'value_6') - self.assertEqual(fragment_file.fragments[0].key_1[5], 'value_7') - self.assertEqual(fragment_file.fragments[0].key_2[0], 'value_a') - self.assertEqual(fragment_file.fragments[0].key_2[1], 'value_b') + self.assertEqual(fragment_file.fragments[0].name, 'test') + self.assertEqual(fragment_file.fragments[0].entries, + {'value_1', 'value_2', 'value_4', 'value_5', 'value_6', 'value_7'}) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: if A = n: value_2 """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(len(fragment_file.fragments[0].key_1), 0) + with self.assertRaises(ParseFatalException): + parse_fragment_file(test_fragment, self.sdkconfig) def test_empty_file(self): test_fragment = self.create_fragment_file(u""" - - - - """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(len(fragment_file.fragments), 0) def test_setting_indent(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 value_2 value_3 """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - self.assertEqual(len(fragment_file.fragments[0].key_1), 3) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[0].key_1[1], 'value_2') - self.assertEqual(fragment_file.fragments[0].key_1[2], 'value_3') + self.assertEqual(fragment_file.fragments[0].name, 'test') + self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_2', 'value_3'}) + def test_settings_unmatch_indent(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: value_1 value_2 # first element dictates indent value_3 """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_values_num_limit(self): - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: - value_a -key_3: - value_1 - value_2 - value_3 -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: - value_a -key_3: - value_1 - value_2 - value_3 - value_4 -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(len(fragment_file.fragments), 1) - - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: - value_a -key_3: - value_1 - value_2 - value_3 - value_4 - value_5 -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(len(fragment_file.fragments), 1) - - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: - value_a -key_3: - value_1 - value_2 -""") - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) - - test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: - value_a -key_3: - value_1 - value_2 - value_3 - value_4 - value_5 - value_6 -""") - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) def test_unsupported_key(self): test_fragment = self.create_fragment_file(u""" -[test:test] +[sections:test] key_1: value_a -key_4: - value_1 """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) def test_empty_fragment(self): test_fragment = self.create_fragment_file(u""" -[test:test] +[sections:test] """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_empty_conditional(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: if B = y: else: value_1 """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: if B = y: value_1 else B = y: """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: if B = y: value_1 elif B = y: else: value_2 """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_out_of_order_conditional(self): test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: elif B = y: value_1 else: value_2 """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" -[test:test] -key_1: +[sections:test] +entries: else: value_2 """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_required_keys(self): - test_fragment = self.create_fragment_file(u""" -[test:test] -key_2: - value_1 -""") - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) def test_multiple_fragments(self): test_fragment = self.create_fragment_file(u""" -[test:test1] -key_1: +[sections:test1] +entries: value_1 -[test:test2] -key_1: - value_2 +[scheme:test2] +entries: + section -> target """) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - - self.assertEqual(len(fragment_file.fragments), 2) - self.assertEqual(fragment_file.fragments[0].key_1[0], 'value_1') - self.assertEqual(fragment_file.fragments[1].key_1[0], 'value_2') + self.assertEqual(fragment_file.fragments[0].name, 'test1') + self.assertEqual(fragment_file.fragments[0].entries, {'value_1'}) + self.assertEqual(fragment_file.fragments[1].name, 'test2') + self.assertEqual(fragment_file.fragments[1].entries, {('section', 'target')}) def test_whole_conditional_fragment(self): test_fragment = self.create_fragment_file(u""" if B = y: - [test:test1] - key_1: + [sections:test1] + entries: value_1 else: - [test:test2] - key_1: + [sections:test2] + entries: value_2 if A = y: - [test:test3] - key_1: + [sections:test3] + entries: value_3 if C = y: value_6 - [test:test4] - key_1: + [sections:test4] + entries: value_4 -[test:test5] -key_1: +[sections:test5] +entries: value_5 """) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) self.assertEqual(len(fragment_file.fragments), 4) self.assertEqual(fragment_file.fragments[0].name, 'test2') + self.assertEqual(fragment_file.fragments[0].entries, {'value_2'}) self.assertEqual(fragment_file.fragments[1].name, 'test3') - self.assertEqual(fragment_file.fragments[1].key_1[1], 'value_6') + self.assertEqual(fragment_file.fragments[1].entries, {'value_3', 'value_6'}) self.assertEqual(fragment_file.fragments[2].name, 'test4') + self.assertEqual(fragment_file.fragments[2].entries, {'value_4'}) self.assertEqual(fragment_file.fragments[3].name, 'test5') + self.assertEqual(fragment_file.fragments[3].entries, {'value_5'}) def test_equivalent_conditional_fragment(self): test_fragment1 = self.create_fragment_file(u""" if A = y: - [test:test1] - key_1: + [sections:test1] + entries: value_1 else: - [test:test2] - key_1: + [sections:test2] + entries: value_2 """) + fragment_file1 = parse_fragment_file(test_fragment1, self.sdkconfig) - fragment_file1 = FragmentFile(test_fragment1, self.sdkconfig) - self.assertEqual(len(fragment_file1.fragments), 1) - self.assertEqual(fragment_file1.fragments[0].key_1[0], 'value_1') + self.assertEqual(fragment_file1.fragments[0].name, 'test1') + self.assertEqual(fragment_file1.fragments[0].entries, {'value_1'}) test_fragment2 = self.create_fragment_file(u""" -[test:test1] -key_1: +[sections:test1] +entries: if A = y: value_1 else: value_2 """) + fragment_file2 = parse_fragment_file(test_fragment2, self.sdkconfig) - fragment_file2 = FragmentFile(test_fragment2, self.sdkconfig) - self.assertEqual(len(fragment_file2.fragments), 1) - self.assertEqual(fragment_file2.fragments[0].key_1[0], 'value_1') + self.assertEqual(fragment_file2.fragments[0].name, 'test1') + self.assertEqual(fragment_file2.fragments[0].entries, {'value_1'}) class SectionsTest(FragmentTest): @@ -477,8 +329,7 @@ entries: .section1 .section2 """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {'.section1', '.section2'}) def test_duplicate_entries(self): @@ -490,8 +341,7 @@ entries: .section3 .section2 """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {'.section1', '.section2', '.section3'}) def test_empty_entries(self): @@ -499,9 +349,8 @@ entries: [sections:test] entries: """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [sections:test] @@ -509,12 +358,10 @@ entries: if B = y: .section1 """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_entries_grammar(self): - test_fragment = self.create_fragment_file(u""" [sections:test] entries: @@ -522,8 +369,7 @@ entries: valid2. .valid3_- """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {'_valid1', 'valid2.', '.valid3_-'}) @@ -533,18 +379,16 @@ entries: entries: 1invalid """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [sections:test] entries: -invalid """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) # + notation test_fragment = self.create_fragment_file(u""" @@ -552,8 +396,7 @@ entries: entries: valid+ """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {'valid+'}) @@ -562,9 +405,8 @@ entries: entries: inva+lid+ """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) class SchemeTest(FragmentTest): @@ -577,7 +419,7 @@ entries: sections2 -> target2 """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {('sections1', 'target1'), ('sections2', 'target2')}) @@ -591,7 +433,7 @@ entries: sections2 -> target2 """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].entries, {('sections1', 'target1'), ('sections2', 'target2')}) @@ -601,9 +443,8 @@ entries: [scheme:test] entries: """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [scheme:test] @@ -611,9 +452,8 @@ entries: if B = y: sections1 -> target1 """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_improper_grammar(self): test_fragment = self.create_fragment_file(u""" @@ -623,7 +463,7 @@ entries: """) with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) class MappingTest(FragmentTest): @@ -646,7 +486,7 @@ entries: ('obj_2', None, 'noflash'), ('*', None, 'noflash')} - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(expected, fragment_file.fragments[0].entries) def test_archive(self): @@ -656,9 +496,8 @@ archive: entries: * (default) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -668,9 +507,8 @@ archive: entries: * (default) """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_archive_allowed_names(self): test_fragment = self.create_fragment_file(u""" @@ -680,8 +518,7 @@ archive: entries: * (default) """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual('libstdc++.a', fragment_file.fragments[0].archive) def test_empty_entries(self): @@ -696,7 +533,7 @@ entries: expected = set() - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(expected, fragment_file.fragments[0].entries) test_fragment = self.create_fragment_file(u""" @@ -707,7 +544,7 @@ entries: """) with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_duplicate_entries(self): test_fragment = self.create_fragment_file(u""" @@ -721,7 +558,7 @@ entries: expected = {('obj', 'symbol', 'noflash')} - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(expected, fragment_file.fragments[0].entries) def test_invalid_grammar(self): @@ -730,18 +567,16 @@ entries: archive: lib.a """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] entries: * (default) """) - - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -749,9 +584,8 @@ archive: lib.a entries: obj: (noflash) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -759,9 +593,8 @@ archive: lib.a entries: obj: () """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -769,9 +602,8 @@ archive: lib.a entries: obj:symbol """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -779,9 +611,8 @@ archive: lib.a entries: (noflash) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -789,9 +620,8 @@ archive: lib.a entries: obj:* (noflash) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -799,9 +629,8 @@ archive: lib.a entries: :symbol (noflash) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) test_fragment = self.create_fragment_file(u""" [mapping:test] @@ -809,9 +638,8 @@ archive: lib.a entries: *:symbol (noflash) """) - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) + parse_fragment_file(test_fragment, self.sdkconfig) def test_keep_flag(self): # Test parsing combinations and orders of flags @@ -823,12 +651,12 @@ entries: text->flash_text KEEP(), rodata->flash_rodata KEEP() KEEP() """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Keep()]), - ('rodata', 'flash_rodata', [Mapping.Keep(), Mapping.Keep()])] + expected = [Flag('text', 'flash_text', [Keep()]), + Flag('rodata', 'flash_rodata', [Keep(), Keep()])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -847,14 +675,14 @@ entries: common->dram0_bss ALIGN(8, pre, post) ALIGN(8) """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Align(8, True, False)]), - ('rodata', 'flash_rodata', [Mapping.Align(8, True, False)]), - ('data', 'dram0_data', [Mapping.Align(8, True, True)]), - ('bss', 'dram0_bss', [Mapping.Align(8, False, True)]), - ('common', 'dram0_bss', [Mapping.Align(8, True, True), Mapping.Align(8, True, False)])] + expected = [Flag('text', 'flash_text', [Align(8, True, False)]), + Flag('rodata', 'flash_rodata', [Align(8, True, False)]), + Flag('data', 'dram0_data', [Align(8, True, True)]), + Flag('bss', 'dram0_bss', [Align(8, False, True)]), + Flag('common', 'dram0_bss', [Align(8, True, True), Align(8, True, False)])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -868,8 +696,8 @@ entries: text->iram0_text ALIGN(8, post, pre) """) - with self.assertRaises(ParseFatalException): - FragmentFile(test_fragment, self.sdkconfig) + with self.assertRaises(ParseException): + parse_fragment_file(test_fragment, self.sdkconfig) def test_sort_flag(self): # Test parsing combinations and orders of flags @@ -887,27 +715,19 @@ entries: dram->dram0_data SORT(alignment, alignment) """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Sort('name')]), - ('rodata', 'flash_rodata', [Mapping.Sort('alignment')]), - ('data', 'dram0_data', [Mapping.Sort('init_priority')]), - ('bss', 'dram0_bss', [Mapping.Sort('name', 'alignment')]), - ('common', 'dram0_bss', [Mapping.Sort('alignment', 'name')]), - ('iram', 'iram0_text', [Mapping.Sort('name', 'name')]), - ('dram', 'dram0_data', [Mapping.Sort('alignment', 'alignment')])] + expected = [Flag('text', 'flash_text', [Sort('name')]), + Flag('rodata', 'flash_rodata', [Sort('alignment')]), + Flag('data', 'dram0_data', [Sort('init_priority')]), + Flag('bss', 'dram0_bss', [Sort('name', 'alignment')]), + Flag('common', 'dram0_bss', [Sort('alignment', 'name')]), + Flag('iram', 'iram0_text', [Sort('name', 'name')]), + Flag('dram', 'dram0_data', [Sort('alignment', 'alignment')])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) - test_fragment = self.create_fragment_file(u""" -[mapping:map] -archive: libmain.a -entries: - obj1 (default) - text->iram0_text SORT(name) SORT(alignment) -""") - def test_surround_flag(self): # Test parsing combinations and orders of flags test_fragment = self.create_fragment_file(u""" @@ -917,11 +737,10 @@ entries: obj1 (default); text->flash_text SURROUND(sym1) """) - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Surround('sym1')])] + expected = [Flag('text', 'flash_text', [Surround('sym1')])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -935,21 +754,21 @@ entries: text->flash_text ALIGN(4) KEEP() SURROUND(sym1) ALIGN(8) SORT(name), rodata->flash_rodata KEEP() ALIGN(4) KEEP() SURROUND(sym1) ALIGN(8) ALIGN(4) SORT(name) """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Align(8, True, False), - Mapping.Sort('name')]), - ('rodata', 'flash_rodata', [Mapping.Keep(), - Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Align(8, True, False), - Mapping.Align(4, True, False), - Mapping.Sort('name')])] + expected = [Flag('text', 'flash_text', [Align(4, True, False), + Keep(), + Surround('sym1'), + Align(8, True, False), + Sort('name')]), + Flag('rodata', 'flash_rodata', [Keep(), + Align(4, True, False), + Keep(), + Surround('sym1'), + Align(8, True, False), + Align(4, True, False), + Sort('name')])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -965,17 +784,17 @@ entries: text->flash_text ALIGN(4) KEEP() SURROUND(sym1) SORT(name), text->flash_text ALIGN(4) KEEP() SURROUND(sym1) SORT(name) """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Sort('name')]), - ('text', 'flash_text', [Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Sort('name')])] + expected = [Flag('text', 'flash_text', [Align(4, True, False), + Keep(), + Surround('sym1'), + Sort('name')]), + Flag('text', 'flash_text', [Align(4, True, False), + Keep(), + Surround('sym1'), + Sort('name')])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -993,261 +812,20 @@ entries: obj1 (default); text->flash_text ALIGN(4) KEEP() SURROUND(sym1) SORT(name) """) - fragment_file = FragmentFile(test_fragment, self.sdkconfig) + fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [('text', 'flash_text', [Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Sort('name')]), - ('text', 'flash_text', [Mapping.Align(4, True, False), - Mapping.Keep(), - Mapping.Surround('sym1'), - Mapping.Sort('name')])] + expected = [Flag('text', 'flash_text', [Align(4, True, False), + Keep(), + Surround('sym1'), + Sort('name')]), + Flag('text', 'flash_text', [Align(4, True, False), + Keep(), + Surround('sym1'), + Sort('name')])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) -class DeprecatedMappingTest(FragmentTest): - - def test_valid_grammar(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - obj:symbol (noflash) - # Comments should not matter - obj (noflash) - # Nor should whitespace - obj : symbol_2 ( noflash ) - obj_2 ( noflash ) - * (noflash) -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual('lib.a', fragment_file.fragments[0].archive) - self.assertEqual('lib_a', fragment_file.fragments[0].name) - - expected = {('obj', 'symbol', 'noflash'), - ('obj', None, 'noflash'), - ('obj', 'symbol_2', 'noflash'), - ('obj_2', None, 'noflash'), - ('*', None, 'noflash') - } - - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_explicit_blank_default_w_others(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = n - obj_a (noflash) - : default -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('*', None, 'default')} - - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_implicit_blank_default_w_others(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = n - obj_a (noflash) -""") - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('*', None, 'default')} - - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_explicit_blank_default(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : default -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('*', None, 'default')} - - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_implicit_blank_default(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : default -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('*', None, 'default')} - - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_multiple_entries(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = n - obj_a1 (noflash) - obj_a2 (noflash) - : B = n - obj_b1 (noflash) - obj_b2 (noflash) - obj_b3 (noflash) - : C = n - obj_c1 (noflash) -""") - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('obj_b1', None, 'noflash'), - ('obj_b2', None, 'noflash'), - ('obj_b3', None, 'noflash')} - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_blank_entries(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = n - obj_a (noflash) - : B = n - : C = n - obj_c (noflash) - : default - obj (noflash) -""") - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - expected = {('*', None, 'default')} - self.assertEqual(expected, fragment_file.fragments[0].entries) - - def test_blank_first_condition(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - obj_a (noflash) - : CONFIG_B = y - obj_b (noflash) -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_nonlast_default_1(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : default - obj_a (noflash) - : CONFIG_A = y - obj_A (noflash) -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_nonlast_default_2(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = y - obj_A (noflash) - : default - obj_a (noflash) - : B = y - obj_B (noflash -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_nonlast_default_3(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = y - obj_A (noflash) - : - obj_a (noflash) - : B = y - obj_B (noflash -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_duplicate_default_1(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : CONFIG_A = y - obj_A (noflash) - : default - obj_a (noflash) - : CONFIG_B = y - obj_B (noflash) - : default - obj_a (noflash) -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_duplicate_default_2(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : CONFIG_A = y - obj_A (noflash) - : CONFIG_B = y - obj_a (noflash) - : default - obj_B (noflash) - : - obj_a (noflash) -""") - - with self.assertRaises(ParseException): - FragmentFile(test_fragment, self.sdkconfig) - - def test_mixed_deprecated_mapping(self): - test_fragment = self.create_fragment_file(u""" -[mapping] -archive: lib.a -entries: - : A = n - obj_A (noflash) - : default - obj_B (noflash) - - -[mapping:test] -archive: lib.a -entries: - if A = n: - obj_A (noflash) - else: - obj_B (noflash) -""") - - fragment_file = FragmentFile(test_fragment, self.sdkconfig) - self.assertEqual(2, len(fragment_file.fragments)) - - self.assertEqual(fragment_file.fragments[0].entries, - fragment_file.fragments[1].entries) - - if __name__ == '__main__': unittest.main() diff --git a/tools/ldgen/test/test_generation.py b/tools/ldgen/test/test_generation.py index 26e038aafe..2ccfd7d70b 100755 --- a/tools/ldgen/test/test_generation.py +++ b/tools/ldgen/test/test_generation.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # @@ -10,20 +10,23 @@ import os import sys import tempfile import unittest - -try: - from generation import Generation, GenerationException -except ImportError: - sys.path.append('../') - from generation import Generation, GenerationException - from io import StringIO -from entity import Entity, EntityDB -from fragments import FragmentFile -from linker_script import LinkerScript -from output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress -from sdkconfig import SDKConfig +try: + from ldgen.entity import Entity, EntityDB + from ldgen.fragments import parse_fragment_file + from ldgen.generation import Generation, GenerationException + from ldgen.linker_script import LinkerScript + from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress + from ldgen.sdkconfig import SDKConfig +except ImportError: + sys.path.append(os.path.dirname(os.path.dirname(__file__))) + from ldgen.entity import Entity, EntityDB + from ldgen.fragments import parse_fragment_file + from ldgen.generation import Generation, GenerationException + from ldgen.linker_script import LinkerScript + from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress + from ldgen.sdkconfig import SDKConfig ROOT = Entity('*') @@ -58,9 +61,8 @@ class GenerationTest(unittest.TestCase): self.sdkconfig = SDKConfig('data/Kconfig', 'data/sdkconfig') - with open('data/base.lf') as fragment_file_obj: - fragment_file = FragmentFile(fragment_file_obj, self.sdkconfig) - self.generation.add_fragments_from_file(fragment_file) + fragment_file = parse_fragment_file('data/base.lf', self.sdkconfig) + self.generation.add_fragments_from_file(fragment_file) self.entities = EntityDB() @@ -78,7 +80,7 @@ class GenerationTest(unittest.TestCase): def add_fragments(self, text): fragment_file = self.create_fragment_file(text) - fragment_file = FragmentFile(fragment_file, self.sdkconfig) + fragment_file = parse_fragment_file(fragment_file, self.sdkconfig) self.generation.add_fragments_from_file(fragment_file) def write(self, expected, actual): @@ -1062,43 +1064,6 @@ entries: with self.assertRaises(GenerationException): self.generation.generate(self.entities) - def test_disambiguated_obj(self): - # Test command generation for disambiguated entry. Should produce similar - # results to test_nondefault_mapping_symbol. - mapping = u""" -[mapping:test] -archive: libfreertos.a -entries: - port.c:xPortGetTickRateHz (noflash) #1 -""" - port = Entity('libfreertos.a', 'port.c') - self.add_fragments(mapping) - actual = self.generation.generate(self.entities) - expected = self.generate_default_rules() - - flash_text = expected['flash_text'] - iram0_text = expected['iram0_text'] - - # Generate exclusion in flash_text A - flash_text[0].exclusions.add(port) - - # Generate intermediate command B - # List all relevant sections except the symbol - # being mapped - port_sections = self.entities.get_sections('libfreertos.a', 'port.c') - filtered_sections = fnmatch.filter(port_sections, '.literal.*') - filtered_sections.extend(fnmatch.filter(port_sections, '.text.*')) - - filtered_sections = [s for s in filtered_sections if not s.endswith('xPortGetTickRateHz')] - filtered_sections.append('.text') - - flash_text.append(InputSectionDesc(port, set(filtered_sections), [])) - - # Input section commands in iram_text for #1 C - iram0_text.append(InputSectionDesc(port, set(['.text.xPortGetTickRateHz', '.literal.xPortGetTickRateHz']), [])) - - self.compare_rules(expected, actual) - def test_root_mapping_fragment_conflict(self): # Test that root mapping fragments are also checked for # conflicts. @@ -1258,84 +1223,6 @@ entries: self.compare_rules(expected, actual) - def test_conditional_on_scheme_legacy_mapping_00(self): - # Test use of conditional scheme on legacy mapping fragment grammar. - mapping = u""" -[mapping] -archive: lib.a -entries: - * (cond_noflash) -""" - self._test_conditional_on_scheme(0, mapping) - - def test_conditional_on_scheme_legacy_mapping_01(self): - # Test use of conditional scheme on legacy mapping fragment grammar. - mapping = u""" -[mapping] -archive: lib.a -entries: - * (cond_noflash) -""" - self._test_conditional_on_scheme(0, mapping) - - def test_conditional_entries_legacy_mapping_fragment(self): - # Test conditional entries on legacy mapping fragment grammar. - mapping = u""" -[mapping:default] -archive: * -entries: - * (default) - -[mapping] -archive: lib.a -entries: - : PERFORMANCE_LEVEL = 0 - : PERFORMANCE_LEVEL = 1 - obj1 (noflash) - : PERFORMANCE_LEVEL = 2 - obj1 (noflash) - obj2 (noflash) - : PERFORMANCE_LEVEL = 3 - obj1 (noflash) - obj2 (noflash) - obj3 (noflash) -""" - self.test_conditional_mapping(mapping) - - def test_multiple_fragment_same_lib_conditional_legacy(self): - # Test conditional entries on legacy mapping fragment grammar - # across multiple fragments. - mapping = u""" -[mapping:default] -archive: * -entries: - * (default) - -[mapping] -archive: lib.a -entries: - : PERFORMANCE_LEVEL = 0 - : PERFORMANCE_LEVEL = 1 - obj1 (noflash) - : PERFORMANCE_LEVEL = 2 - obj1 (noflash) - : PERFORMANCE_LEVEL = 3 - obj1 (noflash) - -[mapping] -archive: lib.a -entries: - : PERFORMANCE_LEVEL = 1 - obj1 (noflash) # ignore duplicate definition - : PERFORMANCE_LEVEL = 2 - obj2 (noflash) - : PERFORMANCE_LEVEL = 3 - obj2 (noflash) - obj3 (noflash) -""" - - self.test_conditional_mapping(mapping) - def test_multiple_fragment_same_lib_conditional(self): # Test conditional entries on new mapping fragment grammar. # across multiple fragments. diff --git a/tools/ldgen/test/test_output_commands.py b/tools/ldgen/test/test_output_commands.py index 96effe52ec..4a6723626d 100755 --- a/tools/ldgen/test/test_output_commands.py +++ b/tools/ldgen/test/test_output_commands.py @@ -1,30 +1,21 @@ #!/usr/bin/env python # -# Copyright 2021 Espressif Systems (Shanghai) CO LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 # +import os import sys import unittest try: - from output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress + from ldgen.entity import Entity + from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress except ImportError: - sys.path.append('../') - from output_commands import InputSectionDesc, SymbolAtAddress, AlignAtAddress + sys.path.append(os.path.dirname(os.path.dirname(__file__))) + from ldgen.entity import Entity + from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress -from entity import Entity SECTIONS = ['.text', '.text.*', '.literal', '.literal.*']