add pytest testing for esp-modbus

.gitlab-ci.yml

@@ -1,11 +1,14 @@
 stages:
   - build
+  - target_test
   - deploy
 
 variables:
   # System environment
+  TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.0:2"
   ESP_DOCS_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:2-2"
   ESP_DOCS_PATH: "$CI_PROJECT_DIR"
+  TEST_DIR: "$CI_PROJECT_DIR/test"
 
   # GitLab-CI environment
   GET_SOURCES_ATTEMPTS: "10"
@@ -28,68 +31,156 @@ after_script:
   # Just for cleaning space, no other causes
   - git clean -ffdx
 
-# This template gets expanded multiple times, once for every IDF version.
-# IDF version is specified by setting the espressif/idf image tag.
-#
-# EXAMPLE_TARGETS sets the list of IDF_TARGET values to build examples for.
-# It should be equal to the list of targets supported by the specific IDF version.
-#
-# TEST_TARGETS sets the list of IDF_TARGET values to build the test_app for.
-# It should contain only the targets with optimized assembly implementations.
-#
 .build_template:
   stage: build
   tags:
     - build
-    - internet
-  script:
-    - pip install idf-component-manager --upgrade
-    - ./build_all.sh
   variables:
-    EXAMPLE_TARGETS: "esp32"
+    SIZE_INFO_LOCATION: "${TEST_DIR}/size_info.txt"
+    IDF_CCACHE_ENABLE: "1"
+  after_script:
+    # Show ccache statistics if enabled globally
+    - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
+  dependencies: []
+
+.before_script_build_jobs:
+  before_script:
+    - pip install idf-component-manager --upgrade
+    - pip install "idf_build_apps~=0.3.0"
+
+# This template gets expanded multiple times, once for every IDF version.
+# IDF version is specified by setting the espressif/idf image tag.
+#
+# TEST_TARGETS sets the list of IDF_TARGET values to build the test for.
+# It should contain only the targets with optimized assembly implementations.
+#
+.build_pytest_template:
+  stage: build
+  extends:
+    - .build_template
+    - .before_script_build_jobs
+  artifacts:
+    paths:
+      - "**/build*/size.json"
+      - "**/build*/build.log"
+      - "**/build*/build_log.txt"
+      - "**/build*/*.bin"
+      - "**/build*/*.elf"
+      - "**/build*/*.map"
+      - "**/build*/flasher_args.json"
+      - "**/build*/flash_project_args"
+      - "**/build*/config/sdkconfig.json"
+      - "**/build*/bootloader/*.bin"
+      - "**/build*/partition_table/*.bin"
+      - $SIZE_INFO_LOCATION
+    when: always
+    expire_in: 3 days
+  script:
+    # CI specific options start from "--collect-size-info xxx". could ignore when running locally
+    # The script below will build all test applications defined in environment variable $TEST_TARGETS
+    - cd ${TEST_DIR}
+    - python -m idf_build_apps build -v -p .
+      --recursive
+      --target all
+      --default-build-targets ${TEST_TARGETS}
+      --config "sdkconfig.ci.*=" --build-dir "build_@t_@w"
+      --check-warnings
+      --ignore-warning-file ../tools/ignore_build_warnings.txt
+      --collect-size-info $SIZE_INFO_LOCATION
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
+  variables:
     TEST_TARGETS: "esp32"
 
-build_idf_v4.1:
-  extends: .build_template
-  image: espressif/idf:release-v4.1
-
-build_idf_v4.2:
-  extends: .build_template
-  image: espressif/idf:release-v4.2
-  variables:
-    EXAMPLE_TARGETS: "esp32 esp32s2"
-    TEST_TARGETS: "esp32 esp32s2"
-
-build_idf_v4.3:
-  extends: .build_template
-  image: espressif/idf:release-v4.3
-  variables:
-    EXAMPLE_TARGETS: "esp32 esp32s2 esp32c3"
-    TEST_TARGETS: "esp32 esp32s2 esp32c3"
-
-build_idf_v4.4:
-  extends: .build_template
-  image: espressif/idf:release-v4.4
-  variables:
-    EXAMPLE_TARGETS: "esp32 esp32s2 esp32s3 esp32c3"
-    TEST_TARGETS: "esp32 esp32s2 esp32s3 esp32c3"
-
-build_idf_v5.0:
-  extends: .build_template
-  image: espressif/idf:release-v5.0
-  variables:
-    EXAMPLE_TARGETS: "esp32 esp32s2 esp32s3 esp32c2 esp32c3"
-    TEST_TARGETS: "esp32 esp32s2 esp32s3 esp32c2 esp32c3"
-    SKIP_GNU_MAKE_BUILD: 1
-
 build_idf_latest:
-  extends: .build_template
+  extends: .build_pytest_template
   image: espressif/idf:latest
   variables:
-    EXAMPLE_TARGETS: "esp32 esp32s2 esp32s3 esp32c3 esp32c2 esp32c6"
+    TEST_TARGETS: "esp32,esp32s2,esp32s3,esp32c2,esp32c3,esp32c6"
-    TEST_TARGETS: "esp32 esp32s2 esp32s3 esp32c3 esp32c2 esp32c6"
+
-    # GNU Make based build system is not supported starting from IDF v5.0
+build_idf_v5.0:
-    SKIP_GNU_MAKE_BUILD: 1
+  extends: .build_pytest_template
+  image: espressif/idf:release-v5.0
+  variables:
+    TEST_TARGETS: "esp32,esp32s2,esp32s3,esp32c2,esp32c3"
+    
+build_idf_v4.4:
+  extends: .build_pytest_template
+  image: espressif/idf:release-v4.4
+  variables:
+    TEST_TARGETS: "esp32,esp32s2,esp32s3,esp32c3"
+
+build_idf_v4.3:
+  extends: .build_pytest_template
+  image: espressif/idf:release-v4.3
+  variables:
+    TEST_TARGETS: "esp32,esp32s2,esp32c3"
+
+build_idf_v4.2:
+  extends: .build_pytest_template
+  image: espressif/idf:release-v4.2
+  variables:
+    TEST_TARGETS: "esp32,esp32s2"
+
+.target_test_template:
+  image: $TARGET_TEST_ENV_IMAGE
+  stage: target_test
+  timeout: 1 hour
+  variables:
+    GIT_DEPTH: 1
+    SUBMODULES_TO_FETCH: "none"
+  cache:
+    # Usually do not need submodule-cache in target_test
+    - key: pip-cache
+      paths:
+        - .cache/pip
+      policy: pull
+
+.before_script_pytest_jobs:
+  before_script:
+    # Get ESP-IDF and install the tools.
+    - cd /opt/
+    - git clone -b ${IDF_BRANCH} --depth 1 https://github.com/espressif/esp-idf.git
+    - cd esp-idf
+    - export IDF_PATH=${PWD}
+    - export IDF_DESCRIBE=`git describe`
+    - export IDF_VERSION=${IDF_DESCRIBE%-*}
+    - tools/idf_tools.py --non-interactive install cmake
+    - tools/idf_tools.py install-python-env
+    - tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
+    - 'echo "ESP-IDF: ${IDF_VERSION}" >> ${TEST_DIR}/idf_version_info.txt'
+    - pip install "pytest-embedded-serial-esp~=1.2.3" "pytest-embedded-idf~=1.2.3"
+
+.test_template:
+  stage: target_test
+  image: ${TARGET_TEST_ENV_IMAGE}
+  extends:
+    - .before_script_pytest_jobs
+  tags:
+    - multi_dut_modbus_${TEST_PORT}
+  artifacts:
+    paths:
+      - "${TEST_DIR}/*/*.log"
+      - "${TEST_DIR}/*.txt"
+      - "${TEST_DIR}/*/results_*.xml"
+      - "${TEST_DIR}/pytest_embedded_log/"
+    reports:
+      junit: ${TEST_DIR}/${TEST_PORT}/results_${IDF_TARGET}_${IDF_VERSION}.xml
+    when: always
+    expire_in: 1 week
+  script:
+    - cd ${TEST_DIR}/${TEST_PORT}
+    - python -m pytest --junit-xml=${TEST_DIR}/${TEST_PORT}/results_${IDF_TARGET}_${IDF_VERSION}.xml --target=${IDF_TARGET}
+    - ls -lh > test_dir.txt
+
+test_examples:
+  extends: .test_template
+  parallel:
+    matrix:
+      - IDF_BRANCH: ["release/v4.4", "release/v5.0", "master"]
+        IDF_TARGET: ["esp32"]
+        TEST_PORT: ["serial", "tcp"]
+  after_script: []
 
 build_docs:
   stage: build
@@ -168,4 +259,3 @@ upload_to_component_manager:
     - pip install idf-component-manager
     - export IDF_COMPONENT_API_TOKEN=${ESP_MODBUS_API_KEY}
     - python -m idf_component_manager upload-component --allow-existing --name=esp-modbus --namespace=espressif
-

pytest.ini

@@ -0,0 +1,46 @@
+[pytest]
+# only the files with prefix `pytest_` would be recognized as pytest test scripts.
+python_files = pytest_*.py
+
+# ignore PytestExperimentalApiWarning for record_xml_attribute
+# set traceback to "short" to prevent the overwhelming tracebacks
+addopts =
+  -s
+  --embedded-services esp,idf
+  --tb short
+  --skip-check-coredump y
+
+# ignore DeprecationWarning
+filterwarnings =
+  ignore::DeprecationWarning:matplotlib.*:
+  ignore::DeprecationWarning:google.protobuf.*:
+  ignore::_pytest.warning_types.PytestExperimentalApiWarning
+
+markers =
+  # target markers
+  esp32: support esp32 target
+  esp32s2: support esp32s2 target
+  esp32s3: support esp32s3 target
+  esp32c3: support esp32c3 target
+  esp32c2: support esp32c2 target
+
+  # env markers
+  generic: tests should be run on generic runners
+
+  # multi-dut markers
+  multi_dut_generic: tests should be run on generic runners, at least have two duts connected.
+  multi_dut_modbus_tcp: Modbus TCP runners with two duts connected
+  multi_dut_modbus_rs485: Modbus RTU/ASCII runners with two duts connected
+
+# log related
+log_cli = True
+log_cli_level = INFO
+log_cli_format = %(asctime)s %(levelname)s %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S
+
+# junit related
+junit_family = xunit1
+
+## log all to `system-out` when case fail
+junit_logging = stdout
+junit_log_passing_tests = False

test/conftest.py

@@ -0,0 +1,319 @@
+# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+# pylint: disable=W0621  # redefined-outer-name
+
+import logging
+import os
+import sys
+from datetime import datetime
+from enum import Enum
+from typing import Any, Callable, Dict, Match, Optional, TextIO, Tuple
+
+import pexpect
+import pytest
+from _pytest.fixtures import FixtureRequest
+from _pytest.monkeypatch import MonkeyPatch
+from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture
+from pytest_embedded_idf.app import IdfApp
+from pytest_embedded_idf.dut import IdfDut
+from pytest_embedded_idf.serial import IdfSerial
+
+
+class Stages(Enum):
+    STACK_DEFAULT = 1
+    STACK_IPV4 = 2
+    STACK_IPV6 = 3
+    STACK_INIT = 4
+    STACK_CONNECT = 5
+    STACK_START = 6
+    STACK_PAR_OK = 7
+    STACK_PAR_FAIL = 8
+    STACK_DESTROY = 9
+
+DEFAULT_SDKCONFIG = 'default'
+ALLOWED_PERCENT_OF_FAILS = 10
+
+class ModbusTestDut(IdfDut):
+
+    TEST_IP_PROMPT = r'Waiting IP([0-9]{1,2}) from stdin:\r\r\n'
+    TEST_IP_SET_CONFIRM = r'.*IP\([0-9]+\) = \[([0-9a-zA-Z\.\:]+)\] set from stdin.*'
+    TEST_IP_ADDRESS_REGEXP = r'.*example_[a-z]+: .* IPv4 [a-z]+:.* ([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}).*'
+    TEST_APP_NAME = r'I \([0-9]+\) cpu_start: Project name:     ([_a-z]*)'
+
+    TEST_EXPECT_STR_TIMEOUT = 120
+    TEST_ACK_TIMEOUT = 60
+    TEST_MAX_CIDS = 8
+
+    app: IdfApp
+    serial: IdfSerial
+
+    def __init__(self, *args, **kwargs) -> None:  # type: ignore
+        super().__init__(*args, **kwargs)
+        self.logger = logging.getLogger()
+        self.test_output: Optional[TextIO] = None
+        self.ip_address: Optional[str] = None
+        self.app_name: Optional[str] = None
+        self.param_fail_count = 0
+        self.param_ok_count = 0
+        self.test_stage = Stages.STACK_DEFAULT
+        self.dictionary = None
+        self.test_finish = False
+        self.test_status = False
+
+    def close(self) -> None:
+        super().close()
+
+    def dut_get_ip(self) -> Optional[str]:
+        if self.ip_address is None:
+            expect_address = self.expect(self.TEST_IP_ADDRESS_REGEXP, timeout=self.TEST_EXPECT_STR_TIMEOUT)
+            if isinstance(expect_address, Match):
+                self.ip_address = expect_address.group(1).decode('ascii')
+        return self.ip_address
+
+    def dut_get_name(self) -> Optional[str]:
+        if self.app_name is None:
+            expect_name = self.expect(self.TEST_APP_NAME, timeout=self.TEST_EXPECT_STR_TIMEOUT)
+            if isinstance(expect_name, Match):
+                self.app_name = expect_name.group(1).decode('ascii')
+        return self.app_name
+
+    def dut_send_ip(self, slave_ip: Optional[str]) -> Optional[int]:
+        ''' The function sends the slave IP address defined as a parameter to master
+        '''
+        addr_num = 0
+        self.expect(self.TEST_IP_PROMPT, timeout=self.TEST_ACK_TIMEOUT)
+        if isinstance(slave_ip, str):
+            for addr_num in range(0, self.TEST_MAX_CIDS):
+                message = r'IP{}={}'.format(addr_num, slave_ip)
+                self.logger.info('{} sent to master'.format(message))
+                self.write(message)
+        return addr_num
+
+    def get_expect_proc(self) -> Optional[object]:
+        expect_proc: object = None
+        try:
+            expect_proc = self.__getattribute__('pexpect_proc')
+        except:
+            expect_proc = self.__getattribute__('_p')
+        finally:
+            if (expect_proc and callable(getattr(expect_proc, 'expect'))):
+                return expect_proc
+            else :
+                return None
+
+    def expect_any(self, *expect_items: Tuple[str, Callable], timeout: Optional[int]) -> None:
+        """
+        expect_any(*expect_items, timeout=DEFAULT_TIMEOUT)
+        expect any of the patterns.
+        will call callback (if provided) if pattern match succeed and then return.
+        will pass match result to the callback.
+
+        :raise ExpectTimeout: failed to match any one of the expect items before timeout
+        :raise UnsupportedExpectItem: pattern in expect_item is not string or compiled RegEx
+
+        :arg expect_items: one or more expect items.
+                           string, compiled RegEx pattern or (string or RegEx(string pattern), callback)
+        :keyword timeout: timeout for expect
+        :return: matched item
+        """
+        def process_expected_item(item_raw: Tuple[str, Callable[..., Any]]) -> Dict[str, Any]:
+            # convert item raw data to standard dict
+            item = {
+                'pattern': item_raw[0] if isinstance(item_raw, tuple) else item_raw,
+                'callback': item_raw[1] if isinstance(item_raw, tuple) else None,
+                'index': -1,
+                'ret': None,
+            }
+            return item
+
+        expect_items_list = [process_expected_item(item) for item in expect_items]
+        expect_patterns = [item['pattern'] for item in expect_items_list if item['pattern'] is not None]
+        match_item = None
+
+        # Workaround: We need to use the original expect method of pexpect process which returns 
+        # index of matched pattern instead of Match object returned by dut.expect()
+        expect_proc: Optional[object] = self.get_expect_proc()
+
+        if expect_proc is not None:
+            match_index = expect_proc.expect(expect_patterns, timeout)
+
+            if isinstance(match_index, int):
+                match_item = expect_items_list[match_index]  # type: ignore
+                match_item['index'] = match_index  # type: ignore , keep match index 
+                if isinstance(expect_proc.match, Match) and len(expect_proc.match.groups()) > 0:
+                    match_item['ret'] = expect_proc.match.groups()
+                if match_item['callback']:
+                    match_item['callback'](match_item['ret'])  # execution of callback function
+        else:
+            self.logger.error('%s: failed to parse output. Please check component versions.', self.app_name)
+            raise RuntimeError from None
+
+    def dut_test_start(self, dictionary: Dict, timeout_value=TEST_EXPECT_STR_TIMEOUT) -> None:  # type: ignore
+        """ The method to initialize and handle test stages
+        """
+        def handle_get_ip4(data: Optional[Any]) -> None:
+            """ Handle get_ip v4
+            """
+            self.logger.info('%s[STACK_IPV4]: %s', self.app_name, str(data))
+            self.test_stage = Stages.STACK_IPV4
+
+        def handle_get_ip6(data: Optional[Any]) -> None:
+            """ Handle get_ip v6
+            """
+            self.logger.info('%s[STACK_IPV6]: %s', self.app_name, str(data))
+            self.test_stage = Stages.STACK_IPV6
+
+        def handle_init(data: Optional[Any]) -> None:
+            """ Handle init
+            """
+            self.logger.info('%s[STACK_INIT]: %s', self.app_name, str(data))
+            self.test_stage = Stages.STACK_INIT
+
+        def handle_connect(data: Optional[Any]) -> None:
+            """ Handle connect
+            """
+            self.logger.info('%s[STACK_CONNECT]: %s', self.app_name, str(data))
+            self.test_stage = Stages.STACK_CONNECT
+
+        def handle_test_start(data: Optional[Any]) -> None:
+            """ Handle connect
+            """
+            self.logger.info('%s[STACK_START]: %s', self.app_name, str(data))
+            self.test_stage = Stages.STACK_START
+
+        def handle_par_ok(data: Optional[Any]) -> None:
+            """ Handle parameter ok
+            """
+            self.logger.info('%s[READ_PAR_OK]: %s', self.app_name, str(data))
+            if self.test_stage.value >= Stages.STACK_START.value:
+                self.param_ok_count += 1
+            self.test_stage = Stages.STACK_PAR_OK
+
+        def handle_par_fail(data: Optional[Any]) -> None:
+            """ Handle parameter fail
+            """
+            self.logger.info('%s[READ_PAR_FAIL]: %s', self.app_name, str(data))
+            self.param_fail_count += 1
+            self.test_stage = Stages.STACK_PAR_FAIL
+
+        def handle_destroy(data: Optional[Any]) -> None:
+            """ Handle destroy
+            """
+            self.logger.info('%s[%s]: %s', self.app_name, Stages.STACK_DESTROY.name, str(data))
+            self.test_stage = Stages.STACK_DESTROY
+            self.test_finish = True
+
+        while not self.test_finish:
+            try:
+                self.expect_any((dictionary[Stages.STACK_IPV4], handle_get_ip4),
+                                (dictionary[Stages.STACK_IPV6], handle_get_ip6),
+                                (dictionary[Stages.STACK_INIT], handle_init),
+                                (dictionary[Stages.STACK_CONNECT], handle_connect),
+                                (dictionary[Stages.STACK_START], handle_test_start),
+                                (dictionary[Stages.STACK_PAR_OK], handle_par_ok),
+                                (dictionary[Stages.STACK_PAR_FAIL], handle_par_fail),
+                                (dictionary[Stages.STACK_DESTROY], handle_destroy),
+                                timeout=timeout_value)
+            except pexpect.TIMEOUT:
+                self.logger.info('%s, expect timeout on stage %s (%s seconds)', self.app_name, self.test_stage.name, timeout_value)
+                self.test_finish = True
+
+    def dut_check_errors(self) -> None:
+        ''' Verify allowed percentage of errors for the dut 
+        '''
+        allowed_ok_percentage = ((self.param_ok_count / (self.param_ok_count + self.param_fail_count + 1)) * 100)
+        if self.param_ok_count and (allowed_ok_percentage > (100 - ALLOWED_PERCENT_OF_FAILS)):
+            self.logger.info('%s: ok_count: %d, fail count: %d', self.app_name, self.param_ok_count, self.param_fail_count)
+        else :
+            self.logger.error('%s: ok_count: %d, number of failed readings %d exceeds %d percent', self.app_name, self.param_ok_count, self.param_fail_count, ALLOWED_PERCENT_OF_FAILS)
+            raise RuntimeError from None
+
+############
+# Fixtures #
+############
+
+@pytest.fixture(scope='session', autouse=True)
+def session_tempdir() -> str:
+    
+    _tmpdir = os.path.join(
+        os.path.dirname(__file__),
+        'pytest_embedded_log',
+        datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
+    )
+    os.makedirs(_tmpdir, exist_ok=True)
+    return _tmpdir
+
+
+@pytest.fixture(autouse=True)
+@multi_dut_fixture
+def junit_properties(
+    test_case_name: str, record_xml_attribute: Callable[[str, object], None]
+) -> None:
+    """
+    This fixture is autoused and will modify the junit report test case name to <target>.<config>.<case_name>
+    """
+    record_xml_attribute('name', test_case_name)
+
+
+@pytest.fixture(scope='module')
+def monkeypatch_module(request: FixtureRequest) -> MonkeyPatch:
+    mp = MonkeyPatch()
+    request.addfinalizer(mp.undo)
+    return mp
+
+
+@pytest.fixture(scope='module', autouse=True)
+def replace_dut_class(monkeypatch_module: MonkeyPatch) -> None:
+    monkeypatch_module.setattr('pytest_embedded_idf.IdfDut', ModbusTestDut)
+
+
+@pytest.fixture
+@multi_dut_argument
+def config(request: FixtureRequest) -> str:
+    return getattr(request, 'param', None) or DEFAULT_SDKCONFIG
+
+
+@pytest.fixture
+@multi_dut_fixture
+def build_dir(app_path: str, target: Optional[str], config: Optional[str]) -> str:
+    """
+    Check local build dir with the following priority:
+
+    1. build_<target>_<config>
+    2. build_<target>
+    3. build_<config>
+    4. build
+
+    Args:
+        app_path: app path
+        target: target
+        config: config
+
+    Returns:
+        valid build directory
+    """
+    check_dirs = []
+    if target is not None and config is not None:
+        check_dirs.append(f'build_{target}_{config}')
+    if target is not None:
+        check_dirs.append(f'build_{target}')
+    if config is not None:
+        check_dirs.append(f'build_{config}')
+    check_dirs.append('build')
+
+    for check_dir in check_dirs:
+        binary_path = os.path.join(app_path, check_dir)
+        if os.path.isdir(binary_path):
+            logging.info(f'find valid binary path: {binary_path}')
+            return check_dir
+
+        logging.warning(
+            'checking binary path: %s... missing... try another place', binary_path
+        )
+
+    recommend_place = check_dirs[0]
+    logging.error(
+        f'no build dir valid. Please build the binary via "idf.py -B {recommend_place} build" and run pytest again'
+    )
+    sys.exit(1)

test/serial/mb_serial_master/sdkconfig.ci.ascii

@@ -0,0 +1,7 @@
+CONFIG_MB_COMM_MODE_ASCII=y
+CONFIG_MB_UART_BAUD_RATE=115200
+CONFIG_FMB_TIMER_PORT_ENABLED=n
+CONFIG_FMB_TIMER_ISR_IN_IRAM=y
+CONFIG_FMB_MASTER_DELAY_MS_CONVERT=200
+CONFIG_FMB_MASTER_TIMEOUT_MS_RESPOND=400
+CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y

test/serial/mb_serial_master/sdkconfig.ci.rtu

@@ -0,0 +1,8 @@
+CONFIG_MB_COMM_MODE_ASCII=n
+CONFIG_MB_COMM_MODE_RTU=y
+CONFIG_MB_UART_BAUD_RATE=115200
+CONFIG_FMB_TIMER_PORT_ENABLED=n
+CONFIG_FMB_TIMER_ISR_IN_IRAM=y
+CONFIG_FMB_MASTER_DELAY_MS_CONVERT=200
+CONFIG_FMB_MASTER_TIMEOUT_MS_RESPOND=400
+CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y

test/serial/mb_serial_slave/sdkconfig.ci.ascii

@@ -0,0 +1,6 @@
+CONFIG_MB_COMM_MODE_ASCII=y
+CONFIG_MB_SLAVE_ADDR=1
+CONFIG_MB_UART_BAUD_RATE=115200
+CONFIG_FMB_TIMER_PORT_ENABLED=y
+CONFIG_FMB_TIMER_ISR_IN_IRAM=y
+CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y

test/serial/mb_serial_slave/sdkconfig.ci.rtu

@@ -0,0 +1,7 @@
+CONFIG_MB_COMM_MODE_ASCII=n
+CONFIG_MB_COMM_MODE_RTU=y
+CONFIG_MB_SLAVE_ADDR=1
+CONFIG_MB_UART_BAUD_RATE=115200
+CONFIG_FMB_TIMER_PORT_ENABLED=y
+CONFIG_FMB_TIMER_ISR_IN_IRAM=y
+CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y

test/serial/pytest_mb_master_slave.py

@@ -0,0 +1,66 @@
+# SPDX-FileCopyrightText: 2016-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+# This is the script to reproduce the issue when the expect() is called from
+# main thread in Multi DUT case.
+
+import logging
+import os
+from typing import Tuple
+
+import pytest
+
+from conftest import ModbusTestDut, Stages
+
+pattern_dict_slave = {Stages.STACK_IPV4: (r'I \([0-9]+\) example_connect: - IPv4 address: ([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})'),
+                      Stages.STACK_IPV6: (r'I \([0-9]+\) example_connect: - IPv6 address: (([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})'),
+                      Stages.STACK_INIT: (r'I \(([0-9]+)\) MB_TCP_SLAVE_PORT: (Protocol stack initialized).'),
+                      Stages.STACK_CONNECT: (r'I\s\(([0-9]+)\) MB_TCP_SLAVE_PORT: Socket \(#[0-9]+\), accept client connection from address: '
+                                             r'([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})'),
+                      Stages.STACK_START: (r'I\s\(([0-9]+)\) SLAVE_TEST: (Start modbus test)'),
+                      Stages.STACK_PAR_OK: (r'I\s\(([0-9]+)\) SLAVE_TEST: ([A-Z]+ [A-Z]+) \([a-zA-Z0-9_]+ us\),\s'
+                                            r'ADDR:([0-9]+), TYPE:[0-9]+, INST_ADDR:0x[a-zA-Z0-9]+, SIZE:[0-9]+'),
+                      Stages.STACK_PAR_FAIL: (r'E \(([0-9]+)\) SLAVE_TEST: Response time exceeds configured [0-9]+ [ms], ignore packet'),
+                      Stages.STACK_DESTROY: (r'I\s\(([0-9]+)\) SLAVE_TEST: (Modbus controller destroyed).')}
+
+pattern_dict_master = {Stages.STACK_IPV4: (r'I \([0-9]+\) example_connect: - IPv4 address: ([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})'),
+                       Stages.STACK_IPV6: (r'I \([0-9]+\) example_connect: - IPv6 address: (([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})'),
+                       Stages.STACK_INIT: (r'I \(([0-9]+)\) MASTER_TEST: (Modbus master stack initialized)'),
+                       Stages.STACK_CONNECT: (r'I\s\(([0-9]+)\) MB_TCP_MASTER_PORT: (Connected [0-9]+ slaves), start polling'),
+                       Stages.STACK_START: (r'I \(([0-9]+)\) MASTER_TEST: (Start modbus test)'),
+                       Stages.STACK_PAR_OK: (r'I \(([0-9]+)\) MASTER_TEST: Characteristic #[0-9]+ ([a-zA-Z0-9_]+)'
+                                             r'\s\([a-zA-Z\_\%\/]+\) value =[a-zA-Z0-9\.\s]* \((0x[a-zA-Z0-9]+)\)[,\sa-z]+ successful'),
+                       Stages.STACK_PAR_FAIL: (r'.*E \(([0-9]+)\) MASTER_TEST: Characteristic #[0-9]+\s\(([a-zA-Z0-9_]+)\)\s'
+                                               r'read fail, err = [0-9]+ \([_A-Z]+\)'),
+                       Stages.STACK_DESTROY: (r'I \(([0-9]+)\) MASTER_TEST: (Destroy master)...')}
+
+LOG_LEVEL = logging.DEBUG
+LOGGER_NAME = 'modbus_test'
+logger = logging.getLogger(LOGGER_NAME)
+
+test_configs = [
+    'rtu',
+    'ascii'
+]
+
+@pytest.mark.esp32
+@pytest.mark.multi_dut_modbus_serial
+@pytest.mark.parametrize('config', test_configs, indirect=True)
+@pytest.mark.parametrize(
+    'count, app_path', [
+        (2, f'{os.path.join(os.path.dirname(__file__), "mb_serial_master")}|{os.path.join(os.path.dirname(__file__), "mb_serial_slave")}')
+    ],
+    indirect=True
+)
+def test_modbus_serial_communication(config: str, dut: Tuple[ModbusTestDut, ModbusTestDut]) -> None:
+    dut_slave = dut[1]
+    dut_master = dut[0]
+
+    logger.info('DUT: %s start.', dut_master.dut_get_name())
+    logger.info('DUT: %s start.', dut_slave.dut_get_name())
+
+    dut_slave.dut_test_start(dictionary=pattern_dict_slave)
+    dut_master.dut_test_start(dictionary=pattern_dict_master)
+
+    dut_slave.dut_check_errors()
+    dut_master.dut_check_errors()

test/tcp/mb_tcp_master/sdkconfig.ci.ethernet

@@ -0,0 +1,33 @@
+#
+# Modbus configuration
+#
+CONFIG_FMB_COMM_MODE_TCP_EN=y
+CONFIG_FMB_TCP_PORT_DEFAULT=502
+CONFIG_FMB_TCP_CONNECTION_TOUT_SEC=20
+CONFIG_FMB_PORT_TASK_STACK_SIZE=4096
+CONFIG_FMB_PORT_TASK_PRIO=10
+CONFIG_FMB_COMM_MODE_RTU_EN=n
+CONFIG_FMB_COMM_MODE_ASCII_EN=n
+CONFIG_FMB_MASTER_TIMEOUT_MS_RESPOND=3000
+CONFIG_FMB_MASTER_DELAY_MS_CONVERT=300
+CONFIG_FMB_TIMER_PORT_ENABLED=y
+CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y
+CONFIG_MB_MDNS_IP_RESOLVER=n
+CONFIG_MB_SLAVE_IP_FROM_STDIN=y
+CONFIG_MB_SLAVE_ADDR=1
+CONFIG_LOG_DEFAULT_LEVEL_DEBUG=y
+CONFIG_EXAMPLE_CONNECT_IPV6=n
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y
+CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y
+CONFIG_EXAMPLE_ETH_PHY_IP101=y
+CONFIG_EXAMPLE_ETH_MDC_GPIO=23
+CONFIG_EXAMPLE_ETH_MDIO_GPIO=18
+CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5
+CONFIG_EXAMPLE_ETH_PHY_ADDR=1
+CONFIG_EXAMPLE_ETHERNET_EMAC_TASK_STACK_SIZE=4096
+
+CONFIG_ETH_ENABLED=y
+CONFIG_ETH_USE_ESP32_EMAC=y
+CONFIG_ETH_PHY_INTERFACE_RMII=y
+CONFIG_ETH_USE_SPI_ETHERNET=n

test/tcp/mb_tcp_master/sdkconfig.ci.wifi

@@ -0,0 +1,16 @@
+CONFIG_FMB_COMM_MODE_TCP_EN=y
+CONFIG_FMB_TCP_PORT_DEFAULT=502
+CONFIG_FMB_TCP_CONNECTION_TOUT_SEC=20
+CONFIG_FMB_PORT_TASK_STACK_SIZE=4096
+CONFIG_FMB_PORT_TASK_PRIO=10
+CONFIG_FMB_COMM_MODE_RTU_EN=n
+CONFIG_FMB_COMM_MODE_ASCII_EN=n
+CONFIG_FMB_MASTER_TIMEOUT_MS_RESPOND=3000
+CONFIG_FMB_MASTER_DELAY_MS_CONVERT=300
+CONFIG_FMB_TIMER_PORT_ENABLED=y
+CONFIG_MB_MDNS_IP_RESOLVER=n
+CONFIG_MB_SLAVE_IP_FROM_STDIN=y
+CONFIG_EXAMPLE_CONNECT_IPV6=n
+CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y
+CONFIG_EXAMPLE_WIFI_SSID="${CI_WIFI_SSID}"
+CONFIG_EXAMPLE_WIFI_PASSWORD="${CI_WIFI_PASSW}"

test/tcp/mb_tcp_master/sdkconfig.defaults

@@ -8,11 +8,16 @@ CONFIG_FMB_PORT_TASK_STACK_SIZE=4096
 CONFIG_FMB_PORT_TASK_PRIO=10
 CONFIG_FMB_COMM_MODE_RTU_EN=n
 CONFIG_FMB_COMM_MODE_ASCII_EN=n
-CONFIG_FMB_MASTER_TIMEOUT_MS_RESPOND=2000
+CONFIG_FMB_MASTER_TIMEOUT_MS_RESPOND=1000
 CONFIG_FMB_MASTER_DELAY_MS_CONVERT=300
 CONFIG_FMB_TIMER_PORT_ENABLED=y
 CONFIG_FMB_TIMER_ISR_IN_IRAM=y
+CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y
 CONFIG_MB_MDNS_IP_RESOLVER=n
 CONFIG_MB_SLAVE_IP_FROM_STDIN=y
+CONFIG_MB_SLAVE_ADDR=1
 CONFIG_EXAMPLE_CONNECT_IPV6=n
-CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y
+CONFIG_EXAMPLE_CONNECT_ETHERNET=n
+CONFIG_EXAMPLE_CONNECT_WIFI=y
+CONFIG_EXAMPLE_WIFI_SSID="${CI_WIFI_SSID}"
+CONFIG_EXAMPLE_WIFI_PASSWORD="${CI_WIFI_PASSW}"

test/tcp/mb_tcp_slave/sdkconfig.ci.ethernet

@@ -0,0 +1,31 @@
+#
+# Modbus configuration
+#
+CONFIG_FMB_COMM_MODE_TCP_EN=y
+CONFIG_FMB_TCP_PORT_DEFAULT=502
+CONFIG_FMB_TCP_CONNECTION_TOUT_SEC=20
+CONFIG_FMB_PORT_TASK_STACK_SIZE=4096
+CONFIG_FMB_PORT_TASK_PRIO=10
+CONFIG_FMB_COMM_MODE_RTU_EN=n
+CONFIG_FMB_COMM_MODE_ASCII_EN=n
+CONFIG_FMB_MASTER_TIMEOUT_MS_RESPOND=1000
+CONFIG_FMB_MASTER_DELAY_MS_CONVERT=300
+CONFIG_FMB_TIMER_PORT_ENABLED=y
+CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y
+CONFIG_MB_MDNS_IP_RESOLVER=n
+CONFIG_MB_SLAVE_IP_FROM_STDIN=y
+CONFIG_MB_SLAVE_ADDR=1
+CONFIG_EXAMPLE_CONNECT_IPV6=n
+CONFIG_EXAMPLE_CONNECT_WIFI=n
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y
+CONFIG_EXAMPLE_USE_INTERNAL_ETHERNET=y
+CONFIG_EXAMPLE_ETH_PHY_IP101=y
+CONFIG_EXAMPLE_ETH_MDC_GPIO=23
+CONFIG_EXAMPLE_ETH_MDIO_GPIO=18
+CONFIG_EXAMPLE_ETH_PHY_RST_GPIO=5
+CONFIG_EXAMPLE_ETH_PHY_ADDR=1
+
+CONFIG_ETH_ENABLED=y
+CONFIG_ETH_USE_ESP32_EMAC=y
+CONFIG_ETH_PHY_INTERFACE_RMII=y
+CONFIG_ETH_USE_SPI_ETHERNET=n

test/tcp/mb_tcp_slave/sdkconfig.ci.wifi

@@ -0,0 +1,19 @@
+CONFIG_FMB_COMM_MODE_TCP_EN=y
+CONFIG_FMB_TCP_PORT_DEFAULT=502
+CONFIG_FMB_TCP_CONNECTION_TOUT_SEC=20
+CONFIG_FMB_PORT_TASK_STACK_SIZE=4096
+CONFIG_FMB_PORT_TASK_PRIO=10
+CONFIG_FMB_COMM_MODE_RTU_EN=n
+CONFIG_FMB_COMM_MODE_ASCII_EN=n
+CONFIG_FMB_MASTER_TIMEOUT_MS_RESPOND=2000
+CONFIG_FMB_MASTER_DELAY_MS_CONVERT=300
+CONFIG_FMB_TIMER_PORT_ENABLED=y
+CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y
+CONFIG_MB_MDNS_IP_RESOLVER=n
+CONFIG_MB_SLAVE_IP_FROM_STDIN=y
+CONFIG_MB_SLAVE_ADDR=1
+CONFIG_EXAMPLE_CONNECT_IPV6=n
+CONFIG_EXAMPLE_CONNECT_ETHERNET=n
+CONFIG_EXAMPLE_CONNECT_WIFI=y
+CONFIG_EXAMPLE_WIFI_SSID="${CI_WIFI_SSID}"
+CONFIG_EXAMPLE_WIFI_PASSWORD="${CI_WIFI_PASSW}"

test/tcp/mb_tcp_slave/sdkconfig.defaults

@@ -8,13 +8,12 @@ CONFIG_FMB_PORT_TASK_STACK_SIZE=4096
 CONFIG_FMB_PORT_TASK_PRIO=10
 CONFIG_FMB_COMM_MODE_RTU_EN=n
 CONFIG_FMB_COMM_MODE_ASCII_EN=n
-CONFIG_FMB_MASTER_TIMEOUT_MS_RESPOND=1000
+CONFIG_FMB_MASTER_TIMEOUT_MS_RESPOND=2000
 CONFIG_FMB_MASTER_DELAY_MS_CONVERT=300
 CONFIG_FMB_TIMER_PORT_ENABLED=y
-CONFIG_FMB_TIMER_ISR_IN_IRAM=y
 CONFIG_MB_MDNS_IP_RESOLVER=n
 CONFIG_MB_SLAVE_IP_FROM_STDIN=y
-CONFIG_MB_SLAVE_ADDR=1
-CONFIG_LOG_DEFAULT_LEVEL_DEBUG=y
 CONFIG_EXAMPLE_CONNECT_IPV6=n
 CONFIG_FMB_TIMER_USE_ISR_DISPATCH_METHOD=y
+CONFIG_EXAMPLE_WIFI_SSID="${CI_WIFI_SSID}"
+CONFIG_EXAMPLE_WIFI_PASSWORD="${CI_WIFI_PASSW}"

test/tcp/pytest_mb_tcp_master_slave.py

@@ -0,0 +1,69 @@
+# SPDX-FileCopyrightText: 2016-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+# This is the script to reproduce the issue when the expect() is called from
+# main thread in Multi DUT case.
+
+import logging
+import os
+from typing import Tuple
+
+import pytest
+
+from conftest import ModbusTestDut, Stages
+
+pattern_dict_slave = {Stages.STACK_IPV4: (r'I \([0-9]+\) example_connect: - IPv4 address: ([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})'),
+                      Stages.STACK_IPV6: (r'I \([0-9]+\) example_connect: - IPv6 address: (([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})'),
+                      Stages.STACK_INIT: (r'I \(([0-9]+)\) MB_TCP_SLAVE_PORT: (Protocol stack initialized).'),
+                      Stages.STACK_CONNECT: (r'I\s\(([0-9]+)\) MB_TCP_SLAVE_PORT: Socket \(#[0-9]+\), accept client connection from address: '
+                                             r'([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})'),
+                      Stages.STACK_START: (r'I\s\(([0-9]+)\) SLAVE_TEST: (Start modbus test)'),
+                      Stages.STACK_PAR_OK: (r'I\s\(([0-9]+)\) SLAVE_TEST: ([A-Z]+ [A-Z]+) \([a-zA-Z0-9_]+ us\),\s'
+                                            r'ADDR:([0-9]+), TYPE:[0-9]+, INST_ADDR:0x[a-zA-Z0-9]+, SIZE:[0-9]+'),
+                      Stages.STACK_PAR_FAIL: (r'E \(([0-9]+)\) SLAVE_TEST: Response time exceeds configured [0-9]+ [ms], ignore packet'),
+                      Stages.STACK_DESTROY: (r'I\s\(([0-9]+)\) SLAVE_TEST: (Modbus controller destroyed).')}
+
+pattern_dict_master = {Stages.STACK_IPV4: (r'I \([0-9]+\) example_connect: - IPv4 address: ([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})'),
+                       Stages.STACK_IPV6: (r'I \([0-9]+\) example_connect: - IPv6 address: (([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})'),
+                       Stages.STACK_INIT: (r'I \(([0-9]+)\) MASTER_TEST: (Modbus master stack initialized)'),
+                       Stages.STACK_CONNECT: (r'I\s\(([0-9]+)\) MB_TCP_MASTER_PORT: (Connected [0-9]+ slaves), start polling'),
+                       Stages.STACK_START: (r'I \(([0-9]+)\) MASTER_TEST: (Start modbus test)'),
+                       Stages.STACK_PAR_OK: (r'I \(([0-9]+)\) MASTER_TEST: Characteristic #[0-9]+ ([a-zA-Z0-9_]+)'
+                                             r'\s\([a-zA-Z\_\%\/]+\) value =[a-zA-Z0-9\.\s]* \((0x[a-zA-Z0-9]+)\)[,\sa-z]+ successful'),
+                       Stages.STACK_PAR_FAIL: (r'.*E \(([0-9]+)\) MASTER_TEST: Characteristic #[0-9]+\s\(([a-zA-Z0-9_]+)\)\s'
+                                               r'read fail, err = [x0-9]+ \([_A-Z]+\)'),
+                       Stages.STACK_DESTROY: (r'I \(([0-9]+)\) MASTER_TEST: (Destroy master)...')}
+
+LOG_LEVEL = logging.DEBUG
+LOGGER_NAME = 'modbus_test'
+logger = logging.getLogger(LOGGER_NAME)
+
+test_configs = [
+    'wifi',
+    'ethernet'
+]
+
+@pytest.mark.esp32
+@pytest.mark.multi_dut_modbus_tcp
+@pytest.mark.parametrize('config', test_configs, indirect=True)
+@pytest.mark.parametrize(
+    'count, app_path', [
+        (2, f'{os.path.join(os.path.dirname(__file__), "mb_tcp_master")}|{os.path.join(os.path.dirname(__file__), "mb_tcp_slave")}')
+    ],
+    indirect=True
+)
+def test_modbus_tcp_communication(dut: Tuple[ModbusTestDut, ModbusTestDut]) -> None:
+    dut_slave = dut[1]
+    dut_master = dut[0]
+
+    logger.info('DUT: %s start.', dut_master.dut_get_name())
+    logger.info('DUT: %s start.', dut_slave.dut_get_name())
+
+    dut_slave_ip_address = dut_slave.dut_get_ip()
+    dut_master.dut_send_ip(dut_slave_ip_address)
+
+    dut_slave.dut_test_start(dictionary=pattern_dict_slave)
+    dut_master.dut_test_start(dictionary=pattern_dict_master)
+    
+    dut_slave.dut_check_errors()
+    dut_master.dut_check_errors()

tools/ignore_build_warnings.txt

@@ -0,0 +1,16 @@
+library/error\.o
+/.*error\S*\.o
+.*error.*\.c\.obj
+.*error.*\.c
+.*error.*\.cpp\.obj
+.*error.*\.cxx\.obj
+.*error.*\.cc\.obj
+-Werror
+error\.d
+/.*error\S*.d
+reassigning to symbol
+changes choice state
+crosstool_version_check\.cmake
+CryptographyDeprecationWarning
+Warning: \d+/\d+ app partitions are too small for binary
+CMake Deprecation Warning at main/lib/tinyxml2/CMakeLists\.txt:11 \(cmake_policy\)