Merge branch 'feature/update_secure_boot_test_app_to_pytest' into 'master'

secure_boot_test_app: Update the test_app to use pytest framework

Closes IDFCI-1135

See merge request espressif/esp-idf!17967

tools/test_apps/.build-test-rules.yml

@@ -70,12 +70,9 @@ tools/test_apps/protocols/netif_components:
       reason: one target is enough to verify netif component dependencies
 
 tools/test_apps/security/secure_boot:
-#  disable:
-#    - if: SOC_SECURE_BOOT_SUPPORTED != 1
   disable:
-    - if: IDF_TARGET == "esp32c2"
+    - if: IDF_ENV_FPGA != 1
-      temporary: true
+      reason: the test can only run on an FPGA as efuses need to be reset during the test.
-      reason: target esp32c2 is not supported yet
 
 tools/test_apps/system/bootloader_sections:
   disable:

tools/test_apps/security/secure_boot/conftest.py

@@ -0,0 +1,252 @@
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+# pylint: disable=W0621  # redefined-outer-name
+
+import collections
+import os
+import tempfile
+import time
+
+import espefuse
+import espsecure
+import pytest
+import serial
+from _pytest.fixtures import FixtureRequest
+from _pytest.monkeypatch import MonkeyPatch
+from pytest_embedded_idf.app import FlashFile
+from pytest_embedded_idf.dut import IdfDut
+from pytest_embedded_idf.serial import IdfSerial
+from pytest_embedded_serial_esp.serial import EspSerial
+
+efuse_reset_port = os.getenv('EFUSEPORT')
+
+
+# This is a custom Serial Class for the FPGA
+class FpgaSerial(IdfSerial):
+    def __init__(self, *args, **kwargs) -> None:  # type: ignore
+        super().__init__(*args, **kwargs)
+        self.efuse_reset_port = efuse_reset_port
+        if self.efuse_reset_port is None:
+            raise RuntimeError('EFUSEPORT not specified')
+
+        self.secure_boot_en = self.app.sdkconfig.get('CONFIG_SECURE_BOOT') and \
+            not self.app.sdkconfig.get('CONFIG_EFUSE_VIRTUAL')
+
+        self.efuses = None
+        self.efuse_operations = None
+
+    @EspSerial.use_esptool(hard_reset_after=False, no_stub=True)
+    def enable_efuses(self) -> None:
+        # We use an extra COM port to reset the efuses on FPGA.
+        # Connect DTR pin of the COM port to the efuse reset pin on daughter board
+        # Set EFUSEPORT env variable to the extra COM port
+        if self.efuse_reset_port is None:
+            raise RuntimeError('EFUSEPORT not specified')
+
+        self.efuses, self.efuse_operations = espefuse.get_efuses(self.esp, False, False, True)
+
+    @EspSerial.use_esptool(hard_reset_after=False, no_stub=True)
+    def bootloader_flash(self, bootloader_path: str) -> None:
+        """
+        Flash bootloader.
+
+        :return: None
+        """
+        offs = int(self.app.sdkconfig.get('BOOTLOADER_OFFSET_IN_FLASH', 0))
+        prev_flash_files = self.app.flash_files
+        flash_files = []
+        flash_files.append(
+            FlashFile(
+                offs,
+                bootloader_path,
+                False,
+            )
+        )
+        self.app.flash_files = flash_files
+        self.app.flash_settings['encrypt'] = False
+        self.app.flash_settings['no_stub'] = True
+        self.app.flash_settings['force'] = True
+        self.flash()
+        # Restore self.app.flash files to original value
+        self.app.flash_files = prev_flash_files
+
+    @EspSerial.use_esptool(hard_reset_after=False, no_stub=True)
+    def app_flash(self, app_path: str) -> None:
+        """
+        Flash App.
+
+        :return: None
+        """
+        offs = self.app.flash_args['app']['offset']
+        prev_flash_files = self.app.flash_files
+        flash_files = []
+        flash_files.append(
+            FlashFile(
+                offs,
+                app_path,
+                False,
+            )
+        )
+        self.app.flash_files = flash_files
+        self.app.flash_settings['encrypt'] = False
+        self.app.flash_settings['no_stub'] = True
+        self.flash()
+        # Restore self.app.flash files to original value
+        self.app.flash_files = prev_flash_files
+
+    @EspSerial.use_esptool(hard_reset_after=True, no_stub=True)
+    def burn_efuse_key_digest(self, key: str, purpose: str, block: str) -> None:
+        if self.efuse_operations is None:
+            self.enable_efuses()
+        BurnDigestArgs = collections.namedtuple('BurnDigestArgs',
+                                                ['keyfile', 'keypurpose', 'block',
+                                                 'force_write_always', 'no_write_protect', 'no_read_protect'])
+        args = BurnDigestArgs([open(key, 'rb')],
+                              [purpose],
+                              [block],
+                              False, False, True)
+
+        self.efuse_operations.burn_key_digest(self.esp, self.efuses, args)   # type: ignore
+
+    @EspSerial.use_esptool(hard_reset_after=False, no_stub=True)
+    def burn_efuse(self, field: str, val: int) -> None:
+        if self.efuse_operations is None:
+            self.enable_efuses()
+        BurnEfuseArgs = collections.namedtuple('BurnEfuseArgs', ['do_not_confirm', 'name_value_pairs'])
+        args = BurnEfuseArgs(True, {field: val})
+
+        self.efuse_operations.burn_efuse(self.esp, self.efuses, args)   # type: ignore
+
+    @EspSerial.use_esptool(hard_reset_after=False, no_stub=True)
+    def burn_efuse_key(self, key: str, purpose: str, block: str) -> None:
+        if self.efuse_operations is None:
+            self.enable_efuses()
+        BurnKeyArgs = collections.namedtuple('BurnKeyArgs',
+                                             ['keyfile', 'keypurpose', 'block',
+                                              'force_write_always', 'no_write_protect', 'no_read_protect'])
+        args = BurnKeyArgs([key],
+                           [purpose],
+                           [block],
+                           False, False, False)
+
+        self.efuse_operations.burn_key(self.esp, self.efuses, args)   # type: ignore
+
+    def reset_efuses(self) -> None:
+        if self.efuse_reset_port is None:
+            raise RuntimeError('EFUSEPORT not specified')
+        with serial.Serial(self.efuse_reset_port) as efuseport:
+            print('Resetting efuses')
+            efuseport.dtr = 0
+            self.proc.setRTS(0)
+            time.sleep(1)
+            efuseport.dtr = 1
+            self.proc.setRTS(1)
+            time.sleep(1)
+            self.proc.setRTS(0)
+            efuseport.dtr = 0
+
+            self.efuse_operations = None
+            self.efuses = None
+
+
+class FpgaDut(IdfDut):
+    ERASE_NVS = True
+    FLASH_ENCRYPT_SCHEME = None             # type: str
+    FLASH_ENCRYPT_CNT_KEY = None            # type: str
+    FLASH_ENCRYPT_CNT_VAL = 0
+    FLASH_ENCRYPT_PURPOSE = None            # type: str
+    SECURE_BOOT_EN_KEY = None               # type: str
+    SECURE_BOOT_EN_VAL = 0
+    FLASH_SECTOR_SIZE = 4096
+
+    def __init__(self, *args, **kwargs) -> None:  # type: ignore
+        super().__init__(*args, **kwargs)
+        self.efuses = None
+        self.efuse_operations = None
+        self.efuse_reset_port = efuse_reset_port
+
+    def sign_data(self, data_file: str, key_files: str, version: str, append_signature: int = 0) -> bytes:
+        SignDataArgs = collections.namedtuple('SignDataArgs',
+                                              ['datafile','keyfile','output', 'version', 'append_signatures'])
+        with tempfile.NamedTemporaryFile() as outfile:
+            args = SignDataArgs(data_file, key_files, outfile.name, str(version), append_signature)
+            espsecure.sign_data(args)
+            outfile.seek(0)
+            return outfile.read()
+
+
+class Esp32c3FpgaDut(FpgaDut):
+    FLASH_ENCRYPT_SCHEME = 'AES-XTS'
+    FLASH_ENCRYPT_CNT_KEY = 'SPI_BOOT_CRYPT_CNT'
+    FLASH_ENCRYPT_CNT_VAL = 1
+    FLASH_ENCRYPT_PURPOSE = 'XTS_AES_128_KEY'
+    SECURE_BOOT_EN_KEY = 'SECURE_BOOT_EN'
+    SECURE_BOOT_EN_VAL = 1
+    WAFER_VERSION = 'WAFER_VERSION_MINOR_LO'
+    WAFER_VERSION_VAL = 3
+
+    def burn_wafer_version(self) -> None:
+        self.serial.burn_efuse(self.WAFER_VERSION, self.WAFER_VERSION_VAL)
+
+    def flash_encrypt_burn_cnt(self) -> None:
+        self.serial.burn_efuse(self.FLASH_ENCRYPT_CNT_KEY, self.FLASH_ENCRYPT_CNT_VAL)
+
+    def flash_encrypt_burn_key(self, key: str, block: int = 0) -> None:
+        self.serial.burn_efuse_key(key, self.FLASH_ENCRYPT_PURPOSE, 'BLOCK_KEY%d' % block)
+
+    def flash_encrypt_get_scheme(self) -> str:
+        return self.FLASH_ENCRYPT_SCHEME
+
+    def secure_boot_burn_en_bit(self) -> None:
+        self.serial.burn_efuse(self.SECURE_BOOT_EN_KEY, self.SECURE_BOOT_EN_VAL)
+
+    def secure_boot_burn_digest(self, digest: str, key_index: int = 0, block: int = 0) -> None:
+        self.serial.burn_efuse_key_digest(digest, 'SECURE_BOOT_DIGEST%d' % key_index, 'BLOCK_KEY%d' % block)
+
+
+class Esp32s3FpgaDut(FpgaDut):
+    FLASH_ENCRYPT_SCHEME = 'AES-XTS'
+    FLASH_ENCRYPT_CNT_KEY = 'SPI_BOOT_CRYPT_CNT'
+    FLASH_ENCRYPT_CNT_VAL = 1
+    FLASH_ENCRYPT_PURPOSE = 'XTS_AES_128_KEY'
+    SECURE_BOOT_EN_KEY = 'SECURE_BOOT_EN'
+    SECURE_BOOT_EN_VAL = 1
+    WAFER_VERSION = 'WAFER_VERSION_MINOR_LO'
+    WAFER_VERSION_VAL = 1
+
+    def burn_wafer_version(self) -> None:
+        self.serial.burn_efuse(self.WAFER_VERSION, self.WAFER_VERSION_VAL)
+
+    def flash_encrypt_burn_cnt(self) -> None:
+        self.serial.burn_efuse(self.FLASH_ENCRYPT_CNT_KEY, self.FLASH_ENCRYPT_CNT_VAL)
+
+    def flash_encrypt_burn_key(self, key: str, block: int = 0) -> None:
+        self.serial.burn_efuse_key(key, self.FLASH_ENCRYPT_PURPOSE, 'BLOCK_KEY%d' % block)
+
+    def flash_encrypt_get_scheme(self) -> str:
+        return self.FLASH_ENCRYPT_SCHEME
+
+    def secure_boot_burn_en_bit(self) -> None:
+        self.serial.burn_efuse(self.SECURE_BOOT_EN_KEY, self.SECURE_BOOT_EN_VAL)
+
+    def secure_boot_burn_digest(self, digest: str, key_index: int = 0, block: int = 0) -> None:
+        self.serial.burn_efuse_key_digest(digest, 'SECURE_BOOT_DIGEST%d' % key_index, 'BLOCK_KEY%d' % block)
+
+
+@pytest.fixture(scope='module')
+def monkeypatch_module(request: FixtureRequest) -> MonkeyPatch:
+    mp = MonkeyPatch()
+    request.addfinalizer(mp.undo)
+    return mp
+
+
+@pytest.fixture(scope='module', autouse=True)
+def replace_dut_class(monkeypatch_module: MonkeyPatch, pytestconfig: pytest.Config) -> None:
+    target = pytestconfig.getoption('target')
+    if target == 'esp32c3':
+        monkeypatch_module.setattr('pytest_embedded_idf.IdfDut', Esp32c3FpgaDut)
+    elif target == 'esp32s3':
+        monkeypatch_module.setattr('pytest_embedded_idf.IdfDut', Esp32s3FpgaDut)
+
+    monkeypatch_module.setattr('pytest_embedded_idf.IdfSerial', FpgaSerial)

tools/test_apps/security/secure_boot/pytest_secure_boot.py

@@ -1,11 +1,13 @@
+# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
 from __future__ import print_function
 
 import os
 import struct
 import zlib
-from io import BytesIO
 
-import ttfw_idf
+import pytest
+from pytest_embedded import Dut
 
 # To prepare a runner for these tests,
 # 1. Connect an FPGA with C3 image
@@ -65,65 +67,67 @@ def corrupt_sig_block(sig_block, seed=0, corrupt_sig=True, corrupt_crc=False):
     return result
 
 
-def dut_start_secure_app(dut):      # type: (ttfw_idf.IDFDUT) -> None
+def dut_start_secure_app(dut: Dut) -> None:
-    dut.reset_efuses()
+    dut.serial.erase_flash()
-    bootloader_bin = os.path.join(dut.app.binary_path, 'bootloader/bootloader.bin')
+    dut.serial.reset_efuses()
-    with open(bootloader_bin, 'rb') as f:
+    dut.burn_wafer_version()
-        dut.write_flash_data([(0x0, f)], None, True, False)
+
-    dut.start_app()
+    bootloader_path = os.path.join(dut.app.binary_path, 'bootloader/bootloader.bin')
+    dut.serial.bootloader_flash(bootloader_path)
+    dut.serial.flash()
 
 
 # Test secure boot flow.
 # Correctly signed bootloader + correctly signed app should work
-@ttfw_idf.idf_custom_test(env_tag='Example_Secure_Boot', target=['esp32c3fpga', 'esp32s3fpga'], ignore=True)
+@pytest.mark.esp32c3
-def test_examples_security_secure_boot(env, _):    # type: (ttfw_idf.TinyFW.Env, None) -> None
+@pytest.mark.esp32s3
-    efuse_port = os.getenv('EFUSEPORT')
+def test_examples_security_secure_boot(dut: Dut) -> None:
-    dut = env.get_dut('secure_boot', 'tools/test_apps/security/secure_boot', efuse_reset_port=efuse_port)
     dut_start_secure_app(dut)
-    dut.expect('Secure Boot is enabled', timeout=2)
+    dut.expect('Secure Boot is enabled', timeout=10)
+    dut.serial.reset_efuses()
 
 
 # Test efuse key index and key block combination.
 # Any key index can be written to any key block and should work
-@ttfw_idf.idf_custom_test(env_tag='Example_Secure_Boot', target=['esp32c3fpga', 'esp32s3fpga'], ignore=True)
+@pytest.mark.esp32c3
-def test_examples_security_secure_boot_key_combo(env, _):      # type: (ttfw_idf.TinyFW.Env, None) -> None
+@pytest.mark.esp32s3
-    efuse_port = os.getenv('EFUSEPORT')
+def test_examples_security_secure_boot_key_combo(dut: Dut) -> None:
-    dut = env.get_dut('secure_boot', 'tools/test_apps/security/secure_boot', efuse_reset_port=efuse_port)
     dut_start_secure_app(dut)
+    dut.expect('Secure Boot is enabled', timeout=10)
     for index in range(3):
+        print(index)
         for block in range(6):
-            dut.reset_efuses()
+            dut.serial.reset_efuses()
+            dut.burn_wafer_version()
             dut.secure_boot_burn_en_bit()
             dut.secure_boot_burn_digest('test_rsa_3072_key.pem', index, block)
-            dut.reset()
+            dut.expect('Secure Boot is enabled', timeout=10)
-            dut.expect('Secure Boot is enabled', timeout=2)
 
 
 # Test secure boot key revoke.
 # If a key is revoked, bootloader signed with that key should fail verification
-@ttfw_idf.idf_custom_test(env_tag='Example_Secure_Boot', target=['esp32c3fpga', 'esp32s3fpga'], ignore=True)
+@pytest.mark.esp32c3
-def test_examples_security_secure_boot_key_revoke(env, _):     # type: (ttfw_idf.TinyFW.Env, None) -> None
+@pytest.mark.esp32s3
-    efuse_port = os.getenv('EFUSEPORT')
+def test_examples_security_secure_boot_key_revoke(dut: Dut) -> None:
-    dut = env.get_dut('secure_boot', 'tools/test_apps/security/secure_boot', efuse_reset_port=efuse_port)
     dut_start_secure_app(dut)
+    dut.expect('Secure Boot is enabled', timeout=10)
     for index in range(3):
-        dut.reset_efuses()
+        dut.serial.reset_efuses()
+        dut.burn_wafer_version()
         dut.secure_boot_burn_en_bit()
+        dut.serial.burn_efuse('SECURE_BOOT_KEY_REVOKE%d' % index, 1)
         dut.secure_boot_burn_digest('test_rsa_3072_key.pem', index, 0)
-        dut.burn_efuse('SECURE_BOOT_KEY_REVOKE%d' % index, 1)
+        dut.expect('secure boot verification failed', timeout=5)
-        dut.reset()
-        dut.expect('secure boot verification failed', timeout=2)
 
 
 # Test bootloader signature corruption.
 # Corrupt one byte at a time of bootloader signature and test that the verification fails
-@ttfw_idf.idf_custom_test(env_tag='Example_Secure_Boot', target=['esp32c3fpga', 'esp32s3fpga'], ignore=True)
+@pytest.mark.esp32c3
-def test_examples_security_secure_boot_corrupt_bl_sig(env, _):     # type: (ttfw_idf.TinyFW.Env, None) -> None
+@pytest.mark.esp32s3
-    efuse_port = os.getenv('EFUSEPORT')
+def test_examples_security_secure_boot_corrupt_bl_sig(dut: Dut) -> None:
-    dut = env.get_dut('secure_boot', 'tools/test_apps/security/secure_boot', efuse_reset_port=efuse_port)
+    dut_start_secure_app(dut)
-    dut.reset_efuses()
+    dut.expect('Secure Boot is enabled', timeout=10)
-    dut.secure_boot_burn_en_bit()
+
-    dut.secure_boot_burn_digest('test_rsa_3072_key.pem', 0, 0)
     bootloader_bin = os.path.join(dut.app.binary_path, 'bootloader/bootloader.bin')
     with open(bootloader_bin, 'rb') as f:
         signed_bl = f.read()
@@ -134,20 +138,27 @@ def test_examples_security_secure_boot_corrupt_bl_sig(env, _):     # type: (ttfw
     for seed in seeds:
         print('Case %d / %d' % (seed, max_seed))
         corrupt_bl = corrupt_signature(signed_bl, seed=seed)
-        dut.write_flash_data([(0x0, BytesIO(corrupt_bl))])
+        with open('corrupt_bl.bin', 'wb') as corrupt_file:
-        dut.expect('Signature Check Failed', timeout=2)
+            corrupt_file.write(corrupt_bl)
+        dut.serial.reset_efuses()
+        dut.burn_wafer_version()
+        dut.serial.bootloader_flash('corrupt_bl.bin')
+        dut.secure_boot_burn_en_bit()
+        dut.secure_boot_burn_digest('test_rsa_3072_key.pem', 0, 0)
+        # Though the order of flashing and burning efuse would not effect the test,
+        # if we flash bootlader before burning en bit, even with no_stub = True
+        # it still calls run_stub() and throws an error as it fails to start stub.
+        dut.expect('Signature Check Failed', timeout=10)
 
 
 # Test app signature corruption.
 # Corrupt app signature, one byte at a time, and test that the verification fails
-@ttfw_idf.idf_custom_test(env_tag='Example_Secure_Boot', target=['esp32c3fpga', 'esp32s3fpga'], ignore=True)
+@pytest.mark.esp32c3
-def test_examples_security_secure_boot_corrupt_app_sig(env, _):    # type: (ttfw_idf.TinyFW.Env, None) -> None
+@pytest.mark.esp32s3
-    efuse_port = os.getenv('EFUSEPORT')
+def test_examples_security_secure_boot_corrupt_app_sig(dut: Dut) -> None:
-    dut = env.get_dut('secure_boot', 'tools/test_apps/security/secure_boot', efuse_reset_port=efuse_port)
     dut_start_secure_app(dut)
-    dut.reset_efuses()
+    dut.expect('Secure Boot is enabled', timeout=10)
-    dut.secure_boot_burn_en_bit()
+
-    dut.secure_boot_burn_digest('test_rsa_3072_key.pem', 0, 0)
     app_bin = os.path.join(dut.app.binary_path, 'secure_boot.bin')
     with open(app_bin, 'rb') as f:
         signed_app = f.read()
@@ -158,26 +169,30 @@ def test_examples_security_secure_boot_corrupt_app_sig(env, _):    # type: (ttfw
     for seed in seeds:
         print('Case %d / %d' % (seed, max_seed))
         corrupt_app = corrupt_signature(signed_app, seed=seed)
-        dut.write_flash_data([(0x20000, BytesIO(corrupt_app))])
+        with open('corrupt_app.bin', 'wb') as corrupt_file:
-        dut.expect('Signature Check Failed', timeout=2)
+            corrupt_file.write(corrupt_app)
+        dut.serial.reset_efuses()
+        dut.burn_wafer_version()
+        dut.serial.app_flash('corrupt_app.bin')
+        dut.secure_boot_burn_en_bit()
+        dut.secure_boot_burn_digest('test_rsa_3072_key.pem', 0, 0)
+        dut.expect('Signature Check Failed', timeout=10)
         dut.expect('image valid, signature bad', timeout=2)
 
     print('Testing invalid CRC...')
     # Valid signature but invalid CRC
-    dut.reset_efuses()
+    dut.serial.reset_efuses()
+    dut.burn_wafer_version()
+
+    corrupt_app = corrupt_signature(signed_app, corrupt_sig=False, corrupt_crc=True)
+    with open('corrupt_app.bin', 'wb') as corrupt_file:
+        corrupt_file.write(corrupt_app)
+
+    dut.serial.app_flash('corrupt_app.bin')
+
     dut.secure_boot_burn_en_bit()
     dut.secure_boot_burn_digest('test_rsa_3072_key.pem', 0, 0)
 
-    corrupt_app = corrupt_signature(signed_app, corrupt_sig=False, corrupt_crc=True)
-    dut.write_flash_data([(0x20000, BytesIO(corrupt_app))])
     dut.expect('Sig block 0 invalid: Stored CRC ends', timeout=2)
     dut.expect('Secure boot signature verification failed', timeout=2)
     dut.expect('No bootable app partitions in the partition table', timeout=2)
-
-
-if __name__ == '__main__':
-    test_examples_security_secure_boot()
-    test_examples_security_secure_boot_key_combo()
-    test_examples_security_secure_boot_key_revoke()
-    test_examples_security_secure_boot_corrupt_bl_sig()
-    test_examples_security_secure_boot_corrupt_app_sig()