Merge branch 'feature/submodules_sbom_v4.3' into 'release/v4.3'

tools: add sbom information for submodules (v4.3)

See merge request espressif/esp-idf!24246

.gitmodules

@@ -2,6 +2,26 @@
 # All the relative URL paths are intended to be GitHub ones
 # For Espressif's public projects please use '../../espressif/proj', not a '../proj'
 #
+# Submodules SBOM information
+# ---------------------------
+# Submodules, which are used directly and not forked into espressif namespace should
+# contain SBOM information here. Other submodules should have the SBOM manifest file
+# included in the root of their project's repository.
+#
+# The sbom-hash entry records the submodule's checkout SHA as presented in git-tree
+# commit object. For example spiffs submodule
+#
+# $ git ls-tree HEAD components/spiffs/spiffs
+# 160000 commit 0dbb3f71c5f6fae3747a9d935372773762baf852	components/spiffs/spiffs
+#
+# The hash can be also obtained with git submodule command
+#
+# $ git submodule status components/spiffs/spiffs
+# 0dbb3f71c5f6fae3747a9d935372773762baf852 components/spiffs/spiffs (0.2-255-g0dbb3f71c5f6)
+#
+# The submodule SHA recorded here has to match with SHA, which is presented in git-tree.
+# This is checked by CI. Also please don't forget to update the submodule version
+# if you are changing the sbom-hash. This is important for SBOM generation.
 
 [submodule "components/esptool_py/esptool"]
 	path = components/esptool_py/esptool
@@ -14,6 +34,12 @@
 [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"]
 	path = components/bootloader/subproject/components/micro-ecc/micro-ecc
 	url = ../../kmackay/micro-ecc.git
+	sbom-version = 1.0
+	sbom-cpe = cpe:2.3:a:micro-ecc_project:micro-ecc:{}:*:*:*:*:*:*:*
+	sbom-supplier = Person: Ken MacKay
+	sbom-url = https://github.com/kmackay/micro-ecc
+	sbom-description = A small and fast ECDH and ECDSA implementation for 8-bit, 32-bit, and 64-bit processors
+	sbom-hash = d037ec89546fad14b5c4d5456c2e23a71e554966
 
 [submodule "components/coap/libcoap"]
 	path = components/coap/libcoap
@@ -30,10 +56,21 @@
 [submodule "components/spiffs/spiffs"]
 	path = components/spiffs/spiffs
 	url = ../../pellepl/spiffs.git
+	sbom-version = 0.2-221-gf5e26c4e9331
+	sbom-supplier = Person: Peter Andersson
+	sbom-url = https://github.com/pellepl/spiffs
+	sbom-description = Wear-leveled SPI flash file system for embedded devices
+	sbom-hash = f5e26c4e933189593a71c6b82cda381a7b21e41c
 
 [submodule "components/json/cJSON"]
 	path = components/json/cJSON
 	url = ../../DaveGamble/cJSON.git
+	sbom-version = 1.7.15
+	sbom-cpe = cpe:2.3:a:cjson_project:cjson:{}:*:*:*:*:*:*:*
+	sbom-supplier = Person: Dave Gamble
+	sbom-url = https://github.com/DaveGamble/cJSON
+	sbom-description = Ultralightweight JSON parser in ANSI C
+	sbom-hash = d348621ca93571343a56862df7de4ff3bc9b5667
 
 [submodule "components/mbedtls/mbedtls"]
 	path = components/mbedtls/mbedtls
@@ -58,10 +95,21 @@
 [submodule "components/protobuf-c/protobuf-c"]
 	path = components/protobuf-c/protobuf-c
 	url = ../../protobuf-c/protobuf-c.git
+	sbom-version = 1.3.0
+	sbom-cpe = cpe:2.3:a:protobuf-c_project:protobuf-c:{}:*:*:*:*:*:*:*
+	sbom-supplier = Organization: protobuf-c community <https://groups.google.com/g/protobuf-c>
+	sbom-url = https://github.com/protobuf-c/protobuf-c
+	sbom-description = Protocol Buffers implementation in C
+	sbom-hash = dac1a65feac4ad72f612aab99f487056fbcf5c1a
 
 [submodule "components/unity/unity"]
 	path = components/unity/unity
 	url = ../../ThrowTheSwitch/Unity.git
+	sbom-version = v2.4.3-51-g7d2bf62b7e6a
+	sbom-supplier = Organization: ThrowTheSwitch community <http://www.throwtheswitch.org>
+	sbom-url = https://github.com/ThrowTheSwitch/Unity
+	sbom-description = Simple Unit Testing for C
+	sbom-hash = 7d2bf62b7e6afaf38153041a9d53c21aeeca9a25
 
 [submodule "examples/build_system/cmake/import_lib/main/lib/tinyxml2"]
 	path = examples/build_system/cmake/import_lib/main/lib/tinyxml2
@@ -90,6 +138,11 @@
 [submodule "components/cmock/CMock"]
 	path = components/cmock/CMock
 	url = ../../ThrowTheSwitch/CMock.git
+	sbom-version = v2.5.2-2-geeecc49ce8af
+	sbom-supplier = Organization: ThrowTheSwitch community <http://www.throwtheswitch.org>
+	sbom-url = https://github.com/ThrowTheSwitch/CMock
+	sbom-description = CMock - Mock/stub generator for C
+	sbom-hash = eeecc49ce8af123cf8ad40efdb9673e37b56230f
 
 [submodule "components/bt/controller/lib_esp32c3_family"]
 	path = components/bt/controller/lib_esp32c3_family

tools/ci/config/host-test.yml

@@ -312,6 +312,13 @@ test_docs:
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py
 
+test_sbom:
+  extends:
+    - .host_test_template
+    - .rules:patterns:sbom
+  script:
+    - python ${IDF_PATH}/tools/test_sbom/test_submodules.py
+
 test_autocomplete:
   extends: .host_test_template
   image: $CI_DOCKER_REGISTRY/linux-shells:1

tools/ci/config/rules.yml

@@ -1,3 +1,40 @@
+# patterns
+.patterns-submodule: &patterns-submodule
+  - "components/asio/asio"
+  - "components/bootloader/subproject/components/micro-ecc/micro-ecc"
+  - "components/bt/controller/lib_esp32"
+  - "components/bt/controller/lib_esp32c3_family"
+  - "components/bt/host/nimble/nimble"
+  - "components/cbor/tinycbor"
+  - "components/cmock/CMock"
+  - "components/cmock/CMock/vendor/c_exception"
+  - "components/cmock/CMock/vendor/unity"
+  - "components/coap/libcoap"
+  - "components/coap/libcoap/ext/tinydtls"
+  - "components/esp_phy/lib"
+  - "components/esp_wifi/lib"
+  - "components/esptool_py/esptool"
+  - "components/expat/expat"
+  - "components/json/cJSON"
+  - "components/libsodium/libsodium"
+  - "components/lwip/lwip"
+  - "components/mbedtls/mbedtls"
+  - "components/mqtt/esp-mqtt"
+  - "components/nghttp/nghttp2"
+  - "components/nghttp/nghttp2/third-party/mruby"
+  - "components/nghttp/nghttp2/third-party/neverbleed"
+  - "components/openthread/lib"
+  - "components/protobuf-c/protobuf-c"
+  - "components/spiffs/spiffs"
+  - "components/tinyusb/tinyusb"
+  - "components/unity/unity"
+  - "examples/build_system/cmake/import_lib/main/lib/tinyxml2"
+  - "examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib"
+  - ".gitmodules"
+
+.patterns-sbom: &patterns-sbom
+  - "tools/test_sbom/*"
+
 # if anchors
 .if-ref-master: &if-ref-master
   if: '$CI_COMMIT_REF_NAME == "master"'
@@ -270,3 +307,13 @@
     - <<: *if-label-custom_test
     - <<: *if-label-unit_test-all_labels
     - <<: *if-label-weekend_test
+
+.rules:patterns:sbom:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-regular_test
+    - <<: *if-label-host_test
+    - <<: *if-dev-push
+      changes: *patterns-sbom
+    - <<: *if-dev-push
+      changes: *patterns-submodule

tools/ci/executable-list.txt

@@ -102,6 +102,7 @@ tools/test_idf_size/test.sh
 tools/test_idf_tools/test_idf_tools.py
 tools/test_mkdfu/test_mkdfu.py
 tools/test_mkuf2/test_mkuf2.py
+tools/test_sbom/test_submodules.py
 tools/unit-test-app/tools/get_available_configs.sh
 tools/unit-test-app/unit_test.py
 tools/windows/eclipse_make.sh

tools/test_sbom/test_submodules.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+import os
+from subprocess import check_output
+
+
+def run_cmd(cmd):  # type: (list[str]) -> str
+    """Simple helper to run command and return it's stdout."""
+    return check_output(cmd, universal_newlines=True).strip()
+
+
+def get_gitwdir():  # type: () -> str
+    """Return absolute path to the current git working tree."""
+    return run_cmd(['git', 'rev-parse', '--show-toplevel'])
+
+
+def get_submodules_info():  # type () -> list[dict[str,str]]
+    """Return list of submodules, where each submodule is represented
+    as dictionary with name, path and hash keys."""
+    cmd = ['git', 'submodule', '--quiet', 'foreach','echo "$name,$sm_path,$sha1"']
+    out = run_cmd(cmd)
+    submodules = []
+    for line in out.splitlines():
+        name, sm_path, sha1 = line.split(',')
+        submodules += [{'name': name, 'path': sm_path, 'hash': sha1}]
+
+    return submodules
+
+
+def get_submodules_config():  # type () -> dict[str,str]
+    """Return dictionary, where key is variable name and value
+    is variable value in git's --list(dot) format. Only variables
+    starting with "submodule." are returned and this prefix is removed
+    to make it simple to match against the submodule info dictionary."""
+    gitmodules_fn = os.path.join(get_gitwdir(), '.gitmodules')
+    gitmodules_data = run_cmd(['git', 'config', '--list', '--file', gitmodules_fn])
+    prefix = 'submodule.'
+    config = {}
+    for line in gitmodules_data.splitlines():
+        var, val = line.split('=', 1)
+        if not var.startswith(prefix):
+            continue
+        # remove "submodule." prefix
+        var = var[len(prefix):]
+        config[var] = val
+
+    return config
+
+
+def test_sha():  # type: () -> None
+    """ Check that submodule SHA1 in git-tree and .gitmodules match
+    if sbom-hash variable is available in the .gitmodules file.
+    """
+    submodules = get_submodules_info()
+    config = get_submodules_config()
+
+    for submodule in submodules:
+        sbom_hash = config.get(submodule['name'] + '.sbom-hash')
+        if not sbom_hash:
+            continue
+        msg = ('Submodule \"{sn}\" SHA \"{sh}\" in git '
+               'tree does not match SHA \"{h}\" recorded in .gitmodules. '
+               'Please update \"sbom-hash\" in .gitmodules for \"{sn}\" '
+               'and also please do not forget to update version and other submodule '
+               'information if necessary. It is important to keep this information '
+               'up-to-date for SBOM generation.').format(sn=submodule['name'], sh=submodule['hash'], h=sbom_hash)
+        assert submodule['hash'] == sbom_hash, msg
+
+
+if __name__ == '__main__':
+    test_sha()