diff --git a/.gitmodules b/.gitmodules index 73dae85efa..8bd5998537 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,6 +2,26 @@ # All the relative URL paths are intended to be GitHub ones # For Espressif's public projects please use '../../espressif/proj', not a '../proj' # +# Submodules SBOM information +# --------------------------- +# Submodules, which are used directly and not forked into espressif namespace should +# contain SBOM information here. Other submodules should have the SBOM manifest file +# included in the root of their project's repository. +# +# The sbom-hash entry records the submodule's checkout SHA as presented in git-tree +# commit object. For example spiffs submodule +# +# $ git ls-tree HEAD components/spiffs/spiffs +# 160000 commit 0dbb3f71c5f6fae3747a9d935372773762baf852 components/spiffs/spiffs +# +# The hash can be also obtained with git submodule command +# +# $ git submodule status components/spiffs/spiffs +# 0dbb3f71c5f6fae3747a9d935372773762baf852 components/spiffs/spiffs (0.2-255-g0dbb3f71c5f6) +# +# The submodule SHA recorded here has to match with SHA, which is presented in git-tree. +# This is checked by CI. Also please don't forget to update the submodule version +# if you are changing the sbom-hash. This is important for SBOM generation. [submodule "components/esptool_py/esptool"] path = components/esptool_py/esptool @@ -14,6 +34,12 @@ [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"] path = components/bootloader/subproject/components/micro-ecc/micro-ecc url = ../../kmackay/micro-ecc.git + sbom-version = 1.0 + sbom-cpe = cpe:2.3:a:micro-ecc_project:micro-ecc:{}:*:*:*:*:*:*:* + sbom-supplier = Person: Ken MacKay + sbom-url = https://github.com/kmackay/micro-ecc + sbom-description = A small and fast ECDH and ECDSA implementation for 8-bit, 32-bit, and 64-bit processors + sbom-hash = d037ec89546fad14b5c4d5456c2e23a71e554966 [submodule "components/coap/libcoap"] path = components/coap/libcoap @@ -30,10 +56,21 @@ [submodule "components/spiffs/spiffs"] path = components/spiffs/spiffs url = ../../pellepl/spiffs.git + sbom-version = 0.2-221-gf5e26c4e9331 + sbom-supplier = Person: Peter Andersson + sbom-url = https://github.com/pellepl/spiffs + sbom-description = Wear-leveled SPI flash file system for embedded devices + sbom-hash = f5e26c4e933189593a71c6b82cda381a7b21e41c [submodule "components/json/cJSON"] path = components/json/cJSON url = ../../DaveGamble/cJSON.git + sbom-version = 1.7.15 + sbom-cpe = cpe:2.3:a:cjson_project:cjson:{}:*:*:*:*:*:*:* + sbom-supplier = Person: Dave Gamble + sbom-url = https://github.com/DaveGamble/cJSON + sbom-description = Ultralightweight JSON parser in ANSI C + sbom-hash = d348621ca93571343a56862df7de4ff3bc9b5667 [submodule "components/mbedtls/mbedtls"] path = components/mbedtls/mbedtls @@ -58,10 +95,21 @@ [submodule "components/protobuf-c/protobuf-c"] path = components/protobuf-c/protobuf-c url = ../../protobuf-c/protobuf-c.git + sbom-version = 1.3.0 + sbom-cpe = cpe:2.3:a:protobuf-c_project:protobuf-c:{}:*:*:*:*:*:*:* + sbom-supplier = Organization: protobuf-c community + sbom-url = https://github.com/protobuf-c/protobuf-c + sbom-description = Protocol Buffers implementation in C + sbom-hash = dac1a65feac4ad72f612aab99f487056fbcf5c1a [submodule "components/unity/unity"] path = components/unity/unity url = ../../ThrowTheSwitch/Unity.git + sbom-version = v2.4.3-51-g7d2bf62b7e6a + sbom-supplier = Organization: ThrowTheSwitch community + sbom-url = https://github.com/ThrowTheSwitch/Unity + sbom-description = Simple Unit Testing for C + sbom-hash = 7d2bf62b7e6afaf38153041a9d53c21aeeca9a25 [submodule "examples/build_system/cmake/import_lib/main/lib/tinyxml2"] path = examples/build_system/cmake/import_lib/main/lib/tinyxml2 @@ -90,6 +138,11 @@ [submodule "components/cmock/CMock"] path = components/cmock/CMock url = ../../ThrowTheSwitch/CMock.git + sbom-version = v2.5.2-2-geeecc49ce8af + sbom-supplier = Organization: ThrowTheSwitch community + sbom-url = https://github.com/ThrowTheSwitch/CMock + sbom-description = CMock - Mock/stub generator for C + sbom-hash = eeecc49ce8af123cf8ad40efdb9673e37b56230f [submodule "components/bt/controller/lib_esp32c3_family"] path = components/bt/controller/lib_esp32c3_family diff --git a/tools/ci/config/host-test.yml b/tools/ci/config/host-test.yml index 9e4eab1570..8a920f6e34 100644 --- a/tools/ci/config/host-test.yml +++ b/tools/ci/config/host-test.yml @@ -312,6 +312,13 @@ test_docs: - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py +test_sbom: + extends: + - .host_test_template + - .rules:patterns:sbom + script: + - python ${IDF_PATH}/tools/test_sbom/test_submodules.py + test_autocomplete: extends: .host_test_template image: $CI_DOCKER_REGISTRY/linux-shells:1 diff --git a/tools/ci/config/rules.yml b/tools/ci/config/rules.yml index 1488f1bf09..14898676df 100644 --- a/tools/ci/config/rules.yml +++ b/tools/ci/config/rules.yml @@ -1,3 +1,40 @@ +# patterns +.patterns-submodule: &patterns-submodule + - "components/asio/asio" + - "components/bootloader/subproject/components/micro-ecc/micro-ecc" + - "components/bt/controller/lib_esp32" + - "components/bt/controller/lib_esp32c3_family" + - "components/bt/host/nimble/nimble" + - "components/cbor/tinycbor" + - "components/cmock/CMock" + - "components/cmock/CMock/vendor/c_exception" + - "components/cmock/CMock/vendor/unity" + - "components/coap/libcoap" + - "components/coap/libcoap/ext/tinydtls" + - "components/esp_phy/lib" + - "components/esp_wifi/lib" + - "components/esptool_py/esptool" + - "components/expat/expat" + - "components/json/cJSON" + - "components/libsodium/libsodium" + - "components/lwip/lwip" + - "components/mbedtls/mbedtls" + - "components/mqtt/esp-mqtt" + - "components/nghttp/nghttp2" + - "components/nghttp/nghttp2/third-party/mruby" + - "components/nghttp/nghttp2/third-party/neverbleed" + - "components/openthread/lib" + - "components/protobuf-c/protobuf-c" + - "components/spiffs/spiffs" + - "components/tinyusb/tinyusb" + - "components/unity/unity" + - "examples/build_system/cmake/import_lib/main/lib/tinyxml2" + - "examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib" + - ".gitmodules" + +.patterns-sbom: &patterns-sbom + - "tools/test_sbom/*" + # if anchors .if-ref-master: &if-ref-master if: '$CI_COMMIT_REF_NAME == "master"' @@ -270,3 +307,13 @@ - <<: *if-label-custom_test - <<: *if-label-unit_test-all_labels - <<: *if-label-weekend_test + +.rules:patterns:sbom: + rules: + - <<: *if-protected + - <<: *if-label-regular_test + - <<: *if-label-host_test + - <<: *if-dev-push + changes: *patterns-sbom + - <<: *if-dev-push + changes: *patterns-submodule diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 5b1e147164..c51127e76d 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -102,6 +102,7 @@ tools/test_idf_size/test.sh tools/test_idf_tools/test_idf_tools.py tools/test_mkdfu/test_mkdfu.py tools/test_mkuf2/test_mkuf2.py +tools/test_sbom/test_submodules.py tools/unit-test-app/tools/get_available_configs.sh tools/unit-test-app/unit_test.py tools/windows/eclipse_make.sh diff --git a/tools/test_sbom/test_submodules.py b/tools/test_sbom/test_submodules.py new file mode 100644 index 0000000000..364dabb9fd --- /dev/null +++ b/tools/test_sbom/test_submodules.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +from subprocess import check_output + + +def run_cmd(cmd): # type: (list[str]) -> str + """Simple helper to run command and return it's stdout.""" + return check_output(cmd, universal_newlines=True).strip() + + +def get_gitwdir(): # type: () -> str + """Return absolute path to the current git working tree.""" + return run_cmd(['git', 'rev-parse', '--show-toplevel']) + + +def get_submodules_info(): # type () -> list[dict[str,str]] + """Return list of submodules, where each submodule is represented + as dictionary with name, path and hash keys.""" + cmd = ['git', 'submodule', '--quiet', 'foreach','echo "$name,$sm_path,$sha1"'] + out = run_cmd(cmd) + submodules = [] + for line in out.splitlines(): + name, sm_path, sha1 = line.split(',') + submodules += [{'name': name, 'path': sm_path, 'hash': sha1}] + + return submodules + + +def get_submodules_config(): # type () -> dict[str,str] + """Return dictionary, where key is variable name and value + is variable value in git's --list(dot) format. Only variables + starting with "submodule." are returned and this prefix is removed + to make it simple to match against the submodule info dictionary.""" + gitmodules_fn = os.path.join(get_gitwdir(), '.gitmodules') + gitmodules_data = run_cmd(['git', 'config', '--list', '--file', gitmodules_fn]) + prefix = 'submodule.' + config = {} + for line in gitmodules_data.splitlines(): + var, val = line.split('=', 1) + if not var.startswith(prefix): + continue + # remove "submodule." prefix + var = var[len(prefix):] + config[var] = val + + return config + + +def test_sha(): # type: () -> None + """ Check that submodule SHA1 in git-tree and .gitmodules match + if sbom-hash variable is available in the .gitmodules file. + """ + submodules = get_submodules_info() + config = get_submodules_config() + + for submodule in submodules: + sbom_hash = config.get(submodule['name'] + '.sbom-hash') + if not sbom_hash: + continue + msg = ('Submodule \"{sn}\" SHA \"{sh}\" in git ' + 'tree does not match SHA \"{h}\" recorded in .gitmodules. ' + 'Please update \"sbom-hash\" in .gitmodules for \"{sn}\" ' + 'and also please do not forget to update version and other submodule ' + 'information if necessary. It is important to keep this information ' + 'up-to-date for SBOM generation.').format(sn=submodule['name'], sh=submodule['hash'], h=sbom_hash) + assert submodule['hash'] == sbom_hash, msg + + +if __name__ == '__main__': + test_sha()