From d4ab79259a1d66411c15d670c3245637d8765468 Mon Sep 17 00:00:00 2001 From: Frantisek Hrbata Date: Tue, 30 May 2023 16:25:02 +0200 Subject: [PATCH] tools: add sbom information for submodules Conflicts: - spiffs submodules version and hash changed to 0.2-221-gf5e26c4e9331 - protobuf-c submodule version and hash changed to v1.3.0 - ci files moved from tools/ci/config/ into .gitlab/ci/ in v4.4, so host-test.yml and rules.yml were changed accordingly in tools/ci/config/. - added patterns-submodule to rules.yml, because they were also added in v4.4 - removed pytest dependency This adds SBOM information for submodules, which are not managed by Espressif. Meaning there is no fork for them in the espressif namespace. Other submodules should add sbom.yml manifest file to the root of their git repository. The SBOM information for submodules is stored in the .gitmodules file. Each SBOM related variable has the "sbom-" prefix and the following variables may be used: sbom-version: submodule version sbom-cpe: CPE record if available in NVD. This will be used by the SBOM tool to check for possible submodule vulnerabilities. The version in the CPE can be replaced with the "{}" placeholder, which will be replaced by the "sbom-version" value from above. sbom-supplier: Person or organization who is providing the submodule. It has to start with "Person:" or "Organization:" prefix as required by the SPDX-2.2 standard. sbom-url: URL to the project if exists, e.g. github. sbom-description: Project description. sbom-hash: Submodule SHA as recorded in the git-tree. This field is used by CI to check that the submodule checkout hash and info in .gitmodules are in sync. IOW if submodule is updated and it has SBOM info in .gitmodules, the .gitmodules has to be updated too. The test is part of this commit. The checkout has of the submodule can be found by using "git submodule status". Example for micro-ecc submodule ---8<--- [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"] path = components/bootloader/subproject/components/micro-ecc/micro-ecc url = ../../kmackay/micro-ecc.git sbom-version = 1.0 sbom-cpe = cpe:2.3:a:micro-ecc_project:micro-ecc:{}:*:*:*:*:*:*:* sbom-supplier = Person: Ken MacKay sbom-url = https://github.com/kmackay/micro-ecc sbom-description = A small and fast ECDH and ECDSA implementation for 8-bit, 32-bit, and 64-bit processors sbom-hash = d037ec89546fad14b5c4d5456c2e23a71e554966 ---8<--- Signed-off-by: Frantisek Hrbata --- .gitmodules | 53 ++++++++++++++++++++++ tools/ci/config/host-test.yml | 7 +++ tools/ci/config/rules.yml | 47 ++++++++++++++++++++ tools/ci/executable-list.txt | 1 + tools/test_sbom/test_submodules.py | 71 ++++++++++++++++++++++++++++++ 5 files changed, 179 insertions(+) create mode 100644 tools/test_sbom/test_submodules.py diff --git a/.gitmodules b/.gitmodules index 73dae85efa..8bd5998537 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,6 +2,26 @@ # All the relative URL paths are intended to be GitHub ones # For Espressif's public projects please use '../../espressif/proj', not a '../proj' # +# Submodules SBOM information +# --------------------------- +# Submodules, which are used directly and not forked into espressif namespace should +# contain SBOM information here. Other submodules should have the SBOM manifest file +# included in the root of their project's repository. +# +# The sbom-hash entry records the submodule's checkout SHA as presented in git-tree +# commit object. For example spiffs submodule +# +# $ git ls-tree HEAD components/spiffs/spiffs +# 160000 commit 0dbb3f71c5f6fae3747a9d935372773762baf852 components/spiffs/spiffs +# +# The hash can be also obtained with git submodule command +# +# $ git submodule status components/spiffs/spiffs +# 0dbb3f71c5f6fae3747a9d935372773762baf852 components/spiffs/spiffs (0.2-255-g0dbb3f71c5f6) +# +# The submodule SHA recorded here has to match with SHA, which is presented in git-tree. +# This is checked by CI. Also please don't forget to update the submodule version +# if you are changing the sbom-hash. This is important for SBOM generation. [submodule "components/esptool_py/esptool"] path = components/esptool_py/esptool @@ -14,6 +34,12 @@ [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"] path = components/bootloader/subproject/components/micro-ecc/micro-ecc url = ../../kmackay/micro-ecc.git + sbom-version = 1.0 + sbom-cpe = cpe:2.3:a:micro-ecc_project:micro-ecc:{}:*:*:*:*:*:*:* + sbom-supplier = Person: Ken MacKay + sbom-url = https://github.com/kmackay/micro-ecc + sbom-description = A small and fast ECDH and ECDSA implementation for 8-bit, 32-bit, and 64-bit processors + sbom-hash = d037ec89546fad14b5c4d5456c2e23a71e554966 [submodule "components/coap/libcoap"] path = components/coap/libcoap @@ -30,10 +56,21 @@ [submodule "components/spiffs/spiffs"] path = components/spiffs/spiffs url = ../../pellepl/spiffs.git + sbom-version = 0.2-221-gf5e26c4e9331 + sbom-supplier = Person: Peter Andersson + sbom-url = https://github.com/pellepl/spiffs + sbom-description = Wear-leveled SPI flash file system for embedded devices + sbom-hash = f5e26c4e933189593a71c6b82cda381a7b21e41c [submodule "components/json/cJSON"] path = components/json/cJSON url = ../../DaveGamble/cJSON.git + sbom-version = 1.7.15 + sbom-cpe = cpe:2.3:a:cjson_project:cjson:{}:*:*:*:*:*:*:* + sbom-supplier = Person: Dave Gamble + sbom-url = https://github.com/DaveGamble/cJSON + sbom-description = Ultralightweight JSON parser in ANSI C + sbom-hash = d348621ca93571343a56862df7de4ff3bc9b5667 [submodule "components/mbedtls/mbedtls"] path = components/mbedtls/mbedtls @@ -58,10 +95,21 @@ [submodule "components/protobuf-c/protobuf-c"] path = components/protobuf-c/protobuf-c url = ../../protobuf-c/protobuf-c.git + sbom-version = 1.3.0 + sbom-cpe = cpe:2.3:a:protobuf-c_project:protobuf-c:{}:*:*:*:*:*:*:* + sbom-supplier = Organization: protobuf-c community + sbom-url = https://github.com/protobuf-c/protobuf-c + sbom-description = Protocol Buffers implementation in C + sbom-hash = dac1a65feac4ad72f612aab99f487056fbcf5c1a [submodule "components/unity/unity"] path = components/unity/unity url = ../../ThrowTheSwitch/Unity.git + sbom-version = v2.4.3-51-g7d2bf62b7e6a + sbom-supplier = Organization: ThrowTheSwitch community + sbom-url = https://github.com/ThrowTheSwitch/Unity + sbom-description = Simple Unit Testing for C + sbom-hash = 7d2bf62b7e6afaf38153041a9d53c21aeeca9a25 [submodule "examples/build_system/cmake/import_lib/main/lib/tinyxml2"] path = examples/build_system/cmake/import_lib/main/lib/tinyxml2 @@ -90,6 +138,11 @@ [submodule "components/cmock/CMock"] path = components/cmock/CMock url = ../../ThrowTheSwitch/CMock.git + sbom-version = v2.5.2-2-geeecc49ce8af + sbom-supplier = Organization: ThrowTheSwitch community + sbom-url = https://github.com/ThrowTheSwitch/CMock + sbom-description = CMock - Mock/stub generator for C + sbom-hash = eeecc49ce8af123cf8ad40efdb9673e37b56230f [submodule "components/bt/controller/lib_esp32c3_family"] path = components/bt/controller/lib_esp32c3_family diff --git a/tools/ci/config/host-test.yml b/tools/ci/config/host-test.yml index 9e4eab1570..8a920f6e34 100644 --- a/tools/ci/config/host-test.yml +++ b/tools/ci/config/host-test.yml @@ -312,6 +312,13 @@ test_docs: - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py +test_sbom: + extends: + - .host_test_template + - .rules:patterns:sbom + script: + - python ${IDF_PATH}/tools/test_sbom/test_submodules.py + test_autocomplete: extends: .host_test_template image: $CI_DOCKER_REGISTRY/linux-shells:1 diff --git a/tools/ci/config/rules.yml b/tools/ci/config/rules.yml index 1488f1bf09..14898676df 100644 --- a/tools/ci/config/rules.yml +++ b/tools/ci/config/rules.yml @@ -1,3 +1,40 @@ +# patterns +.patterns-submodule: &patterns-submodule + - "components/asio/asio" + - "components/bootloader/subproject/components/micro-ecc/micro-ecc" + - "components/bt/controller/lib_esp32" + - "components/bt/controller/lib_esp32c3_family" + - "components/bt/host/nimble/nimble" + - "components/cbor/tinycbor" + - "components/cmock/CMock" + - "components/cmock/CMock/vendor/c_exception" + - "components/cmock/CMock/vendor/unity" + - "components/coap/libcoap" + - "components/coap/libcoap/ext/tinydtls" + - "components/esp_phy/lib" + - "components/esp_wifi/lib" + - "components/esptool_py/esptool" + - "components/expat/expat" + - "components/json/cJSON" + - "components/libsodium/libsodium" + - "components/lwip/lwip" + - "components/mbedtls/mbedtls" + - "components/mqtt/esp-mqtt" + - "components/nghttp/nghttp2" + - "components/nghttp/nghttp2/third-party/mruby" + - "components/nghttp/nghttp2/third-party/neverbleed" + - "components/openthread/lib" + - "components/protobuf-c/protobuf-c" + - "components/spiffs/spiffs" + - "components/tinyusb/tinyusb" + - "components/unity/unity" + - "examples/build_system/cmake/import_lib/main/lib/tinyxml2" + - "examples/peripherals/secure_element/atecc608_ecdsa/components/esp-cryptoauthlib" + - ".gitmodules" + +.patterns-sbom: &patterns-sbom + - "tools/test_sbom/*" + # if anchors .if-ref-master: &if-ref-master if: '$CI_COMMIT_REF_NAME == "master"' @@ -270,3 +307,13 @@ - <<: *if-label-custom_test - <<: *if-label-unit_test-all_labels - <<: *if-label-weekend_test + +.rules:patterns:sbom: + rules: + - <<: *if-protected + - <<: *if-label-regular_test + - <<: *if-label-host_test + - <<: *if-dev-push + changes: *patterns-sbom + - <<: *if-dev-push + changes: *patterns-submodule diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 5b1e147164..c51127e76d 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -102,6 +102,7 @@ tools/test_idf_size/test.sh tools/test_idf_tools/test_idf_tools.py tools/test_mkdfu/test_mkdfu.py tools/test_mkuf2/test_mkuf2.py +tools/test_sbom/test_submodules.py tools/unit-test-app/tools/get_available_configs.sh tools/unit-test-app/unit_test.py tools/windows/eclipse_make.sh diff --git a/tools/test_sbom/test_submodules.py b/tools/test_sbom/test_submodules.py new file mode 100644 index 0000000000..364dabb9fd --- /dev/null +++ b/tools/test_sbom/test_submodules.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +from subprocess import check_output + + +def run_cmd(cmd): # type: (list[str]) -> str + """Simple helper to run command and return it's stdout.""" + return check_output(cmd, universal_newlines=True).strip() + + +def get_gitwdir(): # type: () -> str + """Return absolute path to the current git working tree.""" + return run_cmd(['git', 'rev-parse', '--show-toplevel']) + + +def get_submodules_info(): # type () -> list[dict[str,str]] + """Return list of submodules, where each submodule is represented + as dictionary with name, path and hash keys.""" + cmd = ['git', 'submodule', '--quiet', 'foreach','echo "$name,$sm_path,$sha1"'] + out = run_cmd(cmd) + submodules = [] + for line in out.splitlines(): + name, sm_path, sha1 = line.split(',') + submodules += [{'name': name, 'path': sm_path, 'hash': sha1}] + + return submodules + + +def get_submodules_config(): # type () -> dict[str,str] + """Return dictionary, where key is variable name and value + is variable value in git's --list(dot) format. Only variables + starting with "submodule." are returned and this prefix is removed + to make it simple to match against the submodule info dictionary.""" + gitmodules_fn = os.path.join(get_gitwdir(), '.gitmodules') + gitmodules_data = run_cmd(['git', 'config', '--list', '--file', gitmodules_fn]) + prefix = 'submodule.' + config = {} + for line in gitmodules_data.splitlines(): + var, val = line.split('=', 1) + if not var.startswith(prefix): + continue + # remove "submodule." prefix + var = var[len(prefix):] + config[var] = val + + return config + + +def test_sha(): # type: () -> None + """ Check that submodule SHA1 in git-tree and .gitmodules match + if sbom-hash variable is available in the .gitmodules file. + """ + submodules = get_submodules_info() + config = get_submodules_config() + + for submodule in submodules: + sbom_hash = config.get(submodule['name'] + '.sbom-hash') + if not sbom_hash: + continue + msg = ('Submodule \"{sn}\" SHA \"{sh}\" in git ' + 'tree does not match SHA \"{h}\" recorded in .gitmodules. ' + 'Please update \"sbom-hash\" in .gitmodules for \"{sn}\" ' + 'and also please do not forget to update version and other submodule ' + 'information if necessary. It is important to keep this information ' + 'up-to-date for SBOM generation.').format(sn=submodule['name'], sh=submodule['hash'], h=sbom_hash) + assert submodule['hash'] == sbom_hash, msg + + +if __name__ == '__main__': + test_sha()