From e49d4a83d9e5e133fd5d1bb922f408c19cadb272 Mon Sep 17 00:00:00 2001 From: Alexey Lapshin Date: Tue, 15 Mar 2022 23:02:37 +0400 Subject: [PATCH] tools: Add tool's versions update with checksum file 'idf_tools.py add-version' has new options: --override Override tool versions with new data --checksum-file URL or path to local file with checksum/size for artifacts Usage e.g.: CHECKSUM_URL=https://github.com/espressif/crosstool-NG/releases/download/esp-2021r2/crosstool-NG-esp-2021r2-checksum.sha256 idf_tools.py add-version --tool xtensa-esp32-elf --version esp-2021r2 --override --checksum-file $CHECKSUM_URL Positional argument 'files' moved to optional argument '--artifact-file' Add tests for add-version logic --- tools/idf_tools.py | 109 ++++++++++++++++-- .../artifact_expected_addition.json | 14 +++ .../add_version/artifact_input.json | 12 ++ .../add_version/checksum.sha256 | 98 ++++++++++++++++ .../checksum_expected_addition.json | 44 +++++++ .../checksum_expected_override.json | 44 +++++++ tools/test_idf_tools/test_idf_tools.py | 87 +++++++++++++- 7 files changed, 395 insertions(+), 13 deletions(-) create mode 100644 tools/test_idf_tools/add_version/artifact_expected_addition.json create mode 100644 tools/test_idf_tools/add_version/artifact_input.json create mode 100644 tools/test_idf_tools/add_version/checksum.sha256 create mode 100644 tools/test_idf_tools/add_version/checksum_expected_addition.json create mode 100644 tools/test_idf_tools/add_version/checksum_expected_override.json diff --git a/tools/idf_tools.py b/tools/idf_tools.py index a88b8cdc39..fbc5ccc62e 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -62,8 +62,9 @@ except RuntimeError as e: print(e) raise SystemExit(1) -from typing import IO, Any, Callable, Dict, List, Optional, Set, Tuple, Union # noqa: F401 +from typing import IO, Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, Union # noqa: F401 from urllib.error import ContentTooShortError +from urllib.parse import urljoin, urlparse from urllib.request import urlopen # the following is only for typing annotation from urllib.response import addinfourl # noqa: F401 @@ -116,9 +117,11 @@ PLATFORM_FROM_NAME = { PLATFORM_WIN32: PLATFORM_WIN32, 'Windows-i686': PLATFORM_WIN32, 'Windows-x86': PLATFORM_WIN32, + 'i686-w64-mingw32': PLATFORM_WIN32, PLATFORM_WIN64: PLATFORM_WIN64, 'Windows-x86_64': PLATFORM_WIN64, 'Windows-AMD64': PLATFORM_WIN64, + 'x86_64-w64-mingw32': PLATFORM_WIN64, # macOS PLATFORM_MACOS: PLATFORM_MACOS, 'osx': PLATFORM_MACOS, @@ -131,18 +134,24 @@ PLATFORM_FROM_NAME = { 'linux64': PLATFORM_LINUX64, 'Linux-x86_64': PLATFORM_LINUX64, 'FreeBSD-amd64': PLATFORM_LINUX64, + 'x86_64-linux-gnu': PLATFORM_LINUX64, PLATFORM_LINUX32: PLATFORM_LINUX32, 'linux32': PLATFORM_LINUX32, 'Linux-i686': PLATFORM_LINUX32, 'FreeBSD-i386': PLATFORM_LINUX32, + 'i586-linux-gnu': PLATFORM_LINUX32, + # armhf must be before armel to avoid mismatching + PLATFORM_LINUX_ARMHF: PLATFORM_LINUX_ARMHF, + 'arm-linux-gnueabihf': PLATFORM_LINUX_ARMHF, PLATFORM_LINUX_ARM32: PLATFORM_LINUX_ARM32, 'Linux-arm': PLATFORM_LINUX_ARM32, 'Linux-armv7l': PLATFORM_LINUX_ARM32, - PLATFORM_LINUX_ARMHF: PLATFORM_LINUX_ARMHF, + 'arm-linux-gnueabi': PLATFORM_LINUX_ARM32, PLATFORM_LINUX_ARM64: PLATFORM_LINUX_ARM64, 'Linux-arm64': PLATFORM_LINUX_ARM64, 'Linux-aarch64': PLATFORM_LINUX_ARM64, 'Linux-armv8l': PLATFORM_LINUX_ARM64, + 'aarch64': PLATFORM_LINUX_ARM64, } UNKNOWN_PLATFORM = 'unknown' @@ -531,7 +540,7 @@ class IDFTool(object): # type: (str, str, str, str, str, List[str], str, List[str], Optional[str], int) -> None self.name = name self.description = description - self.versions = OrderedDict() # type: Dict[str, IDFToolVersion] + self.drop_versions() self.version_in_path = None # type: Optional[str] self.versions_installed = [] # type: List[str] if version_regex_replace is None: @@ -557,6 +566,9 @@ class IDFTool(object): del override_dict['platforms'] self._current_options = self._current_options._replace(**override_dict) # type: ignore + def drop_versions(self): # type: () -> None + self.versions = OrderedDict() # type: Dict[str, IDFToolVersion] + def add_version(self, version): # type: (IDFToolVersion) -> None assert(type(version) is IDFToolVersion) self.versions[version.version] = version @@ -1992,6 +2004,76 @@ def action_check_python_dependencies(args): # type: ignore raise SystemExit(1) +class ChecksumCalculator(): + """ + A class used to get size/checksum/basename of local artifact files. + """ + def __init__(self, files): # type: (list[str]) -> None + self.files = files + + def __iter__(self): # type: () -> Iterator[Tuple[int, str, str]] + for f in self.files: + yield (*get_file_size_sha256(f), os.path.basename(f)) + + +class ChecksumParsingError(RuntimeError): + pass + + +class ChecksumFileParser(): + """ + A class used to get size/sha256/filename of artifact using checksum-file with format: + # : bytes + * + ... (2 lines for every artifact) ... + """ + def __init__(self, tool_name, url): # type: (str, str) -> None + self.tool_name = tool_name + + sha256_file_tmp = os.path.join(global_idf_tools_path or '', 'tools', 'add-version.sha256.tmp') + sha256_file = os.path.abspath(url) + + # download sha256 file if URL presented + if urlparse(url).scheme: + sha256_file = sha256_file_tmp + download(url, sha256_file) + + with open(sha256_file, 'r') as f: + self.checksum = f.read().splitlines() + + # remove temp file + if os.path.isfile(sha256_file_tmp): + os.remove(sha256_file_tmp) + + def parseLine(self, regex, line): # type: (str, str) -> str + match = re.search(regex, line) + if not match: + raise ChecksumParsingError(f'Can not parse line "{line}" with regex "{regex}"') + return match.group(1) + + # parse checksum file with formatting used by crosstool-ng, gdb, ... releases + # e.g. https://github.com/espressif/crosstool-NG/releases/download/esp-2021r2/crosstool-NG-esp-2021r2-checksum.sha256 + def __iter__(self): # type: () -> Iterator[Tuple[int, str, str]] + try: + for bytes_str, hash_str in zip(self.checksum[0::2], self.checksum[1::2]): + bytes_filename = self.parseLine(r'^# (\S*):', bytes_str) + hash_filename = self.parseLine(r'^\S* \*(\S*)', hash_str) + if hash_filename != bytes_filename: + fatal('filename in hash-line and in bytes-line are not the same') + raise SystemExit(1) + # crosstool-ng checksum file contains info about few tools + # e.g.: "xtensa-esp32-elf", "xtensa-esp32s2-elf" + # filter records for file by tool_name to avoid mismatch + if not hash_filename.startswith(self.tool_name): + continue + size = self.parseLine(r'^# \S*: (\d*) bytes', bytes_str) + sha256 = self.parseLine(r'^(\S*) ', hash_str) + yield int(size), sha256, hash_filename + except (TypeError, AttributeError) as err: + fatal(f'Error while parsing, check checksum file ({err})') + raise SystemExit(1) + + def action_add_version(args): # type: ignore tools_info = load_tools_info() tool_name = args.tool @@ -2002,14 +2084,18 @@ def action_add_version(args): # type: ignore TODO_MESSAGE, TODO_MESSAGE, [TODO_MESSAGE], TODO_MESSAGE) tools_info[tool_name] = tool_obj version = args.version + version_status = IDFToolVersion.STATUS_SUPPORTED + if args.override and len(tool_obj.versions): + tool_obj.drop_versions() + version_status = IDFToolVersion.STATUS_RECOMMENDED version_obj = tool_obj.versions.get(version) - if version not in tool_obj.versions: + if not version_obj: info('Creating new version {}'.format(version)) - version_obj = IDFToolVersion(version, IDFToolVersion.STATUS_SUPPORTED) + version_obj = IDFToolVersion(version, version_status) tool_obj.versions[version] = version_obj url_prefix = args.url_prefix or 'https://%s/' % TODO_MESSAGE - for file_path in args.files: - file_name = os.path.basename(file_path) + checksum_info = ChecksumFileParser(tool_name, args.checksum_file) if args.checksum_file else ChecksumCalculator(args.artifact_file) + for file_size, file_sha256, file_name in checksum_info: # Guess which platform this file is for found_platform = None for platform_alias, platform_id in PLATFORM_FROM_NAME.items(): @@ -2019,9 +2105,7 @@ def action_add_version(args): # type: ignore if found_platform is None: info('Could not guess platform for file {}'.format(file_name)) found_platform = TODO_MESSAGE - # Get file size and calculate the SHA256 - file_size, file_sha256 = get_file_size_sha256(file_path) - url = url_prefix + file_name + url = urljoin(url_prefix, file_name) info('Adding download for platform {}'.format(found_platform)) info(' size: {}'.format(file_size)) info(' SHA256: {}'.format(file_sha256)) @@ -2283,7 +2367,10 @@ def main(argv): # type: (list[str]) -> None add_version.add_argument('--tool', help='Tool name to set add a version for', required=True) add_version.add_argument('--version', help='Version identifier', required=True) add_version.add_argument('--url-prefix', help='String to prepend to file names to obtain download URLs') - add_version.add_argument('files', help='File names of the download artifacts', nargs='*') + add_version.add_argument('--override', action='store_true', help='Override tool versions with new data') + add_version_files_group = add_version.add_mutually_exclusive_group(required=True) + add_version_files_group.add_argument('--checksum-file', help='URL or path to local file with checksum/size for artifacts') + add_version_files_group.add_argument('--artifact-file', help='File names of the download artifacts', nargs='*') rewrite = subparsers.add_parser('rewrite', help='Load tools.json, validate, and save the result back into JSON') rewrite.add_argument('--output', help='Save new tools.json into this file') diff --git a/tools/test_idf_tools/add_version/artifact_expected_addition.json b/tools/test_idf_tools/add_version/artifact_expected_addition.json new file mode 100644 index 0000000000..76628bac07 --- /dev/null +++ b/tools/test_idf_tools/add_version/artifact_expected_addition.json @@ -0,0 +1,14 @@ +{ + "linux-amd64": { + "sha256": "d2d02ea74de2c9fab1d802db969c18d409a8663a9697977bb1c98ccdd9de4372", + "size": 10, + "url": "http://test.com/xtensa-esp32-elf-test-linux-amd64.tar.gz" + }, + "linux-armhf": { + "sha256": "d1b3707fbdc6a22d16e95bf6b910646f5d9c2b3ed81bd637d454ffb9bb0948e4", + "size": 20, + "url": "http://test.com/xtensa-esp32-elf-test-linux-armhf.tar.gz" + }, + "name": "test", + "status": "supported" +} diff --git a/tools/test_idf_tools/add_version/artifact_input.json b/tools/test_idf_tools/add_version/artifact_input.json new file mode 100644 index 0000000000..eef2d0b285 --- /dev/null +++ b/tools/test_idf_tools/add_version/artifact_input.json @@ -0,0 +1,12 @@ +[ + { + "filename": "xtensa-esp32-elf-test-linux-amd64.tar.gz", + "size": 10, + "sha256": "d2d02ea74de2c9fab1d802db969c18d409a8663a9697977bb1c98ccdd9de4372" + }, + { + "filename": "xtensa-esp32-elf-test-linux-armhf.tar.gz", + "size": 20, + "sha256": "d1b3707fbdc6a22d16e95bf6b910646f5d9c2b3ed81bd637d454ffb9bb0948e4" + } +] diff --git a/tools/test_idf_tools/add_version/checksum.sha256 b/tools/test_idf_tools/add_version/checksum.sha256 new file mode 100644 index 0000000000..79378cf1b4 --- /dev/null +++ b/tools/test_idf_tools/add_version/checksum.sha256 @@ -0,0 +1,98 @@ +# crosstool-NG-esp-2021r2.tar.gz: 3179127 bytes +6b2a40d84bf1d3a0ab71b7df8a265d323c43b4b48de017e30e6262642cb04fb1 *crosstool-NG-esp-2021r2.tar.gz +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-linux-amd64.tar.gz: 106837189 bytes +812d735063da9d063b374b59f55832a96c41fbd27ddaef19000a75de8607ba21 *riscv32-esp-elf-gcc8_4_0-esp-2021r2-linux-amd64.tar.gz +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-linux-arm64.tar.gz: 103273444 bytes +712f1fbc3e08304a6f32aa18b346b16bbcb413b507b3d4c7c3211bf0d7dc4813 *riscv32-esp-elf-gcc8_4_0-esp-2021r2-linux-arm64.tar.gz +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-linux-armel.tar.gz: 103058744 bytes +80a3342cda2cd4b6b75ebb2b36d5d12fce7d375cfadadcff01ec3a907f0a16a2 *riscv32-esp-elf-gcc8_4_0-esp-2021r2-linux-armel.tar.gz +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-linux-armhf.tar.gz: 100134410 bytes +ed4e6bebd34aed77048bd33ae13c0e5ff404b8748f99e4004983ce365309b039 *riscv32-esp-elf-gcc8_4_0-esp-2021r2-linux-armhf.tar.gz +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-linux-i686.tar.gz: 109447789 bytes +7f0162a81558ab0ed09d6c5d356def25b5cb3d5c2d61358f20152fa260ccc8ae *riscv32-esp-elf-gcc8_4_0-esp-2021r2-linux-i686.tar.gz +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-macos.tar.gz: 113672945 bytes +3ff7e5427907cf8e271c1f959b70fb01e39625c3caf61a6567e7b38aa0c11578 *riscv32-esp-elf-gcc8_4_0-esp-2021r2-macos.tar.gz +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-win32.zip: 140809778 bytes +c8ff08883c1456c278fad85e1c43b7c6e251d525683214168655550e85c5b82e *riscv32-esp-elf-gcc8_4_0-esp-2021r2-win32.zip +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-win64.zip: 142365782 bytes +6c04cb4728db928ec6473e63146b695b6dec686a0d40dd73dd3353f05247b19e *riscv32-esp-elf-gcc8_4_0-esp-2021r2-win64.zip +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-amd64.tar.gz: 90565318 bytes +3eb3d68b27fa6ba5af6f88da21cb8face9be0094daaa8960793cfe570ab785ff *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-amd64.tar.gz +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-arm64.tar.gz: 86860292 bytes +aa534be24e45e06b7080a6a3bb8cd9e3cfb818f5f8bce2244d7cfb5e91336541 *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-arm64.tar.gz +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-armel.tar.gz: 86183421 bytes +f0e49ce06fe7833ff5d76961dc2dac5449d320f823bb8c05a302cf85a3a6eb04 *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-armel.tar.gz +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-armhf.tar.gz: 83295350 bytes +b9de7b995630ea000318ee734c33dc1b9c3a9d24b42403e98045a62ccdef1ecf *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-armhf.tar.gz +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-i686.tar.gz: 92582250 bytes +06de09b74652de43e5b22db3b7fc992623044baa75e9faaab68317a986715ba3 *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-i686.tar.gz +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-macos.tar.gz: 97808961 bytes +96443f69c8569417c780ee749d91ef33cffe22153fffa30a0fbf12107d87381b *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-macos.tar.gz +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-win32.zip: 112578260 bytes +076a4171bdc33e5ced3952efffb233d70263dfa760e636704050597a9edf61db *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-win32.zip +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-win64.zip: 115278695 bytes +c35b7998f7f503e0cb22055d1e279ae14b6b0e09bb3ff3846b17d552ece9c247 *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-win64.zip +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-linux-amd64.tar.gz: 90901736 bytes +a6e0947c92b823ca04f062522249f0a428357e0b056f1ff4c6bcabef83cf63a7 *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-linux-amd64.tar.gz +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-linux-arm64.tar.gz: 87176557 bytes +d2e5600fc194b508bd393b236a09fd62ed70afb6c36619d4b106b696a56ca66d *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-linux-arm64.tar.gz +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-linux-armel.tar.gz: 86581102 bytes +3fff4199e986dd74660f17ca27d9414cb98f1b911a7f13bb3b22e784cb1156cf *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-linux-armel.tar.gz +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-linux-armhf.tar.gz: 83722420 bytes +57c37c08e2cb93b300a1b1aeb9ee93350a642832e13e77b6ed4bfa2caddf1e45 *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-linux-armhf.tar.gz +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-linux-i686.tar.gz: 92875986 bytes +7732f9fb371d36b6b324820e300beecc33c2719921a61cf1cdb5bc625016b346 *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-linux-i686.tar.gz +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-macos.tar.gz: 98212907 bytes +e6dd32782fcff8f633299b97d1c671d6b6513390aca2ddbd7543c2cc62e72d7e *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-macos.tar.gz +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-win32.zip: 113022469 bytes +41b917b35f6fbe7d30b7de91c32cf348c406acfa729a1eabc450d040dc46fbe2 *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-win32.zip +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-win64.zip: 115696999 bytes +a764c1a0ee743d69f8cbfadbe4426a2c15c0e233b0894244c7cadf3b4d7dd32a *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-win64.zip +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-linux-amd64.tar.gz: 90887465 bytes +b958eb47f51fc2a91e3beda78a331a380eb8c96d5452f7795adf3f565d7fca2f *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-linux-amd64.tar.gz +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-linux-arm64.tar.gz: 87047917 bytes +5fb122f1109a0b1aa7a42b6b48f56c854c0a84d13047a3bb0a78bdc737bf70e2 *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-linux-arm64.tar.gz +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-linux-armel.tar.gz: 86448074 bytes +d618be508629749110785ce0038b35959cc4e6953629e2dc6d65697425b905e1 *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-linux-armel.tar.gz +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-linux-armhf.tar.gz: 83545136 bytes +c2b129c1979b79cbe5bab78178ac9abe8be66db6dd5ed432a779229d7e87195b *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-linux-armhf.tar.gz +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-linux-i686.tar.gz: 92888291 bytes +9701907da616992079d302acf5a04f97361b39ca3e74112690b2c896875f3a62 *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-linux-i686.tar.gz +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-macos.tar.gz: 98564027 bytes +d417885a5d150d94b3b84f68460b7af399a789cb0c7c632e222feed666c8aaea *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-macos.tar.gz +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-win32.zip: 112979829 bytes +d2d76c69b267767d7caf01f152cf0d1dbb9facba0e9bd2cbcad5130253a14e5f *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-win32.zip +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-win64.zip: 115825020 bytes +9c04d1da09c600b380f323b01c15e3ec511053db7d0c161085081a3fa812dc1e *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-win64.zip +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-patch1-win32.zip: 144692343 bytes +f1ff7d2a87e1f1515371c1e4868b982e6a0958df144e2f1b2bd7e684ec1f9c93 *riscv32-esp-elf-gcc8_4_0-esp-2021r2-patch1-win32.zip +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch1-win32.zip: 116460829 bytes +eba06307022cc659e3c5345ecb3c620c99ec5d0d2a5cb59ac21c831edcbafc45 *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch1-win32.zip +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-patch1-win32.zip: 116905046 bytes +083458aed4e0e1efad3779098b5626dbb41cfe00892daf1ae1fde07f59ac40b9 *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-patch1-win32.zip +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-patch1-win32.zip: 116862406 bytes +0985f5292370daad2bf228d80bcd51aacb060288a24c7d1965fddfb16e4e2613 *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-patch1-win32.zip +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-patch1-win64.zip: 146593717 bytes +791b8c8ed99934a2ec7f42100f2c71fb1ef7042efa7c6267c0d59394175c827a *riscv32-esp-elf-gcc8_4_0-esp-2021r2-patch1-win64.zip +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch1-win64.zip: 119506634 bytes +36a47c80fa79a867244f39794565c391cf4646d221c8f3e228bef45a5de1d32a *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch1-win64.zip +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-patch1-win64.zip: 119924946 bytes +e535084882355d5f7587d79d4c0b6d135a318288acf50a5a2fe1b90dbc934b61 *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-patch1-win64.zip +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-patch1-win64.zip: 120052967 bytes +cb98c854017ffa3222ef1db9e76151364d6f22841b11b07e857363065be91d1f *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-patch1-win64.zip +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-patch2-win32.zip: 144701997 bytes +937566910600d3d5b4ef6f272084fe59ea82dc3711b260a601be7487ef7a4626 *riscv32-esp-elf-gcc8_4_0-esp-2021r2-patch2-win32.zip +# riscv32-esp-elf-gcc8_4_0-esp-2021r2-patch2-win64.zip: 146606360 bytes +40570481ba0d78f7a51e523ce2e7d144b55352071adeeda0d7e81161c6c73245 *riscv32-esp-elf-gcc8_4_0-esp-2021r2-patch2-win64.zip +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch2-win32.zip: 116446514 bytes +c14cc88ddeff6d5494497de33fb5783268c6a171b3bb8c745aafae58507e2356 *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch2-win32.zip +# xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch2-win64.zip: 119516221 bytes +68db46ed4f188e169b922d43215eea781de28f847e7caed3acd5991d0bfb67bd *xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch2-win64.zip +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-patch2-win32.zip: 116913005 bytes +2c6aea1a132c6caa5a71cb5389b43454276bf097c98bb25d5bb778ed65036aef *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-patch2-win32.zip +# xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-patch2-win64.zip: 119924927 bytes +2d57cb5d897592cf0abdae94d1d673cdad294007f6210a96f34e7cd9f26c48a1 *xtensa-esp32s2-elf-gcc8_4_0-esp-2021r2-patch2-win64.zip +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-patch2-win32.zip: 116846719 bytes +cfac4ec95f7cf64b7d81a66799e388062469d53ffb19698c2b30ccf78076e92f *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-patch2-win32.zip +# xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-patch2-win64.zip: 120066549 bytes +31c79edf0df6592da61869d5d85d8e8fd064f0a247f2a3849996facc17a9e972 *xtensa-esp32s3-elf-gcc8_4_0-esp-2021r2-patch2-win64.zip diff --git a/tools/test_idf_tools/add_version/checksum_expected_addition.json b/tools/test_idf_tools/add_version/checksum_expected_addition.json new file mode 100644 index 0000000000..ba82700374 --- /dev/null +++ b/tools/test_idf_tools/add_version/checksum_expected_addition.json @@ -0,0 +1,44 @@ +{ + "linux-amd64":{ + "sha256":"3eb3d68b27fa6ba5af6f88da21cb8face9be0094daaa8960793cfe570ab785ff", + "size":90565318, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-amd64.tar.gz" + }, + "linux-arm64":{ + "sha256":"aa534be24e45e06b7080a6a3bb8cd9e3cfb818f5f8bce2244d7cfb5e91336541", + "size":86860292, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-arm64.tar.gz" + }, + "linux-armel":{ + "sha256":"f0e49ce06fe7833ff5d76961dc2dac5449d320f823bb8c05a302cf85a3a6eb04", + "size":86183421, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-armel.tar.gz" + }, + "linux-armhf":{ + "sha256":"b9de7b995630ea000318ee734c33dc1b9c3a9d24b42403e98045a62ccdef1ecf", + "size":83295350, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-armhf.tar.gz" + }, + "linux-i686":{ + "sha256":"06de09b74652de43e5b22db3b7fc992623044baa75e9faaab68317a986715ba3", + "size":92582250, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-i686.tar.gz" + }, + "macos":{ + "sha256":"96443f69c8569417c780ee749d91ef33cffe22153fffa30a0fbf12107d87381b", + "size":97808961, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-macos.tar.gz" + }, + "name":"test", + "status":"supported", + "win32":{ + "sha256":"c14cc88ddeff6d5494497de33fb5783268c6a171b3bb8c745aafae58507e2356", + "size":116446514, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch2-win32.zip" + }, + "win64":{ + "sha256":"68db46ed4f188e169b922d43215eea781de28f847e7caed3acd5991d0bfb67bd", + "size":119516221, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch2-win64.zip" + } +} diff --git a/tools/test_idf_tools/add_version/checksum_expected_override.json b/tools/test_idf_tools/add_version/checksum_expected_override.json new file mode 100644 index 0000000000..6dce72830f --- /dev/null +++ b/tools/test_idf_tools/add_version/checksum_expected_override.json @@ -0,0 +1,44 @@ +{ + "linux-amd64":{ + "sha256":"3eb3d68b27fa6ba5af6f88da21cb8face9be0094daaa8960793cfe570ab785ff", + "size":90565318, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-amd64.tar.gz" + }, + "linux-arm64":{ + "sha256":"aa534be24e45e06b7080a6a3bb8cd9e3cfb818f5f8bce2244d7cfb5e91336541", + "size":86860292, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-arm64.tar.gz" + }, + "linux-armel":{ + "sha256":"f0e49ce06fe7833ff5d76961dc2dac5449d320f823bb8c05a302cf85a3a6eb04", + "size":86183421, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-armel.tar.gz" + }, + "linux-armhf":{ + "sha256":"b9de7b995630ea000318ee734c33dc1b9c3a9d24b42403e98045a62ccdef1ecf", + "size":83295350, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-armhf.tar.gz" + }, + "linux-i686":{ + "sha256":"06de09b74652de43e5b22db3b7fc992623044baa75e9faaab68317a986715ba3", + "size":92582250, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-linux-i686.tar.gz" + }, + "macos":{ + "sha256":"96443f69c8569417c780ee749d91ef33cffe22153fffa30a0fbf12107d87381b", + "size":97808961, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-macos.tar.gz" + }, + "name":"test", + "status":"recommended", + "win32":{ + "sha256":"c14cc88ddeff6d5494497de33fb5783268c6a171b3bb8c745aafae58507e2356", + "size":116446514, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch2-win32.zip" + }, + "win64":{ + "sha256":"68db46ed4f188e169b922d43215eea781de28f847e7caed3acd5991d0bfb67bd", + "size":119516221, + "url":"http://test.com/xtensa-esp32-elf-gcc8_4_0-esp-2021r2-patch2-win64.zip" + } +} diff --git a/tools/test_idf_tools/test_idf_tools.py b/tools/test_idf_tools/test_idf_tools.py index 3b9355bd31..6385ceb38a 100755 --- a/tools/test_idf_tools/test_idf_tools.py +++ b/tools/test_idf_tools/test_idf_tools.py @@ -386,6 +386,13 @@ class TestUsage(unittest.TestCase): class TestMaintainer(unittest.TestCase): + @classmethod + def setUpClass(cls): + idf_path = os.getenv('IDF_PATH') + cls.tools_old = os.path.join(idf_path, 'tools/tools.json') + cls.tools_new = os.path.join(idf_path, 'tools/tools.new.json') + cls.test_tool_name = 'xtensa-esp32-elf' + def test_validation(self): idf_tools.main(['validate']) @@ -394,12 +401,88 @@ class TestMaintainer(unittest.TestCase): idf_path = os.getenv('IDF_PATH') if not idf_path: self.fail('IDF_PATH needs to be set to run this test') - with open(os.path.join(idf_path, 'tools/tools.json'), 'r') as f: + with open(self.tools_old, 'r') as f: json_old = f.read() - with open(os.path.join(idf_path, 'tools/tools.new.json'), 'r') as f: + with open(self.tools_new, 'r') as f: json_new = f.read() self.assertEqual(json_old, json_new, "Please check 'tools/tools.new.json' to find a cause!") + def add_version_get_expected_json(self, addition_file, replace=False): + with open(self.tools_old, 'r') as f: + expected_json = json.load(f) + with open(addition_file, 'r') as f: + addition_json = json.load(f) + for tool in expected_json['tools']: + if tool['name'] == self.test_tool_name: + if replace: + tool['versions'] = [addition_json] + else: + tool['versions'].append(addition_json) + return expected_json + return None + + def test_add_version_artifact_addition(self): + filenames = [] + with open('add_version/artifact_input.json', 'r') as f: + add_tools_info = json.load(f) + for tool in add_tools_info: + filenames.append(tool['filename']) + with open(tool['filename'], 'w') as f: + self.addCleanup(os.remove, f.name) + f.write('1' * tool['size']) + idf_tools.main( + [ + 'add-version', + '--tool', + self.test_tool_name, + '--url-prefix', + 'http://test.com', + '--version', + 'test', + '--artifact-file' + ] + filenames + ) + expected_json = self.add_version_get_expected_json('add_version/artifact_expected_addition.json') + with open(self.tools_new, 'r') as f1: + self.assertEqual(json.load(f1), expected_json, "Please check 'tools/tools.new.json' to find a cause!") + + def test_add_version_checksum_addition(self): + idf_tools.main( + [ + 'add-version', + '--tool', + self.test_tool_name, + '--url-prefix', + 'http://test.com', + '--version', + 'test', + '--checksum-file', + 'add_version/checksum.sha256', + ] + ) + expected_json = self.add_version_get_expected_json('add_version/checksum_expected_addition.json') + with open(self.tools_new, 'r') as f1: + self.assertEqual(json.load(f1), expected_json, "Please check 'tools/tools.new.json' to find a cause!") + + def test_add_version_checksum_with_override(self): + idf_tools.main( + [ + 'add-version', + '--tool', + self.test_tool_name, + '--url-prefix', + 'http://test.com', + '--version', + 'test', + '--override', + '--checksum-file', + 'add_version/checksum.sha256' + ] + ) + expected_json = self.add_version_get_expected_json('add_version/checksum_expected_override.json', True) + with open(self.tools_new, 'r') as f1: + self.assertEqual(json.load(f1), expected_json, "Please check 'tools/tools.new.json' to find a cause!") + if __name__ == '__main__': unittest.main()