From 7af26fa6d6fa6aabc94ef2a25e6525483755a856 Mon Sep 17 00:00:00 2001 From: "martin.gano" Date: Wed, 4 Nov 2020 23:32:40 +0100 Subject: [PATCH 1/2] Tools: add Python 2 deprecation warning --- components/app_update/otatool.py | 4 + components/efuse/efuse_table_gen.py | 4 + components/spiffs/spiffsgen.py | 537 ++++++++++++++++++++ components/ulp/esp32ulp_mapgen.py | 6 + components/xtensa/trax/traceparse.py | 343 +++++++++++++ docs/en/get-started/linux-setup-scratch.rst | 17 +- docs/en/get-started/linux-setup.rst | 49 +- docs/en/get-started/macos-setup-scratch.rst | 4 + docs/en/get-started/macos-setup.rst | 5 + tools/ci/check_build_warnings.py | 102 ++++ tools/ci/config/build.yml | 502 ++++++++++++++++++ tools/idf.py | 6 + 12 files changed, 1571 insertions(+), 8 deletions(-) create mode 100755 components/spiffs/spiffsgen.py create mode 100644 components/xtensa/trax/traceparse.py create mode 100755 tools/ci/check_build_warnings.py create mode 100644 tools/ci/config/build.yml diff --git a/components/app_update/otatool.py b/components/app_update/otatool.py index aead479d92..2063d3f42e 100755 --- a/components/app_update/otatool.py +++ b/components/app_update/otatool.py @@ -276,6 +276,10 @@ def erase_ota_partition(args): def main(): + if sys.version_info[0] < 3: + print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.") + elif sys.version_info[0] == 3 and sys.version_info[1] < 6: + print("WARNING: Python 3 versions older than 3.6 are not supported.") global quiet parser = argparse.ArgumentParser("ESP-IDF OTA Partitions Tool") diff --git a/components/efuse/efuse_table_gen.py b/components/efuse/efuse_table_gen.py index 02940c27a5..eea308bd23 100755 --- a/components/efuse/efuse_table_gen.py +++ b/components/efuse/efuse_table_gen.py @@ -448,6 +448,10 @@ def create_output_files(name, output_table, debug): def main(): + if sys.version_info[0] < 3: + print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.", file=sys.stderr) + elif sys.version_info[0] == 3 and sys.version_info[1] < 6: + print("WARNING: Python 3 versions older than 3.6 are not supported.", file=sys.stderr) global quiet global max_blk_len diff --git a/components/spiffs/spiffsgen.py b/components/spiffs/spiffsgen.py new file mode 100755 index 0000000000..3314e93f0e --- /dev/null +++ b/components/spiffs/spiffsgen.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python +# +# spiffsgen is a tool used to generate a spiffs image from a directory +# +# Copyright 2019 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division, print_function +import os +import sys +import io +import math +import struct +import argparse +import ctypes + +SPIFFS_PH_FLAG_USED_FINAL_INDEX = 0xF8 +SPIFFS_PH_FLAG_USED_FINAL = 0xFC + +SPIFFS_PH_FLAG_LEN = 1 +SPIFFS_PH_IX_SIZE_LEN = 4 +SPIFFS_PH_IX_OBJ_TYPE_LEN = 1 +SPIFFS_TYPE_FILE = 1 + +# Based on typedefs under spiffs_config.h +SPIFFS_OBJ_ID_LEN = 2 # spiffs_obj_id +SPIFFS_SPAN_IX_LEN = 2 # spiffs_span_ix +SPIFFS_PAGE_IX_LEN = 2 # spiffs_page_ix +SPIFFS_BLOCK_IX_LEN = 2 # spiffs_block_ix + + +class SpiffsBuildConfig(): + def __init__(self, page_size, page_ix_len, block_size, + block_ix_len, meta_len, obj_name_len, obj_id_len, + span_ix_len, packed, aligned, endianness, use_magic, use_magic_len): + if block_size % page_size != 0: + raise RuntimeError("block size should be a multiple of page size") + + self.page_size = page_size + self.block_size = block_size + self.obj_id_len = obj_id_len + self.span_ix_len = span_ix_len + self.packed = packed + self.aligned = aligned + self.obj_name_len = obj_name_len + self.meta_len = meta_len + self.page_ix_len = page_ix_len + self.block_ix_len = block_ix_len + self.endianness = endianness + self.use_magic = use_magic + self.use_magic_len = use_magic_len + + self.PAGES_PER_BLOCK = self.block_size // self.page_size + self.OBJ_LU_PAGES_PER_BLOCK = int(math.ceil(self.block_size / self.page_size * self.obj_id_len / self.page_size)) + self.OBJ_USABLE_PAGES_PER_BLOCK = self.PAGES_PER_BLOCK - self.OBJ_LU_PAGES_PER_BLOCK + + self.OBJ_LU_PAGES_OBJ_IDS_LIM = self.page_size // self.obj_id_len + + self.OBJ_DATA_PAGE_HEADER_LEN = self.obj_id_len + self.span_ix_len + SPIFFS_PH_FLAG_LEN + + pad = 4 - (4 if self.OBJ_DATA_PAGE_HEADER_LEN % 4 == 0 else self.OBJ_DATA_PAGE_HEADER_LEN % 4) + + self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED = self.OBJ_DATA_PAGE_HEADER_LEN + pad + self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD = pad + self.OBJ_DATA_PAGE_CONTENT_LEN = self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN + + self.OBJ_INDEX_PAGES_HEADER_LEN = (self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED + SPIFFS_PH_IX_SIZE_LEN + + SPIFFS_PH_IX_OBJ_TYPE_LEN + self.obj_name_len + self.meta_len) + self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN) // self.block_ix_len + self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) / self.block_ix_len + + +class SpiffsFullError(RuntimeError): + def __init__(self, message=None): + super(SpiffsFullError, self).__init__(message) + + +class SpiffsPage(): + _endianness_dict = { + "little": "<", + "big": ">" + } + + _len_dict = { + 1: "B", + 2: "H", + 4: "I", + 8: "Q" + } + + _type_dict = { + 1: ctypes.c_ubyte, + 2: ctypes.c_ushort, + 4: ctypes.c_uint, + 8: ctypes.c_ulonglong + } + + def __init__(self, bix, build_config): + self.build_config = build_config + self.bix = bix + + +class SpiffsObjLuPage(SpiffsPage): + def __init__(self, bix, build_config): + SpiffsPage.__init__(self, bix, build_config) + + self.obj_ids_limit = self.build_config.OBJ_LU_PAGES_OBJ_IDS_LIM + self.obj_ids = list() + + def _calc_magic(self, blocks_lim): + # Calculate the magic value mirrorring computation done by the macro SPIFFS_MAGIC defined in + # spiffs_nucleus.h + magic = 0x20140529 ^ self.build_config.page_size + if self.build_config.use_magic_len: + magic = magic ^ (blocks_lim - self.bix) + magic = SpiffsPage._type_dict[self.build_config.obj_id_len](magic) + return magic.value + + def register_page(self, page): + if not self.obj_ids_limit > 0: + raise SpiffsFullError() + + obj_id = (page.obj_id, page.__class__) + self.obj_ids.append(obj_id) + self.obj_ids_limit -= 1 + + def to_binary(self): + global test + img = b"" + + for (obj_id, page_type) in self.obj_ids: + if page_type == SpiffsObjIndexPage: + obj_id ^= (1 << ((self.build_config.obj_id_len * 8) - 1)) + img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len], obj_id) + + assert(len(img) <= self.build_config.page_size) + + img += b"\xFF" * (self.build_config.page_size - len(img)) + + return img + + def magicfy(self, blocks_lim): + # Only use magic value if no valid obj id has been written to the spot, which is the + # spot taken up by the last obj id on last lookup page. The parent is responsible + # for determining which is the last lookup page and calling this function. + remaining = self.obj_ids_limit + empty_obj_id_dict = { + 1: 0xFF, + 2: 0xFFFF, + 4: 0xFFFFFFFF, + 8: 0xFFFFFFFFFFFFFFFF + } + if (remaining >= 2): + for i in range(remaining): + if i == remaining - 2: + self.obj_ids.append((self._calc_magic(blocks_lim), SpiffsObjDataPage)) + break + else: + self.obj_ids.append((empty_obj_id_dict[self.build_config.obj_id_len], SpiffsObjDataPage)) + self.obj_ids_limit -= 1 + + +class SpiffsObjIndexPage(SpiffsPage): + def __init__(self, obj_id, span_ix, size, name, build_config): + SpiffsPage.__init__(self, 0, build_config) + self.obj_id = obj_id + self.span_ix = span_ix + self.name = name + self.size = size + + if self.span_ix == 0: + self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM + else: + self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_LIM + + self.pages = list() + + def register_page(self, page): + if not self.pages_lim > 0: + raise SpiffsFullError + + self.pages.append(page.offset) + self.pages_lim -= 1 + + def to_binary(self): + obj_id = self.obj_id ^ (1 << ((self.build_config.obj_id_len * 8) - 1)) + img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len] + + SpiffsPage._len_dict[self.build_config.span_ix_len] + + SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], + obj_id, + self.span_ix, + SPIFFS_PH_FLAG_USED_FINAL_INDEX) + + # Add padding before the object index page specific information + img += b"\xFF" * self.build_config.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD + + # If this is the first object index page for the object, add filname, type + # and size information + if self.span_ix == 0: + img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[SPIFFS_PH_IX_SIZE_LEN] + + SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], + self.size, + SPIFFS_TYPE_FILE) + + img += self.name.encode() + (b"\x00" * ((self.build_config.obj_name_len - len(self.name)) + self.build_config.meta_len)) + + # Finally, add the page index of daa pages + for page in self.pages: + page = page >> int(math.log(self.build_config.page_size, 2)) + img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.page_ix_len], page) + + assert(len(img) <= self.build_config.page_size) + + img += b"\xFF" * (self.build_config.page_size - len(img)) + + return img + + +class SpiffsObjDataPage(SpiffsPage): + def __init__(self, offset, obj_id, span_ix, contents, build_config): + SpiffsPage.__init__(self, 0, build_config) + self.obj_id = obj_id + self.span_ix = span_ix + self.contents = contents + self.offset = offset + + def to_binary(self): + img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + + SpiffsPage._len_dict[self.build_config.obj_id_len] + + SpiffsPage._len_dict[self.build_config.span_ix_len] + + SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], + self.obj_id, + self.span_ix, + SPIFFS_PH_FLAG_USED_FINAL) + + img += self.contents + + assert(len(img) <= self.build_config.page_size) + + img += b"\xFF" * (self.build_config.page_size - len(img)) + + return img + + +class SpiffsBlock(): + def _reset(self): + self.cur_obj_index_span_ix = 0 + self.cur_obj_data_span_ix = 0 + self.cur_obj_id = 0 + self.cur_obj_idx_page = None + + def __init__(self, bix, blocks_lim, build_config): + self.build_config = build_config + self.offset = bix * self.build_config.block_size + self.remaining_pages = self.build_config.OBJ_USABLE_PAGES_PER_BLOCK + self.pages = list() + self.bix = bix + + lu_pages = list() + for i in range(self.build_config.OBJ_LU_PAGES_PER_BLOCK): + page = SpiffsObjLuPage(self.bix, self.build_config) + lu_pages.append(page) + + self.pages.extend(lu_pages) + + self.lu_page_iter = iter(lu_pages) + self.lu_page = next(self.lu_page_iter) + + self._reset() + + def _register_page(self, page): + if isinstance(page, SpiffsObjDataPage): + self.cur_obj_idx_page.register_page(page) # can raise SpiffsFullError + + try: + self.lu_page.register_page(page) + except SpiffsFullError: + self.lu_page = next(self.lu_page_iter) + try: + self.lu_page.register_page(page) + except AttributeError: # no next lookup page + # Since the amount of lookup pages is pre-computed at every block instance, + # this should never occur + raise RuntimeError("invalid attempt to add page to a block when there is no more space in lookup") + + self.pages.append(page) + + def begin_obj(self, obj_id, size, name, obj_index_span_ix=0, obj_data_span_ix=0): + if not self.remaining_pages > 0: + raise SpiffsFullError() + self._reset() + + self.cur_obj_id = obj_id + self.cur_obj_index_span_ix = obj_index_span_ix + self.cur_obj_data_span_ix = obj_data_span_ix + + page = SpiffsObjIndexPage(obj_id, self.cur_obj_index_span_ix, size, name, self.build_config) + self._register_page(page) + + self.cur_obj_idx_page = page + + self.remaining_pages -= 1 + self.cur_obj_index_span_ix += 1 + + def update_obj(self, contents): + if not self.remaining_pages > 0: + raise SpiffsFullError() + page = SpiffsObjDataPage(self.offset + (len(self.pages) * self.build_config.page_size), + self.cur_obj_id, self.cur_obj_data_span_ix, contents, self.build_config) + + self._register_page(page) + + self.cur_obj_data_span_ix += 1 + self.remaining_pages -= 1 + + def end_obj(self): + self._reset() + + def is_full(self): + return self.remaining_pages <= 0 + + def to_binary(self, blocks_lim): + img = b"" + + if self.build_config.use_magic: + for (idx, page) in enumerate(self.pages): + if idx == self.build_config.OBJ_LU_PAGES_PER_BLOCK - 1: + page.magicfy(blocks_lim) + img += page.to_binary() + else: + for page in self.pages: + img += page.to_binary() + + assert(len(img) <= self.build_config.block_size) + + img += b"\xFF" * (self.build_config.block_size - len(img)) + return img + + +class SpiffsFS(): + def __init__(self, img_size, build_config): + if img_size % build_config.block_size != 0: + raise RuntimeError("image size should be a multiple of block size") + + self.img_size = img_size + self.build_config = build_config + + self.blocks = list() + self.blocks_lim = self.img_size // self.build_config.block_size + self.remaining_blocks = self.blocks_lim + self.cur_obj_id = 1 # starting object id + + def _create_block(self): + if self.is_full(): + raise SpiffsFullError("the image size has been exceeded") + + block = SpiffsBlock(len(self.blocks), self.blocks_lim, self.build_config) + self.blocks.append(block) + self.remaining_blocks -= 1 + return block + + def is_full(self): + return self.remaining_blocks <= 0 + + def create_file(self, img_path, file_path): + contents = None + + if len(img_path) > self.build_config.obj_name_len: + raise RuntimeError("object name '%s' too long" % img_path) + + name = img_path + + with open(file_path, "rb") as obj: + contents = obj.read() + + stream = io.BytesIO(contents) + + try: + block = self.blocks[-1] + block.begin_obj(self.cur_obj_id, len(contents), name) + except (IndexError, SpiffsFullError): + block = self._create_block() + block.begin_obj(self.cur_obj_id, len(contents), name) + + contents_chunk = stream.read(self.build_config.OBJ_DATA_PAGE_CONTENT_LEN) + + while contents_chunk: + try: + block = self.blocks[-1] + try: + # This can fail because either (1) all the pages in block have been + # used or (2) object index has been exhausted. + block.update_obj(contents_chunk) + except SpiffsFullError: + # If its (1), use the outer exception handler + if block.is_full(): + raise SpiffsFullError + # If its (2), write another object index page + block.begin_obj(self.cur_obj_id, len(contents), name, + obj_index_span_ix=block.cur_obj_index_span_ix, + obj_data_span_ix=block.cur_obj_data_span_ix) + continue + except (IndexError, SpiffsFullError): + # All pages in the block have been exhausted. Create a new block, copying + # the previous state of the block to a new one for the continuation of the + # current object + prev_block = block + block = self._create_block() + block.cur_obj_id = prev_block.cur_obj_id + block.cur_obj_idx_page = prev_block.cur_obj_idx_page + block.cur_obj_data_span_ix = prev_block.cur_obj_data_span_ix + block.cur_obj_index_span_ix = prev_block.cur_obj_index_span_ix + continue + + contents_chunk = stream.read(self.build_config.OBJ_DATA_PAGE_CONTENT_LEN) + + block.end_obj() + + self.cur_obj_id += 1 + + def to_binary(self): + img = b"" + for block in self.blocks: + img += block.to_binary(self.blocks_lim) + bix = len(self.blocks) + if self.build_config.use_magic: + # Create empty blocks with magic numbers + while self.remaining_blocks > 0: + block = SpiffsBlock(bix, self.blocks_lim, self.build_config) + img += block.to_binary(self.blocks_lim) + self.remaining_blocks -= 1 + bix += 1 + else: + # Just fill remaining spaces FF's + img += "\xFF" * (self.img_size - len(img)) + return img + + +def main(): + if sys.version_info[0] < 3: + print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.") + elif sys.version_info[0] == 3 and sys.version_info[1] < 6: + print("WARNING: Python 3 versions older than 3.6 are not supported.") + parser = argparse.ArgumentParser(description="SPIFFS Image Generator", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument("image_size", + help="Size of the created image") + + parser.add_argument("base_dir", + help="Path to directory from which the image will be created") + + parser.add_argument("output_file", + help="Created image output file path") + + parser.add_argument("--page-size", + help="Logical page size. Set to value same as CONFIG_SPIFFS_PAGE_SIZE.", + type=int, + default=256) + + parser.add_argument("--block-size", + help="Logical block size. Set to the same value as the flash chip's sector size (g_rom_flashchip.sector_size).", + type=int, + default=4096) + + parser.add_argument("--obj-name-len", + help="File full path maximum length. Set to value same as CONFIG_SPIFFS_OBJ_NAME_LEN.", + type=int, + default=32) + + parser.add_argument("--meta-len", + help="File metadata length. Set to value same as CONFIG_SPIFFS_META_LENGTH.", + type=int, + default=4) + + parser.add_argument("--use-magic", + help="Use magic number to create an identifiable SPIFFS image. Specify if CONFIG_SPIFFS_USE_MAGIC.", + action="store_true", + default=True) + + parser.add_argument("--follow-symlinks", + help="Take into account symbolic links during partition image creation.", + action="store_true", + default=False) + + parser.add_argument("--use-magic-len", + help="Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.", + action="store_true", + default=True) + + parser.add_argument("--big-endian", + help="Specify if the target architecture is big-endian. If not specified, little-endian is assumed.", + action="store_true", + default=False) + + args = parser.parse_args() + + if not os.path.exists(args.base_dir): + raise RuntimeError("given base directory %s does not exist" % args.base_dir) + + with open(args.output_file, "wb") as image_file: + image_size = int(args.image_size, 0) + spiffs_build_default = SpiffsBuildConfig(args.page_size, SPIFFS_PAGE_IX_LEN, + args.block_size, SPIFFS_BLOCK_IX_LEN, args.meta_len, + args.obj_name_len, SPIFFS_OBJ_ID_LEN, SPIFFS_SPAN_IX_LEN, + True, True, "big" if args.big_endian else "little", + args.use_magic, args.use_magic_len) + + spiffs = SpiffsFS(image_size, spiffs_build_default) + + for root, dirs, files in os.walk(args.base_dir, followlinks=args.follow_symlinks): + for f in files: + full_path = os.path.join(root, f) + spiffs.create_file("/" + os.path.relpath(full_path, args.base_dir).replace("\\", "/"), full_path) + + image = spiffs.to_binary() + + image_file.write(image) + + +if __name__ == "__main__": + main() diff --git a/components/ulp/esp32ulp_mapgen.py b/components/ulp/esp32ulp_mapgen.py index 52ad8a5f09..c40e137404 100755 --- a/components/ulp/esp32ulp_mapgen.py +++ b/components/ulp/esp32ulp_mapgen.py @@ -5,7 +5,9 @@ # Copyright (c) 2016-2017 Espressif Systems (Shanghai) PTE LTD. # Distributed under the terms of Apache License v2.0 found in the top-level LICENSE file. +from __future__ import print_function from optparse import OptionParser +import sys BASE_ADDR = 0x50000000 @@ -26,6 +28,10 @@ def gen_ld_h_from_sym(f_sym, f_ld, f_h): def main(): + if sys.version_info[0] < 3: + print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.", file=sys.stderr) + elif sys.version_info[0] == 3 and sys.version_info[1] < 6: + print("WARNING: Python 3 versions older than 3.6 are not supported.", file=sys.stderr) description = ("This application generates .h and .ld files for symbols defined in input file. " "The input symbols file can be generated using nm utility like this: " "esp32-ulp-nm -g -f posix > ") diff --git a/components/xtensa/trax/traceparse.py b/components/xtensa/trax/traceparse.py new file mode 100644 index 0000000000..119e4c9e88 --- /dev/null +++ b/components/xtensa/trax/traceparse.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python +# coding=utf-8 +# +# This script decodes Xtensa CPU trace dumps. It allows tracing the program +# execution at instruction level. +# +# Some trivia about the Xtensa CPU trace (TRAX): +# TRAX format mostly follows the IEEE-ISTO 5001-2003 (Nexus) standard. +# The following Nexus Program Trace messages are implemented by TRAX: +# - Indirect Branch Message +# - Syncronization Message +# - Indirect Branch with Synchronization Message +# - Correlation Message +# TRAX outputs compressed traces with 2 MSEO bits (LSB) and 6 MDO bits (MSB), +# packed into a byte. MSEO bits are used to split the stream into packets and messages, +# and MDO bits carry the actual data of the messages. Each message may contain multiple packets. +# +# This script can be used standalone, or loaded into GDB. +# When used standalone, it dumps the list of trace messages to stdout. +# When used from GDB, it also invokes GDB command to dump the list of assembly +# instructions corresponding to each of the messages. +# +# Standalone usage: +# traceparse.py +# +# Usage from GDB: +# xtensa-esp32-elf-gdb -n --batch program.elf -x gdbinit +# with the following gdbinit script: +# set pagination off +# set confirm off +# add-symbol-file rom.elf
+# source traceparse.py +# python parse_and_dump("/path/to/dump_file") +# +# Loading the ROM code is optional; if not loaded, disassembly for ROM sections of code +# will be missing. +# +### +# Copyright 2020 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function +import sys + +# Check if loaded into GDB +try: + assert gdb.__name__ == "gdb" + WITH_GDB = True +except NameError: + WITH_GDB = False + +# MSEO bit masks: +MSEO_PKTEND = 1 << 0 # bit 0: indicates the last byte of a packet +MSEO_MSGEND = 1 << 1 # bit 1: indicates the last byte of the message + +# Message types. The type is stored in the first 6 MDO bits or the first packet. +TVAL_INDBR = 4 # Indirect branch +TVAL_INDBRSYNC = 12 # Indirect branch w/ synchronisation +TVAL_SYNC = 9 # Synchronisation msg +TVAL_CORR = 33 # Correlation message + + +class TraxPacket(object): + def __init__(self, data, truncated=False): + self.data = data + self.size_bytes = len(data) + self.truncated = truncated + + def get_bits(self, start, count=0): + """ + Extract data bits from the packet + :param start: offset, in bits, of the part to be extracted + :param count: number of bits to extract; if omitted or zero, + extracts until the end of the packet + :return: integer containing the extracted bits + """ + start_byte = start // 6 + if count <= 0: + # all remaining bits + count = len(self.data) * 6 - start + bits_remaining = count + result = 0 + shift = 0 + for i, b in enumerate(self.data[start_byte:]): + # which bit in the byte is the starting bit + if i == 0: + # at start_byte: take the offset into account + start_bit = 2 + (start % 6) + else: + # every other byte: start after MSEO bits + start_bit = 2 + # how many bits do we need to copy from this byte + cnt_bits = min(bits_remaining, 8 - start_bit) + mask = (2 ** cnt_bits) - 1 + # take this many bits after the start_bit + bits = (b >> start_bit) & mask + # add these bits to the result + result |= bits << shift + # update the remaining bit count + shift += cnt_bits + bits_remaining -= cnt_bits + if bits_remaining == 0: + break + return result + + def __str__(self): + return "%d byte packet%s" % (self.size_bytes, " (truncated)" if self.truncated else "") + + +class TraxMessage(object): + def __init__(self, packets, truncated=False): + """ + Create and parse a TRAX message from packets + :param packets: list of TraxPacket objects, must not be empty + :param truncated: whether the message was truncated in the stream + """ + assert len(packets) > 0 + self.packets = packets + self.truncated = truncated + if truncated: + self.msg_type = None + else: + self.msg_type = self._get_type() + + # Start and end of the instruction range corresponding to this message + self.pc_start = 0 # inclusive + self.pc_end = 0 # not inclusive + self.pc_target = 0 # PC of the next range + self.is_exception = False # whether the message indicates an exception + self.is_correlation = False # whether this is a correlation message + + # message-specific fields + self.icnt = 0 + self.uaddr = 0 + self.dcont = 0 + + # decode the fields + if not truncated: + self._decode() + + def _get_type(self): + """ + :return: Message type, one of TVAL_XXX values + """ + return self.packets[0].get_bits(0, 6) + + def _decode(self): + """ Parse the packets and fill in the message-specific fields """ + if self.msg_type == TVAL_INDBR: + self.icnt = self.packets[0].get_bits(7, -1) + self.btype = self.packets[0].get_bits(6, 1) + self.uaddr = self.packets[1].get_bits(0) + self.is_exception = self.btype > 0 + elif self.msg_type == TVAL_INDBRSYNC: + self.icnt = self.packets[0].get_bits(8, -1) + self.btype = self.packets[0].get_bits(7, 1) + self.pc_target = self.packets[1].get_bits(0) + self.dcont = self.packets[0].get_bits(6, 1) + self.is_exception = self.btype > 0 + elif self.msg_type == TVAL_SYNC: + self.icnt = self.packets[0].get_bits(7, -1) + self.dcont = self.packets[0].get_bits(6, 1) + self.pc_target = self.packets[1].get_bits(0) + elif self.msg_type == TVAL_CORR: + self.icnt = self.packets[0].get_bits(12, -1) + self.is_correlation = True + else: + raise NotImplementedError("Unknown message type (%d)" % self.msg_type) + + def process_forward(self, cur_pc): + """ + Given the target PC known from the previous message, determine + the PC range corresponding to the current message. + :param cur_pc: previous known PC + :return: target PC after the current message + """ + assert not self.truncated + + next_pc = cur_pc + if self.msg_type == TVAL_INDBR: + next_pc = cur_pc ^ self.uaddr + self.pc_target = next_pc + self.pc_start = cur_pc + self.pc_end = self.pc_start + self.icnt + 1 + if self.msg_type == TVAL_INDBRSYNC: + next_pc = self.pc_target + self.pc_start = cur_pc + self.pc_end = cur_pc + self.icnt + 1 + if self.msg_type == TVAL_SYNC: + next_pc = self.pc_target + self.pc_start = next_pc - self.icnt + self.pc_end = next_pc + 1 + if self.msg_type == TVAL_CORR: + pass + return next_pc + + def process_backward(self, cur_pc): + """ + Given the address of the PC known from the _next_ message, determine + the PC range corresponding to the current message. + :param cur_pc: next known PC + :return: target PC of the _previous_ message + """ + assert not self.truncated + # Backward pass is only used to resolve addresses of messages + # up to the first SYNC/INDBRSYNC message. + # SYNC/INDBRSYNC messages are only handled in the forward pass. + assert self.msg_type != TVAL_INDBRSYNC + assert self.msg_type != TVAL_SYNC + + prev_pc = cur_pc + self.pc_target = cur_pc + if self.msg_type == TVAL_INDBR: + prev_pc ^= self.uaddr + self.pc_start = prev_pc + self.pc_end = prev_pc + self.icnt + 1 + if self.msg_type == TVAL_CORR: + pass + return prev_pc + + def __str__(self): + desc = "Unknown (%d)" % self.msg_type + extra = "" + if self.truncated: + desc = "Truncated" + if self.msg_type == TVAL_INDBR: + desc = "Indirect branch" + extra = ", icnt=%d, uaddr=0x%x, exc=%d" % (self.icnt, self.uaddr, self.is_exception) + if self.msg_type == TVAL_INDBRSYNC: + desc = "Indirect branch w/sync" + extra = ", icnt=%d, dcont=%d, exc=%d" % (self.icnt, self.dcont, self.is_exception) + if self.msg_type == TVAL_SYNC: + desc = "Synchronization" + extra = ", icnt=%d, dcont=%d" % (self.icnt, self.dcont) + if self.msg_type == TVAL_CORR: + desc = "Correlation" + extra = ", icnt=%d" % self.icnt + return "%s message, %d packets, PC range 0x%08x - 0x%08x, target PC 0x%08x" % ( + desc, len(self.packets), self.pc_start, self.pc_end, self.pc_target) + extra + + +def load_messages(data): + """ + Decodes TRAX data and resolves PC ranges. + :param data: input data, bytes + :return: list of TraxMessage objects + """ + messages = [] + packets = [] + packet_start = 0 + msg_cnt = 0 + pkt_cnt = 0 + + # Iterate over the input data, splitting bytes into packets and messages + for i, b in enumerate(data): + if (b & MSEO_MSGEND) and not (b & MSEO_PKTEND): + raise AssertionError("Invalid MSEO bits in b=0x%x. Not a TRAX dump?" % b) + + if b & MSEO_PKTEND: + pkt_cnt += 1 + packets.append(TraxPacket(data[packet_start:i + 1], packet_start == 0)) + packet_start = i + 1 + + if b & MSEO_MSGEND: + msg_cnt += 1 + try: + messages.append(TraxMessage(packets, len(messages) == 0)) + except NotImplementedError as e: + sys.stderr.write("Failed to parse message #%03d (at %d bytes): %s\n" % (msg_cnt, i, str(e))) + packets = [] + + # Resolve PC ranges of messages. + # Forward pass: skip messages until a message with known PC, + # i.e. a SYNC/INDBRSYNC message. Process all messages following it. + pc = 0 + first_sync_index = -1 + for i, m in enumerate(messages): + if pc == 0 and m.pc_target == 0: + continue + if first_sync_index < 0: + first_sync_index = i + pc = m.process_forward(pc) + + # Now process the skipped messages in the reverse direction, + # starting from the first message with known PC. + pc = messages[first_sync_index].pc_start + for m in reversed(messages[0:first_sync_index]): + if m.truncated: + break + pc = m.process_backward(pc) + + return messages + + +def parse_and_dump(filename, disassemble=WITH_GDB): + """ + Decode TRAX data from a file, print out the messages. + :param filename: file to load the dump from + :param disassemble: if True, print disassembly of PC ranges + """ + with open(filename, 'rb') as f: + data = f.read() + + messages = load_messages(data) + sys.stderr.write("Loaded %d messages in %d bytes\n" % (len(messages), len(data))) + + for i, m in enumerate(messages): + if m.truncated: + continue + print("%04d: %s" % (i, str(m))) + if m.is_exception: + print("*** Exception occurred ***") + if disassemble and WITH_GDB: + try: + gdb.execute("disassemble 0x%08x, 0x%08x" % (m.pc_start, m.pc_end)) # noqa: F821 + except gdb.MemoryError: # noqa: F821 + print("Failed to disassemble from 0x%08x to 0x%08x" % (m.pc_start, m.pc_end)) + + +def main(): + if sys.version_info[0] < 3: + print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.", file=sys.stderr) + elif sys.version_info[0] == 3 and sys.version_info[1] < 6: + print("WARNING: Python 3 versions older than 3.6 are not supported.", file=sys.stderr) + if len(sys.argv) < 2: + sys.stderr.write("Usage: %s \n") + raise SystemExit(1) + + parse_and_dump(sys.argv[1]) + + +if __name__ == "__main__" and not WITH_GDB: + main() diff --git a/docs/en/get-started/linux-setup-scratch.rst b/docs/en/get-started/linux-setup-scratch.rst index 1b6d5f9ef5..8700501594 100644 --- a/docs/en/get-started/linux-setup-scratch.rst +++ b/docs/en/get-started/linux-setup-scratch.rst @@ -12,13 +12,17 @@ Install Prerequisites To compile with ESP-IDF you need to get the following packages: +- CentOS 7:: + + sudo yum -y update && sudo yum install git wget flex bison gperf python3 + - Ubuntu and Debian:: - sudo apt-get install gcc git wget make libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-cryptography python-future python-pyparsing libffi-dev libssl-dev + sudo apt-get install git wget libncurses-dev flex bison gperf python3 python3-pip python3-setuptools python3-serial python3-cryptography python3-future python3-pyparsing python3-pyelftools cmake ninja-build ccache libffi-dev libssl-dev - Arch:: - sudo pacman -S --needed gcc git make ncurses flex bison gperf python2-pyserial python2-cryptography python2-future python2-pyparsing + sudo pacman -Sy --needed gcc git make ncurses flex bison gperf python-pyserial python-cryptography python-future python-pyparsing python-pyelftools cmake ninja ccache dfu-util .. note:: @@ -33,7 +37,7 @@ Compile the Toolchain from Source - CentOS 7:: - sudo yum install gawk gperf grep gettext ncurses-devel python python-devel automake bison flex texinfo help2man libtool + sudo yum install gawk gperf grep gettext ncurses-devel python3 python3-devel automake bison flex texinfo help2man libtool make - Ubuntu pre-16.04:: @@ -49,7 +53,7 @@ Compile the Toolchain from Source - Arch:: - TODO + sudo pacman -Sy --needed python-pip Create the working directory and go into it:: @@ -68,6 +72,11 @@ Build the toolchain:: Toolchain will be built in ``~/esp/crosstool-NG/builds/xtensa-esp32-elf``. Follow :ref:`instructions for standard setup ` to add the toolchain to your ``PATH``. +Python 2 deprecation +==================== + +Python 2 reached its `end of life `_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for popular Linux distributions are listed above. + Next Steps ========== diff --git a/docs/en/get-started/linux-setup.rst b/docs/en/get-started/linux-setup.rst index 25d5c1f03f..f16e8710e0 100644 --- a/docs/en/get-started/linux-setup.rst +++ b/docs/en/get-started/linux-setup.rst @@ -10,15 +10,17 @@ To compile with ESP-IDF you need to get the following packages: - CentOS 7:: - sudo yum install gcc git wget make ncurses-devel flex bison gperf python python2-cryptography + sudo yum -y update && sudo yum install git wget flex bison gperf python3 + +CentOS 7 is still supported but CentOS version 8 is recommended for a better user experience. - Ubuntu and Debian:: - sudo apt-get install gcc git wget make libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-cryptography python-future python-pyparsing libffi-dev libssl-dev + sudo apt-get install git wget flex bison gperf python3 python3-pip python3-setuptools libffi-dev libssl-dev - Arch:: - sudo pacman -S --needed gcc git make ncurses flex bison gperf python2-pyserial python2-cryptography python2-future python2-pyparsing + sudo pacman -S --needed gcc git make ncurses flex bison gperf .. note:: @@ -84,7 +86,6 @@ Permission issues /dev/ttyUSB0 With some Linux distributions you may get the ``Failed to open port /dev/ttyUSB0`` error message when flashing the ESP32. :ref:`This can be solved by adding the current user to the dialout group`. - Arch Linux Users ---------------- @@ -99,6 +100,46 @@ Before installing these packages you might need to add the author's public key t Alternatively, use crosstool-NG to compile a gdb that links against ncurses 6. +Setting up Python 3 as default for CentOS +----------------------------------------- + +CentOS 7 and older is providing Python 2.7 as the default interpreter. +Python 3 is recommended instead and can be installed in old distributions as follows, or please consult the documentation of your operating system for other recommended ways to achieve this:: + + sudo yum -y update && sudo yum install python3 python3-pip python3-setuptools + +Making Python 3 the default interpreter is possible by running:: + + sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3 + +Setting up Python 3 as default for Ubuntu and Debian +---------------------------------------------------- + +Ubuntu (version 18.04 and older) and Debian (version 9 and older) are still providing Python 2.7 as the default interpreter. +Python 3 is recommended instead and can be installed in old distributions as follows, or please consult the documentation of your operating system for other recommended ways to achieve this:: + + sudo apt-get install python3 python3-pip python3-setuptools + +Making Python 3 the default interpreter is possible by running:: + + sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3 + +.. note:: + This is system-wide change which may affect all of the applications. + +Fixing broken pip on Ubuntu 16.04 +================================= + +Package ``python3-pip`` could be broken without possibility to upgrade it. +Package has to be removed and installed manually using script `get-pip.py `_.:: + + apt remove python3-pip python3-virtualenv; rm -r ~/.local + rm -r ~/.espressif/python_env && python get-pip.py + +Python 2 deprecation +==================== + +Python 2 reached its `end of life `_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for popular Linux distributions are listed above. Next Steps ========== diff --git a/docs/en/get-started/macos-setup-scratch.rst b/docs/en/get-started/macos-setup-scratch.rst index a4dfe9d7f1..5ec2ac36a0 100644 --- a/docs/en/get-started/macos-setup-scratch.rst +++ b/docs/en/get-started/macos-setup-scratch.rst @@ -65,6 +65,10 @@ Build the toolchain:: Toolchain will be built in ``~/esp/ctng-volume/crosstool-NG/builds/xtensa-esp32-elf``. Follow :ref:`instructions for standard setup ` to add the toolchain to your ``PATH``. +Python 2 deprecation +==================== + +Python 2 reached its `end of life `_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for macOS are listed above. Next Steps ========== diff --git a/docs/en/get-started/macos-setup.rst b/docs/en/get-started/macos-setup.rst index b69ae12374..5a0ff243ff 100644 --- a/docs/en/get-started/macos-setup.rst +++ b/docs/en/get-started/macos-setup.rst @@ -42,6 +42,11 @@ Alternatively, you may create an alias for the above command. This way you can g Then when you need the toolchain you can type ``get_esp32`` on the command line and the toolchain will be added to your ``PATH``. +Python 2 deprecation +==================== + +Python 2 reached its `end of life `_ and support for it in ESP-IDF will be removed soon. Please install Python 3.6 or higher. Instructions for macOS are listed above. + Next Steps ========== diff --git a/tools/ci/check_build_warnings.py b/tools/ci/check_build_warnings.py new file mode 100755 index 0000000000..2e012fa9b4 --- /dev/null +++ b/tools/ci/check_build_warnings.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# coding=utf-8 +# +# CI script to check build logs for warnings. +# Reads the list of builds, in the format produced by find_apps.py or build_apps.py, and finds warnings in the +# log files for every build. +# Exits with a non-zero exit code if any warning is found. + +import argparse +import logging +import os +import re +import sys + +try: + from find_build_apps import BuildItem, setup_logging +except ImportError: + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + from find_build_apps import BuildItem, setup_logging + +WARNING_REGEX = re.compile(r"(?:error|warning)[^\w]", re.MULTILINE | re.IGNORECASE) + +IGNORE_WARNS = [ + re.compile(r_str) for r_str in [ + r"library/error\.o", + r".*error.*\.c\.obj", + r"-Werror", + r"error\.d", + r"reassigning to symbol", + r"changes choice state", + r"crosstool_version_check\.cmake", + r"CryptographyDeprecationWarning", + r"Python 3 versions older than 3.6 are not supported." + ] +] + + +def line_has_warnings(line): # type: (str) -> bool + if not WARNING_REGEX.search(line): + return False + + has_warnings = True + for ignored in IGNORE_WARNS: + if re.search(ignored, line): + has_warnings = False + break + + return has_warnings + + +def main(): + parser = argparse.ArgumentParser(description="ESP-IDF app builder") + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Increase the logging level of the script. Can be specified multiple times.", + ) + parser.add_argument( + "--log-file", + type=argparse.FileType("w"), + help="Write the script log to the specified file, instead of stderr", + ) + parser.add_argument( + "build_list", + type=argparse.FileType("r"), + nargs="?", + default=sys.stdin, + help="Name of the file to read the list of builds from. If not specified, read from stdin.", + ) + args = parser.parse_args() + setup_logging(args) + + build_items = [BuildItem.from_json(line) for line in args.build_list] + if not build_items: + logging.warning("Empty build list") + SystemExit(0) + + found_warnings = 0 + for build_item in build_items: + if not build_item.build_log_path: + logging.debug("No log file for {}".format(build_item.work_dir)) + continue + with open(build_item.build_log_path, "r") as log_file: + for line_no, line in enumerate(log_file): + if line_has_warnings(line): + logging.error("Issue in app {}, config {}:".format(build_item.app_dir, build_item.config_name)) + logging.error(line.rstrip("\n")) + logging.error("See {}:{} for details".format(os.path.basename(build_item.build_log_path), + line_no + 1)) + found_warnings += 1 + break + + if found_warnings: + logging.error("Checked {} builds, found {} warnings".format(len(build_items), found_warnings)) + raise SystemExit(1) + + logging.info("No warnings found") + + +if __name__ == "__main__": + main() diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml new file mode 100644 index 0000000000..21d35a1469 --- /dev/null +++ b/tools/ci/config/build.yml @@ -0,0 +1,502 @@ +.build_template: + stage: build + image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG + tags: + - build + variables: + SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt" + dependencies: [] + +.build_template_app_template: + extends: + - .build_template + - .rules:labels:build + variables: + LOG_PATH: "${CI_PROJECT_DIR}/log_template_app" + BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app" + BUILD_DIR: "@t/@w" + BUILD_LOG_MAKE: "${LOG_PATH}/make_@t_@w.txt" + BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt" + BUILD_COMMAND_ARGS: "" + artifacts: + when: always + paths: + - log_template_app/* + - size_info.txt + - build_template_app/**/size.json + script: + # Set the variable for 'esp-idf-template' testing + - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} + - retry_failed git clone ${ESP_IDF_TEMPLATE_GIT} + # Try to use the same branch name for esp-idf-template that we're + # using on esp-idf. If it doesn't exist then just stick to the default branch + - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template + - export PATH="$IDF_PATH/tools:$PATH" + - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} + - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} + # Only do the default cmake build for each target, remaining part are done in the build_template_app job + - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS} + # Check if there are any stray printf/ets_printf references in WiFi libs + - cd components/esp_wifi/lib + - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done; + - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done; + +# build-related-pre-check-jobs ------------------------------------------------ +# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits +fast_template_app: + extends: + - .build_template_app_template + - .rules:build_tests:target_test-weekend_test + stage: pre_check + variables: + BUILD_COMMAND_ARGS: "-p" + +check_docs_gh_links: + extends: .build_docs_template + stage: pre_check + variables: + SUBMODULES_TO_FETCH: "none" + script: + - cd docs + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck +#------------------------------------------------------------------------------ + +.build_ssc_template: + extends: + - .build_template + - .rules:build_tests:integration_test + artifacts: + paths: + - SSC/ssc_bin + expire_in: 1 week + script: + - retry_failed git clone $SSC_REPOSITORY + - python $CHECKOUT_REF_SCRIPT SSC SSC + - cd SSC + - MAKEFLAGS= ./ci_build_ssc.sh $TARGET_NAME + +build_ssc_esp32: + extends: .build_ssc_template + parallel: 3 + variables: + TARGET_NAME: "ESP32" + +build_ssc_esp32s2: + extends: .build_ssc_template + parallel: 2 + variables: + TARGET_NAME: "ESP32S2" + +.build_esp_idf_tests_cmake: + extends: + - .build_template + - .rules:build_tests:unit_test + dependencies: # set dependencies to null to avoid missing artifacts issue + needs: + - job: fast_template_app + artifacts: false + - scan_tests + artifacts: + paths: + - tools/unit-test-app/output/${IDF_TARGET} + - tools/unit-test-app/builds/*.json + - tools/unit-test-app/builds/${IDF_TARGET}/*/size.json + - components/idf_test/unit_test/*.yml + - $LOG_PATH + - $SIZE_INFO_LOCATION + when: always + expire_in: 4 days + variables: + LOG_PATH: "$CI_PROJECT_DIR/log_ut_cmake" + BUILD_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/builds + OUTPUT_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/output + BUILD_SYSTEM: "cmake" + TEST_TYPE: "unit_test" + PYTHON_VER: 3 + script: + # RISC-V toolchain is optional but ULP may need it, so install: + - $IDF_PATH/tools/idf_tools.py install riscv-none-embed-gcc + - . $IDF_PATH/export.sh + - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh + - cd $CI_PROJECT_DIR/tools/unit-test-app + - python tools/UnitTestParser.py ${BUILD_PATH} + +build_esp_idf_tests_cmake_esp32: + extends: .build_esp_idf_tests_cmake + variables: + IDF_TARGET: esp32 + +build_esp_idf_tests_cmake_esp32s2: + extends: .build_esp_idf_tests_cmake + variables: + IDF_TARGET: esp32s2 + +build_esp_idf_tests_cmake_esp32s3: + extends: .build_esp_idf_tests_cmake + variables: + IDF_TARGET: esp32s3 + +.build_examples_template: + extends: + - .build_template + - .rules:build_tests:example_test-weekend_test + dependencies: # set dependencies to null to avoid missing artifacts issue + needs: + - job: fast_template_app + artifacts: false + - scan_tests + variables: + TEST_PREFIX: examples + TEST_RELATIVE_DIR: examples + SCAN_TEST_JSON: ${CI_PROJECT_DIR}/${TEST_RELATIVE_DIR}/test_configs/scan_${IDF_TARGET}_${BUILD_SYSTEM}.json + TEST_TYPE: example_test + LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX} + BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX} + PYTHON_VER: 3 + script: + # RISC-V toolchain is optional but ULP may need it, so install: + - $IDF_PATH/tools/idf_tools.py install riscv-none-embed-gcc + - . $IDF_PATH/export.sh + # it's not possible to build 100% out-of-tree and have the "artifacts" + # mechanism work, but this is the next best thing + - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh + +build_examples_make: + extends: + - .build_examples_template + # This is a workaround for a rarely encountered issue with building examples in CI. + # Probably related to building of Kconfig in 'make clean' stage + retry: 1 + parallel: 8 + artifacts: + paths: + - $LOG_PATH + - build_${TEST_PREFIX}/*/*/*/build/size.json + - $SIZE_INFO_LOCATION + when: always + expire_in: 4 days + variables: + BUILD_SYSTEM: make + IDF_TARGET: esp32 # currently we only support esp32 + +# same as above, but for CMake +.build_examples_cmake: + extends: .build_examples_template + artifacts: + paths: + - build_${TEST_PREFIX}/list.json + - build_${TEST_PREFIX}/list_job_*.json + - build_${TEST_PREFIX}/*/*/*/sdkconfig + - build_${TEST_PREFIX}/*/*/*/build/size.json + - build_${TEST_PREFIX}/*/*/*/build/*.bin + - build_${TEST_PREFIX}/*/*/*/build/*.elf + - build_${TEST_PREFIX}/*/*/*/build/*.map + - build_${TEST_PREFIX}/*/*/*/build/flasher_args.json + - build_${TEST_PREFIX}/*/*/*/build/bootloader/*.bin + - build_${TEST_PREFIX}/*/*/*/build/partition_table/*.bin + - $LOG_PATH + - $SIZE_INFO_LOCATION + when: always + expire_in: 4 days + variables: + BUILD_SYSTEM: cmake + +build_examples_cmake_esp32: + extends: .build_examples_cmake + parallel: 10 + variables: + IDF_TARGET: esp32 + +build_examples_cmake_esp32s2: + extends: .build_examples_cmake + parallel: 8 + variables: + IDF_TARGET: esp32s2 + +.build_test_apps: + extends: + - .build_examples_cmake + - .rules:build_tests:custom_test-weekend_test + variables: + TEST_PREFIX: test_apps + TEST_RELATIVE_DIR: tools/test_apps + TEST_TYPE: custom_test + script: + - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh + +build_test_apps_esp32: + extends: .build_test_apps + parallel: 8 + variables: + IDF_TARGET: esp32 + +build_test_apps_esp32s2: + extends: .build_test_apps + parallel: 8 + variables: + IDF_TARGET: esp32s2 + +build_test_apps_esp32s3: + extends: .build_test_apps + parallel: 8 + variables: + IDF_TARGET: esp32s3 + +.build_component_ut: + extends: + - .build_test_apps + - .rules:build_tests:unit_test + variables: + TEST_PREFIX: component_ut + TEST_RELATIVE_DIR: component_ut + +build_component_ut_esp32: + extends: .build_component_ut + variables: + IDF_TARGET: esp32 + +build_component_ut_esp32s2: + extends: .build_component_ut + variables: + IDF_TARGET: esp32s2 + +.build_docs_template: + stage: build + image: $ESP_IDF_DOC_ENV_IMAGE + tags: + - build_docs + script: + - cd docs + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs $DOC_BUILDERS -l $DOCLANG -t $DOCTGT build + parallel: + matrix: + - DOCLANG: [ "en", "zh_CN" ] + DOCTGT: [ "esp32", "esp32s2" ] + +build_docs_html: + extends: + - .build_docs_template + - .rules:labels:build_docs + artifacts: + when: always + paths: + - docs/_build/*/*/*.txt + - docs/_build/*/*/html/* + expire_in: 4 days + variables: + DOC_BUILDERS: "html" + +build_docs_pdf: + extends: + - .build_docs_template + - .rules:labels:build_docs-slim + artifacts: + when: always + paths: + - docs/_build/*/*/latex/* + expire_in: 4 days + variables: + DOC_BUILDERS: "latex" + +.test_build_system_template: + extends: + - .build_template + - .rules:build_tests:weekend_test + dependencies: # set dependencies to null to avoid missing artifacts issue + needs: + - job: fast_template_app + artifacts: false + script: + - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh + - rm -rf test_build_system + - mkdir test_build_system + - cd test_build_system + - ${IDF_PATH}/tools/ci/${SHELL_TEST_SCRIPT} + +test_build_system: + extends: .test_build_system_template + variables: + SHELL_TEST_SCRIPT: test_build_system.sh + +test_build_system_cmake: + extends: .test_build_system_template + variables: + SHELL_TEST_SCRIPT: test_build_system_cmake.sh + +test_build_system_cmake_macos: + extends: + - .test_build_system_template + - .before_script_macos + - .rules:os:mac_os + tags: + - macos_shell + variables: + SHELL_TEST_SCRIPT: test_build_system_cmake.sh + +build_docker: + extends: + - .before_script_slim + - .rules:protected-schedule + stage: build + image: espressif/docker-builder:1 + tags: + - build_docker_amd64_brno + variables: + DOCKER_TMP_IMAGE_NAME: "idf_tmp_image" + script: + - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL + - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi + - echo "Using repository at $LOCAL_CI_REPOSITORY_URL" + - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${LOCAL_CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}" + # Build + - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/ + # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227. + # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually. + # Therefore, build a copy of the example located inside the container. + - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build + +.test-on-windows: + extends: + - .before_script_slim + - .rules:protected-schedule + stage: build + image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross + tags: + - build + script: + - cd $TEST_DIR + - mkdir build + - cd build + - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release .. + - cmake --build . + +build_idf_exe: + extends: .test-on-windows + artifacts: + paths: + - tools/windows/idf_exe/build/idf-exe-v*.zip + expire_in: 4 days + variables: + TEST_DIR: tools/windows/idf_exe + +build_cmdlinerunner: + extends: .test-on-windows + artifacts: + paths: + - tools/windows/tool_setup/cmdlinerunner/build/cmdlinerunner.dll + expire_in: 4 days + variables: + TEST_DIR: tools/windows/tool_setup/cmdlinerunner + +build_installer: + extends: + - .before_script_slim + - .rules:protected-schedule + # using a different stage here to be able to use artifacts from build_cmdlinerunner job + stage: host_test + image: $CI_DOCKER_REGISTRY/wine-innosetup:1 + tags: + - build + dependencies: # set dependencies to null to avoid missing artifacts issue + needs: + - build_cmdlinerunner + script: + - cd tools/windows/tool_setup/ + - ./build_installer.sh + +# This job builds template app with permutations of targets and optimization levels +build_template_app: + needs: + - job: fast_template_app + artifacts: false + extends: .build_template_app_template + +# Sonarqube related jobs put here for this reason: +# Here we have two jobs. code_quality_check and code_quality_report. +# +# code_quality_check will analyze the code changes between your MR and +# code repo stored in sonarqube server. The analysis result is only shown in +# the comments under this MR and won't be transferred to the server. +# +# code_quality_report will analyze and transfer both of the newly added code +# and the analysis result to the server. +# +# Put in the front to ensure that the newly merged code can be stored in +# sonarqube server ASAP, in order to avoid reporting unrelated code issues +.sonar_scan_template: + stage: build + image: + name: $CI_DOCKER_REGISTRY/sonarqube-scanner:2 + before_script: + - source tools/ci/utils.sh + - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" + - fetch_submodules + # Exclude the submodules, all paths ends with /** + - export SUBMODULES=$(get_all_submodules) + # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all to + - export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g') + # Exclude the report dir + - export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg" + - python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR + variables: + GIT_DEPTH: 0 + NORMALIZE_CLANGTIDY_PY: $CI_PROJECT_DIR/tools/ci/normalize_clangtidy_path.py + REPORT_DIR: examples/get-started/hello_world/tidybuild/report + tags: + - host_test + dependencies: # set dependencies to null to avoid missing artifacts issue + needs: + - clang_tidy_check_regular + +code_quality_check: + extends: + - .sonar_scan_template + - .rules:trigger + allow_failure: true + script: + - export CI_MR_IID=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py id ${CI_COMMIT_BRANCH}) + - export CI_MR_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_BRANCH} | tr '\n' ',') + # test if this branch have merge request, if not, exit 0 + - test -n "$CI_MR_IID" || exit 0 + - test -n "$CI_MR_COMMITS" || exit 0 + - sonar-scanner + -Dsonar.analysis.mode=preview + -Dsonar.host.url=$SONAR_HOST_URL + -Dsonar.login=$SONAR_LOGIN + -Dsonar.sources=$CI_PROJECT_DIR + -Dsonar.sourceEncoding=UTF-8 + -Dsonar.projectKey=esp-idf + -Dsonar.projectBaseDir=$CI_PROJECT_DIR + -Dsonar.exclusions=$EXCLUSIONS + -Dsonar.gitlab.project_id=$CI_PROJECT_ID + -Dsonar.gitlab.commit_sha=$CI_MR_COMMITS + -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME + -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt + -Dsonar.cxx.includeDirectories=components,/usr/include + -Dsonar.python.pylint_config=.pylintrc + -Dsonar.gitlab.ci_merge_request_iid=$CI_MR_IID + -Dsonar.gitlab.merge_request_discussion=true + -Dsonar.branch.name=$CI_COMMIT_REF_NAME + +code_quality_report: + extends: + - .sonar_scan_template + - .rules:protected-schedule + script: + - sonar-scanner + -Dsonar.host.url=$SONAR_HOST_URL + -Dsonar.login=$SONAR_LOGIN + -Dsonar.sources=$CI_PROJECT_DIR + -Dsonar.sourceEncoding=UTF-8 + -Dsonar.projectKey=esp-idf + -Dsonar.projectBaseDir=$CI_PROJECT_DIR + -Dsonar.exclusions=$EXCLUSIONS + -Dsonar.gitlab.project_id=$CI_PROJECT_ID + -Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA + -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME + -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt + -Dsonar.cxx.includeDirectories=components,/usr/include + -Dsonar.python.pylint_config=.pylintrc + -Dsonar.branch.name=$CI_COMMIT_REF_NAME diff --git a/tools/idf.py b/tools/idf.py index 38a15a6098..03447942f9 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -108,6 +108,12 @@ def check_environment(): print("Setting IDF_PATH environment variable: %s" % detected_idf_path) os.environ["IDF_PATH"] = detected_idf_path + # check Python version + if sys.version_info[0] < 3: + print_warning("WARNING: Support for Python 2 is deprecated and will be removed in future versions.") + elif sys.version_info[0] == 3 and sys.version_info[1] < 6: + print_warning("WARNING: Python 3 versions older than 3.6 are not supported.") + # check Python dependencies print("Checking Python dependencies...") try: From 228bec237104332b7b45c63a769e7a4cb7be1309 Mon Sep 17 00:00:00 2001 From: daiziyan Date: Mon, 30 Nov 2020 20:11:36 +0800 Subject: [PATCH 2/2] add CN translation for adding python2 deprecation warning(MR 11115) --- components/spiffs/spiffsgen.py | 537 ------------------ components/xtensa/trax/traceparse.py | 343 ----------- docs/en/get-started/linux-setup-scratch.rst | 4 - .../zh_CN/get-started/linux-setup-scratch.rst | 14 +- docs/zh_CN/get-started/linux-setup.rst | 49 +- .../zh_CN/get-started/macos-setup-scratch.rst | 7 +- docs/zh_CN/get-started/macos-setup.rst | 7 +- tools/ci/build_examples.sh | 4 +- tools/ci/build_examples_cmake.sh | 2 + tools/ci/check_build_warnings.py | 102 ---- tools/ci/config/build.yml | 502 ---------------- tools/idf.py | 4 +- 12 files changed, 74 insertions(+), 1501 deletions(-) delete mode 100755 components/spiffs/spiffsgen.py delete mode 100644 components/xtensa/trax/traceparse.py delete mode 100755 tools/ci/check_build_warnings.py delete mode 100644 tools/ci/config/build.yml diff --git a/components/spiffs/spiffsgen.py b/components/spiffs/spiffsgen.py deleted file mode 100755 index 3314e93f0e..0000000000 --- a/components/spiffs/spiffsgen.py +++ /dev/null @@ -1,537 +0,0 @@ -#!/usr/bin/env python -# -# spiffsgen is a tool used to generate a spiffs image from a directory -# -# Copyright 2019 Espressif Systems (Shanghai) PTE LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import division, print_function -import os -import sys -import io -import math -import struct -import argparse -import ctypes - -SPIFFS_PH_FLAG_USED_FINAL_INDEX = 0xF8 -SPIFFS_PH_FLAG_USED_FINAL = 0xFC - -SPIFFS_PH_FLAG_LEN = 1 -SPIFFS_PH_IX_SIZE_LEN = 4 -SPIFFS_PH_IX_OBJ_TYPE_LEN = 1 -SPIFFS_TYPE_FILE = 1 - -# Based on typedefs under spiffs_config.h -SPIFFS_OBJ_ID_LEN = 2 # spiffs_obj_id -SPIFFS_SPAN_IX_LEN = 2 # spiffs_span_ix -SPIFFS_PAGE_IX_LEN = 2 # spiffs_page_ix -SPIFFS_BLOCK_IX_LEN = 2 # spiffs_block_ix - - -class SpiffsBuildConfig(): - def __init__(self, page_size, page_ix_len, block_size, - block_ix_len, meta_len, obj_name_len, obj_id_len, - span_ix_len, packed, aligned, endianness, use_magic, use_magic_len): - if block_size % page_size != 0: - raise RuntimeError("block size should be a multiple of page size") - - self.page_size = page_size - self.block_size = block_size - self.obj_id_len = obj_id_len - self.span_ix_len = span_ix_len - self.packed = packed - self.aligned = aligned - self.obj_name_len = obj_name_len - self.meta_len = meta_len - self.page_ix_len = page_ix_len - self.block_ix_len = block_ix_len - self.endianness = endianness - self.use_magic = use_magic - self.use_magic_len = use_magic_len - - self.PAGES_PER_BLOCK = self.block_size // self.page_size - self.OBJ_LU_PAGES_PER_BLOCK = int(math.ceil(self.block_size / self.page_size * self.obj_id_len / self.page_size)) - self.OBJ_USABLE_PAGES_PER_BLOCK = self.PAGES_PER_BLOCK - self.OBJ_LU_PAGES_PER_BLOCK - - self.OBJ_LU_PAGES_OBJ_IDS_LIM = self.page_size // self.obj_id_len - - self.OBJ_DATA_PAGE_HEADER_LEN = self.obj_id_len + self.span_ix_len + SPIFFS_PH_FLAG_LEN - - pad = 4 - (4 if self.OBJ_DATA_PAGE_HEADER_LEN % 4 == 0 else self.OBJ_DATA_PAGE_HEADER_LEN % 4) - - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED = self.OBJ_DATA_PAGE_HEADER_LEN + pad - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD = pad - self.OBJ_DATA_PAGE_CONTENT_LEN = self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN - - self.OBJ_INDEX_PAGES_HEADER_LEN = (self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED + SPIFFS_PH_IX_SIZE_LEN + - SPIFFS_PH_IX_OBJ_TYPE_LEN + self.obj_name_len + self.meta_len) - self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN) // self.block_ix_len - self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) / self.block_ix_len - - -class SpiffsFullError(RuntimeError): - def __init__(self, message=None): - super(SpiffsFullError, self).__init__(message) - - -class SpiffsPage(): - _endianness_dict = { - "little": "<", - "big": ">" - } - - _len_dict = { - 1: "B", - 2: "H", - 4: "I", - 8: "Q" - } - - _type_dict = { - 1: ctypes.c_ubyte, - 2: ctypes.c_ushort, - 4: ctypes.c_uint, - 8: ctypes.c_ulonglong - } - - def __init__(self, bix, build_config): - self.build_config = build_config - self.bix = bix - - -class SpiffsObjLuPage(SpiffsPage): - def __init__(self, bix, build_config): - SpiffsPage.__init__(self, bix, build_config) - - self.obj_ids_limit = self.build_config.OBJ_LU_PAGES_OBJ_IDS_LIM - self.obj_ids = list() - - def _calc_magic(self, blocks_lim): - # Calculate the magic value mirrorring computation done by the macro SPIFFS_MAGIC defined in - # spiffs_nucleus.h - magic = 0x20140529 ^ self.build_config.page_size - if self.build_config.use_magic_len: - magic = magic ^ (blocks_lim - self.bix) - magic = SpiffsPage._type_dict[self.build_config.obj_id_len](magic) - return magic.value - - def register_page(self, page): - if not self.obj_ids_limit > 0: - raise SpiffsFullError() - - obj_id = (page.obj_id, page.__class__) - self.obj_ids.append(obj_id) - self.obj_ids_limit -= 1 - - def to_binary(self): - global test - img = b"" - - for (obj_id, page_type) in self.obj_ids: - if page_type == SpiffsObjIndexPage: - obj_id ^= (1 << ((self.build_config.obj_id_len * 8) - 1)) - img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + - SpiffsPage._len_dict[self.build_config.obj_id_len], obj_id) - - assert(len(img) <= self.build_config.page_size) - - img += b"\xFF" * (self.build_config.page_size - len(img)) - - return img - - def magicfy(self, blocks_lim): - # Only use magic value if no valid obj id has been written to the spot, which is the - # spot taken up by the last obj id on last lookup page. The parent is responsible - # for determining which is the last lookup page and calling this function. - remaining = self.obj_ids_limit - empty_obj_id_dict = { - 1: 0xFF, - 2: 0xFFFF, - 4: 0xFFFFFFFF, - 8: 0xFFFFFFFFFFFFFFFF - } - if (remaining >= 2): - for i in range(remaining): - if i == remaining - 2: - self.obj_ids.append((self._calc_magic(blocks_lim), SpiffsObjDataPage)) - break - else: - self.obj_ids.append((empty_obj_id_dict[self.build_config.obj_id_len], SpiffsObjDataPage)) - self.obj_ids_limit -= 1 - - -class SpiffsObjIndexPage(SpiffsPage): - def __init__(self, obj_id, span_ix, size, name, build_config): - SpiffsPage.__init__(self, 0, build_config) - self.obj_id = obj_id - self.span_ix = span_ix - self.name = name - self.size = size - - if self.span_ix == 0: - self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM - else: - self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_LIM - - self.pages = list() - - def register_page(self, page): - if not self.pages_lim > 0: - raise SpiffsFullError - - self.pages.append(page.offset) - self.pages_lim -= 1 - - def to_binary(self): - obj_id = self.obj_id ^ (1 << ((self.build_config.obj_id_len * 8) - 1)) - img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + - SpiffsPage._len_dict[self.build_config.obj_id_len] + - SpiffsPage._len_dict[self.build_config.span_ix_len] + - SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], - obj_id, - self.span_ix, - SPIFFS_PH_FLAG_USED_FINAL_INDEX) - - # Add padding before the object index page specific information - img += b"\xFF" * self.build_config.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED_PAD - - # If this is the first object index page for the object, add filname, type - # and size information - if self.span_ix == 0: - img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + - SpiffsPage._len_dict[SPIFFS_PH_IX_SIZE_LEN] + - SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], - self.size, - SPIFFS_TYPE_FILE) - - img += self.name.encode() + (b"\x00" * ((self.build_config.obj_name_len - len(self.name)) + self.build_config.meta_len)) - - # Finally, add the page index of daa pages - for page in self.pages: - page = page >> int(math.log(self.build_config.page_size, 2)) - img += struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + - SpiffsPage._len_dict[self.build_config.page_ix_len], page) - - assert(len(img) <= self.build_config.page_size) - - img += b"\xFF" * (self.build_config.page_size - len(img)) - - return img - - -class SpiffsObjDataPage(SpiffsPage): - def __init__(self, offset, obj_id, span_ix, contents, build_config): - SpiffsPage.__init__(self, 0, build_config) - self.obj_id = obj_id - self.span_ix = span_ix - self.contents = contents - self.offset = offset - - def to_binary(self): - img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] + - SpiffsPage._len_dict[self.build_config.obj_id_len] + - SpiffsPage._len_dict[self.build_config.span_ix_len] + - SpiffsPage._len_dict[SPIFFS_PH_FLAG_LEN], - self.obj_id, - self.span_ix, - SPIFFS_PH_FLAG_USED_FINAL) - - img += self.contents - - assert(len(img) <= self.build_config.page_size) - - img += b"\xFF" * (self.build_config.page_size - len(img)) - - return img - - -class SpiffsBlock(): - def _reset(self): - self.cur_obj_index_span_ix = 0 - self.cur_obj_data_span_ix = 0 - self.cur_obj_id = 0 - self.cur_obj_idx_page = None - - def __init__(self, bix, blocks_lim, build_config): - self.build_config = build_config - self.offset = bix * self.build_config.block_size - self.remaining_pages = self.build_config.OBJ_USABLE_PAGES_PER_BLOCK - self.pages = list() - self.bix = bix - - lu_pages = list() - for i in range(self.build_config.OBJ_LU_PAGES_PER_BLOCK): - page = SpiffsObjLuPage(self.bix, self.build_config) - lu_pages.append(page) - - self.pages.extend(lu_pages) - - self.lu_page_iter = iter(lu_pages) - self.lu_page = next(self.lu_page_iter) - - self._reset() - - def _register_page(self, page): - if isinstance(page, SpiffsObjDataPage): - self.cur_obj_idx_page.register_page(page) # can raise SpiffsFullError - - try: - self.lu_page.register_page(page) - except SpiffsFullError: - self.lu_page = next(self.lu_page_iter) - try: - self.lu_page.register_page(page) - except AttributeError: # no next lookup page - # Since the amount of lookup pages is pre-computed at every block instance, - # this should never occur - raise RuntimeError("invalid attempt to add page to a block when there is no more space in lookup") - - self.pages.append(page) - - def begin_obj(self, obj_id, size, name, obj_index_span_ix=0, obj_data_span_ix=0): - if not self.remaining_pages > 0: - raise SpiffsFullError() - self._reset() - - self.cur_obj_id = obj_id - self.cur_obj_index_span_ix = obj_index_span_ix - self.cur_obj_data_span_ix = obj_data_span_ix - - page = SpiffsObjIndexPage(obj_id, self.cur_obj_index_span_ix, size, name, self.build_config) - self._register_page(page) - - self.cur_obj_idx_page = page - - self.remaining_pages -= 1 - self.cur_obj_index_span_ix += 1 - - def update_obj(self, contents): - if not self.remaining_pages > 0: - raise SpiffsFullError() - page = SpiffsObjDataPage(self.offset + (len(self.pages) * self.build_config.page_size), - self.cur_obj_id, self.cur_obj_data_span_ix, contents, self.build_config) - - self._register_page(page) - - self.cur_obj_data_span_ix += 1 - self.remaining_pages -= 1 - - def end_obj(self): - self._reset() - - def is_full(self): - return self.remaining_pages <= 0 - - def to_binary(self, blocks_lim): - img = b"" - - if self.build_config.use_magic: - for (idx, page) in enumerate(self.pages): - if idx == self.build_config.OBJ_LU_PAGES_PER_BLOCK - 1: - page.magicfy(blocks_lim) - img += page.to_binary() - else: - for page in self.pages: - img += page.to_binary() - - assert(len(img) <= self.build_config.block_size) - - img += b"\xFF" * (self.build_config.block_size - len(img)) - return img - - -class SpiffsFS(): - def __init__(self, img_size, build_config): - if img_size % build_config.block_size != 0: - raise RuntimeError("image size should be a multiple of block size") - - self.img_size = img_size - self.build_config = build_config - - self.blocks = list() - self.blocks_lim = self.img_size // self.build_config.block_size - self.remaining_blocks = self.blocks_lim - self.cur_obj_id = 1 # starting object id - - def _create_block(self): - if self.is_full(): - raise SpiffsFullError("the image size has been exceeded") - - block = SpiffsBlock(len(self.blocks), self.blocks_lim, self.build_config) - self.blocks.append(block) - self.remaining_blocks -= 1 - return block - - def is_full(self): - return self.remaining_blocks <= 0 - - def create_file(self, img_path, file_path): - contents = None - - if len(img_path) > self.build_config.obj_name_len: - raise RuntimeError("object name '%s' too long" % img_path) - - name = img_path - - with open(file_path, "rb") as obj: - contents = obj.read() - - stream = io.BytesIO(contents) - - try: - block = self.blocks[-1] - block.begin_obj(self.cur_obj_id, len(contents), name) - except (IndexError, SpiffsFullError): - block = self._create_block() - block.begin_obj(self.cur_obj_id, len(contents), name) - - contents_chunk = stream.read(self.build_config.OBJ_DATA_PAGE_CONTENT_LEN) - - while contents_chunk: - try: - block = self.blocks[-1] - try: - # This can fail because either (1) all the pages in block have been - # used or (2) object index has been exhausted. - block.update_obj(contents_chunk) - except SpiffsFullError: - # If its (1), use the outer exception handler - if block.is_full(): - raise SpiffsFullError - # If its (2), write another object index page - block.begin_obj(self.cur_obj_id, len(contents), name, - obj_index_span_ix=block.cur_obj_index_span_ix, - obj_data_span_ix=block.cur_obj_data_span_ix) - continue - except (IndexError, SpiffsFullError): - # All pages in the block have been exhausted. Create a new block, copying - # the previous state of the block to a new one for the continuation of the - # current object - prev_block = block - block = self._create_block() - block.cur_obj_id = prev_block.cur_obj_id - block.cur_obj_idx_page = prev_block.cur_obj_idx_page - block.cur_obj_data_span_ix = prev_block.cur_obj_data_span_ix - block.cur_obj_index_span_ix = prev_block.cur_obj_index_span_ix - continue - - contents_chunk = stream.read(self.build_config.OBJ_DATA_PAGE_CONTENT_LEN) - - block.end_obj() - - self.cur_obj_id += 1 - - def to_binary(self): - img = b"" - for block in self.blocks: - img += block.to_binary(self.blocks_lim) - bix = len(self.blocks) - if self.build_config.use_magic: - # Create empty blocks with magic numbers - while self.remaining_blocks > 0: - block = SpiffsBlock(bix, self.blocks_lim, self.build_config) - img += block.to_binary(self.blocks_lim) - self.remaining_blocks -= 1 - bix += 1 - else: - # Just fill remaining spaces FF's - img += "\xFF" * (self.img_size - len(img)) - return img - - -def main(): - if sys.version_info[0] < 3: - print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.") - elif sys.version_info[0] == 3 and sys.version_info[1] < 6: - print("WARNING: Python 3 versions older than 3.6 are not supported.") - parser = argparse.ArgumentParser(description="SPIFFS Image Generator", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument("image_size", - help="Size of the created image") - - parser.add_argument("base_dir", - help="Path to directory from which the image will be created") - - parser.add_argument("output_file", - help="Created image output file path") - - parser.add_argument("--page-size", - help="Logical page size. Set to value same as CONFIG_SPIFFS_PAGE_SIZE.", - type=int, - default=256) - - parser.add_argument("--block-size", - help="Logical block size. Set to the same value as the flash chip's sector size (g_rom_flashchip.sector_size).", - type=int, - default=4096) - - parser.add_argument("--obj-name-len", - help="File full path maximum length. Set to value same as CONFIG_SPIFFS_OBJ_NAME_LEN.", - type=int, - default=32) - - parser.add_argument("--meta-len", - help="File metadata length. Set to value same as CONFIG_SPIFFS_META_LENGTH.", - type=int, - default=4) - - parser.add_argument("--use-magic", - help="Use magic number to create an identifiable SPIFFS image. Specify if CONFIG_SPIFFS_USE_MAGIC.", - action="store_true", - default=True) - - parser.add_argument("--follow-symlinks", - help="Take into account symbolic links during partition image creation.", - action="store_true", - default=False) - - parser.add_argument("--use-magic-len", - help="Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.", - action="store_true", - default=True) - - parser.add_argument("--big-endian", - help="Specify if the target architecture is big-endian. If not specified, little-endian is assumed.", - action="store_true", - default=False) - - args = parser.parse_args() - - if not os.path.exists(args.base_dir): - raise RuntimeError("given base directory %s does not exist" % args.base_dir) - - with open(args.output_file, "wb") as image_file: - image_size = int(args.image_size, 0) - spiffs_build_default = SpiffsBuildConfig(args.page_size, SPIFFS_PAGE_IX_LEN, - args.block_size, SPIFFS_BLOCK_IX_LEN, args.meta_len, - args.obj_name_len, SPIFFS_OBJ_ID_LEN, SPIFFS_SPAN_IX_LEN, - True, True, "big" if args.big_endian else "little", - args.use_magic, args.use_magic_len) - - spiffs = SpiffsFS(image_size, spiffs_build_default) - - for root, dirs, files in os.walk(args.base_dir, followlinks=args.follow_symlinks): - for f in files: - full_path = os.path.join(root, f) - spiffs.create_file("/" + os.path.relpath(full_path, args.base_dir).replace("\\", "/"), full_path) - - image = spiffs.to_binary() - - image_file.write(image) - - -if __name__ == "__main__": - main() diff --git a/components/xtensa/trax/traceparse.py b/components/xtensa/trax/traceparse.py deleted file mode 100644 index 119e4c9e88..0000000000 --- a/components/xtensa/trax/traceparse.py +++ /dev/null @@ -1,343 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# -# This script decodes Xtensa CPU trace dumps. It allows tracing the program -# execution at instruction level. -# -# Some trivia about the Xtensa CPU trace (TRAX): -# TRAX format mostly follows the IEEE-ISTO 5001-2003 (Nexus) standard. -# The following Nexus Program Trace messages are implemented by TRAX: -# - Indirect Branch Message -# - Syncronization Message -# - Indirect Branch with Synchronization Message -# - Correlation Message -# TRAX outputs compressed traces with 2 MSEO bits (LSB) and 6 MDO bits (MSB), -# packed into a byte. MSEO bits are used to split the stream into packets and messages, -# and MDO bits carry the actual data of the messages. Each message may contain multiple packets. -# -# This script can be used standalone, or loaded into GDB. -# When used standalone, it dumps the list of trace messages to stdout. -# When used from GDB, it also invokes GDB command to dump the list of assembly -# instructions corresponding to each of the messages. -# -# Standalone usage: -# traceparse.py -# -# Usage from GDB: -# xtensa-esp32-elf-gdb -n --batch program.elf -x gdbinit -# with the following gdbinit script: -# set pagination off -# set confirm off -# add-symbol-file rom.elf
-# source traceparse.py -# python parse_and_dump("/path/to/dump_file") -# -# Loading the ROM code is optional; if not loaded, disassembly for ROM sections of code -# will be missing. -# -### -# Copyright 2020 Espressif Systems (Shanghai) PTE LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import print_function -import sys - -# Check if loaded into GDB -try: - assert gdb.__name__ == "gdb" - WITH_GDB = True -except NameError: - WITH_GDB = False - -# MSEO bit masks: -MSEO_PKTEND = 1 << 0 # bit 0: indicates the last byte of a packet -MSEO_MSGEND = 1 << 1 # bit 1: indicates the last byte of the message - -# Message types. The type is stored in the first 6 MDO bits or the first packet. -TVAL_INDBR = 4 # Indirect branch -TVAL_INDBRSYNC = 12 # Indirect branch w/ synchronisation -TVAL_SYNC = 9 # Synchronisation msg -TVAL_CORR = 33 # Correlation message - - -class TraxPacket(object): - def __init__(self, data, truncated=False): - self.data = data - self.size_bytes = len(data) - self.truncated = truncated - - def get_bits(self, start, count=0): - """ - Extract data bits from the packet - :param start: offset, in bits, of the part to be extracted - :param count: number of bits to extract; if omitted or zero, - extracts until the end of the packet - :return: integer containing the extracted bits - """ - start_byte = start // 6 - if count <= 0: - # all remaining bits - count = len(self.data) * 6 - start - bits_remaining = count - result = 0 - shift = 0 - for i, b in enumerate(self.data[start_byte:]): - # which bit in the byte is the starting bit - if i == 0: - # at start_byte: take the offset into account - start_bit = 2 + (start % 6) - else: - # every other byte: start after MSEO bits - start_bit = 2 - # how many bits do we need to copy from this byte - cnt_bits = min(bits_remaining, 8 - start_bit) - mask = (2 ** cnt_bits) - 1 - # take this many bits after the start_bit - bits = (b >> start_bit) & mask - # add these bits to the result - result |= bits << shift - # update the remaining bit count - shift += cnt_bits - bits_remaining -= cnt_bits - if bits_remaining == 0: - break - return result - - def __str__(self): - return "%d byte packet%s" % (self.size_bytes, " (truncated)" if self.truncated else "") - - -class TraxMessage(object): - def __init__(self, packets, truncated=False): - """ - Create and parse a TRAX message from packets - :param packets: list of TraxPacket objects, must not be empty - :param truncated: whether the message was truncated in the stream - """ - assert len(packets) > 0 - self.packets = packets - self.truncated = truncated - if truncated: - self.msg_type = None - else: - self.msg_type = self._get_type() - - # Start and end of the instruction range corresponding to this message - self.pc_start = 0 # inclusive - self.pc_end = 0 # not inclusive - self.pc_target = 0 # PC of the next range - self.is_exception = False # whether the message indicates an exception - self.is_correlation = False # whether this is a correlation message - - # message-specific fields - self.icnt = 0 - self.uaddr = 0 - self.dcont = 0 - - # decode the fields - if not truncated: - self._decode() - - def _get_type(self): - """ - :return: Message type, one of TVAL_XXX values - """ - return self.packets[0].get_bits(0, 6) - - def _decode(self): - """ Parse the packets and fill in the message-specific fields """ - if self.msg_type == TVAL_INDBR: - self.icnt = self.packets[0].get_bits(7, -1) - self.btype = self.packets[0].get_bits(6, 1) - self.uaddr = self.packets[1].get_bits(0) - self.is_exception = self.btype > 0 - elif self.msg_type == TVAL_INDBRSYNC: - self.icnt = self.packets[0].get_bits(8, -1) - self.btype = self.packets[0].get_bits(7, 1) - self.pc_target = self.packets[1].get_bits(0) - self.dcont = self.packets[0].get_bits(6, 1) - self.is_exception = self.btype > 0 - elif self.msg_type == TVAL_SYNC: - self.icnt = self.packets[0].get_bits(7, -1) - self.dcont = self.packets[0].get_bits(6, 1) - self.pc_target = self.packets[1].get_bits(0) - elif self.msg_type == TVAL_CORR: - self.icnt = self.packets[0].get_bits(12, -1) - self.is_correlation = True - else: - raise NotImplementedError("Unknown message type (%d)" % self.msg_type) - - def process_forward(self, cur_pc): - """ - Given the target PC known from the previous message, determine - the PC range corresponding to the current message. - :param cur_pc: previous known PC - :return: target PC after the current message - """ - assert not self.truncated - - next_pc = cur_pc - if self.msg_type == TVAL_INDBR: - next_pc = cur_pc ^ self.uaddr - self.pc_target = next_pc - self.pc_start = cur_pc - self.pc_end = self.pc_start + self.icnt + 1 - if self.msg_type == TVAL_INDBRSYNC: - next_pc = self.pc_target - self.pc_start = cur_pc - self.pc_end = cur_pc + self.icnt + 1 - if self.msg_type == TVAL_SYNC: - next_pc = self.pc_target - self.pc_start = next_pc - self.icnt - self.pc_end = next_pc + 1 - if self.msg_type == TVAL_CORR: - pass - return next_pc - - def process_backward(self, cur_pc): - """ - Given the address of the PC known from the _next_ message, determine - the PC range corresponding to the current message. - :param cur_pc: next known PC - :return: target PC of the _previous_ message - """ - assert not self.truncated - # Backward pass is only used to resolve addresses of messages - # up to the first SYNC/INDBRSYNC message. - # SYNC/INDBRSYNC messages are only handled in the forward pass. - assert self.msg_type != TVAL_INDBRSYNC - assert self.msg_type != TVAL_SYNC - - prev_pc = cur_pc - self.pc_target = cur_pc - if self.msg_type == TVAL_INDBR: - prev_pc ^= self.uaddr - self.pc_start = prev_pc - self.pc_end = prev_pc + self.icnt + 1 - if self.msg_type == TVAL_CORR: - pass - return prev_pc - - def __str__(self): - desc = "Unknown (%d)" % self.msg_type - extra = "" - if self.truncated: - desc = "Truncated" - if self.msg_type == TVAL_INDBR: - desc = "Indirect branch" - extra = ", icnt=%d, uaddr=0x%x, exc=%d" % (self.icnt, self.uaddr, self.is_exception) - if self.msg_type == TVAL_INDBRSYNC: - desc = "Indirect branch w/sync" - extra = ", icnt=%d, dcont=%d, exc=%d" % (self.icnt, self.dcont, self.is_exception) - if self.msg_type == TVAL_SYNC: - desc = "Synchronization" - extra = ", icnt=%d, dcont=%d" % (self.icnt, self.dcont) - if self.msg_type == TVAL_CORR: - desc = "Correlation" - extra = ", icnt=%d" % self.icnt - return "%s message, %d packets, PC range 0x%08x - 0x%08x, target PC 0x%08x" % ( - desc, len(self.packets), self.pc_start, self.pc_end, self.pc_target) + extra - - -def load_messages(data): - """ - Decodes TRAX data and resolves PC ranges. - :param data: input data, bytes - :return: list of TraxMessage objects - """ - messages = [] - packets = [] - packet_start = 0 - msg_cnt = 0 - pkt_cnt = 0 - - # Iterate over the input data, splitting bytes into packets and messages - for i, b in enumerate(data): - if (b & MSEO_MSGEND) and not (b & MSEO_PKTEND): - raise AssertionError("Invalid MSEO bits in b=0x%x. Not a TRAX dump?" % b) - - if b & MSEO_PKTEND: - pkt_cnt += 1 - packets.append(TraxPacket(data[packet_start:i + 1], packet_start == 0)) - packet_start = i + 1 - - if b & MSEO_MSGEND: - msg_cnt += 1 - try: - messages.append(TraxMessage(packets, len(messages) == 0)) - except NotImplementedError as e: - sys.stderr.write("Failed to parse message #%03d (at %d bytes): %s\n" % (msg_cnt, i, str(e))) - packets = [] - - # Resolve PC ranges of messages. - # Forward pass: skip messages until a message with known PC, - # i.e. a SYNC/INDBRSYNC message. Process all messages following it. - pc = 0 - first_sync_index = -1 - for i, m in enumerate(messages): - if pc == 0 and m.pc_target == 0: - continue - if first_sync_index < 0: - first_sync_index = i - pc = m.process_forward(pc) - - # Now process the skipped messages in the reverse direction, - # starting from the first message with known PC. - pc = messages[first_sync_index].pc_start - for m in reversed(messages[0:first_sync_index]): - if m.truncated: - break - pc = m.process_backward(pc) - - return messages - - -def parse_and_dump(filename, disassemble=WITH_GDB): - """ - Decode TRAX data from a file, print out the messages. - :param filename: file to load the dump from - :param disassemble: if True, print disassembly of PC ranges - """ - with open(filename, 'rb') as f: - data = f.read() - - messages = load_messages(data) - sys.stderr.write("Loaded %d messages in %d bytes\n" % (len(messages), len(data))) - - for i, m in enumerate(messages): - if m.truncated: - continue - print("%04d: %s" % (i, str(m))) - if m.is_exception: - print("*** Exception occurred ***") - if disassemble and WITH_GDB: - try: - gdb.execute("disassemble 0x%08x, 0x%08x" % (m.pc_start, m.pc_end)) # noqa: F821 - except gdb.MemoryError: # noqa: F821 - print("Failed to disassemble from 0x%08x to 0x%08x" % (m.pc_start, m.pc_end)) - - -def main(): - if sys.version_info[0] < 3: - print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.", file=sys.stderr) - elif sys.version_info[0] == 3 and sys.version_info[1] < 6: - print("WARNING: Python 3 versions older than 3.6 are not supported.", file=sys.stderr) - if len(sys.argv) < 2: - sys.stderr.write("Usage: %s \n") - raise SystemExit(1) - - parse_and_dump(sys.argv[1]) - - -if __name__ == "__main__" and not WITH_GDB: - main() diff --git a/docs/en/get-started/linux-setup-scratch.rst b/docs/en/get-started/linux-setup-scratch.rst index 8700501594..accc830d48 100644 --- a/docs/en/get-started/linux-setup-scratch.rst +++ b/docs/en/get-started/linux-setup-scratch.rst @@ -11,10 +11,6 @@ Install Prerequisites ===================== To compile with ESP-IDF you need to get the following packages: - -- CentOS 7:: - - sudo yum -y update && sudo yum install git wget flex bison gperf python3 - Ubuntu and Debian:: diff --git a/docs/zh_CN/get-started/linux-setup-scratch.rst b/docs/zh_CN/get-started/linux-setup-scratch.rst index 8c7dcdadb6..5d4169ba23 100644 --- a/docs/zh_CN/get-started/linux-setup-scratch.rst +++ b/docs/zh_CN/get-started/linux-setup-scratch.rst @@ -15,13 +15,13 @@ - Ubuntu 和 Debian:: - sudo apt-get install gcc git wget make libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-cryptography python-future python-pyparsing libffi-dev libssl-dev + sudo apt-get install git wget libncurses-dev flex bison gperf python3 python3-pip python3-setuptools python3-serial python3-cryptography python3-future python3-pyparsing python3-pyelftools cmake ninja-build ccache libffi-dev libssl-dev - Arch:: - sudo pacman -S --needed gcc git make ncurses flex bison gperf python2-pyserial python2-cryptography python2-future python2-pyparsing + sudo pacman -Sy --needed gcc git make ncurses flex bison gperf python-pyserial python-cryptography python-future python-pyparsing python-pyelftools cmake ninja ccache -.. note:: +.. 注解:: 一些旧的（2014年之前）Linux 发行版中使用的 ``pyserial`` 版本可能是 2.x ， ESP-IDF并不支持。 在这种情况下，请参考 :ref:`安装依赖的 Python 软件包 ` 章节，通过 ``pip`` 工具来安装支持的版本。 @@ -33,7 +33,7 @@ - CentOS 7:: - sudo yum install gawk gperf grep gettext ncurses-devel python python-devel automake bison flex texinfo help2man libtool + sudo yum install gawk gperf grep gettext ncurses-devel python3 python3-devel automake bison flex texinfo help2man libtool - Ubuntu pre-16.04:: @@ -49,7 +49,7 @@ - Arch:: - TODO + sudo pacman -Sy --needed python-pip 新建工作目录，然后进入:: @@ -69,6 +69,10 @@ 编译得到的工具链会被保存到 ``~/esp/crosstool-NG/builds/xtensa-esp32-elf``。根据 :ref:`Linux 下设置环境变量的标准方法 ` 中的介绍，将工具链添加到 ``PATH`` 中。 +停用 Python 2 +==================== + +Python 2 已经 `结束生命周期 `_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的目前主流 Linux 发行版的安装说明。 下一步 ====== diff --git a/docs/zh_CN/get-started/linux-setup.rst b/docs/zh_CN/get-started/linux-setup.rst index 8cd23e928e..c327545763 100644 --- a/docs/zh_CN/get-started/linux-setup.rst +++ b/docs/zh_CN/get-started/linux-setup.rst @@ -12,15 +12,17 @@ Linux 平台工具链的标准设置 - CentOS 7:: - sudo yum install gcc git wget make ncurses-devel flex bison gperf python pyserial + sudo yum -y update && sudo yum install git wget flex bison gperf python3 + +目前仍然支持 CentOS 7，但为了更好的用户体验，建议使用 CentOS 8。 - Ubuntu and Debian:: - sudo apt-get install gcc git wget make libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-cryptography python-future python-pyparsing libffi-dev libssl-dev + sudo apt-get install git wget flex bison gperf python3 python3-pip python3-setuptools cmake ninja-build ccache libffi-dev libssl-dev - Arch:: - sudo pacman -S --needed gcc git make ncurses flex bison gperf python2-pyserial python2-cryptography python2-future python2-pyparsing + sudo pacman -S --needed gcc git make ncurses flex bison gperf .. note:: @@ -98,6 +100,47 @@ Arch Linux 用户 或者，你也可以使用 crosstool-NG 编译一个链接 ncurses 6 的 gdb。 +设置 Python 3 为 CentOS 默认 Python 版本 +---------------------------------------------------- + +CentOS 7 及更早的版本提供 Python 2.7 作为默认解释器。但这里推荐使用 Python 3，您可以运行下方命令安装 Python 3。或者查看当前所用系统的相关文档，按照文档推荐的其它方法安装 Python 3:: + + sudo yum -y update && sudo yum install python3 python3-pip python3-setuptools + +设置 Python 3 为默认 Python 版本:: + + sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3 + + +设置 Python 3 为 Ubuntu 和 Debian 默认 Python 版本 +---------------------------------------------------- + +Ubuntu（v18.04 及之前的版本）和 Debian（v9 及之前的版本）的默认解释器为 Python 2.7，但这里推荐使用 Python 3，您可以运行下方命令安装 Python 3。或者查看当前所用系统的相关文档，按照文档推荐的其它方法安装 Python 3:: + + sudo apt-get install python3 python3-pip python3-setuptools + +设置 Python 3 为默认 Python 版本:: + + sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && alias pip=pip3 + + +.. 注解:: + 上述设置为全局设置，同时会影响到其它应用。 + +修复 Ubuntu 16.04 损坏的 pip +================================= + +``python3-pip`` 包可能已损坏无法升级。需使用脚本 `get-pip.py `_ 手动删除并安装该包:: + + apt remove python3-pip python3-virtualenv; rm -r ~/.local + rm -r ~/.espressif/python_env && python get-pip.py + +停用 Python 2 +==================== + +Python 2 已经 `结束生命周期 `_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的目前主流 Linux 发行版的安装说明。 + + 后续步骤 ========== diff --git a/docs/zh_CN/get-started/macos-setup-scratch.rst b/docs/zh_CN/get-started/macos-setup-scratch.rst index 1588c80c86..ed06aee8cb 100644 --- a/docs/zh_CN/get-started/macos-setup-scratch.rst +++ b/docs/zh_CN/get-started/macos-setup-scratch.rst @@ -1,5 +1,5 @@ ********************************** -从零开始设置 Mac OS 环境下的工具链 +从零开始设置 macOS 环境下的工具链 ********************************** :link_to_translation:`en:[English]` @@ -65,6 +65,11 @@ 编译得到的工具链会被保存到 ``~/esp/ctng-volume/crosstool-NG/builds/xtensa-esp32-elf``。根据 :ref:`Mac OS 下设置环境变量的标准方法 ` 中的介绍，将工具链添加到 ``PATH`` 中。 +停用 Python 2 +==================== + +Python 2 已经 `结束生命周期 `_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的 macOS 安装说明。 + 下一步 ====== diff --git a/docs/zh_CN/get-started/macos-setup.rst b/docs/zh_CN/get-started/macos-setup.rst index 502b37cf92..4c4c6749cd 100644 --- a/docs/zh_CN/get-started/macos-setup.rst +++ b/docs/zh_CN/get-started/macos-setup.rst @@ -44,6 +44,12 @@ Mac OS 版本的 ESP32 工具链可以从以下地址下载： 当需要使用工具链时，在命令行里输入 ``get_esp32``，就可以将工具链添加到 ``PATH`` 中。 +停用 Python 2 +==================== + +Python 2 已经 `结束生命周期 `_，ESP-IDF 很快将不再支持 Python 2。请安装 Python 3.6 或以上版本。可参考上面列出的 macOS 安装说明。 + + 下一步 ========== @@ -57,4 +63,3 @@ Mac OS 版本的 ESP32 工具链可以从以下地址下载： :maxdepth: 1 macos-setup-scratch - diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index a62675cbf1..d96dec3393 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -96,7 +96,7 @@ build_example () { local EXAMPLE_DIR=$(dirname "${MAKE_FILE}") local EXAMPLE_NAME=$(basename "${EXAMPLE_DIR}") - + # Check if the example needs a different base directory. # Path of the Makefile relative to $IDF_PATH local MAKE_FILE_REL=${MAKE_FILE#"${IDF_PATH}/"} @@ -187,6 +187,8 @@ library/error\.o\ \|changes choice state\ \|Compiler version is not supported\ \|Toolchain version is not supported\ +\|Python 3 versions older than 3.6 are not supported\ +\|Python 2 is deprecated and will be removed in future versions\ " sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \ diff --git a/tools/ci/build_examples_cmake.sh b/tools/ci/build_examples_cmake.sh index eabc9c76c1..a1c525c2b3 100755 --- a/tools/ci/build_examples_cmake.sh +++ b/tools/ci/build_examples_cmake.sh @@ -174,6 +174,8 @@ library/error\.o\ \|reassigning to symbol\ \|changes choice state\ \|crosstool_version_check\.cmake\ +\|Python 3 versions older than 3.6 are not supported\ +\|Python 2 is deprecated and will be removed in future versions\ " sort -u "${LOG_SUSPECTED}" | grep -v "${IGNORE_WARNS}" \ diff --git a/tools/ci/check_build_warnings.py b/tools/ci/check_build_warnings.py deleted file mode 100755 index 2e012fa9b4..0000000000 --- a/tools/ci/check_build_warnings.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# -# CI script to check build logs for warnings. -# Reads the list of builds, in the format produced by find_apps.py or build_apps.py, and finds warnings in the -# log files for every build. -# Exits with a non-zero exit code if any warning is found. - -import argparse -import logging -import os -import re -import sys - -try: - from find_build_apps import BuildItem, setup_logging -except ImportError: - sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - from find_build_apps import BuildItem, setup_logging - -WARNING_REGEX = re.compile(r"(?:error|warning)[^\w]", re.MULTILINE | re.IGNORECASE) - -IGNORE_WARNS = [ - re.compile(r_str) for r_str in [ - r"library/error\.o", - r".*error.*\.c\.obj", - r"-Werror", - r"error\.d", - r"reassigning to symbol", - r"changes choice state", - r"crosstool_version_check\.cmake", - r"CryptographyDeprecationWarning", - r"Python 3 versions older than 3.6 are not supported." - ] -] - - -def line_has_warnings(line): # type: (str) -> bool - if not WARNING_REGEX.search(line): - return False - - has_warnings = True - for ignored in IGNORE_WARNS: - if re.search(ignored, line): - has_warnings = False - break - - return has_warnings - - -def main(): - parser = argparse.ArgumentParser(description="ESP-IDF app builder") - parser.add_argument( - "-v", - "--verbose", - action="count", - help="Increase the logging level of the script. Can be specified multiple times.", - ) - parser.add_argument( - "--log-file", - type=argparse.FileType("w"), - help="Write the script log to the specified file, instead of stderr", - ) - parser.add_argument( - "build_list", - type=argparse.FileType("r"), - nargs="?", - default=sys.stdin, - help="Name of the file to read the list of builds from. If not specified, read from stdin.", - ) - args = parser.parse_args() - setup_logging(args) - - build_items = [BuildItem.from_json(line) for line in args.build_list] - if not build_items: - logging.warning("Empty build list") - SystemExit(0) - - found_warnings = 0 - for build_item in build_items: - if not build_item.build_log_path: - logging.debug("No log file for {}".format(build_item.work_dir)) - continue - with open(build_item.build_log_path, "r") as log_file: - for line_no, line in enumerate(log_file): - if line_has_warnings(line): - logging.error("Issue in app {}, config {}:".format(build_item.app_dir, build_item.config_name)) - logging.error(line.rstrip("\n")) - logging.error("See {}:{} for details".format(os.path.basename(build_item.build_log_path), - line_no + 1)) - found_warnings += 1 - break - - if found_warnings: - logging.error("Checked {} builds, found {} warnings".format(len(build_items), found_warnings)) - raise SystemExit(1) - - logging.info("No warnings found") - - -if __name__ == "__main__": - main() diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml deleted file mode 100644 index 21d35a1469..0000000000 --- a/tools/ci/config/build.yml +++ /dev/null @@ -1,502 +0,0 @@ -.build_template: - stage: build - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG - tags: - - build - variables: - SIZE_INFO_LOCATION: "$CI_PROJECT_DIR/size_info.txt" - dependencies: [] - -.build_template_app_template: - extends: - - .build_template - - .rules:labels:build - variables: - LOG_PATH: "${CI_PROJECT_DIR}/log_template_app" - BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app" - BUILD_DIR: "@t/@w" - BUILD_LOG_MAKE: "${LOG_PATH}/make_@t_@w.txt" - BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt" - BUILD_COMMAND_ARGS: "" - artifacts: - when: always - paths: - - log_template_app/* - - size_info.txt - - build_template_app/**/size.json - script: - # Set the variable for 'esp-idf-template' testing - - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} - - retry_failed git clone ${ESP_IDF_TEMPLATE_GIT} - # Try to use the same branch name for esp-idf-template that we're - # using on esp-idf. If it doesn't exist then just stick to the default branch - - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template - - export PATH="$IDF_PATH/tools:$PATH" - - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} - - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} - # Only do the default cmake build for each target, remaining part are done in the build_template_app job - - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS} - # Check if there are any stray printf/ets_printf references in WiFi libs - - cd components/esp_wifi/lib - - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done; - - for dir in esp32 esp32s2; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done; - -# build-related-pre-check-jobs ------------------------------------------------ -# Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits -fast_template_app: - extends: - - .build_template_app_template - - .rules:build_tests:target_test-weekend_test - stage: pre_check - variables: - BUILD_COMMAND_ARGS: "-p" - -check_docs_gh_links: - extends: .build_docs_template - stage: pre_check - variables: - SUBMODULES_TO_FETCH: "none" - script: - - cd docs - - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt - - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck -#------------------------------------------------------------------------------ - -.build_ssc_template: - extends: - - .build_template - - .rules:build_tests:integration_test - artifacts: - paths: - - SSC/ssc_bin - expire_in: 1 week - script: - - retry_failed git clone $SSC_REPOSITORY - - python $CHECKOUT_REF_SCRIPT SSC SSC - - cd SSC - - MAKEFLAGS= ./ci_build_ssc.sh $TARGET_NAME - -build_ssc_esp32: - extends: .build_ssc_template - parallel: 3 - variables: - TARGET_NAME: "ESP32" - -build_ssc_esp32s2: - extends: .build_ssc_template - parallel: 2 - variables: - TARGET_NAME: "ESP32S2" - -.build_esp_idf_tests_cmake: - extends: - - .build_template - - .rules:build_tests:unit_test - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - job: fast_template_app - artifacts: false - - scan_tests - artifacts: - paths: - - tools/unit-test-app/output/${IDF_TARGET} - - tools/unit-test-app/builds/*.json - - tools/unit-test-app/builds/${IDF_TARGET}/*/size.json - - components/idf_test/unit_test/*.yml - - $LOG_PATH - - $SIZE_INFO_LOCATION - when: always - expire_in: 4 days - variables: - LOG_PATH: "$CI_PROJECT_DIR/log_ut_cmake" - BUILD_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/builds - OUTPUT_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/output - BUILD_SYSTEM: "cmake" - TEST_TYPE: "unit_test" - PYTHON_VER: 3 - script: - # RISC-V toolchain is optional but ULP may need it, so install: - - $IDF_PATH/tools/idf_tools.py install riscv-none-embed-gcc - - . $IDF_PATH/export.sh - - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh - - cd $CI_PROJECT_DIR/tools/unit-test-app - - python tools/UnitTestParser.py ${BUILD_PATH} - -build_esp_idf_tests_cmake_esp32: - extends: .build_esp_idf_tests_cmake - variables: - IDF_TARGET: esp32 - -build_esp_idf_tests_cmake_esp32s2: - extends: .build_esp_idf_tests_cmake - variables: - IDF_TARGET: esp32s2 - -build_esp_idf_tests_cmake_esp32s3: - extends: .build_esp_idf_tests_cmake - variables: - IDF_TARGET: esp32s3 - -.build_examples_template: - extends: - - .build_template - - .rules:build_tests:example_test-weekend_test - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - job: fast_template_app - artifacts: false - - scan_tests - variables: - TEST_PREFIX: examples - TEST_RELATIVE_DIR: examples - SCAN_TEST_JSON: ${CI_PROJECT_DIR}/${TEST_RELATIVE_DIR}/test_configs/scan_${IDF_TARGET}_${BUILD_SYSTEM}.json - TEST_TYPE: example_test - LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX} - BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX} - PYTHON_VER: 3 - script: - # RISC-V toolchain is optional but ULP may need it, so install: - - $IDF_PATH/tools/idf_tools.py install riscv-none-embed-gcc - - . $IDF_PATH/export.sh - # it's not possible to build 100% out-of-tree and have the "artifacts" - # mechanism work, but this is the next best thing - - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh - -build_examples_make: - extends: - - .build_examples_template - # This is a workaround for a rarely encountered issue with building examples in CI. - # Probably related to building of Kconfig in 'make clean' stage - retry: 1 - parallel: 8 - artifacts: - paths: - - $LOG_PATH - - build_${TEST_PREFIX}/*/*/*/build/size.json - - $SIZE_INFO_LOCATION - when: always - expire_in: 4 days - variables: - BUILD_SYSTEM: make - IDF_TARGET: esp32 # currently we only support esp32 - -# same as above, but for CMake -.build_examples_cmake: - extends: .build_examples_template - artifacts: - paths: - - build_${TEST_PREFIX}/list.json - - build_${TEST_PREFIX}/list_job_*.json - - build_${TEST_PREFIX}/*/*/*/sdkconfig - - build_${TEST_PREFIX}/*/*/*/build/size.json - - build_${TEST_PREFIX}/*/*/*/build/*.bin - - build_${TEST_PREFIX}/*/*/*/build/*.elf - - build_${TEST_PREFIX}/*/*/*/build/*.map - - build_${TEST_PREFIX}/*/*/*/build/flasher_args.json - - build_${TEST_PREFIX}/*/*/*/build/bootloader/*.bin - - build_${TEST_PREFIX}/*/*/*/build/partition_table/*.bin - - $LOG_PATH - - $SIZE_INFO_LOCATION - when: always - expire_in: 4 days - variables: - BUILD_SYSTEM: cmake - -build_examples_cmake_esp32: - extends: .build_examples_cmake - parallel: 10 - variables: - IDF_TARGET: esp32 - -build_examples_cmake_esp32s2: - extends: .build_examples_cmake - parallel: 8 - variables: - IDF_TARGET: esp32s2 - -.build_test_apps: - extends: - - .build_examples_cmake - - .rules:build_tests:custom_test-weekend_test - variables: - TEST_PREFIX: test_apps - TEST_RELATIVE_DIR: tools/test_apps - TEST_TYPE: custom_test - script: - - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh - -build_test_apps_esp32: - extends: .build_test_apps - parallel: 8 - variables: - IDF_TARGET: esp32 - -build_test_apps_esp32s2: - extends: .build_test_apps - parallel: 8 - variables: - IDF_TARGET: esp32s2 - -build_test_apps_esp32s3: - extends: .build_test_apps - parallel: 8 - variables: - IDF_TARGET: esp32s3 - -.build_component_ut: - extends: - - .build_test_apps - - .rules:build_tests:unit_test - variables: - TEST_PREFIX: component_ut - TEST_RELATIVE_DIR: component_ut - -build_component_ut_esp32: - extends: .build_component_ut - variables: - IDF_TARGET: esp32 - -build_component_ut_esp32s2: - extends: .build_component_ut - variables: - IDF_TARGET: esp32s2 - -.build_docs_template: - stage: build - image: $ESP_IDF_DOC_ENV_IMAGE - tags: - - build_docs - script: - - cd docs - - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt - - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs $DOC_BUILDERS -l $DOCLANG -t $DOCTGT build - parallel: - matrix: - - DOCLANG: [ "en", "zh_CN" ] - DOCTGT: [ "esp32", "esp32s2" ] - -build_docs_html: - extends: - - .build_docs_template - - .rules:labels:build_docs - artifacts: - when: always - paths: - - docs/_build/*/*/*.txt - - docs/_build/*/*/html/* - expire_in: 4 days - variables: - DOC_BUILDERS: "html" - -build_docs_pdf: - extends: - - .build_docs_template - - .rules:labels:build_docs-slim - artifacts: - when: always - paths: - - docs/_build/*/*/latex/* - expire_in: 4 days - variables: - DOC_BUILDERS: "latex" - -.test_build_system_template: - extends: - - .build_template - - .rules:build_tests:weekend_test - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - job: fast_template_app - artifacts: false - script: - - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh - - rm -rf test_build_system - - mkdir test_build_system - - cd test_build_system - - ${IDF_PATH}/tools/ci/${SHELL_TEST_SCRIPT} - -test_build_system: - extends: .test_build_system_template - variables: - SHELL_TEST_SCRIPT: test_build_system.sh - -test_build_system_cmake: - extends: .test_build_system_template - variables: - SHELL_TEST_SCRIPT: test_build_system_cmake.sh - -test_build_system_cmake_macos: - extends: - - .test_build_system_template - - .before_script_macos - - .rules:os:mac_os - tags: - - macos_shell - variables: - SHELL_TEST_SCRIPT: test_build_system_cmake.sh - -build_docker: - extends: - - .before_script_slim - - .rules:protected-schedule - stage: build - image: espressif/docker-builder:1 - tags: - - build_docker_amd64_brno - variables: - DOCKER_TMP_IMAGE_NAME: "idf_tmp_image" - script: - - export LOCAL_CI_REPOSITORY_URL=$CI_REPOSITORY_URL - - if [ -n "$LOCAL_GITLAB_HTTPS_HOST" ]; then export LOCAL_CI_REPOSITORY_URL="https://gitlab-ci-token:${CI_JOB_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"; fi - - echo "Using repository at $LOCAL_CI_REPOSITORY_URL" - - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${LOCAL_CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}" - # Build - - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/ - # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227. - # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually. - # Therefore, build a copy of the example located inside the container. - - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build - -.test-on-windows: - extends: - - .before_script_slim - - .rules:protected-schedule - stage: build - image: $CI_DOCKER_REGISTRY/esp32-toolchain-win-cross - tags: - - build - script: - - cd $TEST_DIR - - mkdir build - - cd build - - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchain-i686-w64-mingw32.cmake -DCMAKE_BUILD_TYPE=Release .. - - cmake --build . - -build_idf_exe: - extends: .test-on-windows - artifacts: - paths: - - tools/windows/idf_exe/build/idf-exe-v*.zip - expire_in: 4 days - variables: - TEST_DIR: tools/windows/idf_exe - -build_cmdlinerunner: - extends: .test-on-windows - artifacts: - paths: - - tools/windows/tool_setup/cmdlinerunner/build/cmdlinerunner.dll - expire_in: 4 days - variables: - TEST_DIR: tools/windows/tool_setup/cmdlinerunner - -build_installer: - extends: - - .before_script_slim - - .rules:protected-schedule - # using a different stage here to be able to use artifacts from build_cmdlinerunner job - stage: host_test - image: $CI_DOCKER_REGISTRY/wine-innosetup:1 - tags: - - build - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - build_cmdlinerunner - script: - - cd tools/windows/tool_setup/ - - ./build_installer.sh - -# This job builds template app with permutations of targets and optimization levels -build_template_app: - needs: - - job: fast_template_app - artifacts: false - extends: .build_template_app_template - -# Sonarqube related jobs put here for this reason: -# Here we have two jobs. code_quality_check and code_quality_report. -# -# code_quality_check will analyze the code changes between your MR and -# code repo stored in sonarqube server. The analysis result is only shown in -# the comments under this MR and won't be transferred to the server. -# -# code_quality_report will analyze and transfer both of the newly added code -# and the analysis result to the server. -# -# Put in the front to ensure that the newly merged code can be stored in -# sonarqube server ASAP, in order to avoid reporting unrelated code issues -.sonar_scan_template: - stage: build - image: - name: $CI_DOCKER_REGISTRY/sonarqube-scanner:2 - before_script: - - source tools/ci/utils.sh - - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" - - fetch_submodules - # Exclude the submodules, all paths ends with /** - - export SUBMODULES=$(get_all_submodules) - # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all to - - export CUSTOM_EXCLUDES=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g') - # Exclude the report dir - - export EXCLUSIONS="$SUBMODULES,$REPORT_DIR/**,docs/_static/**,**/*.png,**/*.jpg" - - python $NORMALIZE_CLANGTIDY_PY $CI_PROJECT_DIR/$REPORT_DIR/warnings.txt $CI_PROJECT_DIR/$REPORT_DIR/clang_tidy_report.txt $CI_PROJECT_DIR - variables: - GIT_DEPTH: 0 - NORMALIZE_CLANGTIDY_PY: $CI_PROJECT_DIR/tools/ci/normalize_clangtidy_path.py - REPORT_DIR: examples/get-started/hello_world/tidybuild/report - tags: - - host_test - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - clang_tidy_check_regular - -code_quality_check: - extends: - - .sonar_scan_template - - .rules:trigger - allow_failure: true - script: - - export CI_MR_IID=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py id ${CI_COMMIT_BRANCH}) - - export CI_MR_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits ${CI_COMMIT_BRANCH} | tr '\n' ',') - # test if this branch have merge request, if not, exit 0 - - test -n "$CI_MR_IID" || exit 0 - - test -n "$CI_MR_COMMITS" || exit 0 - - sonar-scanner - -Dsonar.analysis.mode=preview - -Dsonar.host.url=$SONAR_HOST_URL - -Dsonar.login=$SONAR_LOGIN - -Dsonar.sources=$CI_PROJECT_DIR - -Dsonar.sourceEncoding=UTF-8 - -Dsonar.projectKey=esp-idf - -Dsonar.projectBaseDir=$CI_PROJECT_DIR - -Dsonar.exclusions=$EXCLUSIONS - -Dsonar.gitlab.project_id=$CI_PROJECT_ID - -Dsonar.gitlab.commit_sha=$CI_MR_COMMITS - -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME - -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt - -Dsonar.cxx.includeDirectories=components,/usr/include - -Dsonar.python.pylint_config=.pylintrc - -Dsonar.gitlab.ci_merge_request_iid=$CI_MR_IID - -Dsonar.gitlab.merge_request_discussion=true - -Dsonar.branch.name=$CI_COMMIT_REF_NAME - -code_quality_report: - extends: - - .sonar_scan_template - - .rules:protected-schedule - script: - - sonar-scanner - -Dsonar.host.url=$SONAR_HOST_URL - -Dsonar.login=$SONAR_LOGIN - -Dsonar.sources=$CI_PROJECT_DIR - -Dsonar.sourceEncoding=UTF-8 - -Dsonar.projectKey=esp-idf - -Dsonar.projectBaseDir=$CI_PROJECT_DIR - -Dsonar.exclusions=$EXCLUSIONS - -Dsonar.gitlab.project_id=$CI_PROJECT_ID - -Dsonar.gitlab.commit_sha=$CI_COMMIT_SHA - -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME - -Dsonar.cxx.clangtidy.reportPath=$REPORT_DIR/clang_tidy_report.txt - -Dsonar.cxx.includeDirectories=components,/usr/include - -Dsonar.python.pylint_config=.pylintrc - -Dsonar.branch.name=$CI_COMMIT_REF_NAME diff --git a/tools/idf.py b/tools/idf.py index 03447942f9..5787b47827 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -110,9 +110,9 @@ def check_environment(): # check Python version if sys.version_info[0] < 3: - print_warning("WARNING: Support for Python 2 is deprecated and will be removed in future versions.") + print("WARNING: Support for Python 2 is deprecated and will be removed in future versions.") elif sys.version_info[0] == 3 and sys.version_info[1] < 6: - print_warning("WARNING: Python 3 versions older than 3.6 are not supported.") + print("WARNING: Python 3 versions older than 3.6 are not supported.") # check Python dependencies print("Checking Python dependencies...")