Merge branch 'feature/spiffsgen_py_esp8266_compat' into 'master'

spiffsgen.py: esp8266 compatibility options, performance improvement, type annotations

Closes IDFGH-4925

See merge request espressif/esp-idf!12839

.gitlab/ci/host-test.yml

@@ -110,6 +110,8 @@ test_spiffs_on_host:
   script:
     - cd components/spiffs/test_spiffs_host/
     - make test
+    - cd ../test_spiffsgen
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_spiffsgen.py
 
 test_multi_heap_on_host:
   extends: .host_test_template

components/spiffs/spiffsgen.py

@@ -19,13 +19,21 @@
 from __future__ import division, print_function
 
 import argparse
-import ctypes
 import io
 import math
 import os
 import struct
 import sys
 
+try:
+    import typing
+
+    TSP = typing.TypeVar('TSP', bound='SpiffsObjPageWithIdx')
+    ObjIdsItem = typing.Tuple[int, typing.Type[TSP]]
+except ImportError:
+    pass
+
+
 SPIFFS_PH_FLAG_USED_FINAL_INDEX = 0xF8
 SPIFFS_PH_FLAG_USED_FINAL = 0xFC
 
@@ -41,10 +49,23 @@ SPIFFS_PAGE_IX_LEN = 2  # spiffs_page_ix
 SPIFFS_BLOCK_IX_LEN = 2  # spiffs_block_ix
 
 
-class SpiffsBuildConfig():
+class SpiffsBuildConfig(object):
-    def __init__(self, page_size, page_ix_len, block_size,
+    def __init__(self,
-                 block_ix_len, meta_len, obj_name_len, obj_id_len,
+                 page_size,  # type: int
-                 span_ix_len, packed, aligned, endianness, use_magic, use_magic_len):
+                 page_ix_len,  # type: int
+                 block_size,  # type: int
+                 block_ix_len,  # type: int
+                 meta_len,  # type: int
+                 obj_name_len,  # type: int
+                 obj_id_len,  # type: int
+                 span_ix_len,  # type: int
+                 packed,  # type: bool
+                 aligned,  # type: bool
+                 endianness,  # type: str
+                 use_magic,  # type: bool
+                 use_magic_len,  # type: bool
+                 aligned_obj_ix_tables  # type: bool
+                 ):
         if block_size % page_size != 0:
             raise RuntimeError('block size should be a multiple of page size')
 
@@ -61,6 +82,7 @@ class SpiffsBuildConfig():
         self.endianness = endianness
         self.use_magic = use_magic
         self.use_magic_len = use_magic_len
+        self.aligned_obj_ix_tables = aligned_obj_ix_tables
 
         self.PAGES_PER_BLOCK = self.block_size // self.page_size
         self.OBJ_LU_PAGES_PER_BLOCK = int(math.ceil(self.block_size / self.page_size * self.obj_id_len / self.page_size))
@@ -78,16 +100,22 @@ class SpiffsBuildConfig():
 
         self.OBJ_INDEX_PAGES_HEADER_LEN = (self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED + SPIFFS_PH_IX_SIZE_LEN +
                                            SPIFFS_PH_IX_OBJ_TYPE_LEN + self.obj_name_len + self.meta_len)
-        self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN) // self.block_ix_len
+        if aligned_obj_ix_tables:
-        self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) / self.block_ix_len
+            self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED = (self.OBJ_INDEX_PAGES_HEADER_LEN + SPIFFS_PAGE_IX_LEN - 1) & ~(SPIFFS_PAGE_IX_LEN - 1)
+            self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD = self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED - self.OBJ_INDEX_PAGES_HEADER_LEN
+        else:
+            self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED = self.OBJ_INDEX_PAGES_HEADER_LEN
+            self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD = 0
+
+        self.OBJ_INDEX_PAGES_OBJ_IDS_HEAD_LIM = (self.page_size - self.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED) // self.block_ix_len
+        self.OBJ_INDEX_PAGES_OBJ_IDS_LIM = (self.page_size - self.OBJ_DATA_PAGE_HEADER_LEN_ALIGNED) // self.block_ix_len
 
 
 class SpiffsFullError(RuntimeError):
-    def __init__(self, message=None):
+    pass
-        super(SpiffsFullError, self).__init__(message)
 
 
-class SpiffsPage():
+class SpiffsPage(object):
     _endianness_dict = {
         'little': '<',
         'big': '>'
@@ -100,35 +128,41 @@ class SpiffsPage():
         8: 'Q'
     }
 
-    _type_dict = {
+    def __init__(self, bix, build_config):  # type: (int, SpiffsBuildConfig) -> None
-        1: ctypes.c_ubyte,
-        2: ctypes.c_ushort,
-        4: ctypes.c_uint,
-        8: ctypes.c_ulonglong
-    }
-
-    def __init__(self, bix, build_config):
         self.build_config = build_config
         self.bix = bix
 
+    def to_binary(self):  # type: () -> bytes
+        raise NotImplementedError()
+
+
+class SpiffsObjPageWithIdx(SpiffsPage):
+    def __init__(self, obj_id, build_config):  # type: (int, SpiffsBuildConfig) -> None
+        super(SpiffsObjPageWithIdx, self).__init__(0, build_config)
+        self.obj_id = obj_id
+
+    def to_binary(self):  # type: () -> bytes
+        raise NotImplementedError()
+
 
 class SpiffsObjLuPage(SpiffsPage):
-    def __init__(self, bix, build_config):
+    def __init__(self, bix, build_config):  # type: (int, SpiffsBuildConfig) -> None
         SpiffsPage.__init__(self, bix, build_config)
 
         self.obj_ids_limit = self.build_config.OBJ_LU_PAGES_OBJ_IDS_LIM
-        self.obj_ids = list()
+        self.obj_ids = list()  # type: typing.List[ObjIdsItem]
 
-    def _calc_magic(self, blocks_lim):
+    def _calc_magic(self, blocks_lim):  # type: (int) -> int
-        # Calculate the magic value mirrorring computation done by the macro SPIFFS_MAGIC defined in
+        # Calculate the magic value mirroring computation done by the macro SPIFFS_MAGIC defined in
         # spiffs_nucleus.h
         magic = 0x20140529 ^ self.build_config.page_size
         if self.build_config.use_magic_len:
             magic = magic ^ (blocks_lim - self.bix)
-        magic = SpiffsPage._type_dict[self.build_config.obj_id_len](magic)
+        # narrow the result to build_config.obj_id_len bytes
-        return magic.value
+        mask = (2 << (8 * self.build_config.obj_id_len)) - 1
+        return magic & mask
 
-    def register_page(self, page):
+    def register_page(self, page):  # type: (TSP) -> None
         if not self.obj_ids_limit > 0:
             raise SpiffsFullError()
 
@@ -136,8 +170,7 @@ class SpiffsObjLuPage(SpiffsPage):
         self.obj_ids.append(obj_id)
         self.obj_ids_limit -= 1
 
-    def to_binary(self):
+    def to_binary(self):  # type: () -> bytes
-        global test
         img = b''
 
         for (obj_id, page_type) in self.obj_ids:
@@ -152,7 +185,7 @@ class SpiffsObjLuPage(SpiffsPage):
 
         return img
 
-    def magicfy(self, blocks_lim):
+    def magicfy(self, blocks_lim):  # type: (int) -> None
         # Only use magic value if no valid obj id has been written to the spot, which is the
         # spot taken up by the last obj id on last lookup page. The parent is responsible
         # for determining which is the last lookup page and calling this function.
@@ -163,7 +196,7 @@ class SpiffsObjLuPage(SpiffsPage):
             4: 0xFFFFFFFF,
             8: 0xFFFFFFFFFFFFFFFF
         }
-        if (remaining >= 2):
+        if remaining >= 2:
             for i in range(remaining):
                 if i == remaining - 2:
                     self.obj_ids.append((self._calc_magic(blocks_lim), SpiffsObjDataPage))
@@ -173,10 +206,10 @@ class SpiffsObjLuPage(SpiffsPage):
                 self.obj_ids_limit -= 1
 
 
-class SpiffsObjIndexPage(SpiffsPage):
+class SpiffsObjIndexPage(SpiffsObjPageWithIdx):
-    def __init__(self, obj_id, span_ix, size, name, build_config):
+    def __init__(self, obj_id, span_ix, size, name, build_config
-        SpiffsPage.__init__(self, 0, build_config)
+                 ):  # type: (int, int, int, str, SpiffsBuildConfig) -> None
-        self.obj_id = obj_id
+        super(SpiffsObjIndexPage, self).__init__(obj_id, build_config)
         self.span_ix = span_ix
         self.name = name
         self.size = size
@@ -186,16 +219,16 @@ class SpiffsObjIndexPage(SpiffsPage):
         else:
             self.pages_lim = self.build_config.OBJ_INDEX_PAGES_OBJ_IDS_LIM
 
-        self.pages = list()
+        self.pages = list()  # type: typing.List[int]
 
-    def register_page(self, page):
+    def register_page(self, page):  # type: (SpiffsObjDataPage) -> None
         if not self.pages_lim > 0:
             raise SpiffsFullError
 
         self.pages.append(page.offset)
         self.pages_lim -= 1
 
-    def to_binary(self):
+    def to_binary(self):  # type: () -> bytes
         obj_id = self.obj_id ^ (1 << ((self.build_config.obj_id_len * 8) - 1))
         img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] +
                           SpiffsPage._len_dict[self.build_config.obj_id_len] +
@@ -217,7 +250,10 @@ class SpiffsObjIndexPage(SpiffsPage):
                                self.size,
                                SPIFFS_TYPE_FILE)
 
-            img += self.name.encode() + (b'\x00' * ((self.build_config.obj_name_len - len(self.name)) + self.build_config.meta_len))
+            img += self.name.encode() + (b'\x00' * (
+                (self.build_config.obj_name_len - len(self.name))
+                + self.build_config.meta_len
+                + self.build_config.OBJ_INDEX_PAGES_HEADER_LEN_ALIGNED_PAD))
 
         # Finally, add the page index of daa pages
         for page in self.pages:
@@ -232,15 +268,15 @@ class SpiffsObjIndexPage(SpiffsPage):
         return img
 
 
-class SpiffsObjDataPage(SpiffsPage):
+class SpiffsObjDataPage(SpiffsObjPageWithIdx):
-    def __init__(self, offset, obj_id, span_ix, contents, build_config):
+    def __init__(self, offset, obj_id, span_ix, contents, build_config
-        SpiffsPage.__init__(self, 0, build_config)
+                 ):  # type: (int, int, int, bytes, SpiffsBuildConfig) -> None
-        self.obj_id = obj_id
+        super(SpiffsObjDataPage, self).__init__(obj_id, build_config)
         self.span_ix = span_ix
         self.contents = contents
         self.offset = offset
 
-    def to_binary(self):
+    def to_binary(self):  # type: () -> bytes
         img = struct.pack(SpiffsPage._endianness_dict[self.build_config.endianness] +
                           SpiffsPage._len_dict[self.build_config.obj_id_len] +
                           SpiffsPage._len_dict[self.build_config.span_ix_len] +
@@ -258,18 +294,18 @@ class SpiffsObjDataPage(SpiffsPage):
         return img
 
 
-class SpiffsBlock():
+class SpiffsBlock(object):
-    def _reset(self):
+    def _reset(self):  # type: () -> None
         self.cur_obj_index_span_ix = 0
         self.cur_obj_data_span_ix = 0
         self.cur_obj_id = 0
-        self.cur_obj_idx_page = None
+        self.cur_obj_idx_page = None  # type: typing.Optional[SpiffsObjIndexPage]
 
-    def __init__(self, bix, blocks_lim, build_config):
+    def __init__(self, bix, build_config):  # type: (int, SpiffsBuildConfig) -> None
         self.build_config = build_config
         self.offset = bix * self.build_config.block_size
         self.remaining_pages = self.build_config.OBJ_USABLE_PAGES_PER_BLOCK
-        self.pages = list()
+        self.pages = list()  # type: typing.List[SpiffsPage]
         self.bix = bix
 
         lu_pages = list()
@@ -284,8 +320,9 @@ class SpiffsBlock():
 
         self._reset()
 
-    def _register_page(self, page):
+    def _register_page(self, page):  # type: (TSP) -> None
         if isinstance(page, SpiffsObjDataPage):
+            assert self.cur_obj_idx_page is not None
             self.cur_obj_idx_page.register_page(page)  # can raise SpiffsFullError
 
         try:
@@ -301,7 +338,8 @@ class SpiffsBlock():
 
         self.pages.append(page)
 
-    def begin_obj(self, obj_id, size, name, obj_index_span_ix=0, obj_data_span_ix=0):
+    def begin_obj(self, obj_id, size, name, obj_index_span_ix=0, obj_data_span_ix=0
+                  ):  # type: (int, int, str, int, int) -> None
         if not self.remaining_pages > 0:
             raise SpiffsFullError()
         self._reset()
@@ -318,7 +356,7 @@ class SpiffsBlock():
         self.remaining_pages -= 1
         self.cur_obj_index_span_ix += 1
 
-    def update_obj(self, contents):
+    def update_obj(self, contents):  # type: (bytes) -> None
         if not self.remaining_pages > 0:
             raise SpiffsFullError()
         page = SpiffsObjDataPage(self.offset + (len(self.pages) * self.build_config.page_size),
@@ -329,18 +367,19 @@ class SpiffsBlock():
         self.cur_obj_data_span_ix += 1
         self.remaining_pages -= 1
 
-    def end_obj(self):
+    def end_obj(self):  # type: () -> None
         self._reset()
 
-    def is_full(self):
+    def is_full(self):  # type: () -> bool
         return self.remaining_pages <= 0
 
-    def to_binary(self, blocks_lim):
+    def to_binary(self, blocks_lim):  # type: (int) -> bytes
         img = b''
 
         if self.build_config.use_magic:
             for (idx, page) in enumerate(self.pages):
                 if idx == self.build_config.OBJ_LU_PAGES_PER_BLOCK - 1:
+                    assert isinstance(page, SpiffsObjLuPage)
                     page.magicfy(blocks_lim)
                 img += page.to_binary()
         else:
@@ -353,34 +392,32 @@ class SpiffsBlock():
         return img
 
 
-class SpiffsFS():
+class SpiffsFS(object):
-    def __init__(self, img_size, build_config):
+    def __init__(self, img_size, build_config):  # type: (int, SpiffsBuildConfig) -> None
         if img_size % build_config.block_size != 0:
             raise RuntimeError('image size should be a multiple of block size')
 
         self.img_size = img_size
         self.build_config = build_config
 
-        self.blocks = list()
+        self.blocks = list()  # type: typing.List[SpiffsBlock]
         self.blocks_lim = self.img_size // self.build_config.block_size
         self.remaining_blocks = self.blocks_lim
         self.cur_obj_id = 1  # starting object id
 
-    def _create_block(self):
+    def _create_block(self):  # type: () -> SpiffsBlock
         if self.is_full():
             raise SpiffsFullError('the image size has been exceeded')
 
-        block = SpiffsBlock(len(self.blocks), self.blocks_lim, self.build_config)
+        block = SpiffsBlock(len(self.blocks), self.build_config)
         self.blocks.append(block)
         self.remaining_blocks -= 1
         return block
 
-    def is_full(self):
+    def is_full(self):  # type: () -> bool
         return self.remaining_blocks <= 0
 
-    def create_file(self, img_path, file_path):
+    def create_file(self, img_path, file_path):  # type: (str, str) -> None
-        contents = None
-
         if len(img_path) > self.build_config.obj_name_len:
             raise RuntimeError("object name '%s' too long" % img_path)
 
@@ -434,31 +471,51 @@ class SpiffsFS():
 
         self.cur_obj_id += 1
 
-    def to_binary(self):
+    def to_binary(self):  # type: () -> bytes
         img = b''
+        all_blocks = []
         for block in self.blocks:
-            img += block.to_binary(self.blocks_lim)
+            all_blocks.append(block.to_binary(self.blocks_lim))
         bix = len(self.blocks)
         if self.build_config.use_magic:
             # Create empty blocks with magic numbers
             while self.remaining_blocks > 0:
-                block = SpiffsBlock(bix, self.blocks_lim, self.build_config)
+                block = SpiffsBlock(bix, self.build_config)
-                img += block.to_binary(self.blocks_lim)
+                all_blocks.append(block.to_binary(self.blocks_lim))
                 self.remaining_blocks -= 1
                 bix += 1
         else:
             # Just fill remaining spaces FF's
-            img += '\xFF' * (self.img_size - len(img))
+            all_blocks.append(b'\xFF' * (self.img_size - len(all_blocks) * self.build_config.block_size))
+        img += b''.join([blk for blk in all_blocks])
         return img
 
 
-def main():
+class CustomHelpFormatter(argparse.HelpFormatter):
+    """
+    Similar to argparse.ArgumentDefaultsHelpFormatter, except it
+    doesn't add the default value if "(default:" is already present.
+    This helps in the case of options with action="store_false", like
+    --no-magic or --no-magic-len.
+    """
+    def _get_help_string(self, action):  # type: (argparse.Action) -> str
+        if action.help is None:
+            return ''
+        if '%(default)' not in action.help and '(default:' not in action.help:
+            if action.default is not argparse.SUPPRESS:
+                defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE]
+                if action.option_strings or action.nargs in defaulting_nargs:
+                    return action.help + ' (default: %(default)s)'
+        return action.help
+
+
+def main():  # type: () -> None
     if sys.version_info[0] < 3:
         print('WARNING: Support for Python 2 is deprecated and will be removed in future versions.', file=sys.stderr)
     elif sys.version_info[0] == 3 and sys.version_info[1] < 6:
         print('WARNING: Python 3 versions older than 3.6 are not supported.', file=sys.stderr)
     parser = argparse.ArgumentParser(description='SPIFFS Image Generator',
-                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+                                     formatter_class=CustomHelpFormatter)
 
     parser.add_argument('image_size',
                         help='Size of the created image')
@@ -490,24 +547,38 @@ def main():
                         default=4)
 
     parser.add_argument('--use-magic',
+                        dest='use_magic',
                         help='Use magic number to create an identifiable SPIFFS image. Specify if CONFIG_SPIFFS_USE_MAGIC.',
-                        action='store_true',
+                        action='store_true')
-                        default=True)
+
+    parser.add_argument('--no-magic',
+                        dest='use_magic',
+                        help='Inverse of --use-magic (default: --use-magic is enabled)',
+                        action='store_false')
+
+    parser.add_argument('--use-magic-len',
+                        dest='use_magic_len',
+                        help='Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.',
+                        action='store_true')
+
+    parser.add_argument('--no-magic-len',
+                        dest='use_magic_len',
+                        help='Inverse of --use-magic-len (default: --use-magic-len is enabled)',
+                        action='store_false')
 
     parser.add_argument('--follow-symlinks',
                         help='Take into account symbolic links during partition image creation.',
-                        action='store_true',
+                        action='store_true')
-                        default=False)
-
-    parser.add_argument('--use-magic-len',
-                        help='Use position in memory to create different magic numbers for each block. Specify if CONFIG_SPIFFS_USE_MAGIC_LENGTH.',
-                        action='store_true',
-                        default=True)
 
     parser.add_argument('--big-endian',
                         help='Specify if the target architecture is big-endian. If not specified, little-endian is assumed.',
+                        action='store_true')
+
+    parser.add_argument('--aligned-obj-ix-tables',
                         action='store_true',
-                        default=False)
+                        help='Use aligned object index tables. Specify if SPIFFS_ALIGNED_OBJECT_INDEX_TABLES is set.')
+
+    parser.set_defaults(use_magic=True, use_magic_len=True)
 
     args = parser.parse_args()
 
@@ -520,7 +591,7 @@ def main():
                                                  args.block_size, SPIFFS_BLOCK_IX_LEN, args.meta_len,
                                                  args.obj_name_len, SPIFFS_OBJ_ID_LEN, SPIFFS_SPAN_IX_LEN,
                                                  True, True, 'big' if args.big_endian else 'little',
-                                                 args.use_magic, args.use_magic_len)
+                                                 args.use_magic, args.use_magic_len, args.aligned_obj_ix_tables)
 
         spiffs = SpiffsFS(image_size, spiffs_build_default)
 

components/spiffs/test_spiffsgen/test_spiffsgen.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+import os
+import sys
+import unittest
+
+try:
+    import typing
+except ImportError:
+    pass
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+try:
+    import spiffsgen
+except ImportError:
+    raise
+
+
+class SpiffsgenTest(unittest.TestCase):
+    def test_configs(self):  # type: () -> None
+        """Run spiffsgen with different configs, and check that
+        an image is generated (there is no exception), and the image size
+        is as expected.
+        """
+        default_config = dict(
+            page_size=256,
+            page_ix_len=spiffsgen.SPIFFS_PAGE_IX_LEN,
+            block_size=4096,
+            block_ix_len=spiffsgen.SPIFFS_BLOCK_IX_LEN,
+            meta_len=4,
+            obj_name_len=32,
+            obj_id_len=spiffsgen.SPIFFS_BLOCK_IX_LEN,
+            span_ix_len=spiffsgen.SPIFFS_SPAN_IX_LEN,
+            packed=True,
+            aligned=True,
+            endianness='little',
+            use_magic=True,
+            use_magic_len=True,
+            aligned_obj_ix_tables=False
+        )
+
+        def make_config(**kwargs):  # type: (typing.Any) -> spiffsgen.SpiffsBuildConfig
+            """Return SpiffsBuildConfig object with configuration set
+            by default_config plus any options overridden in kwargs.
+            """
+            new_config = dict(default_config)
+            new_config.update(**kwargs)
+            return spiffsgen.SpiffsBuildConfig(**new_config)
+
+        configs = [
+            make_config(),
+            make_config(use_magic_len=False, use_magic=False, aligned_obj_ix_tables=True),
+            make_config(meta_len=4, obj_name_len=16),
+            make_config(block_size=8192),
+            make_config(page_size=512)
+        ]
+
+        image_size = 64 * 1024
+        for config in configs:
+            spiffs = spiffsgen.SpiffsFS(image_size, config)
+            spiffs.create_file('/test', __file__)
+            image = spiffs.to_binary()
+            self.assertEqual(len(image), image_size)
+            # Note: it would be nice to compile spiffs for host with the given
+            # config, and verify that the image is parsed correctly.
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/ci/executable-list.txt

@@ -17,6 +17,7 @@ components/partition_table/parttool.py
 components/partition_table/test_gen_esp32part_host/check_sizes_test.py
 components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py
 components/spiffs/spiffsgen.py
+components/spiffs/test_spiffsgen/test_spiffsgen.py
 components/ulp/esp32ulp_mapgen.py
 docs/build_docs.py
 docs/check_lang_folder_sync.sh

tools/ci/mypy_ignore_list.txt

@@ -19,7 +19,6 @@ components/protocomm/python/constants_pb2.py
 components/protocomm/python/sec0_pb2.py
 components/protocomm/python/sec1_pb2.py
 components/protocomm/python/session_pb2.py
-components/spiffs/spiffsgen.py
 components/ulp/esp32ulp_mapgen.py
 components/wifi_provisioning/python/wifi_config_pb2.py
 components/wifi_provisioning/python/wifi_constants_pb2.py