From df2ae4689cd1056bfda67c9dbe08c015da1202aa Mon Sep 17 00:00:00 2001 From: Levente Tamas Date: Thu, 30 Dec 2021 00:22:13 +0200 Subject: [PATCH 1/3] fix fatfs_create_spiflash_image() feature (IDFGH-6383) Open the files in binary mode, and remove the redundant encode --- components/fatfs/fatfsgen.py | 2 +- components/fatfs/fatfsgen_utils/fs_object.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/fatfs/fatfsgen.py b/components/fatfs/fatfsgen.py index c53602fd68..ac9c46fdad 100755 --- a/components/fatfs/fatfsgen.py +++ b/components/fatfs/fatfsgen.py @@ -167,7 +167,7 @@ class FATFS: normal_path = os.path.normpath(folder_relative_path) split_path = normal_path.split(os.sep) if os.path.isfile(real_path): - with open(real_path) as file: + with open(real_path, "rb") as file: content = file.read() file_name, extension = os.path.splitext(split_path[-1]) extension = extension[1:] # remove the dot from the extension diff --git a/components/fatfs/fatfsgen_utils/fs_object.py b/components/fatfs/fatfsgen_utils/fs_object.py index e5137ca9a2..2b725e373c 100644 --- a/components/fatfs/fatfsgen_utils/fs_object.py +++ b/components/fatfs/fatfsgen_utils/fs_object.py @@ -47,7 +47,7 @@ class File: # we assume that the correct amount of clusters is allocated current_cluster = self._first_cluster for content_part in split_content_into_sectors(content, self.fatfs_state.sector_size): - content_as_list = content_part.encode() + content_as_list = content_part if current_cluster is None: raise FatalError('No free space left!') From f7983fd3131e6f10ecdc1f236ad76d1b24f9ff3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ga=C5=88o?= Date: Mon, 3 Jan 2022 00:47:00 +0100 Subject: [PATCH 2/3] Use bytes instead of strings in tests --- .../fatfs/test_fatfsgen/test_fatfsgen.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/components/fatfs/test_fatfsgen/test_fatfsgen.py b/components/fatfs/test_fatfsgen/test_fatfsgen.py index 074ce579a3..f7d0c0f826 100755 --- a/components/fatfs/test_fatfsgen/test_fatfsgen.py +++ b/components/fatfs/test_fatfsgen/test_fatfsgen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os @@ -56,7 +56,7 @@ class FatFSGen(unittest.TestCase): def test_write_to_file_with_extension_sn_fat12(self) -> None: fatfs = fatfsgen.FATFS() fatfs.create_file('WRITEF', extension='TXT') - fatfs.write_content(path_from_root=['WRITEF.TXT'], content='testcontent') + fatfs.write_content(path_from_root=['WRITEF.TXT'], content=b'testcontent') fatfs.write_filesystem(CFG['output_file']) file_system = fatfs.read_filesystem(CFG['output_file']) @@ -69,7 +69,7 @@ class FatFSGen(unittest.TestCase): fatfs = fatfsgen.FATFS() fatfs.create_directory('TESTFOLD') fatfs.create_file('WRITEF', extension='TXT', path_from_root=['TESTFOLD']) - fatfs.write_content(path_from_root=['TESTFOLD', 'WRITEF.TXT'], content='testcontent') + fatfs.write_content(path_from_root=['TESTFOLD', 'WRITEF.TXT'], content=b'testcontent') fatfs.write_filesystem(CFG['output_file']) file_system = fatfs.read_filesystem(CFG['output_file']) @@ -100,7 +100,7 @@ class FatFSGen(unittest.TestCase): def test_full_sector_file(self) -> None: fatfs = fatfsgen.FATFS() fatfs.create_file('WRITEF', extension='TXT') - fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * 'a') + fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * b'a') fatfs.write_filesystem(CFG['output_file']) file_system = fatfs.read_filesystem(CFG['output_file']) self.assertEqual(file_system[0x1000: 0x100e], b'\xf8\xff\xff\xff\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00') @@ -109,7 +109,7 @@ class FatFSGen(unittest.TestCase): def test_file_chaining(self) -> None: fatfs = fatfsgen.FATFS() fatfs.create_file('WRITEF', extension='TXT') - fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * 'a' + 'a') + fatfs.write_content(path_from_root=['WRITEF.TXT'], content=CFG['sector_size'] * b'a' + b'a') fatfs.write_filesystem(CFG['output_file']) file_system = fatfs.read_filesystem(CFG['output_file']) self.assertEqual(file_system[0x1000: 0x100e], b'\xf8\xff\xff\x03\xf0\xff\x00\x00\x00\x00\x00\x00\x00\x00') @@ -121,8 +121,8 @@ class FatFSGen(unittest.TestCase): for i in range(CFG['sector_size'] // CFG['entry_size']): fatfs.create_file(f'A{str(i).upper()}', path_from_root=['TESTFOLD']) - fatfs.write_content(path_from_root=['TESTFOLD', 'A0'], content='first') - fatfs.write_content(path_from_root=['TESTFOLD', 'A126'], content='later') + fatfs.write_content(path_from_root=['TESTFOLD', 'A0'], content=b'first') + fatfs.write_content(path_from_root=['TESTFOLD', 'A126'], content=b'later') fatfs.write_filesystem(CFG['output_file']) file_system = fatfs.read_filesystem(CFG['output_file']) self.assertEqual(file_system[0x1000: 0x10d0], @@ -136,13 +136,13 @@ class FatFSGen(unittest.TestCase): fatfs.create_directory('TESTFOLD') fatfs.create_directory('TESTFOLL', path_from_root=['TESTFOLD']) self.assertRaises(WriteDirectoryException, fatfs.write_content, path_from_root=['TESTFOLD', 'TESTFOLL'], - content='testcontent') + content=b'testcontent') def test_write_non_existing_file_in_folder_sn_fat12(self) -> None: fatfs = fatfsgen.FATFS() fatfs.create_directory('TESTFOLD') self.assertRaises(FileNotFoundError, fatfs.write_content, path_from_root=['TESTFOLD', 'AHOJ'], - content='testcontent') + content=b'testcontent') @staticmethod def create_too_many_files() -> None: @@ -160,8 +160,8 @@ class FatFSGen(unittest.TestCase): for i in range(2 * CFG['sector_size'] // CFG['entry_size']): fatfs.create_file(f'A{str(i).upper()}', path_from_root=['TESTFOLD']) - fatfs.write_content(path_from_root=['TESTFOLD', 'A253'], content='later') - fatfs.write_content(path_from_root=['TESTFOLD', 'A255'], content='last') + fatfs.write_content(path_from_root=['TESTFOLD', 'A253'], content=b'later') + fatfs.write_content(path_from_root=['TESTFOLD', 'A255'], content=b'last') fatfs.write_filesystem(CFG['output_file']) file_system = fatfs.read_filesystem(CFG['output_file']) self.assertEqual(file_system[0x105000:0x105010], b'later\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') @@ -192,7 +192,7 @@ class FatFSGen(unittest.TestCase): fatfs.create_directory('TESTFOLL', path_from_root=['TESTFOLD']) fatfs.create_directory('TESTFOLO', path_from_root=['TESTFOLD', 'TESTFOLL']) fatfs.create_file('WRITEF', extension='TXT', path_from_root=['TESTFOLD', 'TESTFOLL', 'TESTFOLO']) - fatfs.write_content(path_from_root=['TESTFOLD', 'TESTFOLL', 'TESTFOLO', 'WRITEF.TXT'], content='later') + fatfs.write_content(path_from_root=['TESTFOLD', 'TESTFOLL', 'TESTFOLO', 'WRITEF.TXT'], content=b'later') fatfs.write_filesystem(CFG['output_file']) file_system = fatfs.read_filesystem(CFG['output_file']) @@ -204,7 +204,7 @@ class FatFSGen(unittest.TestCase): fatfs.create_directory('TESTFOLD', path_from_root=['TESTFOLD']) fatfs.create_directory('TESTFOLD', path_from_root=['TESTFOLD', 'TESTFOLD']) fatfs.create_file('WRITEF', extension='TXT', path_from_root=['TESTFOLD', 'TESTFOLD', 'TESTFOLD']) - fatfs.write_content(path_from_root=['TESTFOLD', 'TESTFOLD', 'TESTFOLD', 'WRITEF.TXT'], content='later') + fatfs.write_content(path_from_root=['TESTFOLD', 'TESTFOLD', 'TESTFOLD', 'WRITEF.TXT'], content=b'later') fatfs.write_filesystem(CFG['output_file']) file_system = fatfs.read_filesystem(CFG['output_file']) From 140f5be289c2d9d919288f8827db43ac811b0ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ga=C5=88o?= Date: Mon, 3 Jan 2022 01:02:16 +0100 Subject: [PATCH 3/3] Updated typehints from str values to bytes --- components/fatfs/fatfsgen.py | 6 +++--- components/fatfs/fatfsgen_utils/fs_object.py | 6 +++--- components/fatfs/fatfsgen_utils/utils.py | 6 +++--- components/fatfs/test_fatfsgen/test_fatfsgen.py | 1 - components/fatfs/test_fatfsgen/test_wl_fatfsgen.py | 3 ++- components/fatfs/wl_fatfsgen.py | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/components/fatfs/fatfsgen.py b/components/fatfs/fatfsgen.py index ac9c46fdad..9db556827b 100755 --- a/components/fatfs/fatfsgen.py +++ b/components/fatfs/fatfsgen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os @@ -107,7 +107,7 @@ class FATFS: parent_dir = self.root_directory.recursive_search(path_from_root, self.root_directory) self.root_directory.new_directory(name=name, parent=parent_dir, path_from_root=path_from_root) - def write_content(self, path_from_root: List[str], content: str) -> None: + def write_content(self, path_from_root: List[str], content: bytes) -> None: """ fat fs invokes root directory to recursively find the required file and writes the content """ @@ -167,7 +167,7 @@ class FATFS: normal_path = os.path.normpath(folder_relative_path) split_path = normal_path.split(os.sep) if os.path.isfile(real_path): - with open(real_path, "rb") as file: + with open(real_path, 'rb') as file: content = file.read() file_name, extension = os.path.splitext(split_path[-1]) extension = extension[1:] # remove the dot from the extension diff --git a/components/fatfs/fatfsgen_utils/fs_object.py b/components/fatfs/fatfsgen_utils/fs_object.py index 2b725e373c..8f78a8a17c 100644 --- a/components/fatfs/fatfsgen_utils/fs_object.py +++ b/components/fatfs/fatfsgen_utils/fs_object.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os @@ -42,7 +42,7 @@ class File: def name_equals(self, name: str, extension: str) -> bool: return self.name == name and self.extension == extension - def write(self, content: str) -> None: + def write(self, content: bytes) -> None: self.entry.update_content_size(len(content)) # we assume that the correct amount of clusters is allocated current_cluster = self._first_cluster @@ -207,7 +207,7 @@ class Directory: directory.init_directory() target_dir.entities.append(directory) - def write_to_file(self, path: List[str], content: str) -> None: + def write_to_file(self, path: List[str], content: bytes) -> None: """ Writes to file existing in the directory structure. diff --git a/components/fatfs/fatfsgen_utils/utils.py b/components/fatfs/fatfsgen_utils/utils.py index 78ae3edcd3..8c6dae058e 100644 --- a/components/fatfs/fatfsgen_utils/utils.py +++ b/components/fatfs/fatfsgen_utils/utils.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse @@ -18,7 +18,7 @@ def crc32(input_values: List[int], crc: int) -> int: return binascii.crc32(bytearray(input_values), crc) -def required_clusters_count(cluster_size: int, content: str) -> int: +def required_clusters_count(cluster_size: int, content: bytes) -> int: # compute number of required clusters for file text return (len(content) + cluster_size - 1) // cluster_size @@ -66,7 +66,7 @@ def clean_second_half_byte(bytes_array: bytearray, address: int) -> None: bytes_array[address] &= 0x0f -def split_content_into_sectors(content: str, sector_size: int) -> List[str]: +def split_content_into_sectors(content: bytes, sector_size: int) -> List[bytes]: result = [] clusters_cnt = required_clusters_count(cluster_size=sector_size, content=content) diff --git a/components/fatfs/test_fatfsgen/test_fatfsgen.py b/components/fatfs/test_fatfsgen/test_fatfsgen.py index f7d0c0f826..b6a8bfb488 100755 --- a/components/fatfs/test_fatfsgen/test_fatfsgen.py +++ b/components/fatfs/test_fatfsgen/test_fatfsgen.py @@ -49,7 +49,6 @@ class FatFSGen(unittest.TestCase): fatfs.create_file('TESTF', extension='TXT') fatfs.write_filesystem(CFG['output_file']) file_system = fatfs.read_filesystem(CFG['output_file']) - self.assertEqual(file_system[0x2000:0x200c], b'TESTF TXT\x20') # check entry name and type self.assertEqual(file_system[0x1000:0x1006], b'\xf8\xff\xff\xff\x0f\x00') # check fat diff --git a/components/fatfs/test_fatfsgen/test_wl_fatfsgen.py b/components/fatfs/test_fatfsgen/test_wl_fatfsgen.py index 8f7e49f452..a5c64be669 100755 --- a/components/fatfs/test_fatfsgen/test_wl_fatfsgen.py +++ b/components/fatfs/test_fatfsgen/test_wl_fatfsgen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os @@ -132,6 +132,7 @@ class WLFatFSGen(unittest.TestCase): fatfs.wl_write_filesystem(CFG['output_file']) with open(CFG['output_file'], 'rb') as fs_file: file_system = bytearray(fs_file.read()) + self.assertEqual(file_system[0x7060:0x7070], b'TESTFIL2 \x00\x00\x01\x00') self.assertEqual(file_system[0x7070:0x7080], b'!\x00\x00\x00\x00\x00\x01\x00\x01\x00\x05\x00\x0b\x00\x00\x00') self.assertEqual(file_system[0x8040:0x8050], b'LASTFILE \x00\x00\x01\x00') diff --git a/components/fatfs/wl_fatfsgen.py b/components/fatfs/wl_fatfsgen.py index ff55a47e46..e762bb8a9d 100755 --- a/components/fatfs/wl_fatfsgen.py +++ b/components/fatfs/wl_fatfsgen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 from typing import List, Optional @@ -180,7 +180,7 @@ class WLFATFS: def wl_create_directory(self, name: str, path_from_root: Optional[List[str]] = None) -> None: self.plain_fatfs.create_directory(name, path_from_root) - def wl_write_content(self, path_from_root: List[str], content: str) -> None: + def wl_write_content(self, path_from_root: List[str], content: bytes) -> None: self.plain_fatfs.write_content(path_from_root, content)