diff --git a/components/nvs_flash/nvs_partition_generator/README.rst b/components/nvs_flash/nvs_partition_generator/README.rst index 97a83b7308..cf31261366 100644 --- a/components/nvs_flash/nvs_partition_generator/README.rst +++ b/components/nvs_flash/nvs_partition_generator/README.rst @@ -12,7 +12,7 @@ Prerequisites To use this utility in encryption mode, the following packages need to be installed: - cryptography package -This dependency is already captured by including these packages in `requirement.txt` in top level IDF directory. +These dependencies is already captured by including these packages in `requirement.txt` in top level IDF directory. CSV file format --------------- @@ -28,7 +28,7 @@ Type Encoding Supported values are: ``u8``, ``i8``, ``u16``, ``u32``, ``i32``, ``string``, ``hex2bin``, ``base64`` and ``binary``. This specifies how actual data values are encoded in the resultant binary file. Difference between ``string`` and ``binary`` encoding is that ``string`` data is terminated with a NULL character, whereas ``binary`` data is not. - .. note:: For ``file`` type, only ``hex2bin``, ``base64``, ``string`` and ``binary`` is supported as of now. +.. note:: For ``file`` type, only ``hex2bin``, ``base64``, ``string`` and ``binary`` is supported as of now. Value Data value. @@ -44,7 +44,7 @@ Below is an example dump of such CSV file:: key1,data,u8,1 key2,file,string,/path/to/file -.. note:: Make sure there are no spaces before and after ',' in CSV file. +.. note:: Make sure there are no spaces before and after ',' or at the end of each line in CSV file. NVS Entry and Namespace association ----------------------------------- @@ -71,7 +71,7 @@ Running the utility python nvs_partition_gen.py [-h] [--input INPUT] [--output OUTPUT] [--size SIZE] [--version {v1,v2}] [--keygen {true,false}] [--encrypt {true,false}] - [--keyfile KEYFILE] + [--keyfile KEYFILE] [--outdir OUTDIR] +------------------------+----------------------------------------------------------------------------------------------+ @@ -85,15 +85,14 @@ Running the utility +------------------------+----------------------------------------------------------------------------------------------+ | --version {v1,v2} | Set version. Default: v2 | +------------------------+----------------------------------------------------------------------------------------------+ -| --keygen {true,false} | Generate keys for encryption. Creates an `encryption_keys.bin` file (in current directory). | -| | Default: false | +| --keygen {true,false} | Generate keys for encryption. | +------------------------+----------------------------------------------------------------------------------------------+ | --encrypt {true,false} | Set encryption mode. Default: false | +------------------------+----------------------------------------------------------------------------------------------+ | --keyfile KEYFILE | File having key for encryption (Applicable only if encryption mode is true) | +------------------------+----------------------------------------------------------------------------------------------+ - - +| --outdir OUTDIR | The output directory to store the files created (Default: current directory) | ++------------------------+----------------------------------------------------------------------------------------------+ You can run this utility in two modes: - Default mode - Binary generated in this mode is an unencrypted binary file. @@ -108,7 +107,7 @@ You can run this utility in two modes: python nvs_partition_gen.py [-h] --input INPUT --output OUTPUT --size SIZE [--version {v1,v2}] [--keygen {true,false}] [--encrypt {true,false}] - [--keyfile KEYFILE] + [--keyfile KEYFILE] [--outdir OUTDIR] You can run the utility using below command:: @@ -123,28 +122,34 @@ You can run the utility using below command:: python nvs_partition_gen.py [-h] --input INPUT --output OUTPUT --size SIZE --encrypt {true,false} - --keygen {true,false} | --keyfile KEYFILE - [--version {v1,v2}] + --keygen {true,false} --keyfile KEYFILE + [--version {v1,v2}] [--outdir OUTDIR] You can run the utility using below commands: - - By taking encryption keys as an input file. A sample encryption keys binary file is provided with the utility:: - - python nvs_partition_gen.py --input sample.csv --output sample_encrypted.bin --size 0x3000 --encrypt true --keyfile testdata/sample_encryption_keys.bin - - By enabling generation of encryption keys:: python nvs_partition_gen.py --input sample.csv --output sample_encrypted.bin --size 0x3000 --encrypt true --keygen true + - By taking encryption keys as an input file. A sample encryption keys binary file is provided with the utility:: + + python nvs_partition_gen.py --input sample.csv --output sample_encrypted.bin --size 0x3000 --encrypt true --keyfile testdata/sample_encryption_keys.bin + + - By enabling generation of encryption keys and storing the keys in custom filename:: + + python nvs_partition_gen.py --input sample.csv --output sample_encrypted.bin --size 0x3000 --encrypt true --keygen true --keyfile encryption_keys_generated.bin + +.. note:: If `--keygen` is given with `--keyfile` argument, generated keys will be stored in `--keyfile` file. If `--keygen` argument is absent, `--keyfile` is taken as input file having key for encryption. -*To generate* **only** *encryption keys with this utility* ( Creates an `encryption_keys.bin` file in current directory ): :: +*To generate* **only** *encryption keys with this utility*:: python nvs_partition_gen.py --keygen true -.. note:: This `encryption_keys.bin` file is compatible with NVS key-partition structure. Refer to :ref:`nvs_key_partition` for more details. +This creates an `encryption_keys_.bin` file. +.. note:: This newly created file having encryption keys in `keys/` directory is compatible with NVS key-partition structure. Refer to :ref:`nvs_key_partition` for more details. You can also provide the format version number (in any of the two modes): @@ -179,3 +184,4 @@ Caveats - Utility doesn't check for duplicate keys and will write data pertaining to both keys. User needs to make sure keys are distinct. - Once a new page is created, no data will be written in the space left in previous page. Fields in the CSV file need to be ordered in such a way so as to optimize memory. - 64-bit datatype is not yet supported. + diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py index 99ee53ff68..bf423334bb 100755 --- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py +++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py @@ -31,8 +31,14 @@ import array import csv import zlib import codecs -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.hazmat.backends import default_backend +import datetime +import distutils.dir_util +try: + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend +except ImportError: + print("cryptography package needs to be installed.\nRun: pip install cryptography>=2.1.4") + sys.exit(0) VERSION1_PRINT = "v1 - Multipage Blob Support Disabled" VERSION2_PRINT = "v2 - Multipage Blob Support Enabled" @@ -637,7 +643,8 @@ def nvs_close(nvs_instance): def check_input_args(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None, - encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_encrypt_arg_str=None): + encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_encrypt_arg_str=None, + output_dir=None): global version, is_encrypt_data, input_size, key_gen @@ -646,6 +653,12 @@ def check_input_args(input_filename=None, output_filename=None, input_part_size= key_gen = is_key_gen input_size = input_part_size + if not output_dir == os.getcwd() and (key_file and os.path.isabs(key_file)): + sys.exit("Error. Cannot provide --outdir argument as --keyfile is absolute path.") + + if not os.path.isdir(output_dir): + distutils.dir_util.mkpath(output_dir) + if is_encrypt_data.lower() == 'true': is_encrypt_data = True elif is_encrypt_data.lower() == 'false': @@ -668,18 +681,25 @@ def check_input_args(input_filename=None, output_filename=None, input_part_size= elif any(arg is not None for arg in [input_filename, output_filename, input_size]): sys.exit(print_arg_str) else: - if not input_size: - if not all(arg is not None for arg in [input_filename, output_filename]): - sys.exit(print_arg_str) + if not (input_filename and output_filename and input_size): + sys.exit(print_arg_str) - if is_encrypt_data and not key_gen and not key_file: - sys.exit(print_encrypt_arg_str) + if is_encrypt_data and not key_gen and not key_file: + sys.exit(print_encrypt_arg_str) - if is_encrypt_data and key_gen and key_file: - sys.exit(print_encrypt_arg_str) + if not is_encrypt_data and key_file: + sys.exit("Invalid. Cannot give --keyfile as --encrypt is set to false.") - if not is_encrypt_data and key_file: - sys.exit("Invalid. Cannot give --keyfile as --encrypt is set to false.") + if key_file: + key_file_name, key_file_ext = os.path.splitext(key_file) + if key_file_ext: + if not key_file_ext == '.bin': + sys.exit("--keyfile argument can be a filename with no extension or .bin extension only") + + # If only one of the arguments - input_filename, output_filename, input_size is given + if ((any(arg is None for arg in [input_filename, output_filename, input_size])) is True) and \ + ((all(arg is None for arg in [input_filename, output_filename, input_size])) is False): + sys.exit(print_arg_str) if input_size: # Set size @@ -695,7 +715,8 @@ def check_input_args(input_filename=None, output_filename=None, input_part_size= sys.exit("Minimum NVS partition size needed is 0x3000 bytes.") -def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None, encrypt_mode=None, key_file=None, version_no=None): +def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None, encrypt_mode=None, + key_file=None, encr_key_prefix=None, version_no=None, output_dir=None): """ Wrapper to generate nvs partition binary :param input_filename: Name of input file containing data @@ -709,6 +730,9 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None """ global key_input, key_len_needed + encr_key_bin_file = None + encr_keys_dir = None + backslash = ['/','\\'] key_len_needed = 64 key_input = bytearray() @@ -720,6 +744,8 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None key_input = key_f.read(64) if all(arg is not None for arg in [input_filename, output_filename, input_size]): + if not os.path.isabs(output_filename) and not any(ch in output_filename for ch in backslash): + output_filename = os.path.join(output_dir, '') + output_filename input_file = open(input_filename, 'rt', encoding='utf8') output_file = open(output_filename, 'wb') @@ -737,6 +763,8 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None input_file.close() output_file.close() + print("NVS binary created: " + output_filename) + if key_gen: keys_page_buf = bytearray(b'\xff') * Page.PAGE_PARAMS["max_size"] key_bytes = bytearray() @@ -750,57 +778,87 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None crc_data = bytes(crc_data) crc = zlib.crc32(crc_data, 0xFFFFFFFF) struct.pack_into(' #include #include +#include #include #include +#include #define TEST_ESP_ERR(rc, res) CHECK((rc) == (res)) #define TEST_ESP_OK(rc) CHECK((rc) == ESP_OK) @@ -2358,32 +2360,68 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena { int childpid = fork(); int status; - if (childpid == 0) { - exit(execlp("python", "python", - "../nvs_partition_generator/nvs_partition_gen.py", - "--input", - "../nvs_partition_generator/sample_multipage_blob.csv", - "--output", - "../nvs_partition_generator/partition_encrypted_using_keygen.bin", - "--size", - "0x4000", - "--encrypt", - "True", - "--keygen", - "true",NULL)); + if (childpid == 0) { + exit(execlp("rm", " rm", + "-rf", + "keys",NULL)); } else { CHECK(childpid > 0); waitpid(childpid, &status, 0); CHECK(WEXITSTATUS(status) != -1); + + childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "--input", + "../nvs_partition_generator/sample_multipage_blob.csv", + "--output", + "../nvs_partition_generator/partition_encrypted_using_keygen.bin", + "--size", + "0x4000", + "--encrypt", + "True", + "--keygen", + "true",NULL)); + + } else { + CHECK(childpid > 0); + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) != -1); + + } } + + DIR *dir; + struct dirent *file; + char *filename; + char *files; + char *file_ext; + + dir = opendir("keys"); + while ((file = readdir(dir)) != NULL) + { + filename = file->d_name; + files = strrchr(filename, '.'); + if (files != NULL) + { + file_ext = files+1; + if (strncmp(file_ext,"bin",3) == 0) + { + break; + } + } + } + + std::string encr_file = std::string("keys/") + std::string(filename); SpiFlashEmulator emu("../nvs_partition_generator/partition_encrypted_using_keygen.bin"); char buffer[64]; FILE *fp; - fp = fopen("encryption_keys.bin","rb"); + fp = fopen(encr_file.c_str(),"rb"); fread(buffer,sizeof(buffer),1,fp); fclose(fp); @@ -2398,14 +2436,37 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, 4, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg); - } TEST_CASE("test nvs apis for nvs partition generator utility with encryption enabled using keyfile", "[nvs_part_gen]") { int childpid = fork(); int status; - if (childpid == 0) { + + DIR *dir; + struct dirent *file; + char *filename; + char *files; + char *file_ext; + + dir = opendir("keys"); + while ((file = readdir(dir)) != NULL) + { + filename = file->d_name; + files = strrchr(filename, '.'); + if (files != NULL) + { + file_ext = files+1; + if (strncmp(file_ext,"bin",3) == 0) + { + break; + } + } + } + + std::string encr_file = std::string("keys/") + std::string(filename); + + if (childpid == 0) { exit(execlp("python", "python", "../nvs_partition_generator/nvs_partition_gen.py", "--input", @@ -2417,7 +2478,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena "--encrypt", "True", "--keyfile", - "encryption_keys.bin",NULL)); + encr_file.c_str(),NULL)); } else { CHECK(childpid > 0); @@ -2430,7 +2491,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena char buffer[64]; FILE *fp; - fp = fopen("encryption_keys.bin","rb"); + fp = fopen(encr_file.c_str(),"rb"); fread(buffer,sizeof(buffer),1,fp); fclose(fp); @@ -2448,7 +2509,8 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena childpid = fork(); if (childpid == 0) { exit(execlp("rm", " rm", - "encryption_keys.bin",NULL)); + "-rf", + "keys",NULL)); } else { CHECK(childpid > 0); waitpid(childpid, &status, 0);