From cae5f1f5532fc34cb541cb3aaed83c752389ef4e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 28 Dec 2015 01:15:06 +0200 Subject: [PATCH] Implement uploading files to file system SPIFFS for ESP8266 // Issue #382 --- HISTORY.rst | 2 + docs/envvars.rst | 4 + docs/platforms/creating_platform.rst | 3 + docs/platforms/espressif.rst | 21 +++-- docs/platforms/espressif_extra.rst | 15 +++- docs/projectconf.rst | 17 +++- docs/userguide/cmd_run.rst | 1 + platformio/__init__.py | 2 +- platformio/builder/main.py | 1 + platformio/builder/scripts/espressif.py | 113 +++++++++++++++++++----- platformio/builder/tools/piomisc.py | 16 +++- platformio/platforms/base.py | 4 + platformio/platforms/espressif.py | 4 + platformio/util.py | 7 ++ 14 files changed, 171 insertions(+), 39 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2169836a..5bc94036 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,8 @@ PlatformIO 2.0 * Moved SCons to PlatformIO packages. PlatformIO does not require SCons to be installed in your system. Significantly simplified installation process of PlatformIO. ``pip install platformio`` rocks! +* Implemented uploading files to file system SPIFFS for ESP8266 + (`issue #382 `_) * Added support for the new Adafruit boards Bluefruit Micro and Feather (`issue #403 `_) * Added support for RFDuino diff --git a/docs/envvars.rst b/docs/envvars.rst index c495e8c8..508786ad 100644 --- a/docs/envvars.rst +++ b/docs/envvars.rst @@ -55,6 +55,10 @@ Allows to override :ref:`projectconf` option :ref:`projectconf_pio_src_dir`. Allows to override :ref:`projectconf` option :ref:`projectconf_pio_envs_dir`. +.. envvar:: PLATFORMIO_DATA_DIR + +Allows to override :ref:`projectconf` option :ref:`projectconf_pio_data_dir`. + Builder ------- diff --git a/docs/platforms/creating_platform.rst b/docs/platforms/creating_platform.rst index 56304512..884aed97 100644 --- a/docs/platforms/creating_platform.rst +++ b/docs/platforms/creating_platform.rst @@ -113,6 +113,9 @@ Packages * - ``tool-micronucleus`` - `Micronucleus `_ + * - ``tool-mkspiffs`` + - `Tool to build and unpack SPIFFS images `_ + * - ``tool-mspdebug`` - `MSPDebug `_ diff --git a/docs/platforms/espressif.rst b/docs/platforms/espressif.rst index c745b855..7a7f8acc 100644 --- a/docs/platforms/espressif.rst +++ b/docs/platforms/espressif.rst @@ -28,21 +28,24 @@ Packages * - Name - Contents + * - ``toolchain-xtensa`` + - `xtensa-gcc `_, `GDB `_ + + * - ``tool-esptool`` + - `esptool-ck `_ + + * - ``tool-mkspiffs`` + - `Tool to build and unpack SPIFFS images `_ + + * - ``framework-arduinoespressif`` + - `Arduino Wiring-based Framework (ESP8266 Core) `_ + * - ``ldscripts`` - `Linker Scripts `_ * - ``sdk-esp8266`` - `ESP8266 SDK `_ - * - ``tool-esptool`` - - `esptool-ck `_ - - * - ``framework-arduinoespressif`` - - `Arduino Wiring-based Framework (ESP8266 Core) `_ - - * - ``toolchain-xtensa`` - - `xtensa-gcc `_, `GDB `_ - .. warning:: **Linux Users:** Don't forget to install "udev" rules file `99-platformio-udev.rules `_ (an instruction is located in the file). diff --git a/docs/platforms/espressif_extra.rst b/docs/platforms/espressif_extra.rst index 2fe42b73..b786d7ae 100644 --- a/docs/platforms/espressif_extra.rst +++ b/docs/platforms/espressif_extra.rst @@ -23,6 +23,7 @@ from :ref:`projectconf` upload_speed = 9600 +.. _platform_espressif_customflash: Custom Flash Size ----------------- @@ -39,8 +40,6 @@ The list with preconfigured LD scripts is located in public repository * ``esp8266.flash.2m.ld`` 2M (1M SPIFFS) * ``esp8266.flash.4m1.ld`` 4M (1M SPIFFS) * ``esp8266.flash.4m.ld`` 4M (3M SPIFFS) -* ``esp8266.flash.8m.ld`` 8M (7M SPIFFS) -* ``esp8266.flash.16m.ld`` 16M (15M SPIFFS) To override default LD script please use :ref:`projectconf_build_flags` from :ref:`projectconf`. @@ -50,6 +49,18 @@ To override default LD script please use :ref:`projectconf_build_flags` from [env:myenv] build_flags = -Wl,-Tesp8266.flash.4m.ld +.. _platform_espressif_uploadfs: + +Uploading files to file system SPIFFS +------------------------------------- + +1. Put files to :ref:`projectconf_pio_data_dir` +2. Run target ``uploadfs`` via :option:`platformio run --target` command. + +By default, will be used default LD Script for the board where is specified +SPIFFS flash data (start, end, page, block). You can override it using +:ref:`platform_espressif_customflash`. + Over-the-Air (OTA) update ------------------------- diff --git a/docs/projectconf.rst b/docs/projectconf.rst index ee79d7e8..8e809dc1 100644 --- a/docs/projectconf.rst +++ b/docs/projectconf.rst @@ -115,6 +115,18 @@ This option can be overridden by global environment variable are defined in :ref:`projectconf`, then **TRY TO DELETE** this folder. In this situation you will remove all cached files without any risk. +.. _projectconf_pio_data_dir: + +``data_dir`` +^^^^^^^^^^^^ + +Data directory to store contents and :ref:`platform_espressif_uploadfs`. + +A default value is ``%project_dir%/data``. + +This option can be overridden by global environment variable +:envvar:`PLATFORMIO_DATA_DIR`. + [env:NAME] ---------- @@ -463,8 +475,9 @@ Example, specify own upload command for :ref:`platform_atmelavr`: ^^^^^^^^^^^ A list with targets which will be processed by :ref:`cmd_run` command by -default. You can enter more than one target separated with "space". Which -targets are supported is described in :option:`platformio run --target`. +default. You can enter more than one target separated with "space". + +The list with available targets is located in :option:`platformio run --target`. **Tip!** You can use these targets like an option to :option:`platformio run --target` command. For example: diff --git a/docs/userguide/cmd_run.rst b/docs/userguide/cmd_run.rst index 0e24018b..ce6f43b7 100644 --- a/docs/userguide/cmd_run.rst +++ b/docs/userguide/cmd_run.rst @@ -53,6 +53,7 @@ Pre-built targets: * ``program`` firmware "auto-uploading" for embedded platforms using external programmer (available only for :ref:`platform_atmelavr`) * ``uploadlazy`` upload existing firmware without project rebuilding +* ``uploadfs`` :ref:`platform_espressif_uploadfs` * ``envdump`` dump current build environment * ``size`` print the size of the sections in a firmware/program diff --git a/platformio/__init__.py b/platformio/__init__.py index 914802b5..71776dd6 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (2, 7, "0.dev1") +VERSION = (2, 7, "0.dev2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/builder/main.py b/platformio/builder/main.py index e0b7235a..0c66ff6d 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -76,6 +76,7 @@ DefaultEnvironment( PROJECT_DIR=util.get_project_dir(), PROJECTLIB_DIR=util.get_projectlib_dir(), PROJECTSRC_DIR=util.get_projectsrc_dir(), + PROJECTDATA_DIR=util.get_projectdata_dir(), PIOENVS_DIR=util.get_pioenvs_dir(), PIOBUILDER_DIR=join(util.get_source_dir(), "builder"), diff --git a/platformio/builder/scripts/espressif.py b/platformio/builder/scripts/espressif.py index 4e3c62b3..2cb1a73e 100644 --- a/platformio/builder/scripts/espressif.py +++ b/platformio/builder/scripts/espressif.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=redefined-outer-name + """ Builder for Espressif MCUs """ @@ -24,22 +26,14 @@ from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild, Builder, Default, DefaultEnvironment) -def BeforeUpload(target, source, env): # pylint: disable=W0613,W0621 - env.AutodetectUploadPort() - - -def _get_flash_size(env): # pylint: disable=redefined-outer-name +def _get_flash_size(env): # use board's flash size by default board_max_size = int( env.get("BOARD_OPTIONS", {}).get("upload", {}).get("maximum_size", 0)) - # check if user overrides - for f in env.get("LINKFLAGS", []): - if "-Wl,-T" not in f: - continue - match = re.search(r"-Wl,-T.*\.flash\.(\d+)(m|k).*\.ld", env.subst(f)) - if not match: - continue + # check if user overrides LD Script + match = re.search(r"\.flash\.(\d+)(m|k).*\.ld", env.GetActualLDScript()) + if match: if match.group(2) == "k": board_max_size = int(match.group(1)) * 1024 elif match.group(2) == "m": @@ -106,9 +100,13 @@ env.Replace( "-Wl,--gc-sections" ], - SIZEPRINTCMD='"$SIZETOOL" -B -d $SOURCES', + # + # Upload + # UPLOADER=join("$PIOPACKAGES_DIR", "tool-esptool", "esptool"), + UPLOADEROTA=join("$PLATFORMFW_DIR", "tools", "espota.py"), + UPLOADERFLAGS=[ "-vv", "-cd", "${BOARD_OPTIONS['upload']['resetmethod']}", @@ -116,7 +114,27 @@ env.Replace( "-cp", "$UPLOAD_PORT", "-cf", "$SOURCE" ], + UPLOADERFSFLAGS=[ + "$UPLOADERFLAGS", + "-ca", "${int(SPIFFS_START, 16)}" + ], + UPLOADEROTAFLAGS=[ + "--debug", + "--progress", + "-i", "$UPLOAD_PORT", + "-f", "$SOURCE" + ], + UPLOADCMD='"$UPLOADER" $UPLOADERFLAGS', + UPLOADFSCMD='"$UPLOADER" $UPLOADERFSFLAGS', + UPLOADOTACMD='"$UPLOADEROTA" $UPLOADEROTAFLAGS', + + # + # Misc + # + + MKSPIFFSTOOL=join("$PIOPACKAGES_DIR", "tool-mkspiffs", "mkspiffs"), + SIZEPRINTCMD='"$SIZETOOL" -B -d $SOURCES', PROGNAME="firmware", PROGSUFFIX=".elf" @@ -149,6 +167,48 @@ env.Append( ) ) + +# +# SPIFFS +# + +def _fetch_spiffs_size(target, source, env): + spiffs_re = re.compile( + r"PROVIDE\s*\(\s*_SPIFFS_(\w+)\s*=\s*(0x[\dA-F]+)\s*\)") + with open(env.GetActualLDScript()) as f: + for line in f.readlines(): + match = spiffs_re.search(line) + if not match: + continue + env["SPIFFS_%s" % match.group(1).upper()] = match.group(2) + + assert all([k in env for k in ["SPIFFS_START", "SPIFFS_END", "SPIFFS_PAGE", + "SPIFFS_BLOCK"]]) + return (target, source) + + +env.Append( + BUILDERS=dict( + DataToBin=Builder( + action=" ".join([ + '"$MKSPIFFSTOOL"', + "-c", "$SOURCES", + "-p", "${int(SPIFFS_PAGE, 16)}", + "-b", "${int(SPIFFS_BLOCK, 16)}", + "-s", "${int(SPIFFS_END, 16) - int(SPIFFS_START, 16)}", + "$TARGET" + ]), + emitter=_fetch_spiffs_size, + source_factory=env.Dir, + suffix=".bin" + ) + ) +) + +# +# Framework and SDK specific configuration +# + if "FRAMEWORK" in env: env.Append( LINKFLAGS=[ @@ -161,14 +221,7 @@ if "FRAMEWORK" in env: try: if env.get("UPLOAD_PORT") and socket.inet_aton(env.get("UPLOAD_PORT")): env.Replace( - UPLOADEROTA=join("$PLATFORMFW_DIR", "tools", "espota.py"), - UPLOADERFLAGS=[ - "--debug", - "--progress", - "-i", "$UPLOAD_PORT", - "-f", "$SOURCE" - ], - UPLOADCMD='"$UPLOADEROTA" $UPLOADERFLAGS' + UPLOADCMD="$UPLOADOTACMD" ) except socket.error: pass @@ -256,9 +309,21 @@ AlwaysBuild(target_size) # Target: Upload firmware # -upload = env.Alias(["upload", "uploadlazy"], target_firm, - [BeforeUpload, "$UPLOADCMD"]) -AlwaysBuild(upload) +target_upload = env.Alias( + ["upload", "uploadlazy"], target_firm, + [lambda target, source, env: env.AutodetectUploadPort(), "$UPLOADCMD"]) +env.AlwaysBuild(target_upload) + +# +# Target: Upload SPIFFS image +# + +target_mkspiffs = env.DataToBin(join("$BUILD_DIR", "spiffs_image"), + "$PROJECTDATA_DIR") +target_uploadfs = env.Alias( + "uploadfs", target_mkspiffs, + [lambda target, source, env: env.AutodetectUploadPort(), "$UPLOADFSCMD"]) +env.AlwaysBuild(target_mkspiffs, target_uploadfs) # # Target: Define targets diff --git a/platformio/builder/tools/piomisc.py b/platformio/builder/tools/piomisc.py index 94743cec..ee6a20eb 100644 --- a/platformio/builder/tools/piomisc.py +++ b/platformio/builder/tools/piomisc.py @@ -18,7 +18,7 @@ import atexit import re from glob import glob from os import environ, remove -from os.path import basename, join +from os.path import basename, isfile, join from platformio.util import exec_command, where_is_program @@ -190,6 +190,19 @@ def GetCompilerType(env): return None +def GetActualLDScript(env): + for f in env.get("LINKFLAGS", []): + if f.startswith("-Wl,-T"): + script = env.subst(f[6:].replace('"', "").strip()) + if isfile(script): + return script + for d in env.get("LIBPATH", []): + path = join(env.subst(d), script) + if isfile(path): + return path + return None + + def exists(_): return True @@ -198,4 +211,5 @@ def generate(env): env.AddMethod(ConvertInoToCpp) env.AddMethod(DumpIDEData) env.AddMethod(GetCompilerType) + env.AddMethod(GetActualLDScript) return env diff --git a/platformio/platforms/base.py b/platformio/platforms/base.py index fc27e13a..bd7b4c10 100644 --- a/platformio/platforms/base.py +++ b/platformio/platforms/base.py @@ -146,6 +146,10 @@ PLATFORM_PACKAGES = { ], "tool-rfdloader": [ ("rfdloader", "https://github.com/RFduino/RFduino") + ], + "tool-mkspiffs": [ + ("Tool to build and unpack SPIFFS images", + "https://github.com/igrr/mkspiffs") ] } diff --git a/platformio/platforms/espressif.py b/platformio/platforms/espressif.py index a6784e1a..abd93dcb 100644 --- a/platformio/platforms/espressif.py +++ b/platformio/platforms/espressif.py @@ -41,6 +41,10 @@ class EspressifPlatform(BasePlatform): "default": True }, + "tool-mkspiffs": { + "alias": "uploader" + }, + "sdk-esp8266": { }, diff --git a/platformio/util.py b/platformio/util.py index 816627d8..8a8bce42 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -200,6 +200,13 @@ def get_pioenvs_dir(): ) +def get_projectdata_dir(): + return _get_projconf_option_dir( + "data_dir", + join(get_project_dir(), "data") + ) + + def get_project_config(): path = join(get_project_dir(), "platformio.ini") if not isfile(path):