Implement uploading files to file system SPIFFS for ESP8266 // Issue #382

HISTORY.rst

@@ -10,6 +10,8 @@ PlatformIO 2.0
 * Moved SCons to PlatformIO packages. PlatformIO does not require SCons to be
   installed in your system. Significantly simplified installation process of
   PlatformIO. ``pip install platformio`` rocks!
+* Implemented uploading files to file system SPIFFS for ESP8266
+  (`issue #382 <https://github.com/platformio/platformio/issues/382>`_)
 * Added support for the new Adafruit boards Bluefruit Micro and Feather
   (`issue #403 <https://github.com/platformio/platformio/issues/403>`_)
 * Added support for RFDuino

docs/envvars.rst

@@ -55,6 +55,10 @@ Allows to override :ref:`projectconf` option :ref:`projectconf_pio_src_dir`.
 
 Allows to override :ref:`projectconf` option :ref:`projectconf_pio_envs_dir`.
 
+.. envvar:: PLATFORMIO_DATA_DIR
+
+Allows to override :ref:`projectconf` option :ref:`projectconf_pio_data_dir`.
+
 
 Builder
 -------

docs/platforms/creating_platform.rst

@@ -113,6 +113,9 @@ Packages
     * - ``tool-micronucleus``
       - `Micronucleus <https://github.com/micronucleus/micronucleus>`_
 
+    * - ``tool-mkspiffs``
+      - `Tool to build and unpack SPIFFS images <https://github.com/igrr/mkspiffs>`_
+
     * - ``tool-mspdebug``
       - `MSPDebug <http://mspdebug.sourceforge.net/>`_
 

docs/platforms/espressif.rst

@@ -28,21 +28,24 @@ Packages
     * - Name
       - Contents
 
+    * - ``toolchain-xtensa``
+      - `xtensa-gcc <https://github.com/jcmvbkbc/gcc-xtensa>`_, `GDB <http://www.gnu.org/software/gdb/>`_
+
+    * - ``tool-esptool``
+      - `esptool-ck <https://github.com/igrr/esptool-ck>`_
+
+    * - ``tool-mkspiffs``
+      - `Tool to build and unpack SPIFFS images <https://github.com/igrr/mkspiffs>`_
+
+    * - ``framework-arduinoespressif``
+      - `Arduino Wiring-based Framework (ESP8266 Core) <https://github.com/esp8266/Arduino>`_
+
     * - ``ldscripts``
       - `Linker Scripts <https://sourceware.org/binutils/docs/ld/Scripts.html>`_
 
     * - ``sdk-esp8266``
       - `ESP8266 SDK <http://bbs.espressif.com>`_
 
-    * - ``tool-esptool``
-      - `esptool-ck <https://github.com/igrr/esptool-ck>`_
-
-    * - ``framework-arduinoespressif``
-      - `Arduino Wiring-based Framework (ESP8266 Core) <https://github.com/esp8266/Arduino>`_
-
-    * - ``toolchain-xtensa``
-      - `xtensa-gcc <https://github.com/jcmvbkbc/gcc-xtensa>`_, `GDB <http://www.gnu.org/software/gdb/>`_
-
 .. warning::
     **Linux Users:** Don't forget to install "udev" rules file
     `99-platformio-udev.rules <https://github.com/platformio/platformio/blob/develop/scripts/99-platformio-udev.rules>`_ (an instruction is located in the file).

docs/platforms/espressif_extra.rst

@@ -23,6 +23,7 @@ from :ref:`projectconf`
 
     upload_speed = 9600
 
+.. _platform_espressif_customflash:
 
 Custom Flash Size
 -----------------
@@ -39,8 +40,6 @@ The list with preconfigured LD scripts is located in public repository
 * ``esp8266.flash.2m.ld`` 2M (1M SPIFFS)
 * ``esp8266.flash.4m1.ld`` 4M (1M SPIFFS)
 * ``esp8266.flash.4m.ld`` 4M (3M SPIFFS)
-* ``esp8266.flash.8m.ld`` 8M (7M SPIFFS)
-* ``esp8266.flash.16m.ld`` 16M (15M SPIFFS)
 
 To override default LD script please use :ref:`projectconf_build_flags` from
 :ref:`projectconf`.
@@ -50,6 +49,18 @@ To override default LD script please use :ref:`projectconf_build_flags` from
     [env:myenv]
     build_flags = -Wl,-Tesp8266.flash.4m.ld
 
+.. _platform_espressif_uploadfs:
+
+Uploading files to file system SPIFFS
+-------------------------------------
+
+1. Put files to :ref:`projectconf_pio_data_dir`
+2. Run target ``uploadfs`` via  :option:`platformio run --target` command.
+
+By default, will be used default LD Script for the board where is specified
+SPIFFS flash data (start, end, page, block). You can override it using
+:ref:`platform_espressif_customflash`.
+
 Over-the-Air (OTA) update
 -------------------------
 

docs/projectconf.rst

@@ -115,6 +115,18 @@ This option can be overridden by global environment variable
     are defined in :ref:`projectconf`, then **TRY TO DELETE** this folder. In
     this situation you will remove all cached files without any risk.
 
+.. _projectconf_pio_data_dir:
+
+``data_dir``
+^^^^^^^^^^^^
+
+Data directory to store contents and :ref:`platform_espressif_uploadfs`.
+
+A default value is ``%project_dir%/data``.
+
+This option can be overridden by global environment variable
+:envvar:`PLATFORMIO_DATA_DIR`.
+
 [env:NAME]
 ----------
 
@@ -463,8 +475,9 @@ Example, specify own upload command for :ref:`platform_atmelavr`:
 ^^^^^^^^^^^
 
 A list with targets which will be processed by :ref:`cmd_run` command by
-default. You can enter more than one target separated with "space". Which
-targets are supported is described in :option:`platformio run --target`.
+default. You can enter more than one target separated with "space".
+
+The list with available targets is located in :option:`platformio run --target`.
 
 **Tip!** You can use these targets like an option to
 :option:`platformio run --target` command. For example:

docs/userguide/cmd_run.rst

@@ -53,6 +53,7 @@ Pre-built targets:
 * ``program`` firmware "auto-uploading" for embedded platforms using external
   programmer (available only for :ref:`platform_atmelavr`)
 * ``uploadlazy`` upload existing firmware without project rebuilding
+* ``uploadfs`` :ref:`platform_espressif_uploadfs`
 * ``envdump`` dump current build environment
 * ``size`` print the size of the sections in a firmware/program
 

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (2, 7, "0.dev1")
+VERSION = (2, 7, "0.dev2")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/builder/main.py

@@ -76,6 +76,7 @@ DefaultEnvironment(
     PROJECT_DIR=util.get_project_dir(),
     PROJECTLIB_DIR=util.get_projectlib_dir(),
     PROJECTSRC_DIR=util.get_projectsrc_dir(),
+    PROJECTDATA_DIR=util.get_projectdata_dir(),
     PIOENVS_DIR=util.get_pioenvs_dir(),
 
     PIOBUILDER_DIR=join(util.get_source_dir(), "builder"),

platformio/builder/scripts/espressif.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# pylint: disable=redefined-outer-name
+
 """
     Builder for Espressif MCUs
 """
@@ -24,22 +26,14 @@ from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild, Builder, Default,
                           DefaultEnvironment)
 
 
-def BeforeUpload(target, source, env):  # pylint: disable=W0613,W0621
-    env.AutodetectUploadPort()
-
-
-def _get_flash_size(env):  # pylint: disable=redefined-outer-name
+def _get_flash_size(env):
     # use board's flash size by default
     board_max_size = int(
         env.get("BOARD_OPTIONS", {}).get("upload", {}).get("maximum_size", 0))
 
-    # check if user overrides
-    for f in env.get("LINKFLAGS", []):
-        if "-Wl,-T" not in f:
-            continue
-        match = re.search(r"-Wl,-T.*\.flash\.(\d+)(m|k).*\.ld", env.subst(f))
-        if not match:
-            continue
+    # check if user overrides LD Script
+    match = re.search(r"\.flash\.(\d+)(m|k).*\.ld", env.GetActualLDScript())
+    if match:
         if match.group(2) == "k":
             board_max_size = int(match.group(1)) * 1024
         elif match.group(2) == "m":
@@ -106,9 +100,13 @@ env.Replace(
         "-Wl,--gc-sections"
     ],
 
-    SIZEPRINTCMD='"$SIZETOOL" -B -d $SOURCES',
+    #
+    # Upload
+    #
 
     UPLOADER=join("$PIOPACKAGES_DIR", "tool-esptool", "esptool"),
+    UPLOADEROTA=join("$PLATFORMFW_DIR", "tools", "espota.py"),
+
     UPLOADERFLAGS=[
         "-vv",
         "-cd", "${BOARD_OPTIONS['upload']['resetmethod']}",
@@ -116,7 +114,27 @@ env.Replace(
         "-cp", "$UPLOAD_PORT",
         "-cf", "$SOURCE"
     ],
+    UPLOADERFSFLAGS=[
+        "$UPLOADERFLAGS",
+        "-ca", "${int(SPIFFS_START, 16)}"
+    ],
+    UPLOADEROTAFLAGS=[
+        "--debug",
+        "--progress",
+        "-i", "$UPLOAD_PORT",
+        "-f", "$SOURCE"
+    ],
+
     UPLOADCMD='"$UPLOADER" $UPLOADERFLAGS',
+    UPLOADFSCMD='"$UPLOADER" $UPLOADERFSFLAGS',
+    UPLOADOTACMD='"$UPLOADEROTA" $UPLOADEROTAFLAGS',
+
+    #
+    # Misc
+    #
+
+    MKSPIFFSTOOL=join("$PIOPACKAGES_DIR", "tool-mkspiffs", "mkspiffs"),
+    SIZEPRINTCMD='"$SIZETOOL" -B -d $SOURCES',
 
     PROGNAME="firmware",
     PROGSUFFIX=".elf"
@@ -149,6 +167,48 @@ env.Append(
     )
 )
 
+
+#
+# SPIFFS
+#
+
+def _fetch_spiffs_size(target, source, env):
+    spiffs_re = re.compile(
+        r"PROVIDE\s*\(\s*_SPIFFS_(\w+)\s*=\s*(0x[\dA-F]+)\s*\)")
+    with open(env.GetActualLDScript()) as f:
+        for line in f.readlines():
+            match = spiffs_re.search(line)
+            if not match:
+                continue
+            env["SPIFFS_%s" % match.group(1).upper()] = match.group(2)
+
+    assert all([k in env for k in ["SPIFFS_START", "SPIFFS_END", "SPIFFS_PAGE",
+                                   "SPIFFS_BLOCK"]])
+    return (target, source)
+
+
+env.Append(
+    BUILDERS=dict(
+        DataToBin=Builder(
+            action=" ".join([
+                '"$MKSPIFFSTOOL"',
+                "-c", "$SOURCES",
+                "-p", "${int(SPIFFS_PAGE, 16)}",
+                "-b", "${int(SPIFFS_BLOCK, 16)}",
+                "-s", "${int(SPIFFS_END, 16) - int(SPIFFS_START, 16)}",
+                "$TARGET"
+            ]),
+            emitter=_fetch_spiffs_size,
+            source_factory=env.Dir,
+            suffix=".bin"
+        )
+    )
+)
+
+#
+# Framework and SDK specific configuration
+#
+
 if "FRAMEWORK" in env:
     env.Append(
         LINKFLAGS=[
@@ -161,14 +221,7 @@ if "FRAMEWORK" in env:
     try:
         if env.get("UPLOAD_PORT") and socket.inet_aton(env.get("UPLOAD_PORT")):
             env.Replace(
-                UPLOADEROTA=join("$PLATFORMFW_DIR", "tools", "espota.py"),
-                UPLOADERFLAGS=[
-                    "--debug",
-                    "--progress",
-                    "-i", "$UPLOAD_PORT",
-                    "-f", "$SOURCE"
-                ],
-                UPLOADCMD='"$UPLOADEROTA" $UPLOADERFLAGS'
+                UPLOADCMD="$UPLOADOTACMD"
             )
     except socket.error:
         pass
@@ -256,9 +309,21 @@ AlwaysBuild(target_size)
 # Target: Upload firmware
 #
 
-upload = env.Alias(["upload", "uploadlazy"], target_firm,
-                   [BeforeUpload, "$UPLOADCMD"])
-AlwaysBuild(upload)
+target_upload = env.Alias(
+    ["upload", "uploadlazy"], target_firm,
+    [lambda target, source, env: env.AutodetectUploadPort(), "$UPLOADCMD"])
+env.AlwaysBuild(target_upload)
+
+#
+# Target: Upload SPIFFS image
+#
+
+target_mkspiffs = env.DataToBin(join("$BUILD_DIR", "spiffs_image"),
+                                "$PROJECTDATA_DIR")
+target_uploadfs = env.Alias(
+    "uploadfs", target_mkspiffs,
+    [lambda target, source, env: env.AutodetectUploadPort(), "$UPLOADFSCMD"])
+env.AlwaysBuild(target_mkspiffs, target_uploadfs)
 
 #
 # Target: Define targets

platformio/builder/tools/piomisc.py

@@ -18,7 +18,7 @@ import atexit
 import re
 from glob import glob
 from os import environ, remove
-from os.path import basename, join
+from os.path import basename, isfile, join
 
 from platformio.util import exec_command, where_is_program
 
@@ -190,6 +190,19 @@ def GetCompilerType(env):
     return None
 
 
+def GetActualLDScript(env):
+    for f in env.get("LINKFLAGS", []):
+        if f.startswith("-Wl,-T"):
+            script = env.subst(f[6:].replace('"', "").strip())
+            if isfile(script):
+                return script
+            for d in env.get("LIBPATH", []):
+                path = join(env.subst(d), script)
+                if isfile(path):
+                    return path
+    return None
+
+
 def exists(_):
     return True
 
@@ -198,4 +211,5 @@ def generate(env):
     env.AddMethod(ConvertInoToCpp)
     env.AddMethod(DumpIDEData)
     env.AddMethod(GetCompilerType)
+    env.AddMethod(GetActualLDScript)
     return env

platformio/platforms/base.py

@@ -146,6 +146,10 @@ PLATFORM_PACKAGES = {
     ],
     "tool-rfdloader": [
         ("rfdloader", "https://github.com/RFduino/RFduino")
+    ],
+    "tool-mkspiffs": [
+        ("Tool to build and unpack SPIFFS images",
+         "https://github.com/igrr/mkspiffs")
     ]
 }
 

platformio/platforms/espressif.py

@@ -41,6 +41,10 @@ class EspressifPlatform(BasePlatform):
             "default": True
         },
 
+        "tool-mkspiffs": {
+            "alias": "uploader"
+        },
+
         "sdk-esp8266": {
         },
 

platformio/util.py

@@ -200,6 +200,13 @@ def get_pioenvs_dir():
     )
 
 
+def get_projectdata_dir():
+    return _get_projconf_option_dir(
+        "data_dir",
+        join(get_project_dir(), "data")
+    )
+
+
 def get_project_config():
     path = join(get_project_dir(), "platformio.ini")
     if not isfile(path):