Merge branch 'feature/pio-remote' into develop

HISTORY.rst

@@ -4,7 +4,7 @@ Release Notes
 PlatformIO 3.0
 --------------
 
-3.1.1 (2016-??-??)
+3.2.0 (2016-??-??)
 ~~~~~~~~~~~~~~~~~~
 
 * Improved detecting of ARM mbed media disk for uploading
@@ -14,13 +14,40 @@ PlatformIO 3.0
 
 -------
 
+* Development platform `Atmel SAM <https://github.com/platformio/platform-atmelsam>`__
+
+  + Updated ARM mbed OS to 5.1.4/rev126
+
 * Development platform `Espressif 8266 <https://github.com/platformio/platform-espressif8266>`__
 
   + Add support for ESPrectro board
+  + Additional target "buildfs" to accompany "uploadfs"
+    (`issue #6 <https://github.com/platformio/platform-espressif8266/issues/6>`__)
+
+* Development platform `Freescale Kinetis <https://github.com/platformio/platform-freescalekinetis>`__
+
+  + Updated ARM mbed OS to 5.1.4/rev126
+
+* Development platform `Nordic nRF51 <https://github.com/platformio/platform-nordicnrf51>`__
+
+  + Updated ARM mbed OS to 5.1.4/rev126
+
+* Development platform `NXP LPC <https://github.com/platformio/platform-nxplpc>`__
+
+  + Updated ARM mbed OS to 5.1.4/rev126
+
+* Development platform `Silicon Labs EFM32 <https://github.com/platformio/platform-siliconlabsefm32>`__
+
+  + Updated ARM mbed OS to 5.1.4/rev126
 
 * Development platform `ST STM32 <https://github.com/platformio/platform-ststm32>`__
 
   + Added support for new boards: ST 32F769IDISCOVERY
+  + Updated ARM mbed OS to 5.1.4/rev126
+
+* Development platform `Teensy <https://github.com/platformio/platform-teensy>`__
+
+  + Updated ARM mbed OS to 5.1.4/rev126
 
 3.1.0 (2016-09-19)
 ~~~~~~~~~~~~~~~~~~

docs/platforms/espressif8266_extra.rst

@@ -117,10 +117,10 @@ Uploading files to file system SPIFFS
     Please make sure to read `ESP8266 Flash layout <https://github.com/esp8266/Arduino/blob/master/doc/filesystem.md#flash-layout>`_
     information first.
 
-1. Initialise project :ref:`cmd_init` (if you have not initialized yet)
+1. Initialize project :ref:`cmd_init` (if you have not initialized yet)
 2. Create ``data`` folder (it should be on the same level as ``src`` folder)
    and put files here. Also, you can specify own location for :ref:`projectconf_pio_data_dir`
-3. Run target ``uploadfs`` using  :option:`platformio run --target` command.
+3. Run ``buildfs`` or ``uploadfs`` target using :option:`platformio run --target` command.
 
 To upload SPIFFS image using OTA update please specify ``upload_port`` /
 ``--upload-port`` as IP address or mDNS host name (ending with the ``*.local``).

docs/userguide/cmd_run.rst

@@ -55,7 +55,6 @@ Pre-built targets:
 * ``upload`` firmware "auto-uploading" for embedded platforms
 * ``program`` firmware "auto-uploading" for embedded platforms using external
   programmer (available only for :ref:`platform_atmelavr`)
-* ``uploadlazy`` upload existing firmware without project rebuilding
 * ``uploadfs`` :ref:`platform_espressif_uploadfs`
 * ``envdump`` dump current build environment
 * ``size`` print the size of the sections in a firmware/program

platformio/__init__.py

@@ -14,7 +14,7 @@
 
 import sys
 
-VERSION = (3, 1, "1a2")
+VERSION = (3, 2, "0a2")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/builder/tools/piomisc.py

@@ -292,16 +292,23 @@ def GetCompilerType(env):
 
 
 def GetActualLDScript(env):
+
+    def _lookup_in_ldpath(script):
+        for d in env.get("LIBPATH", []):
+            path = join(env.subst(d), script)
+            if isfile(path):
+                return path
+        return None
+
     script = None
     for f in env.get("LINKFLAGS", []):
         if f.startswith("-Wl,-T"):
             script = env.subst(f[6:].replace('"', "").strip())
             if isfile(script):
                 return script
-            for d in env.get("LIBPATH", []):
+            path = _lookup_in_ldpath(script)
-                path = join(env.subst(d), script)
+            if path:
-                if isfile(path):
+                return path
-                    return path
 
     if script:
         sys.stderr.write(
@@ -309,7 +316,13 @@ def GetActualLDScript(env):
             (script, env.subst("$LIBPATH")))
         env.Exit(1)
 
-    return None
+    if not script and "LDSCRIPT_PATH" in env:
+        path = _lookup_in_ldpath(env['LDSCRIPT_PATH'])
+        if path:
+            return path
+
+    sys.stderr.write("Error: Could not find LD script\n")
+    env.Exit(1)
 
 
 def VerboseAction(_, act, actstr):

platformio/builder/tools/pioupload.py

@@ -21,6 +21,7 @@ from platform import system
 from shutil import copyfile
 from time import sleep
 
+from SCons.Node.Alias import Alias
 from serial import Serial
 
 from platformio import util
@@ -160,7 +161,8 @@ def CheckUploadSize(_, target, source, env):  # pylint: disable=W0613,W0621
 
     sysenv = environ.copy()
     sysenv['PATH'] = str(env['ENV']['PATH'])
-    cmd = [env.subst("$SIZETOOL"), "-B", str(target[0])]
+    cmd = [env.subst("$SIZETOOL"), "-B",
+           str(source[0] if isinstance(target[0], Alias) else target[0])]
     result = util.exec_command(cmd, env=sysenv)
     if result['returncode'] != 0:
         return

platformio/builder/tools/platformio.py

@@ -21,7 +21,8 @@ from os import sep, walk
 from os.path import basename, dirname, isdir, join, realpath
 
 from SCons.Action import Action
-from SCons.Script import COMMAND_LINE_TARGETS, DefaultEnvironment, SConscript
+from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild,
+                          DefaultEnvironment, SConscript)
 from SCons.Util import case_sensitive_suffixes
 
 from platformio.util import pioversion_to_intstr
@@ -97,9 +98,10 @@ def BuildProgram(env):
     program = env.Program(
         join("$BUILD_DIR", env.subst("$PROGNAME")), env['PIOBUILDFILES'])
 
-    if set(["upload", "uploadlazy", "program"]) & set(COMMAND_LINE_TARGETS):
+    checksize_action = Action(env.CheckUploadSize, "Checking program size")
-        env.AddPostAction(program, Action(env.CheckUploadSize,
+    AlwaysBuild(env.Alias("checkprogsize", program, checksize_action))
-                                          "Checking program size $TARGET"))
+    if set(["upload", "program"]) & set(COMMAND_LINE_TARGETS):
+        env.AddPostAction(program, checksize_action)
 
     return program
 
@@ -226,7 +228,7 @@ def CollectBuildFiles(env,
 
 
 def BuildFrameworks(env, frameworks):
-    if not frameworks or "uploadlazy" in COMMAND_LINE_TARGETS:
+    if not frameworks:
         return
 
     if "BOARD" not in env:

platformio/commands/remote.py

@@ -0,0 +1,245 @@
+# Copyright 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import threading
+from os import getcwd
+from os.path import isfile, join
+from tempfile import mkdtemp
+from time import sleep
+
+import click
+from serial import VERSION as PYSERIAL_VERSION
+
+from platformio import util
+from platformio.commands.device import device_monitor as cmd_device_monitor
+from platformio.pioplus import pioplus_call
+
+# pylint: disable=unused-argument
+
+
+@click.group("remote", short_help="PIO Remote")
+@click.option("-a", "--agent", multiple=True)
+def cli(**kwargs):
+    pass
+
+
+@cli.group("agent", short_help="Start new agent or list active")
+def remote_agent():
+    pass
+
+
+@remote_agent.command("start", short_help="Start agent")
+@click.option("-n", "--name")
+@click.option("-s", "--share", multiple=True, metavar="E-MAIL")
+def remote_agent_start(**kwargs):
+    pioplus_call(sys.argv[1:])
+
+
+@remote_agent.command("list", short_help="List active agents")
+def remote_agent_list():
+    pioplus_call(sys.argv[1:])
+
+
+@cli.command("run", short_help="Process project environments")
+@click.option("-e", "--environment", multiple=True)
+@click.option("-t", "--target", multiple=True)
+@click.option("--upload-port")
+@click.option(
+    "-d",
+    "--project-dir",
+    default=getcwd,
+    type=click.Path(
+        exists=True,
+        file_okay=True,
+        dir_okay=True,
+        writable=True,
+        resolve_path=True))
+@click.option("-s", "--silent", is_flag=True)
+@click.option("-v", "--verbose", is_flag=True)
+@click.option("-r", "--build-remotely", is_flag=True)
+def remote_run(**kwargs):
+    pioplus_call(sys.argv[1:])
+
+
+@cli.group("device", short_help="Monitor device or list existing")
+def remote_device():
+    pass
+
+
+@remote_device.command("list", short_help="List devices")
+@click.option("--json-output", is_flag=True)
+def device_list(json_output):
+    pioplus_call(sys.argv[1:])
+
+
+if int(PYSERIAL_VERSION[0]) == 3:
+
+    @remote_device.command("monitor", short_help="Monitor device (Serial)")
+    @click.option("--port", "-p", help="Port, a number or a device name")
+    @click.option(
+        "--baud",
+        "-b",
+        type=int,
+        default=9600,
+        help="Set baud rate, default=9600")
+    @click.option(
+        "--parity",
+        default="N",
+        type=click.Choice(["N", "E", "O", "S", "M"]),
+        help="Set parity, default=N")
+    @click.option(
+        "--rtscts",
+        is_flag=True,
+        help="Enable RTS/CTS flow control, default=Off")
+    @click.option(
+        "--xonxoff",
+        is_flag=True,
+        help="Enable software flow control, default=Off")
+    @click.option(
+        "--rts",
+        default=None,
+        type=click.Choice(["0", "1"]),
+        help="Set initial RTS line state")
+    @click.option(
+        "--dtr",
+        default=None,
+        type=click.Choice(["0", "1"]),
+        help="Set initial DTR line state")
+    @click.option(
+        "--echo", is_flag=True, help="Enable local echo, default=Off")
+    @click.option(
+        "--encoding",
+        default="UTF-8",
+        help="Set the encoding for the serial port (e.g. hexlify, "
+        "Latin1, UTF-8), default: UTF-8")
+    @click.option(
+        "--filter", "-f", multiple=True, help="Add text transformation")
+    @click.option(
+        "--eol",
+        default="CRLF",
+        type=click.Choice(["CR", "LF", "CRLF"]),
+        help="End of line mode, default=CRLF")
+    @click.option(
+        "--raw",
+        is_flag=True,
+        help="Do not apply any encodings/transformations")
+    @click.option(
+        "--exit-char",
+        type=int,
+        default=29,
+        help="ASCII code of special character that is used to exit "
+        "the application, default=29 (DEC)")
+    @click.option(
+        "--menu-char",
+        type=int,
+        default=20,
+        help="ASCII code of special character that is used to "
+        "control miniterm (menu), default=20 (DEC)")
+    @click.option(
+        "--quiet",
+        is_flag=True,
+        help="Diagnostics: suppress non-error messages, default=Off")
+    @click.pass_context
+    def device_monitor(ctx, **kwargs):
+        _device_monitor(ctx, **kwargs)
+else:
+
+    @remote_device.command("monitor", short_help="Monitor device (Serial)")
+    @click.option("--port", "-p", help="Port, a number or a device name")
+    @click.option(
+        "--baud",
+        "-b",
+        type=int,
+        default=9600,
+        help="Set baud rate, default=9600")
+    @click.option(
+        "--parity",
+        default="N",
+        type=click.Choice(["N", "E", "O", "S", "M"]),
+        help="Set parity, default=N")
+    @click.option(
+        "--rtscts",
+        is_flag=True,
+        help="Enable RTS/CTS flow control, default=Off")
+    @click.option(
+        "--xonxoff",
+        is_flag=True,
+        help="Enable software flow control, default=Off")
+    @click.option(
+        "--rts",
+        default=None,
+        type=click.Choice(["0", "1"]),
+        help="Set initial RTS line state, default=0")
+    @click.option(
+        "--dtr",
+        default=None,
+        type=click.Choice(["0", "1"]),
+        help="Set initial DTR line state, default=0")
+    @click.option(
+        "--echo", is_flag=True, help="Enable local echo, default=Off")
+    @click.option(
+        "--cr",
+        is_flag=True,
+        help="Do not send CR+LF, send CR only, default=Off")
+    @click.option(
+        "--lf",
+        is_flag=True,
+        help="Do not send CR+LF, send LF only, default=Off")
+    @click.option(
+        "--debug",
+        "-d",
+        count=True,
+        help="""Debug received data (escape non-printable chars)
+    # --debug can be given multiple times:
+    # 0: just print what is received
+    # 1: escape non-printable characters, do newlines as unusual
+    # 2: escape non-printable characters, newlines too
+    # 3: hex dump everything""")
+    @click.option(
+        "--exit-char",
+        type=int,
+        default=29,
+        help="ASCII code of special character that is used to exit "
+        "the application, default=29 (DEC)")
+    @click.option(
+        "--menu-char",
+        type=int,
+        default=20,
+        help="ASCII code of special character that is used to "
+        "control miniterm (menu), default=20 (DEC)")
+    @click.option(
+        "--quiet",
+        is_flag=True,
+        help="Diagnostics: suppress non-error messages, default=Off")
+    @click.pass_context
+    def device_monitor(ctx, **kwargs):
+        _device_monitor(ctx, **kwargs)
+
+
+def _device_monitor(ctx, **kwargs):
+    sock_dir = mkdtemp(suffix="pioplus")
+    sock_file = join(sock_dir, "sock")
+    try:
+        t = threading.Thread(
+            target=pioplus_call, args=(sys.argv[1:] + ["--sock", sock_dir], ))
+        t.start()
+        while t.is_alive() and not isfile(sock_file):
+            sleep(0.1)
+        if not t.is_alive():
+            return
+        ctx.invoke(cmd_device_monitor, port=open(sock_file).read())
+        t.join(2)
+    finally:
+        util.rmtree_(sock_dir)

platformio/commands/run.py

@@ -109,28 +109,7 @@ def cli(ctx, environment, target, upload_port, project_dir, silent, verbose,
 
         if len(results) > 1:
             click.echo()
-            print_header("[%s]" % click.style("SUMMARY"))
+            print_summary(results, start_time)
-
-            successed = True
-            for envname, status in results.items():
-                status_str = click.style("SUCCESS", fg="green")
-                if status is False:
-                    successed = False
-                    status_str = click.style("ERROR", fg="red")
-                elif status is None:
-                    status_str = click.style("SKIP", fg="yellow")
-
-                click.echo(
-                    "Environment %s\t[%s]" % (click.style(
-                        envname, fg="cyan"), status_str),
-                    err=status is False)
-
-            print_header(
-                "[%s] Took %.2f seconds" % ((click.style(
-                    "SUCCESS", fg="green",
-                    bold=True) if successed else click.style(
-                        "ERROR", fg="red", bold=True)), time() - start_time),
-                is_error=not successed)
 
         if any([r is False for r in results.values()]):
             raise exception.ReturnErrorCode()
@@ -326,6 +305,38 @@ def print_header(label, is_error=False):
     click.echo("%s %s %s" % (half_line, label, half_line), err=is_error)
 
 
+def print_summary(results, start_time):
+    print_header("[%s]" % click.style("SUMMARY"))
+
+    envname_max_len = 0
+    for envname in results:
+        if len(envname) > envname_max_len:
+            envname_max_len = len(envname)
+
+    successed = True
+    for envname, status in results.items():
+        status_str = click.style("SUCCESS", fg="green")
+        if status is False:
+            successed = False
+            status_str = click.style("ERROR", fg="red")
+        elif status is None:
+            status_str = click.style("SKIP", fg="yellow")
+
+        format_str = (
+            "Environment {0:<" + str(envname_max_len + 9) + "}\t[{1}]")
+        click.echo(
+            format_str.format(
+                click.style(
+                    envname, fg="cyan"), status_str),
+            err=status is False)
+
+    print_header(
+        "[%s] Took %.2f seconds" % ((click.style(
+            "SUCCESS", fg="green", bold=True) if successed else click.style(
+                "ERROR", fg="red", bold=True)), time() - start_time),
+        is_error=not successed)
+
+
 def check_project_defopts(config):
     if not config.has_section("platformio"):
         return True

platformio/exception.py

@@ -200,13 +200,6 @@ class CIBuildEnvsEmpty(PlatformioException):
         "predefined environments using `--project-conf` option"
 
 
-class TestDirEmpty(PlatformioException):
-
-    MESSAGE = "Test directory '{0}' is empty. More details about Unit "\
-              "Testing:\n http://docs.platformio.org/en/stable/platforms/"\
-              "unit_testing.html"
-
-
 class UpgradeError(PlatformioException):
 
     MESSAGE = """{0}

platformio/maintenance.py

@@ -51,8 +51,11 @@ def on_platformio_start(ctx, force, caller):
     if not caller:
         if getenv("PLATFORMIO_CALLER"):
             caller = getenv("PLATFORMIO_CALLER")
-        elif getenv("C9_UID"):
+        elif util.is_container():
-            caller = "C9"
+            if getenv("C9_UID"):
+                caller = "C9"
+            elif getenv("USER") == "cabox":
+                caller = "CA"
 
     app.set_session_var("command_ctx", ctx)
     app.set_session_var("force_option", force)
@@ -244,7 +247,7 @@ def check_platformio_upgrade():
         click.secho("pip install -U platformio", fg="cyan", nl=False)
         click.secho("` command.", fg="yellow")
     click.secho("Changes: ", fg="yellow", nl=False)
-    click.secho("http://docs.platformio.org/en/stable/history.html", fg="cyan")
+    click.secho("http://docs.platformio.org/en/latest/history.html", fg="cyan")
     click.echo("*" * terminal_width)
     click.echo("")
 

platformio/managers/platform.py

@@ -438,7 +438,7 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
         return names
 
     def configure_default_packages(self, variables, targets):
-        # enbale used frameworks
+        # enable used frameworks
         for framework in variables.get("pioframework", "").split(","):
             if not self.frameworks:
                 continue
@@ -453,7 +453,7 @@ class PlatformBase(PlatformPackagesMixin, PlatformRunMixin):
             for _name, _opts in self.packages.iteritems():
                 if _opts.get("type") == "uploader":
                     self.packages[_name]['optional'] = False
-                elif "uploadlazy" in targets:
+                elif "nobuild" in targets:
                     # skip all packages, allow only upload tools
                     self.packages[_name]['optional'] = True
 

platformio/pioplus.py

@@ -20,39 +20,48 @@ from platform import system
 from platformio import exception, util
 from platformio.managers.package import PackageManager
 
-PACKAGE_PIOPLUS_NAME = "tool-pioplus"
+PACKAGE_DEPS = {"pysite": {"name": "pysite-pioplus",
+                           "requirements": ">=0.1.0"},
+                "tool": {"name": "tool-pioplus",
+                         "requirements": ">=0.2.0"}}
 
 
 class PioPlusPackageManager(PackageManager):
 
     def __init__(self):
-        PackageManager.__init__(
+        PackageManager.__init__(self, join(util.get_home_dir(), "packages"), [
-            self, join(util.get_home_dir(), "packages"),
+            "https://dl.bintray.com/platformio/dl-packages/manifest.json",
-            ["https://dl.bintray.com/platformio/dl-packages/manifest.json",
+            "https://sourceforge.net/projects/platformio-storage/files/"
-             "https://dl.platformio.org/packages/manifest.json"])
+            "packages/manifest.json/download",
+            "https://dl.platformio.org/packages/manifest.json"
+        ])
 
 
-def get_pioplusexe_path():
+def pioplus_install():
     pm = PioPlusPackageManager()
-    package_dir = pm.get_package_dir(PACKAGE_PIOPLUS_NAME)
+    for item in PACKAGE_DEPS.values():
-    if not package_dir:
+        pm.install(item['name'], item['requirements'], silent=True)
-        pm.install(PACKAGE_PIOPLUS_NAME)
-        package_dir = pm.get_package_dir(PACKAGE_PIOPLUS_NAME)
-    assert package_dir
-    return join(package_dir, "pioplus")
 
 
 def pioplus_update():
     pm = PioPlusPackageManager()
-    if pm.get_package_dir(PACKAGE_PIOPLUS_NAME):
+    for item in PACKAGE_DEPS.values():
-        pm.update(PACKAGE_PIOPLUS_NAME)
+        package_dir = pm.get_package_dir(item['name'], item['requirements'])
+        if package_dir:
+            pm.update(item['name'], item['requirements'])
 
 
 def pioplus_call(args, **kwargs):
-    pioplus_path = get_pioplusexe_path()
+    pioplus_install()
+    pm = PioPlusPackageManager()
+    pioplus_path = join(
+        pm.get_package_dir(PACKAGE_DEPS['tool']['name'],
+                           PACKAGE_DEPS['tool']['requirements']), "pioplus")
     if system() == "Linux":
         os.environ['LD_LIBRARY_PATH'] = dirname(pioplus_path)
     os.environ['PYTHONEXEPATH'] = util.get_pythonexe_path()
+    os.environ['PYTHONPYSITEDIR'] = pm.get_package_dir(
+        PACKAGE_DEPS['pysite']['name'], PACKAGE_DEPS['pysite']['requirements'])
     util.copy_pythonpath_to_osenv()
     if subprocess.call([pioplus_path] + args, **kwargs) != 0:
         raise exception.ReturnErrorCode()

platformio/telemetry.py

@@ -92,12 +92,14 @@ class MeasurementProtocol(TelemetryBase):
         self['an'] = " ".join(dpdata)
 
     def _prefill_custom_data(self):
+        caller_id = str(app.get_session_var("caller_id"))
         self['cd1'] = util.get_systype()
         self['cd2'] = "Python/%s %s" % (platform.python_version(),
                                         platform.platform())
-        self['cd4'] = 1 if not util.is_ci() else 0
+        self['cd4'] = 1 if (not util.is_ci() and
-        if app.get_session_var("caller_id"):
+                            (caller_id or not util.is_container())) else 0
-            self['cd5'] = str(app.get_session_var("caller_id")).lower()
+        if caller_id:
+            self['cd5'] = caller_id.lower()
 
     def _prefill_screen_name(self):
         self['cd3'] = " ".join([str(s).lower() for s in sys.argv[1:]])
@@ -108,10 +110,13 @@ class MeasurementProtocol(TelemetryBase):
         args = [str(s).lower() for s in ctx_args if not str(s).startswith("-")]
         if not args:
             return
-        if args[0] in ("lib", "platform", "serialports", "settings"):
+        cmd_path = args[:1]
+        if args[0] in ("lib", "platform", "platforms", "serialports", "device",
+                       "settings", "remote"):
             cmd_path = args[:2]
-        else:
+            if args[0] == "remote":
-            cmd_path = args[:1]
+                if len(args) > 2 and args[1] in ("agent", "device"):
+                    cmd_path = args[:3]
         self['screen_name'] = " ".join([p.title() for p in cmd_path])
 
     def send(self, hittype):

platformio/util.py

@@ -298,6 +298,17 @@ def is_ci():
     return os.getenv("CI", "").lower() == "true"
 
 
+def is_container():
+    if not isfile("/proc/1/cgroup"):
+        return False
+    with open("/proc/1/cgroup") as fp:
+        for line in fp:
+            line = line.strip()
+            if ":" in line and not line.endswith(":/"):
+                return True
+    return False
+
+
 def exec_command(*args, **kwargs):
     result = {"out": None, "err": None, "returncode": None}