PlatformIO as Continuous Integration tool for embedded projects // Resolve #108

HISTORY.rst

@@ -4,9 +4,11 @@ Release History
 2.0.0 (2015-??-??)
 ------------------
 
-* Implemented PlatformIO CLI 2.0: "platform" related commands have been
+* PlatformIO CLI 2.0: "platform" related commands have been
   moved to ``platformio platforms`` subcommand
   (`issue #158 <https://github.com/platformio/platformio/issues/158>`_)
+* PlatformIO as Continuous Integration (CI) tool for embedded projects
+  (`issue #108 <https://github.com/platformio/platformio/issues/108>`_)
 * Created `PlatformIO gitter.im <https://gitter.im/platformio/platformio>`_ room
   (`issue #174 <https://github.com/platformio/platformio/issues/174>`_)
 * Added global ``-f, --force`` option which will force to accept any

docs/userguide/cmd_ci.rst

@@ -0,0 +1,125 @@
+.. _cmd_ci:
+
+platformio ci
+=============
+
+.. contents::
+
+Usage
+-----
+
+.. code-block:: bash
+
+    platformio ci [OPTIONS] [SRC]
+
+
+Description
+-----------
+
+`Continuous integration (CI, wiki) <http://en.wikipedia.org/wiki/Continuous_integration>`_
+is the practice, in software engineering, of merging all developer working
+copies with a shared mainline several times a day.
+
+:ref:`cmd_ci` command is conceived of as "hot key" for building project with
+arbitrary source code structure. In a nutshell, using ``SRC`` and
+:option:`platformio ci --lib` contents PlatformIO initialises via
+:ref:`cmd_init` new project in :option:`platformio ci --build-dir`
+with the build environments (using :option:`platformio ci --board` or
+:option:`platformio ci --project-conf`) and processes them via :ref:`cmd_run`
+command.
+
+:ref:`cmd_ci` command is intended to be used in combination with the build
+servers and the popular
+`Continuous Integration Software <http://en.wikipedia.org/wiki/Comparison_of_continuous_integration_software>`_.
+
+By integrating regularly, you can detect errors quickly, and locate them more
+easily.
+
+.. note::
+    :ref:`cmd_ci` command accepts **multiple** ``SRC`` arguments,
+    :option:`platformio ci --lib` and :option:`platformio ci --exclude` options
+    which can be a path to directory, file or
+    `Glob Pattern <http://en.wikipedia.org/wiki/Glob_(programming)>`_.
+
+Options
+-------
+
+.. program:: platformio ci
+
+.. option::
+    -l, --lib
+
+Source code which will be copied to ``%build_dir%/lib`` directly.
+
+If :option:`platformio ci --lib` is a path to file (not to directory), then
+PlatformIO will create temporary directory within ``%build_dir%/lib`` and copy
+the rest files into it.
+
+
+.. option::
+    --exclude
+
+Exclude directories and/-or files from :option:`platformio ci --build-dir`. The
+path must be relative to PlatformIO project within
+:option:`platformio ci --build-dir`.
+
+For example, exclude from project ``src`` directory:
+
+* ``examples`` folder
+* ``*.h`` files from ``foo`` folder
+
+.. code-block:: bash
+
+    platformio ci --exclude=src/examples --exclude=src/foo/*.h [SRC]
+
+.. option::
+    --board, -b
+
+Build project with automatically pre-generated environments based on board
+settings.
+
+For more details please look into :option:`platformio init --board`.
+
+.. option::
+    --build-dir
+
+Path to directory where PlatformIO will initialise new project. By default it's
+temporary directory within your operation system.
+
+.. note::
+
+    This directory will be removed at the end of build process. If you want to
+    keep it, please use :option:`platformio ci --keep-build-dir`.
+
+.. option::
+    --keep-build-dir
+
+Don't remove :option:`platformio ci --build-dir` after build process.
+
+.. option::
+    --project-conf
+
+Buid project using pre-configured :ref:`projectconf`.
+
+Examples
+--------
+
+1. Integration `Travis.CI <http://travis-ci.org/>`_ for GitHub
+   `USB_Host_Shield_2.0 <https://github.com/felis/USB_Host_Shield_2.0>`_
+   project. The ``.travis.yml`` configuration file:
+
+.. code-block:: yaml
+
+    language: python
+    python:
+        - "2.7"
+
+    env:
+        - PLATFORMIO_CI_SRC=examples/Bluetooth/PS3SPP/PS3SPP.ino
+        - PLATFORMIO_CI_SRC=examples/pl2303/pl2303_gps/pl2303_gps.ino
+
+    install:
+        - python -c "$(curl -fsSL https://raw.githubusercontent.com/platformio/platformio/master/scripts/get-platformio.py)"
+
+    script:
+        - platformio ci --lib="." --board=uno --board=teensy31 --board=due

docs/userguide/index.rst

@@ -44,6 +44,7 @@ Commands
     :maxdepth: 2
 
     cmd_boards
+    cmd_ci
     cmd_init
     platformio lib <lib/index>
     platformio platforms <platforms/index>

platformio/__init__.py

@@ -1,7 +1,7 @@
 # Copyright (C) Ivan Kravets <me@ikravets.com>
 # See LICENSE for details.
 
-VERSION = (2, 0, "0.dev4")
+VERSION = (2, 0, "0.dev5")
 __version__ = ".".join([str(s) for s in VERSION])
 
 __title__ = "platformio"

platformio/commands/ci.py

@@ -0,0 +1,139 @@
+# Copyright (C) Ivan Kravets <me@ikravets.com>
+# See LICENSE for details.
+
+from glob import glob
+from os import environ, makedirs, remove
+from os.path import basename, isdir, isfile, join
+from shutil import copyfile, copytree, rmtree
+from tempfile import mkdtemp
+
+import click
+
+from platformio import app
+from platformio.commands.init import cli as cmd_init
+from platformio.commands.run import cli as cmd_run
+from platformio.exception import CIBuildEnvsEmpty
+from platformio.util import get_boards
+
+
+def validate_path(ctx, param, value):  # pylint: disable=W0613
+    invalid_path = None
+    for p in value:
+        if not glob(p):
+            invalid_path = p
+            break
+    try:
+        assert invalid_path is None
+        return value
+    except AssertionError:
+        raise click.BadParameter("Found invalid path: %s" % invalid_path)
+
+
+def validate_boards(ctx, param, value):  # pylint: disable=W0613
+    unknown_boards = set(value) - set(get_boards().keys())
+    try:
+        assert not unknown_boards
+        return value
+    except AssertionError:
+        raise click.BadParameter(
+            "%s. Please search for the board types using "
+            "`platformio boards` command" % ", ".join(unknown_boards))
+
+
+@click.command("ci", short_help="Continuous Integration")
+@click.argument("src", nargs=-1, callback=validate_path)
+@click.option("--lib", "-l", multiple=True, callback=validate_path)
+@click.option("--exclude", multiple=True)
+@click.option("--board", "-b", multiple=True, callback=validate_boards)
+@click.option("--build-dir", default=mkdtemp,
+              type=click.Path(exists=True, file_okay=False, dir_okay=True,
+                              writable=True, resolve_path=True))
+@click.option("--keep-build-dir", is_flag=True)
+@click.option("--project-conf",
+              type=click.Path(exists=True, file_okay=True, dir_okay=False,
+                              readable=True, resolve_path=True))
+@click.pass_context
+def cli(ctx, src, lib, exclude, board,  # pylint: disable=R0913
+        build_dir, keep_build_dir, project_conf):
+
+    if not src:
+        src = environ.get("PLATFORMIO_CI_SRC", "").split(":")
+    if not src:
+        raise click.BadParameter("Missing argument 'src'")
+
+    try:
+        app.set_session_var("force_option", True)
+        _clean_dir(build_dir)
+
+        for dir_name, patterns in dict(lib=lib, src=src).iteritems():
+            if not patterns:
+                continue
+            contents = []
+            for p in patterns:
+                contents += glob(p)
+            _copy_contents(join(build_dir, dir_name), contents)
+
+        if project_conf and isfile(project_conf):
+            copyfile(project_conf, join(build_dir, "platformio.ini"))
+        elif not board:
+            raise CIBuildEnvsEmpty()
+
+        if exclude:
+            _exclude_contents(build_dir, exclude)
+
+        # initialise project
+        ctx.invoke(cmd_init, project_dir=build_dir, board=board,
+                   disable_auto_uploading=True)
+
+        # process project
+        ctx.invoke(cmd_run, project_dir=build_dir)
+    finally:
+        if not keep_build_dir:
+            rmtree(build_dir)
+
+
+def _clean_dir(dirpath):
+    rmtree(dirpath)
+    makedirs(dirpath)
+
+
+def _copy_contents(dst_dir, contents):
+    items = {
+        "dirs": set(),
+        "files": set()
+    }
+
+    for path in contents:
+        if isdir(path):
+            items['dirs'].add(path)
+        elif isfile(path):
+            items['files'].add(path)
+
+    dst_dir_name = basename(dst_dir)
+
+    if dst_dir_name == "src" and len(items['dirs']) == 1:
+        copytree(list(items['dirs']).pop(), dst_dir)
+    else:
+        makedirs(dst_dir)
+        for d in items['dirs']:
+            copytree(d, join(dst_dir, basename(d)))
+
+    if not items['files']:
+        return
+
+    if dst_dir_name == "lib":
+        dst_dir = join(dst_dir, mkdtemp(dir=dst_dir))
+
+    for f in items['files']:
+        copyfile(f, join(dst_dir, basename(f)))
+
+
+def _exclude_contents(dst_dir, patterns):
+    contents = []
+    for p in patterns:
+        contents += glob(join(dst_dir, p))
+    for path in contents:
+        if isdir(path):
+            rmtree(path)
+        elif isfile(path):
+            remove(path)

platformio/exception.py

@@ -165,6 +165,15 @@ class UpgraderFailed(PlatformioException):
     MESSAGE = "An error occurred while upgrading PlatformIO"
 
 
+class CIBuildEnvsEmpty(PlatformioException):
+
+    MESSAGE = (
+        "Can't find PlatformIO build environments.\nPlease specify `--board` "
+        "or path to `platformio.ini` with predefined environments using "
+        "`--project-conf` option"
+    )
+
+
 class SConsNotInstalled(PlatformioException):
 
     MESSAGE = (