Allow to ignore tests using glob patterns // Issue #408

docs/platforms/unit_testing.rst

@@ -44,7 +44,7 @@ Design
 PlatformIO Test System design is based on a few isolated components:
 
 1. **Main program**. Contains the independent modules, procedures,
-   functions or methods that will be the target candidates (TC) for testing
+   functions or methods that will be the target candidates (TC) for testing.
 2. **Unit test**. This a small independent program that is intended to
    re-use TC from the main program and apply tests for them.
 3. **Test processor**. The set of approaches and tools that will be used
@@ -53,8 +53,8 @@ PlatformIO Test System design is based on a few isolated components:
 Workflow
 --------
 
-1. Create PlatformIO project using :ref:`cmd_init` command
-2. Place source code of main program to ``src`` directory
+1. Create PlatformIO project using :ref:`cmd_init` command.
+2. Place source code of main program to ``src`` directory.
 3. Wrap ``main()`` or ``setup()/loop()`` methods of main program in ``UNIT_TEST``
    guard:
 
@@ -86,10 +86,10 @@ Workflow
        }
        #endif
 
-4. Create ``test`` directory in the root of project. See :ref:`projectconf_pio_test_dir`
+4. Create ``test`` directory in the root of project. See :ref:`projectconf_pio_test_dir`.
 5. Write test using :ref:`unit_testing_api`. The each test is a small
    independent program with own ``main()`` or ``setup()/loop()`` methods. Also,
-   test should start from ``UNITY_BEGIN()`` and finish with ``UNITY_END()``
+   test should start from ``UNITY_BEGIN()`` and finish with ``UNITY_END()``.
 6. Place test to ``test`` directory. If you have more than one test, split them
    into sub-folders. For example, ``test/test_1/*.[c,cpp,h]``,
    ``test_N/*.[c,cpp,h]``, etc. If no such directory in ``test`` folder, then
@@ -185,11 +185,11 @@ Example
 -------
 
 1. Please follow to :ref:`quickstart` and create "Blink Project". According
-   to the Unit Testing :ref:`unit_testing_design` it is the **Main program**
+   to the Unit Testing :ref:`unit_testing_design` it is the **Main program**.
 2. Create ``test`` directory in that project (on the same level as ``src``)
-   and place ``test_main.cpp`` file to it (the source code is located below)
+   and place ``test_main.cpp`` file to it (the source code is located below).
 3. Wrap ``setup()`` and ``loop()`` methods of main program in ``UNIT_TEST``
-   guard
+   guard.
 4. Run tests using :ref:`cmd_test` command.
 
 Project structure
@@ -313,11 +313,11 @@ Source files
               delay(500);
               RUN_TEST(test_led_state_low);
               delay(500);
+              i++;
           }
           else if (i == max_blinks) {
-            UNITY_END();  // IMPORTANT LINE!
+            UNITY_END(); // stop unit testing
           }
-          i++;
       }
 
       #endif

docs/userguide/cmd_test.rst

@@ -42,6 +42,32 @@ Options
 
 Process specified environments. More details :option:`platformio run --environment`
 
+.. option::
+    -i, --ignore
+
+Ignore tests where the name matches with specified pattern. More than one
+option/pattern is allowed.
+
+.. list-table::
+    :header-rows:  1
+
+    * - Pattern
+      - Meaning
+
+    * - ``*``
+      - matches everything
+
+    * - ``?``
+      - matches any single character
+
+    * - ``[seq]``
+      - matches any character in seq
+
+    * - ``[!seq]``
+      - matches any character not in seq
+
+For example, ``platformio test --ignore "mytest*" -i "test[13]"``
+
 .. option::
     --upload-port
 

platformio/commands/test.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from fnmatch import fnmatch
 from os import getcwd, listdir
 from os.path import isdir, join
 from time import sleep, time
@@ -27,22 +28,21 @@ from platformio.managers.platform import PlatformFactory
 
 @click.command("test", short_help="Unit Testing")
 @click.option("--environment", "-e", multiple=True, metavar="<environment>")
+@click.option("--ignore", "-i", multiple=True, metavar="<pattern>")
 @click.option("--upload-port", metavar="<upload port>")
 @click.option("--project-dir", "-d", default=getcwd,
               type=click.Path(exists=True, file_okay=False, dir_okay=True,
                               writable=True, resolve_path=True))
 @click.option("--verbose", "-v", count=True, default=3)
 @click.pass_context
-def cli(ctx, environment, upload_port,  # pylint: disable=R0913,R0914
+def cli(ctx, environment, ignore, upload_port,  # pylint: disable=R0913,R0914
         project_dir, verbose):
     assert check_project_envs(project_dir, environment)
     with util.cd(project_dir):
         test_dir = util.get_projecttest_dir()
         if not isdir(test_dir):
             raise exception.TestDirEmpty(test_dir)
-        config = util.get_project_config()
-        env_names = set(
-            [s[4:] for s in config.sections() if s.startswith("env:")])
+        projectconf = util.get_project_config()
 
     test_names = []
     for item in sorted(listdir(test_dir)):
@@ -56,11 +56,20 @@ def cli(ctx, environment, upload_port,  # pylint: disable=R0913,R0914
     start_time = time()
     results = []
     for testname in test_names:
-        for envname in env_names:
+        for envname in projectconf.sections():
+            if not envname.startswith("env:"):
+                continue
+            envname = envname[4:]
             if environment and envname not in environment:
                 continue
+
+            # check ignore patterns
+            if testname != "*" and any([fnmatch(testname, i) for i in ignore]):
+                results.append((None, testname, envname))
+                continue
+
             tp = TestProcessor(ctx, testname, envname, {
-                "project_config": config,
+                "project_config": projectconf,
                 "project_dir": project_dir,
                 "upload_port": upload_port,
                 "verbose": verbose
@@ -72,13 +81,18 @@ def cli(ctx, environment, upload_port,  # pylint: disable=R0913,R0914
 
     passed = True
     for result in results:
-        if not result[0]:
+        status, testname, envname = result
+        status_str = click.style("PASSED", fg="green")
+        if status is False:
             passed = False
+            status_str = click.style("FAILED", fg="red")
+        elif status is None:
+            status_str = click.style("IGNORED", fg="yellow")
+
         click.echo("test:%s/env:%s\t%s" % (
-            click.style(result[1], fg="yellow"),
-            click.style(result[2], fg="cyan"),
-            click.style("PASSED" if passed else "FAILED", fg="green"
-                        if passed else "red")), err=not passed)
+            click.style(testname, fg="yellow"),
+            click.style(envname, fg="cyan"),
+            status_str), err=status is False)
 
     print_header("[%s] Took %.2f seconds" % (
         (click.style("PASSED", fg="green", bold=True) if passed
@@ -144,7 +158,8 @@ class TestProcessor(object):
                     line[:-5], click.style("PASSED", fg="green")))
             elif ":FAIL:" in line:
                 passed = False
-                click.secho(line, fg="red")
+                click.echo("%s\t%s" % (
+                    line, click.style("FAILED", fg="red")))
             else:
                 click.echo(line)
             if all([l in line for l in ("Tests", "Failures", "Ignored")]):