Clean autogenerated files before running tests // Resolve #3523

Fixes possible conflicts between auxiliary test transport files when
project contains multiple environments with different platforms

HISTORY.rst

@@ -17,6 +17,7 @@ PlatformIO Core 4
 
 * Added support for `custom targets <https://docs.platformio.org/page/projectconf/advanced_scripting.html#custom-targets>`__ (user cases: command shortcuts, pre/post processing based on dependencies, custom command launcher with options, etc.)
 * Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci <https://docs.platformio.org/page/core/userguide/cmd_ci.html>`__, `src_filter <https://docs.platformio.org/page/projectconf/section_env_build.html#src-filter>`__, `check_patterns <https://docs.platformio.org/page/projectconf/section_env_check.html#check-patterns>`__, `library.json > srcFilter <https://docs.platformio.org/page/librarymanager/config.html#srcfilter>`__). Python 3.5+ is required.
+* Fixed an issue with PIO Unit Testing when running multiple environments (`issue #3523 <https://github.com/platformio/platformio-core/issues/3523>`_)
 
 4.3.4 (2020-05-23)
 ~~~~~~~~~~~~~~~~~~

platformio/commands/test/processor.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import atexit
-from os import remove
+from os import remove, listdir
 from os.path import isdir, isfile, join
 from string import Template
 
@@ -194,24 +194,29 @@ class TestProcessorBase(object):
             ]
         )
 
-        def delete_tmptest_file(file_):
-            try:
-                remove(file_)
-            except:  # pylint: disable=bare-except
-                if isfile(file_):
-                    click.secho(
-                        "Warning: Could not remove temporary file '%s'. "
-                        "Please remove it manually." % file_,
-                        fg="yellow",
-                    )
+        tmp_file_prefix = "tmp_pio_test_transport"
+
+        def delete_tmptest_files(test_dir):
+            for item in listdir(test_dir):
+                if item.startswith(tmp_file_prefix) and isfile(join(test_dir, item)):
+                    try:
+                        remove(join(test_dir, item))
+                    except:  # pylint: disable=bare-except
+                        click.secho(
+                            "Warning: Could not remove temporary file '%s'. "
+                            "Please remove it manually." % join(test_dir, item),
+                            fg="yellow",
+                        )
 
         transport_options = TRANSPORT_OPTIONS[self.get_transport()]
         tpl = Template(file_tpl).substitute(transport_options)
         data = Template(tpl).substitute(baudrate=self.get_baudrate())
+
+        delete_tmptest_files(test_dir)
         tmp_file = join(
-            test_dir, "output_export." + transport_options.get("language", "c")
+            test_dir, "%s.%s" % (tmp_file_prefix, transport_options.get("language", "c"))
         )
         with open(tmp_file, "w") as fp:
             fp.write(data)
 
-        atexit.register(delete_tmptest_file, tmp_file)
+        atexit.register(delete_tmptest_files, test_dir)

tests/commands/test_test.py

@@ -17,6 +17,7 @@ from os.path import join
 import pytest
 
 from platformio import util
+from platformio.commands.test.command import cli as cmd_test
 
 
 def test_local_env():
@@ -31,7 +32,49 @@ def test_local_env():
         ]
     )
     if result["returncode"] != 1:
-        pytest.fail(result)
+        pytest.fail(str(result))
     assert all([s in result["err"] for s in ("PASSED", "IGNORED", "FAILED")]), result[
         "out"
     ]
+
+
+def test_multiple_env_build(clirunner, validate_cliresult, tmpdir):
+
+    project_dir = tmpdir.mkdir("project")
+    project_dir.join("platformio.ini").write(
+        """
+[env:teensy31]
+platform = teensy
+framework = mbed
+board = teensy31
+
+[env:native]
+platform = native
+
+[env:espressif32]
+platform = espressif32
+framework = arduino
+board = esp32dev
+"""
+    )
+
+    project_dir.mkdir("test").join("test_main.cpp").write(
+        """
+#ifdef ARDUINO
+void setup() {}
+void loop() {}
+#else
+int main() {
+    UNITY_BEGIN();
+    UNITY_END();
+}
+#endif
+"""
+    )
+
+    result = clirunner.invoke(
+        cmd_test, ["-d", str(project_dir), "--without-testing", "--without-uploading"],
+    )
+
+    validate_cliresult(result)
+    assert "Multiple ways to build" not in result.output