Control a number of parallel build jobs with a new -j, --jobs option

HISTORY.rst

@@ -45,6 +45,7 @@ PlatformIO 4.0
   - Switch between `Build Configurations <http://docs.platformio.org/page/projectconf/build_configurations.html>`__ (``release`` and ``debug``) with a new project configuration option `build_type <http://docs.platformio.org/page/projectconf/section_env_build.html#build-type>`__
   - Custom `platform_packages <http://docs.platformio.org/page/projectconf/section_env_general.html#platform>`__ per a build environment with an option to override default (`issue #1367 <https://github.com/platformio/platformio-core/issues/1367>`_)
   - Print platform package details, such as version, VSC source and commit (`issue #2155 <https://github.com/platformio/platformio-core/issues/2155>`_)
+  - Control a number of parallel build jobs with a new `-j, --jobs <http://docs.platformio.org/page/userguide/cmd_run.html#cmdoption-platformio-run-j>`__ option
   - Override default `"platformio.ini" (Project Configuration File) <https://docs.platformio.org/page/projectconf.html>`__ with a custom using ``-c, --project-conf`` option for `platformio run <http://docs.platformio.org/page/userguide/cmd_run.html>`__, `platformio debug <http://docs.platformio.org/page/userguide/cmd_debug.html>`__, or `platformio test <http://docs.platformio.org/page/userguide/cmd_test.html>`__ commands (`issue #1913 <https://github.com/platformio/platformio-core/issues/1913>`_)
   - Override default development platform upload command with a custom `upload_command <http://docs.platformio.org/page/projectconf/section_env_upload.html#upload-command>`__ (`issue #2599 <https://github.com/platformio/platformio-core/issues/2599>`_)
   - Configure a shared folder for the derived files (objects, firmwares, ELFs) from a build system using `build_cache_dir <http://docs.platformio.org/page/projectconf/section_platformio.html#build-cache-dir>`__ option (`issue #2674 <https://github.com/platformio/platformio-core/issues/2674>`_)

platformio/commands/run/command.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from multiprocessing import cpu_count
 from os import getcwd
 from os.path import isfile, join
 from time import time
@@ -30,6 +31,11 @@ from platformio.project.helpers import (find_project_dir_above,
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
 
+try:
+    DEFAULT_JOB_NUMS = cpu_count()
+except NotImplementedError:
+    DEFAULT_JOB_NUMS = 1
+
 
 @click.command("run", short_help="Process project environments")
 @click.option("-e", "--environment", multiple=True)
@@ -50,11 +56,18 @@ from platformio.project.helpers import (find_project_dir_above,
                               dir_okay=False,
                               readable=True,
                               resolve_path=True))
+@click.option("-j",
+              "--jobs",
+              type=int,
+              default=DEFAULT_JOB_NUMS,
+              help=("Allow N jobs at once. "
+                    "Default is a number of CPUs in a system (N=%d)" %
+                    DEFAULT_JOB_NUMS))
 @click.option("-s", "--silent", is_flag=True)
 @click.option("-v", "--verbose", is_flag=True)
 @click.option("--disable-auto-clean", is_flag=True)
 @click.pass_context
-def cli(ctx, environment, target, upload_port, project_dir, project_conf,
+def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs,
         silent, verbose, disable_auto_clean):
     # find project directory on upper level
     if isfile(project_dir):
@@ -95,7 +108,7 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf,
                 click.echo()
 
             ep = EnvironmentProcessor(ctx, envname, config, target,
-                                      upload_port, silent, verbose)
+                                      upload_port, silent, verbose, jobs)
             result = (envname, ep.process())
             results.append(result)
 

platformio/commands/run/processor.py

@@ -32,8 +32,8 @@ class EnvironmentProcessor(object):
     DEFAULT_PRINT_OPTIONS = ("platform", "framework", "board")
 
     def __init__(  # pylint: disable=too-many-arguments
-            self, cmd_ctx, name, config, targets, upload_port, silent,
+            self, cmd_ctx, name, config, targets, upload_port, silent, verbose,
-            verbose):
+            jobs):
         self.cmd_ctx = cmd_ctx
         self.name = name
         self.config = config
@@ -41,6 +41,7 @@ class EnvironmentProcessor(object):
         self.upload_port = upload_port
         self.silent = silent
         self.verbose = verbose
+        self.jobs = jobs
         self.options = config.items(env=name, as_dict=True)
 
     def process(self):
@@ -112,4 +113,5 @@ class EnvironmentProcessor(object):
                                 skip_default_package=True)
             p = PlatformFactory.newPlatform(self.options['platform'])
 
-        return p.run(build_vars, build_targets, self.silent, self.verbose)
+        return p.run(build_vars, build_targets, self.silent, self.verbose,
+                     self.jobs)

platformio/managers/platform.py

@@ -17,7 +17,6 @@ import os
 import re
 import sys
 from imp import load_source
-from multiprocessing import cpu_count
 from os.path import basename, dirname, isdir, isfile, join
 
 import click
@@ -368,7 +367,8 @@ class PlatformRunMixin(object):
         value = base64.urlsafe_b64decode(data)
         return value.decode() if is_bytes(value) else value
 
-    def run(self, variables, targets, silent, verbose):
+    def run(  # pylint: disable=too-many-arguments
+            self, variables, targets, silent, verbose, jobs):
         assert isinstance(variables, dict)
         assert isinstance(targets, list)
 
@@ -393,28 +393,28 @@ class PlatformRunMixin(object):
         if not isfile(variables['build_script']):
             raise exception.BuildScriptNotFound(variables['build_script'])
 
-        result = self._run_scons(variables, targets)
+        result = self._run_scons(variables, targets, jobs)
         assert "returncode" in result
 
         return result
 
-    def _run_scons(self, variables, targets):
+    def _run_scons(self, variables, targets, jobs):
-        cmd = [
+        args = [
             get_pythonexe_path(),
-            join(get_core_package_dir("tool-scons"), "script", "scons"), "-Q",
+            join(get_core_package_dir("tool-scons"), "script", "scons"),
-            "-j %d" % self.get_job_nums(), "--warn=no-no-parallel-support",
+            "-Q", "--warn=no-no-parallel-support",
-            "-f",
+            "--jobs", str(jobs),
-            join(util.get_source_dir(), "builder", "main.py")
+            "--sconstruct", join(util.get_source_dir(), "builder", "main.py")
-        ]
+        ]  # yapf: disable
-        cmd.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
+        args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
         # pylint: disable=protected-access
-        cmd.append("ISATTY=%d" %
+        args.append("ISATTY=%d" %
-                   (1 if click._compat.isatty(sys.stdout) else 0))
+                    (1 if click._compat.isatty(sys.stdout) else 0))
-        cmd += targets
+        args += targets
 
         # encode and append variables
         for key, value in variables.items():
-            cmd.append("%s=%s" % (key.upper(), self.encode_scons_arg(value)))
+            args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value)))
 
         def _write_and_flush(stream, data):
             try:
@@ -425,7 +425,7 @@ class PlatformRunMixin(object):
 
         copy_pythonpath_to_osenv()
         result = exec_command(
-            cmd,
+            args,
             stdout=BuildAsyncPipe(
                 line_callback=self._on_stdout_line,
                 data_callback=lambda data: _write_and_flush(sys.stdout, data)),
@@ -481,13 +481,6 @@ class PlatformRunMixin(object):
            dots="*" * (56 + len(filename)))
         click.echo(banner, err=True)
 
-    @staticmethod
-    def get_job_nums():
-        try:
-            return cpu_count()
-        except NotImplementedError:
-            return 1
-
 
 class PlatformBase(  # pylint: disable=too-many-public-methods
         PlatformPackagesMixin, PlatformRunMixin):