Improved computing of project check sum (structure, configuration) and avoid unnecessary rebuilding

HISTORY.rst

@@ -11,6 +11,7 @@ PlatformIO 4.0
 
 * Print `debug tool <http://docs.platformio.org/en/latest/plus/debugging.html#tools-debug-probes>`__ name for the active debugging session
 * Do not shutdown PIO Home Server for "upgrade" operations (`issue #2784 <https://github.com/platformio/platformio-core/issues/2784>`_)
+* Improved computing of project check sum (structure, configuration) and avoid unnecessary rebuilding
 * Fixed an issue with incorrect escaping of Windows slashes when using `PIO Unified Debugger <http://docs.platformio.org/en/latest/plus/debugging.html>`__ and "piped" openOCD
 
 4.0.0 (2019-07-10)

platformio/commands/run/command.py

@@ -74,10 +74,14 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs,
         project_dir = find_project_dir_above(project_dir)
 
     with util.cd(project_dir):
+        config = ProjectConfig.get_instance(
+            project_conf or join(project_dir, "platformio.ini"))
+        config.validate(environment)
+
         # clean obsolete build dir
         if not disable_auto_clean:
             try:
-                clean_build_dir(get_project_build_dir())
+                clean_build_dir(get_project_build_dir(), config)
             except:  # pylint: disable=bare-except
                 click.secho(
                     "Can not remove temporary directory `%s`. Please remove "
@@ -85,10 +89,6 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs,
                     get_project_build_dir(force=True),
                     fg="yellow")
 
-        config = ProjectConfig.get_instance(
-            project_conf or join(project_dir, "platformio.ini"))
-        config.validate(environment)
-
         handle_legacy_libdeps(project_dir, config)
 
         results = []

platformio/commands/run/helpers.py

@@ -13,13 +13,13 @@
 # limitations under the License.
 
 from os import makedirs
-from os.path import getmtime, isdir, isfile, join
+from os.path import isdir, isfile, join
 from time import time
 
 import click
 
 from platformio import util
-from platformio.project.helpers import (calculate_project_hash,
+from platformio.project.helpers import (compute_project_checksum,
                                         get_project_dir,
                                         get_project_libdeps_dir)
 
@@ -43,32 +43,26 @@ def handle_legacy_libdeps(project_dir, config):
         fg="yellow")
 
 
-def clean_build_dir(build_dir):
+def clean_build_dir(build_dir, config):
     # remove legacy ".pioenvs" folder
     legacy_build_dir = join(get_project_dir(), ".pioenvs")
     if isdir(legacy_build_dir) and legacy_build_dir != build_dir:
         util.rmtree_(legacy_build_dir)
 
-    structhash_file = join(build_dir, "structure.hash")
-    proj_hash = calculate_project_hash()
+    checksum_file = join(build_dir, "project.checksum")
+    checksum = compute_project_checksum(config)
 
-    # if project's config is modified
-    if (isdir(build_dir) and getmtime(join(
-            get_project_dir(), "platformio.ini")) > getmtime(build_dir)):
+    if isdir(build_dir):
+        # check project structure
+        if isfile(checksum_file):
+            with open(checksum_file) as f:
+                if f.read() == checksum:
+                    return
         util.rmtree_(build_dir)
 
-    # check project structure
-    if isdir(build_dir) and isfile(structhash_file):
-        with open(structhash_file) as f:
-            if f.read() == proj_hash:
-                return
-        util.rmtree_(build_dir)
-
-    if not isdir(build_dir):
-        makedirs(build_dir)
-
-    with open(structhash_file, "w") as f:
-        f.write(proj_hash)
+    makedirs(build_dir)
+    with open(checksum_file, "w") as f:
+        f.write(checksum)
 
 
 def print_header(label, is_error=False, fg=None):

platformio/project/config.py

@@ -16,7 +16,7 @@ import glob
 import json
 import os
 import re
-from os.path import isfile
+from os.path import expanduser, isfile
 
 import click
 
@@ -106,6 +106,8 @@ class ProjectConfig(object):
 
         # load extra configs
         for pattern in self.get("platformio", "extra_configs", []):
+            if pattern.startswith("~"):
+                pattern = expanduser(pattern)
             for item in glob.glob(pattern):
                 self.read(item)
 

platformio/project/helpers.py

@@ -165,23 +165,33 @@ def get_project_shared_dir():
                                     join(get_project_dir(), "shared"))
 
 
-def calculate_project_hash():
+def compute_project_checksum(config):
+    # rebuild when PIO Core version changes
+    checksum = sha1(__version__)
+
+    # configuration file state
+    checksum.update(hashlib_encode_data(config.to_json()))
+
+    # project file structure
     check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S")
-    chunks = [__version__]
-    for d in (get_project_src_dir(), get_project_lib_dir()):
+    for d in (get_project_include_dir(), get_project_src_dir(),
+              get_project_lib_dir()):
         if not isdir(d):
             continue
+        chunks = []
         for root, _, files in walk(d):
             for f in files:
                 path = join(root, f)
                 if path.endswith(check_suffixes):
                     chunks.append(path)
-    chunks_to_str = ",".join(sorted(chunks))
-    if WINDOWS:
-        # Fix issue with useless project rebuilding for case insensitive FS.
-        # A case of disk drive can differ...
-        chunks_to_str = chunks_to_str.lower()
-    return sha1(hashlib_encode_data(chunks_to_str)).hexdigest()
+        if not chunks:
+            continue
+        chunks_to_str = ",".join(sorted(chunks))
+        if WINDOWS:  # case insensitive OS
+            chunks_to_str = chunks_to_str.lower()
+        checksum.update(hashlib_encode_data(chunks_to_str))
+
+    return checksum.hexdigest()
 
 
 def load_project_ide_data(project_dir, env_name):