From 8059e04499fd4d54195c3d75b74a2804aadb6be8 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 16 Jul 2019 15:47:33 +0300 Subject: [PATCH] Improved computing of project check sum (structure, configuration) and avoid unnecessary rebuilding --- HISTORY.rst | 1 + platformio/commands/run/command.py | 10 ++++----- platformio/commands/run/helpers.py | 34 ++++++++++++------------------ platformio/project/config.py | 4 +++- platformio/project/helpers.py | 28 ++++++++++++++++-------- 5 files changed, 42 insertions(+), 35 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e19a7e76..3fd03d2e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ PlatformIO 4.0 * Print `debug tool `__ name for the active debugging session * Do not shutdown PIO Home Server for "upgrade" operations (`issue #2784 `_) +* Improved computing of project check sum (structure, configuration) and avoid unnecessary rebuilding * Fixed an issue with incorrect escaping of Windows slashes when using `PIO Unified Debugger `__ and "piped" openOCD 4.0.0 (2019-07-10) diff --git a/platformio/commands/run/command.py b/platformio/commands/run/command.py index 84d19857..543ec424 100644 --- a/platformio/commands/run/command.py +++ b/platformio/commands/run/command.py @@ -74,10 +74,14 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, project_dir = find_project_dir_above(project_dir) with util.cd(project_dir): + config = ProjectConfig.get_instance( + project_conf or join(project_dir, "platformio.ini")) + config.validate(environment) + # clean obsolete build dir if not disable_auto_clean: try: - clean_build_dir(get_project_build_dir()) + clean_build_dir(get_project_build_dir(), config) except: # pylint: disable=bare-except click.secho( "Can not remove temporary directory `%s`. Please remove " @@ -85,10 +89,6 @@ def cli(ctx, environment, target, upload_port, project_dir, project_conf, jobs, get_project_build_dir(force=True), fg="yellow") - config = ProjectConfig.get_instance( - project_conf or join(project_dir, "platformio.ini")) - config.validate(environment) - handle_legacy_libdeps(project_dir, config) results = [] diff --git a/platformio/commands/run/helpers.py b/platformio/commands/run/helpers.py index 0c51012b..3058acc1 100644 --- a/platformio/commands/run/helpers.py +++ b/platformio/commands/run/helpers.py @@ -13,13 +13,13 @@ # limitations under the License. from os import makedirs -from os.path import getmtime, isdir, isfile, join +from os.path import isdir, isfile, join from time import time import click from platformio import util -from platformio.project.helpers import (calculate_project_hash, +from platformio.project.helpers import (compute_project_checksum, get_project_dir, get_project_libdeps_dir) @@ -43,32 +43,26 @@ def handle_legacy_libdeps(project_dir, config): fg="yellow") -def clean_build_dir(build_dir): +def clean_build_dir(build_dir, config): # remove legacy ".pioenvs" folder legacy_build_dir = join(get_project_dir(), ".pioenvs") if isdir(legacy_build_dir) and legacy_build_dir != build_dir: util.rmtree_(legacy_build_dir) - structhash_file = join(build_dir, "structure.hash") - proj_hash = calculate_project_hash() + checksum_file = join(build_dir, "project.checksum") + checksum = compute_project_checksum(config) - # if project's config is modified - if (isdir(build_dir) and getmtime(join( - get_project_dir(), "platformio.ini")) > getmtime(build_dir)): + if isdir(build_dir): + # check project structure + if isfile(checksum_file): + with open(checksum_file) as f: + if f.read() == checksum: + return util.rmtree_(build_dir) - # check project structure - if isdir(build_dir) and isfile(structhash_file): - with open(structhash_file) as f: - if f.read() == proj_hash: - return - util.rmtree_(build_dir) - - if not isdir(build_dir): - makedirs(build_dir) - - with open(structhash_file, "w") as f: - f.write(proj_hash) + makedirs(build_dir) + with open(checksum_file, "w") as f: + f.write(checksum) def print_header(label, is_error=False, fg=None): diff --git a/platformio/project/config.py b/platformio/project/config.py index dc455809..e98ace55 100644 --- a/platformio/project/config.py +++ b/platformio/project/config.py @@ -16,7 +16,7 @@ import glob import json import os import re -from os.path import isfile +from os.path import expanduser, isfile import click @@ -106,6 +106,8 @@ class ProjectConfig(object): # load extra configs for pattern in self.get("platformio", "extra_configs", []): + if pattern.startswith("~"): + pattern = expanduser(pattern) for item in glob.glob(pattern): self.read(item) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 3c017666..0078fe74 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -165,23 +165,33 @@ def get_project_shared_dir(): join(get_project_dir(), "shared")) -def calculate_project_hash(): +def compute_project_checksum(config): + # rebuild when PIO Core version changes + checksum = sha1(__version__) + + # configuration file state + checksum.update(hashlib_encode_data(config.to_json())) + + # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") - chunks = [__version__] - for d in (get_project_src_dir(), get_project_lib_dir()): + for d in (get_project_include_dir(), get_project_src_dir(), + get_project_lib_dir()): if not isdir(d): continue + chunks = [] for root, _, files in walk(d): for f in files: path = join(root, f) if path.endswith(check_suffixes): chunks.append(path) - chunks_to_str = ",".join(sorted(chunks)) - if WINDOWS: - # Fix issue with useless project rebuilding for case insensitive FS. - # A case of disk drive can differ... - chunks_to_str = chunks_to_str.lower() - return sha1(hashlib_encode_data(chunks_to_str)).hexdigest() + if not chunks: + continue + chunks_to_str = ",".join(sorted(chunks)) + if WINDOWS: # case insensitive OS + chunks_to_str = chunks_to_str.lower() + checksum.update(hashlib_encode_data(chunks_to_str)) + + return checksum.hexdigest() def load_project_ide_data(project_dir, env_name):