From f8db1d11a703d5ebcc41a621109c4dc46046ac5b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 22 Jun 2016 21:25:44 +0300 Subject: [PATCH] New Library Build System: intelligent dependency finder that interprets C Preprocessor conditional macros // Resolve #432 --- HISTORY.rst | 3 + platformio/__init__.py | 2 +- platformio/builder/main.py | 3 +- platformio/builder/tools/piolib.py | 242 +++++++++++++++++++++++++ platformio/builder/tools/platformio.py | 162 +---------------- 5 files changed, 256 insertions(+), 156 deletions(-) create mode 100644 platformio/builder/tools/piolib.py diff --git a/HISTORY.rst b/HISTORY.rst index d7fb834a..a337c560 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,9 @@ PlatformIO 3.0 (`issue #479 `_) * Unit Testing for Embedded (`docs `__) (`issue #408 `_) +* New Library Build System: intelligent dependency finder that interprets + C Preprocessor conditional macros + (`issue #432 `_) PlatformIO 2.0 -------------- diff --git a/platformio/__init__.py b/platformio/__init__.py index 1e9074a2..facbf112 100644 --- a/platformio/__init__.py +++ b/platformio/__init__.py @@ -14,7 +14,7 @@ import sys -VERSION = (3, 0, "0.dev1") +VERSION = (3, 0, "0.dev2") __version__ = ".".join([str(s) for s in VERSION]) __title__ = "platformio" diff --git a/platformio/builder/main.py b/platformio/builder/main.py index ab957d57..94b1e41b 100644 --- a/platformio/builder/main.py +++ b/platformio/builder/main.py @@ -63,7 +63,8 @@ commonvars.AddVariables( DefaultEnvironment( tools=[ "gcc", "g++", "as", "ar", "gnulink", - "platformio", "devplatform", "piotest", "pioupload", "pioar", "piomisc" + "platformio", "devplatform", + "piolib", "piotest", "pioupload", "pioar", "piomisc" ], toolpath=[join(util.get_source_dir(), "builder", "tools")], variables=commonvars, diff --git a/platformio/builder/tools/piolib.py b/platformio/builder/tools/piolib.py new file mode 100644 index 00000000..a0836a8a --- /dev/null +++ b/platformio/builder/tools/piolib.py @@ -0,0 +1,242 @@ +# Copyright 2014-present Ivan Kravets +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os +from os.path import basename, commonprefix, isdir, isfile, join +from sys import modules + +import SCons.Scanner + +from platformio import util + + +class LibBuilderFactory(object): + + @staticmethod + def new(env, path): + clsname = "UnknownLibBuilder" + if isfile(join(path, "library.json")): + clsname = "PlatformIOLibBuilder" + else: + env_frameworks = [ + f.lower().strip() for f in env.get("FRAMEWORK", "").split(",")] + used_frameworks = LibBuilderFactory.get_used_frameworks(env, path) + common_frameworks = set(env_frameworks) & set(used_frameworks) + if common_frameworks: + clsname = "%sLibBuilder" % list(common_frameworks)[0].title() + elif used_frameworks: + clsname = "%sLibBuilder" % used_frameworks[0].title() + + obj = getattr(modules[__name__], clsname)(env, path) + assert isinstance(obj, LibBuilderBase) + return obj + + @staticmethod + def get_used_frameworks(env, path): + if any([isfile(join(path, fname)) + for fname in ("library.properties", "keywords.txt")]): + return ["arduino"] + + if isfile(join(path, "module.json")): + return ["mbed"] + + # check source files + for root, _, files in os.walk(path, followlinks=True): + for fname in files: + if not env.IsFileWithExt(fname, ("c", "cpp", "h")): + continue + with open(join(root, fname)) as f: + content = f.read() + if "Arduino.h" in content: + return ["arduino"] + elif "mbed.h" in content: + return ["mbed"] + return [] + + +class LibBuilderBase(object): + + def __init__(self, env, path): + self.env = env + self.path = path + self._is_built = False + + def __repr__(self): + return "%s(%r)" % (self.__class__, self.path) + + def __contains__(self, path): + return commonprefix((self.path, path)) == self.path + + @property + def name(self): + return basename(self.path) + + @property + def src_filter(self): + return " ".join([ + "+<*>", "-<.git%s>" % os.sep, "-" % os.sep, + "-" % os.sep, "-" % os.sep, + "-" % os.sep, "-" % os.sep + ]) + + @property + def src_dir(self): + return (join(self.path, "src") if isdir(join(self.path, "src")) + else self.path) + + @property + def build_dir(self): + return join("$BUILD_DIR", "lib", self.name) + + @property + def is_built(self): + return self._is_built + + def get_path_dirs(self, use_build_dir=False): + return [self.build_dir if use_build_dir else self.src_dir] + + def build(self): + print "Depends on: %s" % self.name + assert self._is_built is False + self._is_built = True + return self.env.BuildLibrary(self.build_dir, self.src_dir) + + +class UnknownLibBuilder(LibBuilderBase): + pass + + +class ArduinoLibBuilder(LibBuilderBase): + + def get_path_dirs(self, use_build_dir=False): + path_dirs = LibBuilderBase.get_path_dirs(self, use_build_dir) + if not isdir(join(self.src_dir, "utility")): + return path_dirs + path_dirs.append(join(self.build_dir if use_build_dir + else self.src_dir, "utility")) + return path_dirs + + +class MbedLibBuilder(LibBuilderBase): + + def __init__(self, env, path): + self.module_json = {} + if isfile(join(path, "module.json")): + self.module_json = util.load_json(join(path, "module.json")) + + LibBuilderBase.__init__(self, env, path) + + @property + def src_dir(self): + if isdir(join(self.path, "source")): + return join(self.path, "source") + return LibBuilderBase.src_dir.fget(self) + + def get_path_dirs(self, use_build_dir=False): + path_dirs = LibBuilderBase.get_path_dirs(self, use_build_dir) + for p in self.module_json.get("extraIncludes", []): + if p.startswith("source/"): + p = p[7:] + path_dirs.append( + join(self.build_dir if use_build_dir else self.src_dir, p)) + return path_dirs + + +class PlatformIOLibBuilder(LibBuilderBase): + + def __init__(self, env, path): + self.library_json = {} + if isfile(join(path, "library.json")): + self.library_json = util.load_json(join(path, "library.json")) + + LibBuilderBase.__init__(self, env, path) + + +def find_deps(env, scanner, path_dirs, src_dir, src_filter): + result = [] + for item in env.MatchSourceFiles(src_dir, src_filter): + result.extend(env.File(join(src_dir, item)).get_implicit_deps( + env, scanner, path_dirs)) + return result + + +def find_and_build_deps(env, lib_builders, scanner, + src_dir, src_filter): + path_dirs = tuple() + built_path_dirs = tuple() + for lb in lib_builders: + items = [env.Dir(d) for d in lb.get_path_dirs()] + if lb.is_built: + built_path_dirs += tuple(items) + else: + path_dirs += tuple(items) + path_dirs = built_path_dirs + path_dirs + + target_lbs = [] + deps = find_deps(env, scanner, path_dirs, src_dir, src_filter) + for d in deps: + for lb in lib_builders: + if d.get_abspath() in lb: + if lb not in target_lbs and not lb.is_built: + target_lbs.append(lb) + break + + libs = [] + # add build dirs to global CPPPATH + for lb in target_lbs: + env.Append( + CPPPATH=lb.get_path_dirs(use_build_dir=True) + ) + # start builder + for lb in target_lbs: + libs.append(lb.build()) + + if env.get("LIB_DFCYCLIC", "").lower() == "true": + for lb in target_lbs: + libs.extend(find_and_build_deps( + env, lib_builders, scanner, lb.src_dir, lb.src_filter)) + + return libs + + +def BuildDependentLibraries(env, src_dir): + lib_builders = [] + libs_dirs = [env.subst(d) for d in env.get("LIBSOURCE_DIRS", []) + if isdir(env.subst(d))] + for libs_dir in libs_dirs: + if not isdir(libs_dir): + continue + for item in sorted(os.listdir(libs_dir)): + if isdir(join(libs_dir, item)): + lib_builders.append( + LibBuilderFactory.new(env, join(libs_dir, item))) + + print "Looking for dependencies..." + print "Library locations: " + ", ".join(libs_dirs) + print "Collecting %d libraries" % len(lib_builders) + + return find_and_build_deps( + env, lib_builders, SCons.Scanner.C.CScanner(), + src_dir, env.get("SRC_FILTER")) + + +def exists(_): + return True + + +def generate(env): + env.AddMethod(BuildDependentLibraries) + return env diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index ac9306a3..138ba0b7 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -16,8 +16,8 @@ from __future__ import absolute_import import re from glob import glob -from os import listdir, sep, walk -from os.path import basename, dirname, isdir, isfile, join, normpath, realpath +from os import sep, walk +from os.path import basename, dirname, isdir, join, normpath, realpath from SCons.Script import COMMAND_LINE_TARGETS, DefaultEnvironment, SConscript from SCons.Util import case_sensitive_suffixes @@ -26,11 +26,7 @@ from platformio.util import pioversion_to_intstr SRC_BUILD_EXT = ["c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"] SRC_HEADER_EXT = ["h", "hpp"] -SRC_DEFAULT_FILTER = " ".join([ - "+<*>", "-<.git%s>" % sep, "-" % sep, - "-" % sep, "-" % sep, - "-" % sep, "-" % sep -]) +SRC_FILTER_DEFAULT = " ".join(["+<*>", "-<.git%s>" % sep, "-" % sep]) def BuildProgram(env): @@ -173,7 +169,7 @@ def IsFileWithExt(env, file_, ext): # pylint: disable=W0613 return False -def MatchSourceFiles(env, src_dir, src_filter): +def MatchSourceFiles(env, src_dir, src_filter=None): SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>") @@ -181,6 +177,9 @@ def MatchSourceFiles(env, src_dir, src_filter): if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT): items.add(item.replace(src_dir + sep, "")) + src_dir = env.subst(src_dir) + src_filter = src_filter or SRC_FILTER_DEFAULT + matches = set() # correct fs directory separator src_filter = src_filter.replace("/", sep).replace("\\", sep) @@ -214,8 +213,7 @@ def CollectBuildFiles(env, variant_dir, src_dir, if src_dir.endswith(sep): src_dir = src_dir[:-1] - for item in env.MatchSourceFiles( - src_dir, src_filter or SRC_DEFAULT_FILTER): + for item in env.MatchSourceFiles(src_dir, src_filter): _reldir = dirname(item) _src_dir = join(src_dir, _reldir) if _reldir else src_dir _var_dir = join(variant_dir, _reldir) if _reldir else variant_dir @@ -265,149 +263,6 @@ def BuildLibrary(env, variant_dir, src_dir, src_filter=None): ) -def BuildDependentLibraries(env, src_dir): # pylint: disable=R0914 - - INCLUDES_RE = re.compile( - r"^\s*#include\s+(\<|\")([^\>\"\']+)(?:\>|\")", re.M) - LIBSOURCE_DIRS = [env.subst(d) for d in env.get("LIBSOURCE_DIRS", [])] - - # start internal prototypes - - class IncludeFinder(object): - - def __init__(self, base_dir, name, is_system=False): - self.base_dir = base_dir - self.name = name - self.is_system = is_system - - self._inc_path = None - self._lib_dir = None - self._lib_name = None - - def getIncPath(self): - return self._inc_path - - def getLibDir(self): - return self._lib_dir - - def getLibName(self): - return self._lib_name - - def run(self): - if not self.is_system and self._find_in_local(): - return True - return self._find_in_system() - - def _find_in_local(self): - if isfile(join(self.base_dir, self.name)): - self._inc_path = join(self.base_dir, self.name) - return True - else: - return False - - def _find_in_system(self): - for lsd_dir in LIBSOURCE_DIRS: - if not isdir(lsd_dir): - continue - - for ld in env.get("LIB_USE", []) + sorted(listdir(lsd_dir)): - if not isdir(join(lsd_dir, ld)): - continue - - inc_path = normpath(join(lsd_dir, ld, self.name)) - try: - lib_dir = inc_path[:inc_path.index( - sep, len(lsd_dir) + 1)] - except ValueError: - continue - lib_name = basename(lib_dir) - - # ignore user's specified libs - if lib_name in env.get("LIB_IGNORE", []): - continue - - if not isfile(inc_path): - # if source code is in "src" dir - lib_dir = join(lsd_dir, lib_name, "src") - inc_path = join(lib_dir, self.name) - - if isfile(inc_path): - self._lib_dir = lib_dir - self._lib_name = lib_name - self._inc_path = inc_path - return True - return False - - def _get_dep_libs(src_dir): - state = { - "paths": set(), - "libs": set(), - "ordered": set() - } - - state = _process_src_dir(state, env.subst(src_dir)) - - result = [] - for item in sorted(state['ordered'], key=lambda s: s[0]): - result.append((item[1], item[2])) - return result - - def _process_src_dir(state, src_dir): - for root, _, files in walk(src_dir, followlinks=True): - for f in files: - if env.IsFileWithExt(f, SRC_BUILD_EXT + SRC_HEADER_EXT): - state = _parse_includes(state, env.File(join(root, f))) - return state - - def _parse_includes(state, node): - skip_includes = ("arduino.h", "energia.h") - matches = INCLUDES_RE.findall(node.get_text_contents()) - for (inc_type, inc_name) in matches: - base_dir = dirname(node.get_abspath()) - if inc_name.lower() in skip_includes: - continue - if join(base_dir, inc_name) in state['paths']: - continue - else: - state['paths'].add(join(base_dir, inc_name)) - - finder = IncludeFinder(base_dir, inc_name, inc_type == "<") - if finder.run(): - _parse_includes(state, env.File(finder.getIncPath())) - - _lib_dir = finder.getLibDir() - if _lib_dir and _lib_dir not in state['libs']: - state['ordered'].add(( - len(state['ordered']) + 1, finder.getLibName(), - _lib_dir)) - state['libs'].add(_lib_dir) - - if env.subst("$LIB_DFCYCLIC").lower() == "true": - state = _process_src_dir(state, _lib_dir) - return state - - # end internal prototypes - - deplibs = _get_dep_libs(src_dir) - for l, ld in deplibs: - env.Append( - CPPPATH=[join("$BUILD_DIR", l)] - ) - # add automatically "utility" dir from the lib (Arduino issue) - if isdir(join(ld, "utility")): - env.Append( - CPPPATH=[join("$BUILD_DIR", l, "utility")] - ) - - libs = [] - for (libname, inc_dir) in deplibs: - lib = env.BuildLibrary( - join("$BUILD_DIR", libname), inc_dir) - env.Clean(libname, lib) - libs.append(lib) - return libs - - def exists(_): return True @@ -422,5 +277,4 @@ def generate(env): env.AddMethod(CollectBuildFiles) env.AddMethod(BuildFrameworks) env.AddMethod(BuildLibrary) - env.AddMethod(BuildDependentLibraries) return env