mirror of
https://github.com/platformio/platformio-core.git
synced 2025-07-29 17:47:14 +02:00
New Library Build System: intelligent dependency finder that interprets C Preprocessor conditional macros // Resolve #432
This commit is contained in:
Ivan Kravets
@ -13,6 +13,9 @@ PlatformIO 3.0
|
||||
(`issue #479 <https://github.com/platformio/platformio/issues/479>`_)
|
||||
* Unit Testing for Embedded (`docs <http://docs.platformio.org/en/latest/platforms/unit_testing.html>`__)
|
||||
(`issue #408 <https://github.com/platformio/platformio/issues/408>`_)
|
||||
* New Library Build System: intelligent dependency finder that interprets
|
||||
C Preprocessor conditional macros
|
||||
(`issue #432 <https://github.com/platformio/platformio/issues/432>`_)
|
||||
|
||||
PlatformIO 2.0
|
||||
--------------
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
|
||||
import sys
|
||||
|
||||
VERSION = (3, 0, "0.dev1")
|
||||
VERSION = (3, 0, "0.dev2")
|
||||
__version__ = ".".join([str(s) for s in VERSION])
|
||||
|
||||
__title__ = "platformio"
|
||||
|
||||
@ -63,7 +63,8 @@ commonvars.AddVariables(
|
||||
DefaultEnvironment(
|
||||
tools=[
|
||||
"gcc", "g++", "as", "ar", "gnulink",
|
||||
"platformio", "devplatform", "piotest", "pioupload", "pioar", "piomisc"
|
||||
"platformio", "devplatform",
|
||||
"piolib", "piotest", "pioupload", "pioar", "piomisc"
|
||||
],
|
||||
toolpath=[join(util.get_source_dir(), "builder", "tools")],
|
||||
variables=commonvars,
|
||||
|
||||
242
platformio/builder/tools/piolib.py
Normal file
242
platformio/builder/tools/piolib.py
Normal file
@ -0,0 +1,242 @@
|
||||
# Copyright 2014-present Ivan Kravets <me@ikravets.com>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
from os.path import basename, commonprefix, isdir, isfile, join
|
||||
from sys import modules
|
||||
|
||||
import SCons.Scanner
|
||||
|
||||
from platformio import util
|
||||
|
||||
|
||||
class LibBuilderFactory(object):
|
||||
|
||||
@staticmethod
|
||||
def new(env, path):
|
||||
clsname = "UnknownLibBuilder"
|
||||
if isfile(join(path, "library.json")):
|
||||
clsname = "PlatformIOLibBuilder"
|
||||
else:
|
||||
env_frameworks = [
|
||||
f.lower().strip() for f in env.get("FRAMEWORK", "").split(",")]
|
||||
used_frameworks = LibBuilderFactory.get_used_frameworks(env, path)
|
||||
common_frameworks = set(env_frameworks) & set(used_frameworks)
|
||||
if common_frameworks:
|
||||
clsname = "%sLibBuilder" % list(common_frameworks)[0].title()
|
||||
elif used_frameworks:
|
||||
clsname = "%sLibBuilder" % used_frameworks[0].title()
|
||||
|
||||
obj = getattr(modules[__name__], clsname)(env, path)
|
||||
assert isinstance(obj, LibBuilderBase)
|
||||
return obj
|
||||
|
||||
@staticmethod
|
||||
def get_used_frameworks(env, path):
|
||||
if any([isfile(join(path, fname))
|
||||
for fname in ("library.properties", "keywords.txt")]):
|
||||
return ["arduino"]
|
||||
|
||||
if isfile(join(path, "module.json")):
|
||||
return ["mbed"]
|
||||
|
||||
# check source files
|
||||
for root, _, files in os.walk(path, followlinks=True):
|
||||
for fname in files:
|
||||
if not env.IsFileWithExt(fname, ("c", "cpp", "h")):
|
||||
continue
|
||||
with open(join(root, fname)) as f:
|
||||
content = f.read()
|
||||
if "Arduino.h" in content:
|
||||
return ["arduino"]
|
||||
elif "mbed.h" in content:
|
||||
return ["mbed"]
|
||||
return []
|
||||
|
||||
|
||||
class LibBuilderBase(object):
|
||||
|
||||
def __init__(self, env, path):
|
||||
self.env = env
|
||||
self.path = path
|
||||
self._is_built = False
|
||||
|
||||
def __repr__(self):
|
||||
return "%s(%r)" % (self.__class__, self.path)
|
||||
|
||||
def __contains__(self, path):
|
||||
return commonprefix((self.path, path)) == self.path
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return basename(self.path)
|
||||
|
||||
@property
|
||||
def src_filter(self):
|
||||
return " ".join([
|
||||
"+<*>", "-<.git%s>" % os.sep, "-<svn%s>" % os.sep,
|
||||
"-<example%s>" % os.sep, "-<examples%s>" % os.sep,
|
||||
"-<test%s>" % os.sep, "-<tests%s>" % os.sep
|
||||
])
|
||||
|
||||
@property
|
||||
def src_dir(self):
|
||||
return (join(self.path, "src") if isdir(join(self.path, "src"))
|
||||
else self.path)
|
||||
|
||||
@property
|
||||
def build_dir(self):
|
||||
return join("$BUILD_DIR", "lib", self.name)
|
||||
|
||||
@property
|
||||
def is_built(self):
|
||||
return self._is_built
|
||||
|
||||
def get_path_dirs(self, use_build_dir=False):
|
||||
return [self.build_dir if use_build_dir else self.src_dir]
|
||||
|
||||
def build(self):
|
||||
print "Depends on: %s" % self.name
|
||||
assert self._is_built is False
|
||||
self._is_built = True
|
||||
return self.env.BuildLibrary(self.build_dir, self.src_dir)
|
||||
|
||||
|
||||
class UnknownLibBuilder(LibBuilderBase):
|
||||
pass
|
||||
|
||||
|
||||
class ArduinoLibBuilder(LibBuilderBase):
|
||||
|
||||
def get_path_dirs(self, use_build_dir=False):
|
||||
path_dirs = LibBuilderBase.get_path_dirs(self, use_build_dir)
|
||||
if not isdir(join(self.src_dir, "utility")):
|
||||
return path_dirs
|
||||
path_dirs.append(join(self.build_dir if use_build_dir
|
||||
else self.src_dir, "utility"))
|
||||
return path_dirs
|
||||
|
||||
|
||||
class MbedLibBuilder(LibBuilderBase):
|
||||
|
||||
def __init__(self, env, path):
|
||||
self.module_json = {}
|
||||
if isfile(join(path, "module.json")):
|
||||
self.module_json = util.load_json(join(path, "module.json"))
|
||||
|
||||
LibBuilderBase.__init__(self, env, path)
|
||||
|
||||
@property
|
||||
def src_dir(self):
|
||||
if isdir(join(self.path, "source")):
|
||||
return join(self.path, "source")
|
||||
return LibBuilderBase.src_dir.fget(self)
|
||||
|
||||
def get_path_dirs(self, use_build_dir=False):
|
||||
path_dirs = LibBuilderBase.get_path_dirs(self, use_build_dir)
|
||||
for p in self.module_json.get("extraIncludes", []):
|
||||
if p.startswith("source/"):
|
||||
p = p[7:]
|
||||
path_dirs.append(
|
||||
join(self.build_dir if use_build_dir else self.src_dir, p))
|
||||
return path_dirs
|
||||
|
||||
|
||||
class PlatformIOLibBuilder(LibBuilderBase):
|
||||
|
||||
def __init__(self, env, path):
|
||||
self.library_json = {}
|
||||
if isfile(join(path, "library.json")):
|
||||
self.library_json = util.load_json(join(path, "library.json"))
|
||||
|
||||
LibBuilderBase.__init__(self, env, path)
|
||||
|
||||
|
||||
def find_deps(env, scanner, path_dirs, src_dir, src_filter):
|
||||
result = []
|
||||
for item in env.MatchSourceFiles(src_dir, src_filter):
|
||||
result.extend(env.File(join(src_dir, item)).get_implicit_deps(
|
||||
env, scanner, path_dirs))
|
||||
return result
|
||||
|
||||
|
||||
def find_and_build_deps(env, lib_builders, scanner,
|
||||
src_dir, src_filter):
|
||||
path_dirs = tuple()
|
||||
built_path_dirs = tuple()
|
||||
for lb in lib_builders:
|
||||
items = [env.Dir(d) for d in lb.get_path_dirs()]
|
||||
if lb.is_built:
|
||||
built_path_dirs += tuple(items)
|
||||
else:
|
||||
path_dirs += tuple(items)
|
||||
path_dirs = built_path_dirs + path_dirs
|
||||
|
||||
target_lbs = []
|
||||
deps = find_deps(env, scanner, path_dirs, src_dir, src_filter)
|
||||
for d in deps:
|
||||
for lb in lib_builders:
|
||||
if d.get_abspath() in lb:
|
||||
if lb not in target_lbs and not lb.is_built:
|
||||
target_lbs.append(lb)
|
||||
break
|
||||
|
||||
libs = []
|
||||
# add build dirs to global CPPPATH
|
||||
for lb in target_lbs:
|
||||
env.Append(
|
||||
CPPPATH=lb.get_path_dirs(use_build_dir=True)
|
||||
)
|
||||
# start builder
|
||||
for lb in target_lbs:
|
||||
libs.append(lb.build())
|
||||
|
||||
if env.get("LIB_DFCYCLIC", "").lower() == "true":
|
||||
for lb in target_lbs:
|
||||
libs.extend(find_and_build_deps(
|
||||
env, lib_builders, scanner, lb.src_dir, lb.src_filter))
|
||||
|
||||
return libs
|
||||
|
||||
|
||||
def BuildDependentLibraries(env, src_dir):
|
||||
lib_builders = []
|
||||
libs_dirs = [env.subst(d) for d in env.get("LIBSOURCE_DIRS", [])
|
||||
if isdir(env.subst(d))]
|
||||
for libs_dir in libs_dirs:
|
||||
if not isdir(libs_dir):
|
||||
continue
|
||||
for item in sorted(os.listdir(libs_dir)):
|
||||
if isdir(join(libs_dir, item)):
|
||||
lib_builders.append(
|
||||
LibBuilderFactory.new(env, join(libs_dir, item)))
|
||||
|
||||
print "Looking for dependencies..."
|
||||
print "Library locations: " + ", ".join(libs_dirs)
|
||||
print "Collecting %d libraries" % len(lib_builders)
|
||||
|
||||
return find_and_build_deps(
|
||||
env, lib_builders, SCons.Scanner.C.CScanner(),
|
||||
src_dir, env.get("SRC_FILTER"))
|
||||
|
||||
|
||||
def exists(_):
|
||||
return True
|
||||
|
||||
|
||||
def generate(env):
|
||||
env.AddMethod(BuildDependentLibraries)
|
||||
return env
|
||||
@ -16,8 +16,8 @@ from __future__ import absolute_import
|
||||
|
||||
import re
|
||||
from glob import glob
|
||||
from os import listdir, sep, walk
|
||||
from os.path import basename, dirname, isdir, isfile, join, normpath, realpath
|
||||
from os import sep, walk
|
||||
from os.path import basename, dirname, isdir, join, normpath, realpath
|
||||
|
||||
from SCons.Script import COMMAND_LINE_TARGETS, DefaultEnvironment, SConscript
|
||||
from SCons.Util import case_sensitive_suffixes
|
||||
@ -26,11 +26,7 @@ from platformio.util import pioversion_to_intstr
|
||||
|
||||
SRC_BUILD_EXT = ["c", "cpp", "S", "spp", "SPP", "sx", "s", "asm", "ASM"]
|
||||
SRC_HEADER_EXT = ["h", "hpp"]
|
||||
SRC_DEFAULT_FILTER = " ".join([
|
||||
"+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep,
|
||||
"-<example%s>" % sep, "-<examples%s>" % sep,
|
||||
"-<test%s>" % sep, "-<tests%s>" % sep
|
||||
])
|
||||
SRC_FILTER_DEFAULT = " ".join(["+<*>", "-<.git%s>" % sep, "-<svn%s>" % sep])
|
||||
|
||||
|
||||
def BuildProgram(env):
|
||||
@ -173,7 +169,7 @@ def IsFileWithExt(env, file_, ext): # pylint: disable=W0613
|
||||
return False
|
||||
|
||||
|
||||
def MatchSourceFiles(env, src_dir, src_filter):
|
||||
def MatchSourceFiles(env, src_dir, src_filter=None):
|
||||
|
||||
SRC_FILTER_PATTERNS_RE = re.compile(r"(\+|\-)<([^>]+)>")
|
||||
|
||||
@ -181,6 +177,9 @@ def MatchSourceFiles(env, src_dir, src_filter):
|
||||
if env.IsFileWithExt(item, SRC_BUILD_EXT + SRC_HEADER_EXT):
|
||||
items.add(item.replace(src_dir + sep, ""))
|
||||
|
||||
src_dir = env.subst(src_dir)
|
||||
src_filter = src_filter or SRC_FILTER_DEFAULT
|
||||
|
||||
matches = set()
|
||||
# correct fs directory separator
|
||||
src_filter = src_filter.replace("/", sep).replace("\\", sep)
|
||||
@ -214,8 +213,7 @@ def CollectBuildFiles(env, variant_dir, src_dir,
|
||||
if src_dir.endswith(sep):
|
||||
src_dir = src_dir[:-1]
|
||||
|
||||
for item in env.MatchSourceFiles(
|
||||
src_dir, src_filter or SRC_DEFAULT_FILTER):
|
||||
for item in env.MatchSourceFiles(src_dir, src_filter):
|
||||
_reldir = dirname(item)
|
||||
_src_dir = join(src_dir, _reldir) if _reldir else src_dir
|
||||
_var_dir = join(variant_dir, _reldir) if _reldir else variant_dir
|
||||
@ -265,149 +263,6 @@ def BuildLibrary(env, variant_dir, src_dir, src_filter=None):
|
||||
)
|
||||
|
||||
|
||||
def BuildDependentLibraries(env, src_dir): # pylint: disable=R0914
|
||||
|
||||
INCLUDES_RE = re.compile(
|
||||
r"^\s*#include\s+(\<|\")([^\>\"\']+)(?:\>|\")", re.M)
|
||||
LIBSOURCE_DIRS = [env.subst(d) for d in env.get("LIBSOURCE_DIRS", [])]
|
||||
|
||||
# start internal prototypes
|
||||
|
||||
class IncludeFinder(object):
|
||||
|
||||
def __init__(self, base_dir, name, is_system=False):
|
||||
self.base_dir = base_dir
|
||||
self.name = name
|
||||
self.is_system = is_system
|
||||
|
||||
self._inc_path = None
|
||||
self._lib_dir = None
|
||||
self._lib_name = None
|
||||
|
||||
def getIncPath(self):
|
||||
return self._inc_path
|
||||
|
||||
def getLibDir(self):
|
||||
return self._lib_dir
|
||||
|
||||
def getLibName(self):
|
||||
return self._lib_name
|
||||
|
||||
def run(self):
|
||||
if not self.is_system and self._find_in_local():
|
||||
return True
|
||||
return self._find_in_system()
|
||||
|
||||
def _find_in_local(self):
|
||||
if isfile(join(self.base_dir, self.name)):
|
||||
self._inc_path = join(self.base_dir, self.name)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def _find_in_system(self):
|
||||
for lsd_dir in LIBSOURCE_DIRS:
|
||||
if not isdir(lsd_dir):
|
||||
continue
|
||||
|
||||
for ld in env.get("LIB_USE", []) + sorted(listdir(lsd_dir)):
|
||||
if not isdir(join(lsd_dir, ld)):
|
||||
continue
|
||||
|
||||
inc_path = normpath(join(lsd_dir, ld, self.name))
|
||||
try:
|
||||
lib_dir = inc_path[:inc_path.index(
|
||||
sep, len(lsd_dir) + 1)]
|
||||
except ValueError:
|
||||
continue
|
||||
lib_name = basename(lib_dir)
|
||||
|
||||
# ignore user's specified libs
|
||||
if lib_name in env.get("LIB_IGNORE", []):
|
||||
continue
|
||||
|
||||
if not isfile(inc_path):
|
||||
# if source code is in "src" dir
|
||||
lib_dir = join(lsd_dir, lib_name, "src")
|
||||
inc_path = join(lib_dir, self.name)
|
||||
|
||||
if isfile(inc_path):
|
||||
self._lib_dir = lib_dir
|
||||
self._lib_name = lib_name
|
||||
self._inc_path = inc_path
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_dep_libs(src_dir):
|
||||
state = {
|
||||
"paths": set(),
|
||||
"libs": set(),
|
||||
"ordered": set()
|
||||
}
|
||||
|
||||
state = _process_src_dir(state, env.subst(src_dir))
|
||||
|
||||
result = []
|
||||
for item in sorted(state['ordered'], key=lambda s: s[0]):
|
||||
result.append((item[1], item[2]))
|
||||
return result
|
||||
|
||||
def _process_src_dir(state, src_dir):
|
||||
for root, _, files in walk(src_dir, followlinks=True):
|
||||
for f in files:
|
||||
if env.IsFileWithExt(f, SRC_BUILD_EXT + SRC_HEADER_EXT):
|
||||
state = _parse_includes(state, env.File(join(root, f)))
|
||||
return state
|
||||
|
||||
def _parse_includes(state, node):
|
||||
skip_includes = ("arduino.h", "energia.h")
|
||||
matches = INCLUDES_RE.findall(node.get_text_contents())
|
||||
for (inc_type, inc_name) in matches:
|
||||
base_dir = dirname(node.get_abspath())
|
||||
if inc_name.lower() in skip_includes:
|
||||
continue
|
||||
if join(base_dir, inc_name) in state['paths']:
|
||||
continue
|
||||
else:
|
||||
state['paths'].add(join(base_dir, inc_name))
|
||||
|
||||
finder = IncludeFinder(base_dir, inc_name, inc_type == "<")
|
||||
if finder.run():
|
||||
_parse_includes(state, env.File(finder.getIncPath()))
|
||||
|
||||
_lib_dir = finder.getLibDir()
|
||||
if _lib_dir and _lib_dir not in state['libs']:
|
||||
state['ordered'].add((
|
||||
len(state['ordered']) + 1, finder.getLibName(),
|
||||
_lib_dir))
|
||||
state['libs'].add(_lib_dir)
|
||||
|
||||
if env.subst("$LIB_DFCYCLIC").lower() == "true":
|
||||
state = _process_src_dir(state, _lib_dir)
|
||||
return state
|
||||
|
||||
# end internal prototypes
|
||||
|
||||
deplibs = _get_dep_libs(src_dir)
|
||||
for l, ld in deplibs:
|
||||
env.Append(
|
||||
CPPPATH=[join("$BUILD_DIR", l)]
|
||||
)
|
||||
# add automatically "utility" dir from the lib (Arduino issue)
|
||||
if isdir(join(ld, "utility")):
|
||||
env.Append(
|
||||
CPPPATH=[join("$BUILD_DIR", l, "utility")]
|
||||
)
|
||||
|
||||
libs = []
|
||||
for (libname, inc_dir) in deplibs:
|
||||
lib = env.BuildLibrary(
|
||||
join("$BUILD_DIR", libname), inc_dir)
|
||||
env.Clean(libname, lib)
|
||||
libs.append(lib)
|
||||
return libs
|
||||
|
||||
|
||||
def exists(_):
|
||||
return True
|
||||
|
||||
@ -422,5 +277,4 @@ def generate(env):
|
||||
env.AddMethod(CollectBuildFiles)
|
||||
env.AddMethod(BuildFrameworks)
|
||||
env.AddMethod(BuildLibrary)
|
||||
env.AddMethod(BuildDependentLibraries)
|
||||
return env
|
||||
|
||||
Reference in New Issue
Block a user