From f84b113b5e44c2011f170025b924dd78edede10a Mon Sep 17 00:00:00 2001 From: Arne Augenstein Date: Sat, 24 Jan 2015 08:53:34 +0100 Subject: [PATCH 1/8] fixing handling of relative paths --- platformio/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index a5c015f3..0cfb389c 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -4,7 +4,7 @@ import json from os import name as os_name from os import getcwd, getenv, listdir, makedirs, utime -from os.path import dirname, expanduser, isdir, isfile, join, realpath +from os.path import dirname, expanduser, isdir, isfile, join, realpath, abspath from platform import system, uname from subprocess import PIPE, Popen @@ -51,7 +51,7 @@ def get_lib_dir(): config = get_project_config() if (config.has_section("platformio") and config.has_option("platformio", "lib_dir")): - lib_dir = config.get("platformio", "lib_dir") + lib_dir = abspath(config.get("platformio", "lib_dir")) if lib_dir.startswith("~"): return expanduser(lib_dir) else: From 87e1b09088b68102ee1d72c98b678781844db00b Mon Sep 17 00:00:00 2001 From: Arne Augenstein Date: Sat, 24 Jan 2015 16:04:53 +0100 Subject: [PATCH 2/8] sort imports --- platformio/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/util.py b/platformio/util.py index 0cfb389c..fe850d0d 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -4,7 +4,7 @@ import json from os import name as os_name from os import getcwd, getenv, listdir, makedirs, utime -from os.path import dirname, expanduser, isdir, isfile, join, realpath, abspath +from os.path import abspath, dirname, expanduser, isdir, isfile, join, realpath from platform import system, uname from subprocess import PIPE, Popen From ef76822773be82a3f5c8570614c2aad50db58bfb Mon Sep 17 00:00:00 2001 From: Arne Augenstein Date: Sat, 24 Jan 2015 16:14:44 +0100 Subject: [PATCH 3/8] expand path first, then get absolute path --- platformio/util.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index fe850d0d..83c3bdf3 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -51,11 +51,10 @@ def get_lib_dir(): config = get_project_config() if (config.has_section("platformio") and config.has_option("platformio", "lib_dir")): - lib_dir = abspath(config.get("platformio", "lib_dir")) + lib_dir = config.get("platformio", "lib_dir") if lib_dir.startswith("~"): - return expanduser(lib_dir) - else: - return lib_dir + lib_dir = expanduser(lib_dir) + return abspath(lib_dir) except NotPlatformProject: pass return join(get_home_dir(), "lib") From abd6ff9145cda7d32bb4977689003526b9f6dfea Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Jan 2015 12:45:10 +0200 Subject: [PATCH 4/8] Apply "isort" --- platformio/builder/scripts/atmelavr.py | 2 +- platformio/builder/scripts/timsp430.py | 2 +- platformio/builder/scripts/titiva.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/platformio/builder/scripts/atmelavr.py b/platformio/builder/scripts/atmelavr.py index 2b2584a4..dc02c28b 100644 --- a/platformio/builder/scripts/atmelavr.py +++ b/platformio/builder/scripts/atmelavr.py @@ -8,7 +8,7 @@ from os.path import join from time import sleep -from SCons.Script import (AlwaysBuild, Builder, COMMAND_LINE_TARGETS, Default, +from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild, Builder, Default, DefaultEnvironment, Exit) from platformio.util import get_serialports diff --git a/platformio/builder/scripts/timsp430.py b/platformio/builder/scripts/timsp430.py index 98ac204f..5af0f0c8 100644 --- a/platformio/builder/scripts/timsp430.py +++ b/platformio/builder/scripts/timsp430.py @@ -9,7 +9,7 @@ from os.path import join from platform import system -from SCons.Script import (AlwaysBuild, Builder, COMMAND_LINE_TARGETS, Default, +from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild, Builder, Default, DefaultEnvironment) env = DefaultEnvironment() diff --git a/platformio/builder/scripts/titiva.py b/platformio/builder/scripts/titiva.py index 046d11b0..8a23fedd 100644 --- a/platformio/builder/scripts/titiva.py +++ b/platformio/builder/scripts/titiva.py @@ -8,7 +8,7 @@ from os.path import join -from SCons.Script import (AlwaysBuild, Builder, COMMAND_LINE_TARGETS, Default, +from SCons.Script import (COMMAND_LINE_TARGETS, AlwaysBuild, Builder, Default, DefaultEnvironment) env = DefaultEnvironment() From d2ad023b5259beb2ed4e4004aec912b30efd744e Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Jan 2015 12:45:26 +0200 Subject: [PATCH 5/8] Apply "isort" --- platformio/commands/lib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/commands/lib.py b/platformio/commands/lib.py index 96b750e5..cb6e89a1 100644 --- a/platformio/commands/lib.py +++ b/platformio/commands/lib.py @@ -9,7 +9,6 @@ from platformio import app, exception from platformio.libmanager import LibraryManager from platformio.util import get_api_result, get_lib_dir - LIBLIST_TPL = ("[{id:^14}] {name:<25} {compatibility:<30} " "\"{authornames}\": {description}") From 1ae23085d4f23316e1d7eeb69f71ae92749aaa7a Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Jan 2015 12:49:46 +0200 Subject: [PATCH 6/8] Allow to ignore some libs from "Library Dependency Finder" --- HISTORY.rst | 2 ++ docs/projectconf.rst | 13 +++++++++++++ platformio/builder/main.py | 5 ++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 52dcf346..079af8c7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ Release History command which allows to return the output in `JSON `_ format (`issue #42 `_) * Fixed an issue with the libraries that are git repositories (`issue #49 `_) +* Allowed to ignore some libs from "Library Dependency Finder" via + `ignore_libs Date: Thu, 29 Jan 2015 14:05:02 +0200 Subject: [PATCH 7/8] Expand warning for all OS about obsolate "setuptools" --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 287188da..386b5532 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -62,7 +62,7 @@ On *Windows OS* it may look like: C:\Python27\python.exe get-platformio.py .. warning:: - **Mac OS Users**: If you have an error ``pkg_resources.DistributionNotFound`` please + If you have an error ``pkg_resources.DistributionNotFound`` please upgrade *SetupTools* package: ``$ [sudo] pip uninstall setuptools`` and ``$ [sudo] pip install setuptools``. Then re-install *PlatformIO*: ``$ [sudo] pip uninstall platformio`` From 1921164ba36eed9ae1e938c9982805f16272018b Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 29 Jan 2015 16:09:17 +0200 Subject: [PATCH 8/8] Refactored *Library Dependency Finder* // Resolve #48 #50 #55 --- HISTORY.rst | 12 +- platformio/builder/tools/platformio.py | 223 ++++++++++++++++--------- 2 files changed, 153 insertions(+), 82 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 079af8c7..5d6194ad 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,16 +1,20 @@ Release History =============== -0.11.0 (2015-01-?) ------------------- +0.11.0 (2015-?) +--------------- +* Refactored *Library Dependency Finder* (issues + `#48 `_, + `#50 `_, + `#55 `_) * Added ``--json-output`` option to `platformio boards `__ command which allows to return the output in `JSON `_ format (`issue #42 `_) -* Fixed an issue with the libraries that are git repositories (`issue #49 `_) -* Allowed to ignore some libs from "Library Dependency Finder" via +* Allowed to ignore some libs from *Library Dependency Finder* via `ignore_libs `_) 0.10.2 (2015-01-06) diff --git a/platformio/builder/tools/platformio.py b/platformio/builder/tools/platformio.py index 10db16f4..7bb942b6 100644 --- a/platformio/builder/tools/platformio.py +++ b/platformio/builder/tools/platformio.py @@ -4,8 +4,8 @@ import atexit import platform import re -from os import getenv, listdir, remove, walk -from os.path import basename, isdir, isfile, join +from os import getenv, listdir, remove, sep, walk +from os.path import basename, dirname, isdir, isfile, join, normpath from time import sleep from SCons.Script import Exit, SConscript, SConscriptChdir @@ -35,7 +35,7 @@ def BuildFirmware(env, corelibs): join("$BUILD_DIR", "src"), join("$PROJECT_DIR", "src")) # build dependent libs - deplibs = src.BuildDependentLibraries(vdirs) + deplibs = src.BuildDependentLibraries(join("$PROJECT_DIR", "src")) src.MergeFlags(getenv("PIOSRCBUILD_FLAGS", "$SRCBUILD_FLAGS")) @@ -56,6 +56,20 @@ def GlobCXXFiles(env, path): return files +def VariantDirRecursive(env, variant_dir, src_dir, duplicate=True): + ignore_pattern = (".git", ".svn", "examples") + variants = [] + src_dir = env.subst(src_dir) + for root, _, _ in walk(src_dir): + _src_dir = root + _var_dir = variant_dir + root.replace(src_dir, "") + if any([s in _src_dir.lower() for s in ignore_pattern]): + continue + env.VariantDir(_var_dir, _src_dir, duplicate) + variants.append(_var_dir) + return variants + + def BuildLibrary(env, variant_dir, library_dir): lib = env.Clone() vdirs = lib.VariantDirRecursive(variant_dir, library_dir) @@ -65,11 +79,135 @@ def BuildLibrary(env, variant_dir, library_dir): ) -def BuildDependentLibraries(env, src_dirs): - libs = [] - deplibs = env.GetDependentLibraries(src_dirs) +def BuildDependentLibraries(env, src_dir): # pylint: disable=R0914 + + INCLUDES_RE = re.compile(r"^\s*#include\s+(\<|\")([^\>\"\']+)(?:\>|\")", + re.M) + LIBSOURCE_DIRS = [env.subst(d) for d in env.get("LIBSOURCE_DIRS", [])] + + # start internal prototypes + + class IncludeFinder(object): + + def __init__(self, base_dir, name, is_system=False): + self.base_dir = base_dir + self.name = name + self.is_system = is_system + + self._inc_path = None + self._lib_dir = None + self._lib_name = None + + def getIncPath(self): + return self._inc_path + + def getLibDir(self): + return self._lib_dir + + def getLibName(self): + return self._lib_name + + def run(self): + if not self.is_system and self._find_in_local(): + return True + return self._find_in_system() + + def _find_in_local(self): + if isfile(join(self.base_dir, self.name)): + self._inc_path = join(self.base_dir, self.name) + return True + else: + return False + + def _find_in_system(self): + for lsd_dir in LIBSOURCE_DIRS: + if not isdir(lsd_dir): + continue + + for ld in listdir(lsd_dir): + inc_path = normpath(join(lsd_dir, ld, self.name)) + lib_dir = inc_path[:inc_path.index(sep, len(lsd_dir) + 1)] + lib_name = basename(lib_dir) + + # ignore user's specified libs + if "IGNORE_LIBS" in env and lib_name in env['IGNORE_LIBS']: + continue + + if not isfile(inc_path): + # if source code is in "src" dir + lib_dir = join(lsd_dir, lib_name, "src") + inc_path = join(lib_dir, self.name) + + if isfile(inc_path): + self._lib_dir = lib_dir + self._lib_name = lib_name + self._inc_path = inc_path + return True + return False + + def _get_dep_libs(src_dir): + state = { + "paths": set(), + "libs": set(), + "ordered": set() + } + state = _process_src_dir(state, env.subst(src_dir)) + + result = [] + for item in sorted(state['ordered'], key=lambda s: s[0]): + result.append((item[1], item[2])) + return result + + def _process_src_dir(state, src_dir): + for root, _, _ in walk(src_dir): + for node in (env.GlobCXXFiles(root) + + env.Glob(join(root, "*.h"))): + state = _parse_includes(state, node) + return state + + def _parse_includes(state, node): + if node.path in state['paths']: + return state + else: + state['paths'].add(node.path) + + skip_includes = ("arduino.h", "energia.h") + matches = INCLUDES_RE.findall(node.get_text_contents()) + for (inc_type, inc_name) in matches: + base_dir = dirname(node.path) + if inc_name.lower() in skip_includes: + continue + if join(base_dir, inc_name) in state['paths']: + continue + else: + state['paths'].add(join(base_dir, inc_name)) + + finder = IncludeFinder(base_dir, inc_name, inc_type == "<") + if finder.run(): + _lib_dir = finder.getLibDir() + + if _lib_dir and _lib_dir not in state['libs']: + state['ordered'].add(( + len(state['ordered']) + 1, finder.getLibName(), + _lib_dir)) + + _parse_includes(state, env.File(finder.getIncPath())) + + if _lib_dir and _lib_dir not in state['libs']: + state['libs'].add(_lib_dir) + state = _process_src_dir(state, _lib_dir) + return state + + # end internal prototypes + + deplibs = _get_dep_libs(src_dir) env.Append(CPPPATH=[join("$BUILD_DIR", l) for (l, _) in deplibs]) + # add automatically "utility" dir from the lib (Arduino issue) + env.Append(CPPPATH=[join("$BUILD_DIR", l, "utility") for (l, ld) in deplibs + if isdir(join(ld, "utility"))]) + + libs = [] for (libname, inc_dir) in deplibs: lib = env.BuildLibrary( join("$BUILD_DIR", libname), inc_dir) @@ -78,75 +216,6 @@ def BuildDependentLibraries(env, src_dirs): return libs -def GetDependentLibraries(env, src_dirs): - includes = {} - nodes = [] - regexp = re.compile(r"^\s*#include\s+(?:\<|\")([^\>\"\']+)(?:\>|\")", re.M) - - for item in src_dirs: - nodes += env.GlobCXXFiles(item) - nodes += env.Glob(join(item, "*.h")) - - for node in nodes: - env.ParseIncludesRecurive(regexp, node, includes) - includes = sorted(includes.items(), key=lambda s: s[0]) - - result = [] - for i in includes: - items = [(i[1][1], i[1][2])] - - if isdir(join(items[0][1], "utility")): - items.append(("%sUtility" % items[0][0], - join(items[0][1], "utility"))) - - for item in items: - if item in result: - continue - result.append(item) - return result - - -def ParseIncludesRecurive(env, regexp, source_file, includes): - matches = regexp.findall(source_file.get_text_contents()) - for inc_fname in matches: - if inc_fname in includes: - continue - - for lsd_dir in env['LIBSOURCE_DIRS']: - lsd_dir = env.subst(lsd_dir) - if not isdir(lsd_dir): - continue - - for libname in listdir(lsd_dir): - inc_dir = join(lsd_dir, libname) - inc_file = join(inc_dir, inc_fname) - if not isfile(inc_file): - # if source code is in "src" dir - inc_dir = join(lsd_dir, libname, "src") - inc_file = join(inc_dir, inc_fname) - if not isfile(inc_file): - continue - - includes[inc_fname] = (len(includes) + 1, libname, inc_dir) - env.ParseIncludesRecurive(regexp, env.File(inc_file), includes) - - -def VariantDirRecursive(env, variant_dir, src_dir, duplicate=True): - # add root dir by default - variants = [variant_dir] - env.VariantDir(variant_dir, src_dir, duplicate) - for root, dirnames, _ in walk(env.subst(src_dir)): - if not dirnames: - continue - for dn in dirnames: - source_dir = join(root, dn) - if ".git" in source_dir or ".svn" in source_dir: - continue - env.VariantDir(join(variant_dir, dn), source_dir, duplicate) - variants.append(join(variant_dir, dn)) - return variants - - def ConvertInoToCpp(env): def delete_tmpcpp(files): @@ -244,11 +313,9 @@ def generate(env): env.AddMethod(ProcessGeneral) env.AddMethod(BuildFirmware) env.AddMethod(GlobCXXFiles) + env.AddMethod(VariantDirRecursive) env.AddMethod(BuildLibrary) env.AddMethod(BuildDependentLibraries) - env.AddMethod(GetDependentLibraries) - env.AddMethod(ParseIncludesRecurive) - env.AddMethod(VariantDirRecursive) env.AddMethod(ConvertInoToCpp) env.AddMethod(FlushSerialBuffer) env.AddMethod(TouchSerialPort)