CLion: Add paths to libraries specified via lib_extra_dirs option (#3463)

* Add paths to libraries specified via lib_extra_dirs option

Besides, global folders in SRC_LIST seem a bit unnecessary
since there might be unused libraries in these folders

* Refactor processing of includes when exporting IDE/Editor projects

Split includes according to their source. That will help export includes in a more flexible way.
For example some IDEs don't need include paths from toolchains

* Add new record to history log

* Typo fix

HISTORY.rst

@@ -15,6 +15,7 @@ PlatformIO Core 4
 * Fixed UnicodeDecodeError on Windows when network drive (NAS) is used (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_)
 * Fixed an issue when saving libraries in new project results in error "No option 'lib_deps' in section" (`issue #3442 <https://github.com/platformio/platformio-core/issues/3442>`_)
 * Fixed an incorrect node path used for pattern matching when processing middleware nodes
+* Fixed an issue with missing ``lib_extra_dirs`` option in SRC_LIST for CLion (`issue #3460 <https://github.com/platformio/platformio-core/issues/3460>`_)
 
 4.3.1 (2020-03-20)
 ~~~~~~~~~~~~~~~~~~

platformio/builder/tools/pioide.py

@@ -25,17 +25,26 @@ from platformio.proc import exec_command, where_is_program
 
 
 def _dump_includes(env):
-    includes = []
+    includes = {}
 
-    for item in env.get("CPPPATH", []):
+    includes["build"] = [
-        includes.append(env.subst(item))
+        env.subst("$PROJECT_INCLUDE_DIR"),
+        env.subst("$PROJECT_SRC_DIR"),
+    ]
+    includes["build"].extend(
+        [os.path.realpath(env.subst(item)) for item in env.get("CPPPATH", [])]
+    )
 
     # installed libs
+    includes["compatlib"] = []
     for lb in env.GetLibBuilders():
-        includes.extend(lb.get_include_dirs())
+        includes["compatlib"].extend(
+            [os.path.realpath(inc) for inc in lb.get_include_dirs()]
+        )
 
     # includes from toolchains
     p = env.PioPlatform()
+    includes["toolchain"] = []
     for name in p.get_installed_packages():
         if p.get_package_type(name) != "toolchain":
             continue
@@ -47,22 +56,14 @@ def _dump_includes(env):
             os.path.join(toolchain_dir, "lib", "gcc", "*", "*", "include*"),
         ]
         for g in toolchain_incglobs:
-            includes.extend(glob(g))
+            includes["toolchain"].extend([os.path.realpath(inc) for inc in glob(g)])
 
+    includes["unity"] = []
     unity_dir = get_core_package_dir("tool-unity")
     if unity_dir:
-        includes.append(unity_dir)
+        includes["unity"].append(unity_dir)
 
-    includes.extend([env.subst("$PROJECT_INCLUDE_DIR"), env.subst("$PROJECT_SRC_DIR")])
+    return includes
-
-    # remove duplicates
-    result = []
-    for item in includes:
-        item = os.path.realpath(item)
-        if item not in result:
-            result.append(item)
-
-    return result
 
 
 def _get_gcc_defines(env):
@@ -158,8 +159,6 @@ def DumpIDEData(env):
         "libsource_dirs": [env.subst(l) for l in env.GetLibSourceDirs()],
         "defines": _dump_defines(env),
         "includes": _dump_includes(env),
-        "cc_flags": env.subst(LINTCCOM),
-        "cxx_flags": env.subst(LINTCXXCOM),
         "cc_path": where_is_program(env.subst("$CC"), env.subst("${ENV['PATH']}")),
         "cxx_path": where_is_program(env.subst("$CXX"), env.subst("${ENV['PATH']}")),
         "gdb_path": where_is_program(env.subst("$GDB"), env.subst("${ENV['PATH']}")),

platformio/ide/projectgenerator.py

@@ -57,6 +57,20 @@ class ProjectGenerator(object):
 
         return envname
 
+    @staticmethod
+    def filter_includes(includes_map, ignore_scopes=None, to_unix_path=True):
+        ignore_scopes = ignore_scopes or []
+        result = []
+        for scope, includes in includes_map.items():
+            if scope in ignore_scopes:
+                continue
+            for include in includes:
+                if to_unix_path:
+                    include = fs.to_unix_path(include)
+                if include not in result:
+                    result.append(include)
+        return result
+
     def _load_tplvars(self):
         tpl_vars = {
             "config": self.config,
@@ -92,12 +106,13 @@ class ProjectGenerator(object):
         for key, value in tpl_vars.items():
             if key.endswith(("_path", "_dir")):
                 tpl_vars[key] = fs.to_unix_path(value)
-        for key in ("includes", "src_files", "libsource_dirs"):
+        for key in ("src_files", "libsource_dirs"):
             if key not in tpl_vars:
                 continue
             tpl_vars[key] = [fs.to_unix_path(inc) for inc in tpl_vars[key]]
 
         tpl_vars["to_unix_path"] = fs.to_unix_path
+        tpl_vars["filter_includes"] = self.filter_includes
         return tpl_vars
 
     def get_src_files(self):

platformio/ide/tpls/atom/.clang_complete.tpl

@@ -1,4 +1,4 @@
-% for include in includes:
+% for include in filter_includes(includes):
 -I{{include}}
 % end
 % for define in defines:

platformio/ide/tpls/atom/.gcc-flags.json.tpl

@@ -4,6 +4,6 @@
   "gccDefaultCFlags": "-fsyntax-only {{! to_unix_path(cc_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
   "gccDefaultCppFlags": "-fsyntax-only {{! to_unix_path(cxx_flags).replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
   "gccErrorLimit": 15,
-  "gccIncludePaths": "{{ ','.join(includes) }}",
+  "gccIncludePaths": "{{ ','.join(filter_includes(includes)) }}",
   "gccSuppressWarnings": false
 }

platformio/ide/tpls/clion/CMakeListsPrivate.txt.tpl

@@ -5,10 +5,12 @@
 # please create `CMakeListsUser.txt` in the root of project.
 # The `CMakeListsUser.txt` will not be overwritten by PlatformIO.
 
-% from platformio.project.helpers import (load_project_ide_data)
+% import os
-%
 % import re
 %
+% from platformio.compat import WINDOWS
+% from platformio.project.helpers import (load_project_ide_data)
+%
 % def _normalize_path(path):
 %   if project_dir in path:
 %     path = path.replace(project_dir, "${CMAKE_CURRENT_LIST_DIR}")
@@ -22,12 +24,24 @@
 %   return path
 % end
 %
+% def _fix_lib_dirs(lib_dirs):
+%   result = []
+%   for lib_dir in lib_dirs:
+%     if not os.path.isabs(lib_dir):
+%       lib_dir = os.path.join(project_dir, lib_dir)
+%     end
+%     result.append(to_unix_path(os.path.normpath(lib_dir)))
+%   end
+%   return result
+% end
+%
 % def _escape(text):
 %   return to_unix_path(text).replace('"', '\\"')
 % end
 %
 % envs = config.envs()
 
+
 % if len(envs) > 1:
 set(CMAKE_CONFIGURATION_TYPES "{{ ";".join(envs) }};" CACHE STRING "Build Types reflect PlatformIO Environments" FORCE)
 % else:
@@ -54,13 +68,13 @@ set(CMAKE_CXX_STANDARD {{ cxx_stds[-1] }})
 % end
 
 if (CMAKE_BUILD_TYPE MATCHES "{{ env_name }}")
-%for define in defines:
+% for define in defines:
     add_definitions(-D'{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}')
-%end
+% end
 
-%for include in includes:
+% for include in filter_includes(includes):
-    include_directories("{{ _normalize_path(to_unix_path(include)) }}")
+    include_directories("{{ _normalize_path(include) }}")
-%end
+% end
 endif()
 
 % leftover_envs = list(set(envs) ^ set([env_name]))
@@ -76,9 +90,16 @@ if (CMAKE_BUILD_TYPE MATCHES "{{ env }}")
     add_definitions(-D'{{!re.sub(r"([\"\(\)#])", r"\\\1", define)}}')
 %   end
 
-%   for include in data["includes"]:
+%   for include in filter_includes(data["includes"]):
     include_directories("{{ _normalize_path(to_unix_path(include)) }}")
 %   end
 endif()
 % end
-FILE(GLOB_RECURSE SRC_LIST "{{ _normalize_path(project_src_dir) }}/*.*" "{{ _normalize_path(project_lib_dir) }}/*.*" "{{ _normalize_path(project_libdeps_dir) }}/*.*")
+%
+% lib_extra_dirs = _fix_lib_dirs(config.get("env:" + env_name, "lib_extra_dirs", []))
+% src_paths = [project_src_dir, project_lib_dir, project_libdeps_dir] + lib_extra_dirs
+FILE(GLOB_RECURSE SRC_LIST
+%   for path in src_paths:
+    {{  _normalize_path(path) + "/*.*" }}
+%   end
+)

platformio/ide/tpls/codeblocks/platformio.cbp.tpl

@@ -52,7 +52,7 @@
 			% for define in defines:
 			<Add option="-D{{define}}"/>
 			% end
-			% for include in includes:
+			% for include in filter_includes(includes):
 			<Add directory="{{include}}"/>
 			% end
 		</Compiler>

platformio/ide/tpls/eclipse/.cproject.tpl

@@ -23,10 +23,8 @@
 							<tool id="org.eclipse.cdt.build.core.settings.holder.libs.1409095472" name="holder for library settings" superClass="org.eclipse.cdt.build.core.settings.holder.libs"/>
 							<tool id="org.eclipse.cdt.build.core.settings.holder.1624502120" name="Assembly" superClass="org.eclipse.cdt.build.core.settings.holder">
 								<option id="org.eclipse.cdt.build.core.settings.holder.incpaths.239157887" name="Include Paths" superClass="org.eclipse.cdt.build.core.settings.holder.incpaths" valueType="includePath">
-									% for include in includes:
+									% cleaned_includes = filter_includes(includes, ["toolchain"])
-									% if "toolchain" in include:
+									% for include in cleaned_includes:
-									% continue
-									% end
                                     % if include.startswith(user_home_dir):
                                     % if "windows" in systype:
 									<listOptionValue builtIn="false" value="${USERPROFILE}{{include.replace(user_home_dir, '')}}"/>
@@ -47,10 +45,7 @@
 							</tool>
 							<tool id="org.eclipse.cdt.build.core.settings.holder.54121539" name="GNU C++" superClass="org.eclipse.cdt.build.core.settings.holder">
 								<option id="org.eclipse.cdt.build.core.settings.holder.incpaths.1096940598" name="Include Paths" superClass="org.eclipse.cdt.build.core.settings.holder.incpaths" valueType="includePath">
-									% for include in includes:
+									% for include in cleaned_includes:
-									% if "toolchain" in include:
-									% continue
-									% end
                                     % if include.startswith(user_home_dir):
                                     % if "windows" in systype:
 									<listOptionValue builtIn="false" value="${USERPROFILE}{{include.replace(user_home_dir, '')}}"/>
@@ -71,10 +66,7 @@
 							</tool>
 							<tool id="org.eclipse.cdt.build.core.settings.holder.1310559623" name="GNU C" superClass="org.eclipse.cdt.build.core.settings.holder">
 								<option id="org.eclipse.cdt.build.core.settings.holder.incpaths.41298875" name="Include Paths" superClass="org.eclipse.cdt.build.core.settings.holder.incpaths" valueType="includePath">
-									% for include in includes:
+									% for include in cleaned_includes:
-									% if "toolchain" in include:
-									% continue
-									% end
                                     % if include.startswith(user_home_dir):
                                     % if "windows" in systype:
 									<listOptionValue builtIn="false" value="${USERPROFILE}{{include.replace(user_home_dir, '')}}"/>
@@ -121,10 +113,7 @@
 							<tool id="org.eclipse.cdt.build.core.settings.holder.libs.1855678035" name="holder for library settings" superClass="org.eclipse.cdt.build.core.settings.holder.libs"/>
 							<tool id="org.eclipse.cdt.build.core.settings.holder.30528994" name="Assembly" superClass="org.eclipse.cdt.build.core.settings.holder">
 								<option id="org.eclipse.cdt.build.core.settings.holder.incpaths.794801023" name="Include Paths" superClass="org.eclipse.cdt.build.core.settings.holder.incpaths" valueType="includePath">
-									% for include in includes:
+									% for include in cleaned_includes:
-									% if "toolchain" in include:
-									% continue
-									% end
                                     % if include.startswith(user_home_dir):
                                     % if "windows" in systype:
 									<listOptionValue builtIn="false" value="${USERPROFILE}{{include.replace(user_home_dir, '')}}"/>
@@ -145,10 +134,7 @@
 							</tool>
 							<tool id="org.eclipse.cdt.build.core.settings.holder.1146422798" name="GNU C++" superClass="org.eclipse.cdt.build.core.settings.holder">
 								<option id="org.eclipse.cdt.build.core.settings.holder.incpaths.650084869" name="Include Paths" superClass="org.eclipse.cdt.build.core.settings.holder.incpaths" useByScannerDiscovery="false" valueType="includePath">
-									% for include in includes:
+									% for include in cleaned_includes:
-									% if "toolchain" in include:
-									% continue
-									% end
                                     % if include.startswith(user_home_dir):
                                     % if "windows" in systype:
 									<listOptionValue builtIn="false" value="${USERPROFILE}{{include.replace(user_home_dir, '')}}"/>
@@ -169,10 +155,7 @@
 							</tool>
 							<tool id="org.eclipse.cdt.build.core.settings.holder.1637357529" name="GNU C" superClass="org.eclipse.cdt.build.core.settings.holder">
 								<option id="org.eclipse.cdt.build.core.settings.holder.incpaths.1246337321" name="Include Paths" superClass="org.eclipse.cdt.build.core.settings.holder.incpaths" useByScannerDiscovery="false" valueType="includePath">
-									% for include in includes:
+									% for include in cleaned_includes:
-									% if "toolchain" in include:
-									% continue
-									% end
                                     % if include.startswith(user_home_dir):
                                     % if "windows" in systype:
 									<listOptionValue builtIn="false" value="${USERPROFILE}{{include.replace(user_home_dir, '')}}"/>

platformio/ide/tpls/emacs/.ccls.tpl

@@ -13,7 +13,7 @@ clang
 {{"%cpp"}} -std=c++{{ cxx_stds[-1] }}
 % end
 
-% for include in includes:
+% for include in filter_includes(includes):
 -I{{ include }}
 % end
 

platformio/ide/tpls/emacs/.clang_complete.tpl

@@ -1,4 +1,4 @@
-% for include in includes:
+% for include in filter_includes(includes):
 -I{{include}}
 % end
 % for define in defines:

platformio/ide/tpls/netbeans/nbproject/configurations.xml.tpl

@@ -34,9 +34,10 @@
           <cleanCommand>{{platformio_path}} -f -c netbeans run --target clean</cleanCommand>
           <executablePath></executablePath>
           <cTool>
+            % cleaned_includes = filter_includes(includes)
             <incDir>
               <pElem>src</pElem>
-              % for include in includes:
+              % for include in cleaned_includes:
               <pElem>{{include}}</pElem>
               % end
             </incDir>
@@ -49,7 +50,7 @@
           <ccTool>
             <incDir>
               <pElem>src</pElem>
-              % for include in includes:
+              % for include in cleaned_includes:
               <pElem>{{include}}</pElem>
               % end
             </incDir>

platformio/ide/tpls/qtcreator/platformio.pro.tpl

@@ -5,7 +5,7 @@ else {
     HOMEDIR += $$(HOME)
 }
 
-% for include in includes:
+% for include in filter_includes(includes):
 % if include.startswith(user_home_dir):
 INCLUDEPATH += "$${HOMEDIR}{{include.replace(user_home_dir, "")}}"
 % else:

platformio/ide/tpls/vim/.ccls.tpl

@@ -13,7 +13,7 @@ clang
 {{"%cpp"}} -std=c++{{ cxx_stds[-1] }}
 % end
 
-% for include in includes:
+% for include in filter_includes(includes):
 -I{{ include }}
 % end
 

platformio/ide/tpls/vim/.clang_complete.tpl

@@ -1,4 +1,4 @@
-% for include in includes:
+% for include in filter_includes(includes):
 -I"{{include}}"
 % end
 % for define in defines:

platformio/ide/tpls/vim/.gcc-flags.json.tpl

@@ -4,6 +4,6 @@
   "gccDefaultCFlags": "-fsyntax-only {{! cc_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
   "gccDefaultCppFlags": "-fsyntax-only {{! cxx_flags.replace(' -MMD ', ' ').replace('"', '\\"') }} {{ !_defines.replace('"', '\\"') }}",
   "gccErrorLimit": 15,
-  "gccIncludePaths": "{{ ','.join(includes) }}",
+  "gccIncludePaths": "{{ ','.join(filter_includes(includes)) }}",
   "gccSuppressWarnings": false
 }

platformio/ide/tpls/visualstudio/platformio.vcxproj.tpl

@@ -42,13 +42,14 @@
     <NMakeBuildCommandLine>platformio -f -c visualstudio run</NMakeBuildCommandLine>
     <NMakeCleanCommandLine>platformio -f -c visualstudio run --target clean</NMakeCleanCommandLine>
     <NMakePreprocessorDefinitions>{{!";".join(defines)}}</NMakePreprocessorDefinitions>
-    <NMakeIncludeSearchPath>{{";".join(["$(HOMEDRIVE)$(HOMEPATH)%s" % i.replace(user_home_dir, "") if i.startswith(user_home_dir) else i for i in includes])}}</NMakeIncludeSearchPath>
+    % cleaned_includes = filter_includes(includes)
+    <NMakeIncludeSearchPath>{{";".join(["$(HOMEDRIVE)$(HOMEPATH)%s" % i.replace(user_home_dir, "") if i.startswith(user_home_dir) else i for i in cleaned_includes])}}</NMakeIncludeSearchPath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <NMakeBuildCommandLine>platformio run</NMakeBuildCommandLine>
     <NMakeCleanCommandLine>platformio run --target clean</NMakeCleanCommandLine>
     <NMakePreprocessorDefinitions>{{!";".join(defines)}}</NMakePreprocessorDefinitions>
-    <NMakeIncludeSearchPath>{{";".join(["$(HOMEDRIVE)$(HOMEPATH)%s" % i.replace(user_home_dir, "") if i.startswith(user_home_dir) else i for i in includes])}}</NMakeIncludeSearchPath>
+    <NMakeIncludeSearchPath>{{";".join(["$(HOMEDRIVE)$(HOMEPATH)%s" % i.replace(user_home_dir, "") if i.startswith(user_home_dir) else i for i in cleaned_includes])}}</NMakeIncludeSearchPath>
   </PropertyGroup>
   <ItemDefinitionGroup>
   </ItemDefinitionGroup>

platformio/ide/tpls/vscode/.vscode/c_cpp_properties.json.tpl

@@ -66,13 +66,7 @@
 %   return result
 % end
 %
-% cleaned_includes = []
+% cleaned_includes = filter_includes(includes, ["toolchain"])
-% for include in includes:
-%   if "toolchain-" not in os.path.dirname(os.path.commonprefix(
-%       [include, cc_path])) and os.path.isdir(include):
-%     cleaned_includes.append(include)
-%   end
-% end
 %
 % STD_RE = re.compile(r"\-std=[a-z\+]+(\d+)")
 % cc_stds = STD_RE.findall(cc_flags)