Add support for “globstar/**” (recursive) pattern

HISTORY.rst

@@ -6,6 +6,11 @@ Release Notes
 PlatformIO Core 4
 -----------------
 
+4.3.5 (2020-??-??)
+~~~~~~~~~~~~~~~~~~
+
+* Added support for "globstar/`**`" (recursive) pattern for the different commands and configuration options (`platformio ci <https://docs.platformio.org/page/core/userguide/cmd_ci.html>`__, `src_filter <https://docs.platformio.org/page/projectconf/section_env_build.html#src-filter>`__, `check_patterns <https://docs.platformio.org/page/projectconf/section_env_check.html#check-patterns>`__, `library.json > srcFilter <https://docs.platformio.org/page/librarymanager/config.html#srcfilter>`__). Python 3.5+ is required.
+
 4.3.4 (2020-05-23)
 ~~~~~~~~~~~~~~~~~~
 

platformio/commands/check/tools/base.py

@@ -12,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import glob
 import os
 from tempfile import NamedTemporaryFile
 
 import click
 
-from platformio import fs, proc
+from platformio import compat, fs, proc
 from platformio.commands.check.defect import DefectItem
 from platformio.project.helpers import load_project_ide_data
 
@@ -183,7 +182,7 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
                 result["c++"].append(os.path.realpath(path))
 
         for pattern in patterns:
-            for item in glob.glob(pattern):
+            for item in compat.glob_recursive(pattern):
                 if not os.path.isdir(item):
                     _add_file(item)
                 for root, _, files in os.walk(item, followlinks=True):

platformio/commands/ci.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from glob import glob
 from os import getenv, makedirs, remove
 from os.path import basename, isdir, isfile, join, realpath
 from shutil import copyfile, copytree
@@ -20,11 +19,10 @@ from tempfile import mkdtemp
 
 import click
 
-from platformio import app, fs
+from platformio import app, compat, fs
 from platformio.commands.project import project_init as cmd_project_init
 from platformio.commands.project import validate_boards
 from platformio.commands.run.command import cli as cmd_run
-from platformio.compat import glob_escape
 from platformio.exception import CIBuildEnvsEmpty
 from platformio.project.config import ProjectConfig
 
@@ -36,7 +34,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
         if p.startswith("~"):
             value[i] = fs.expanduser(p)
         value[i] = realpath(value[i])
-        if not glob(value[i]):
+        if not compat.glob_recursive(value[i]):
             invalid_path = p
             break
     try:
@@ -98,7 +96,7 @@ def cli(  # pylint: disable=too-many-arguments, too-many-branches
                 continue
             contents = []
             for p in patterns:
-                contents += glob(p)
+                contents += compat.glob_recursive(p)
             _copy_contents(join(build_dir, dir_name), contents)
 
         if project_conf and isfile(project_conf):
@@ -159,7 +157,7 @@ def _copy_contents(dst_dir, contents):
 def _exclude_contents(dst_dir, patterns):
     contents = []
     for p in patterns:
-        contents += glob(join(glob_escape(dst_dir), p))
+        contents += compat.glob_recursive(join(compat.glob_escape(dst_dir), p))
     for path in contents:
         path = realpath(path)
         if isdir(path):

platformio/commands/home/rpc/handlers/os.py

@@ -14,7 +14,6 @@
 
 from __future__ import absolute_import
 
-import glob
 import io
 import os
 import shutil
@@ -25,7 +24,7 @@ from twisted.internet import defer  # pylint: disable=import-error
 
 from platformio import app, fs, util
 from platformio.commands.home import helpers
-from platformio.compat import PY2, get_filesystem_encoding
+from platformio.compat import PY2, get_filesystem_encoding, glob_recursive
 
 
 class OSRPC(object):
@@ -115,7 +114,9 @@ class OSRPC(object):
             pathnames = [pathnames]
         result = set()
         for pathname in pathnames:
-            result |= set(glob.glob(os.path.join(root, pathname) if root else pathname))
+            result |= set(
+                glob_recursive(os.path.join(root, pathname) if root else pathname)
+            )
         return list(result)
 
     @staticmethod

platformio/compat.py

@@ -15,6 +15,7 @@
 # pylint: disable=unused-import, no-name-in-module, import-error,
 # pylint: disable=no-member, undefined-variable
 
+import glob
 import inspect
 import json
 import locale
@@ -81,6 +82,9 @@ if PY2:
     _magic_check = re.compile("([*?[])")
     _magic_check_bytes = re.compile(b"([*?[])")
 
+    def glob_recursive(pathname):
+        return glob.glob(pathname)
+
     def glob_escape(pathname):
         """Escape all special characters."""
         # https://github.com/python/cpython/blob/master/Lib/glob.py#L161
@@ -122,6 +126,9 @@ else:
             return obj
         return json.dumps(obj, ensure_ascii=False, sort_keys=True)
 
+    def glob_recursive(pathname):
+        return glob.glob(pathname, recursive=True)
+
     def load_python_module(name, pathname):
         spec = importlib.util.spec_from_file_location(name, pathname)
         module = importlib.util.module_from_spec(spec)

platformio/fs.py

@@ -18,12 +18,11 @@ import re
 import shutil
 import stat
 import sys
-from glob import glob
 
 import click
 
 from platformio import exception
-from platformio.compat import WINDOWS, glob_escape
+from platformio.compat import WINDOWS, glob_escape, glob_recursive
 
 
 class cd(object):
@@ -135,7 +134,7 @@ def match_src_files(src_dir, src_filter=None, src_exts=None, followlinks=True):
     src_filter = src_filter.replace("/", os.sep).replace("\\", os.sep)
     for (action, pattern) in re.findall(r"(\+|\-)<([^>]+)>", src_filter):
         items = set()
-        for item in glob(os.path.join(glob_escape(src_dir), pattern)):
+        for item in glob_recursive(os.path.join(glob_escape(src_dir), pattern)):
             if os.path.isdir(item):
                 for root, _, files in os.walk(item, followlinks=followlinks):
                     for f in files:

platformio/project/config.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import glob
 import json
 import os
 import re
@@ -21,7 +20,13 @@ from hashlib import sha1
 import click
 
 from platformio import fs
-from platformio.compat import PY2, WINDOWS, hashlib_encode_data, string_types
+from platformio.compat import (
+    PY2,
+    WINDOWS,
+    glob_recursive,
+    hashlib_encode_data,
+    string_types,
+)
 from platformio.project import exception
 from platformio.project.options import ProjectOptions
 
@@ -117,7 +122,7 @@ class ProjectConfigBase(object):
         for pattern in self.get("platformio", "extra_configs", []):
             if pattern.startswith("~"):
                 pattern = fs.expanduser(pattern)
-            for item in glob.glob(pattern):
+            for item in glob_recursive(pattern):
                 self.read(item)
 
     def _maintain_renaimed_options(self):