Improved support for projects located on a network share // Resolve #3417 , Resolve #3926 , Resolve #4102

HISTORY.rst

@@ -11,6 +11,7 @@ PlatformIO Core 5
 5.2.4 (2021-??-??)
 ~~~~~~~~~~~~~~~~~~
 
+- Improved support for projects located on a network share (`issue #3417 <https://github.com/platformio/platformio-core/issues/3417>`_, `issue #3926 <https://github.com/platformio/platformio-core/issues/3926>`_, `issue #4099 <https://github.com/platformio/platformio-core/issues/4099>`_)
 - Fixed an issue with the CLion project generator when a macro contains a space (`issue #4102 <https://github.com/platformio/platformio-core/issues/4102>`_)
 
 5.2.3 (2021-11-05)

platformio/app.py

@@ -31,7 +31,7 @@ from platformio.project.helpers import get_default_projects_dir
 
 def projects_dir_validate(projects_dir):
     assert os.path.isdir(projects_dir)
-    return os.path.realpath(projects_dir)
+    return os.path.abspath(projects_dir)
 
 
 DEFAULT_SETTINGS = {

platformio/builder/main.py

@@ -128,15 +128,20 @@ env.Replace(
     ],
 )
 
-if (
+if int(ARGUMENTS.get("ISATTY", 0)):
-    compat.IS_WINDOWS
+    # pylint: disable=protected-access
-    and sys.version_info >= (3, 8)
+    click._compat.isatty = lambda stream: True
-    and env["PROJECT_DIR"].startswith("\\\\")
+
-):
+if compat.IS_WINDOWS and sys.version_info >= (3, 8) and os.getcwd().startswith("\\\\"):
+    click.secho("!!! WARNING !!!\t\t" * 3, fg="red")
     click.secho(
-        "There is a known issue with Python 3.8+ and mapped network drives on "
+        "Your project is located on a mapped network drive but the "
-        "Windows.\nSee a solution at:\n"
+        "current command-line shell does not support the UNC paths.",
-        "https://github.com/platformio/platformio-core/issues/3417",
+        fg="yellow",
+    )
+    click.secho(
+        "Please move your project to a physical drive or check this workaround: "
+        "https://bit.ly/3kuU5mP\n",
         fg="yellow",
     )
 
@@ -145,10 +150,6 @@ if env.subst("$BUILD_CACHE_DIR"):
         os.makedirs(env.subst("$BUILD_CACHE_DIR"))
     env.CacheDir("$BUILD_CACHE_DIR")
 
-if int(ARGUMENTS.get("ISATTY", 0)):
-    # pylint: disable=protected-access
-    click._compat.isatty = lambda stream: True
-
 is_clean_all = "cleanall" in COMMAND_LINE_TARGETS
 if env.GetOption("clean") or is_clean_all:
     env.PioClean(is_clean_all)

platformio/builder/tools/pioide.py

@@ -32,14 +32,14 @@ def _dump_includes(env):
         env.subst("$PROJECT_SRC_DIR"),
     ]
     includes["build"].extend(
-        [os.path.realpath(env.subst(item)) for item in env.get("CPPPATH", [])]
+        [os.path.abspath(env.subst(item)) for item in env.get("CPPPATH", [])]
     )
 
     # installed libs
     includes["compatlib"] = []
     for lb in env.GetLibBuilders():
         includes["compatlib"].extend(
-            [os.path.realpath(inc) for inc in lb.get_include_dirs()]
+            [os.path.abspath(inc) for inc in lb.get_include_dirs()]
         )
 
     # includes from toolchains
@@ -56,9 +56,7 @@ def _dump_includes(env):
             os.path.join(toolchain_dir, "*", "include*"),
         ]
         for g in toolchain_incglobs:
-            includes["toolchain"].extend(
+            includes["toolchain"].extend([os.path.abspath(inc) for inc in glob.glob(g)])
-                [os.path.realpath(inc) for inc in glob.glob(g)]
-            )
 
     # include Unity framework if there are tests in project
     includes["unity"] = []
@@ -132,7 +130,7 @@ def _dump_defines(env):
 def _get_svd_path(env):
     svd_path = env.GetProjectOption("debug_svd_path")
     if svd_path:
-        return os.path.realpath(svd_path)
+        return os.path.abspath(svd_path)
 
     if "BOARD" not in env:
         return None
@@ -147,7 +145,7 @@ def _get_svd_path(env):
     # default file from ./platform/misc/svd folder
     p = env.PioPlatform()
     if os.path.isfile(os.path.join(p.get_dir(), "misc", "svd", svd_path)):
-        return os.path.realpath(os.path.join(p.get_dir(), "misc", "svd", svd_path))
+        return os.path.abspath(os.path.join(p.get_dir(), "misc", "svd", svd_path))
     return None
 
 

platformio/builder/tools/piolib.py

@@ -125,7 +125,7 @@ class LibBuilderBase(object):
     def __init__(self, env, path, manifest=None, verbose=False):
         self.env = env.Clone()
         self.envorigin = env.Clone()
-        self.path = os.path.realpath(env.subst(path))
+        self.path = os.path.abspath(env.subst(path))
         self.verbose = verbose
 
         try:
@@ -290,7 +290,7 @@ class LibBuilderBase(object):
             if self.extra_script:
                 self.env.SConscriptChdir(1)
                 self.env.SConscript(
-                    os.path.realpath(self.extra_script),
+                    os.path.abspath(self.extra_script),
                     exports={"env": self.env, "pio_lib_builder": self},
                 )
             self.env.ProcessUnFlags(self.build_unflags)
@@ -750,14 +750,14 @@ class PlatformIOLibBuilder(LibBuilderBase):
     def include_dir(self):
         if "includeDir" in self._manifest.get("build", {}):
             with fs.cd(self.path):
-                return os.path.realpath(self._manifest.get("build").get("includeDir"))
+                return os.path.abspath(self._manifest.get("build").get("includeDir"))
         return LibBuilderBase.include_dir.fget(self)  # pylint: disable=no-member
 
     @property
     def src_dir(self):
         if "srcDir" in self._manifest.get("build", {}):
             with fs.cd(self.path):
-                return os.path.realpath(self._manifest.get("build").get("srcDir"))
+                return os.path.abspath(self._manifest.get("build").get("srcDir"))
         return LibBuilderBase.src_dir.fget(self)  # pylint: disable=no-member
 
     @property
@@ -1024,7 +1024,7 @@ def GetLibBuilders(env):  # pylint: disable=too-many-branches
     found_incompat = False
 
     for storage_dir in env.GetLibSourceDirs():
-        storage_dir = os.path.realpath(storage_dir)
+        storage_dir = os.path.abspath(storage_dir)
         if not os.path.isdir(storage_dir):
             continue
         for item in sorted(os.listdir(storage_dir)):

platformio/builder/tools/piomisc.py

@@ -376,7 +376,7 @@ def GetExtraScripts(env, scope):
     if not items:
         return items
     with fs.cd(env.subst("$PROJECT_DIR")):
-        return [os.path.realpath(item) for item in items]
+        return [os.path.abspath(item) for item in items]
 
 
 def exists(_):

platformio/builder/tools/platformio.py

@@ -207,12 +207,12 @@ def ParseFlagsExtended(env, flags):  # pylint: disable=too-many-branches
     for k in ("CPPPATH", "LIBPATH"):
         for i, p in enumerate(result.get(k, [])):
             if os.path.isdir(p):
-                result[k][i] = os.path.realpath(p)
+                result[k][i] = os.path.abspath(p)
 
     # fix relative path for "-include"
     for i, f in enumerate(result.get("CCFLAGS", [])):
         if isinstance(f, tuple) and f[0] == "-include":
-            result["CCFLAGS"][i] = (f[0], env.File(os.path.realpath(f[1].get_path())))
+            result["CCFLAGS"][i] = (f[0], env.File(os.path.abspath(f[1].get_path())))
 
     return result
 

platformio/commands/check/defect.py

@@ -86,7 +86,7 @@ class DefectItem(object):
             "severity": self.SEVERITY_LABELS[self.severity],
             "category": self.category,
             "message": self.message,
-            "file": os.path.realpath(self.file),
+            "file": os.path.abspath(self.file),
             "line": self.line,
             "column": self.column,
             "callstack": self.callstack,

platformio/commands/check/tools/base.py

@@ -201,11 +201,11 @@ class CheckToolBase(object):  # pylint: disable=too-many-instance-attributes
 
         def _add_file(path):
             if path.endswith(header_extensions):
-                result["headers"].append(os.path.realpath(path))
+                result["headers"].append(os.path.abspath(path))
             elif path.endswith(c_extension):
-                result["c"].append(os.path.realpath(path))
+                result["c"].append(os.path.abspath(path))
             elif path.endswith(cpp_extensions):
-                result["c++"].append(os.path.realpath(path))
+                result["c++"].append(os.path.abspath(path))
 
         for pattern in patterns:
             for item in glob.glob(pattern, recursive=True):

platformio/commands/ci.py

@@ -33,7 +33,7 @@ def validate_path(ctx, param, value):  # pylint: disable=unused-argument
     for i, p in enumerate(value):
         if p.startswith("~"):
             value[i] = fs.expanduser(p)
-        value[i] = os.path.realpath(value[i])
+        value[i] = os.path.abspath(value[i])
         if not glob.glob(value[i], recursive=True):
             invalid_path = p
             break
@@ -162,7 +162,7 @@ def _exclude_contents(dst_dir, patterns):
     for p in patterns:
         contents += glob.glob(os.path.join(glob.escape(dst_dir), p), recursive=True)
     for path in contents:
-        path = os.path.realpath(path)
+        path = os.path.abspath(path)
         if os.path.isdir(path):
             fs.rmtree(path)
         elif os.path.isfile(path):

platformio/commands/home/rpc/handlers/project.py

@@ -93,7 +93,7 @@ class ProjectRPC:
             # skip non existing folders and resolve full path
             for key in ("envLibdepsDirs", "libExtraDirs"):
                 data[key] = [
-                    fs.expanduser(d) if d.startswith("~") else os.path.realpath(d)
+                    fs.expanduser(d) if d.startswith("~") else os.path.abspath(d)
                     for d in data[key]
                     if os.path.isdir(d)
                 ]

platformio/debug/process/gdb.py

@@ -61,7 +61,7 @@ class GDBClientProcess(DebugClientProcess):
     def _get_data_dir(gdb_path):
         if "msp430" in gdb_path:
             return None
-        gdb_data_dir = os.path.realpath(
+        gdb_data_dir = os.path.abspath(
             os.path.join(os.path.dirname(gdb_path), "..", "share", "gdb")
         )
         return gdb_data_dir if os.path.isdir(gdb_data_dir) else None

platformio/fs.py

@@ -24,7 +24,7 @@ import sys
 
 import click
 
-from platformio import exception
+from platformio import exception, proc
 from platformio.compat import IS_WINDOWS
 
 
@@ -41,7 +41,7 @@ class cd(object):
 
 
 def get_source_dir():
-    curpath = os.path.realpath(__file__)
+    curpath = os.path.abspath(__file__)
     if not os.path.isfile(curpath):
         for p in sys.path:
             if os.path.isfile(os.path.join(p, __file__)):
@@ -119,7 +119,7 @@ def ensure_udev_rules():
     if not any(os.path.isfile(p) for p in installed_rules):
         raise exception.MissedUdevRules
 
-    origin_path = os.path.realpath(
+    origin_path = os.path.abspath(
         os.path.join(get_source_dir(), "..", "scripts", "99-platformio-udev.rules")
     )
     if not os.path.isfile(origin_path):
@@ -181,6 +181,25 @@ def to_unix_path(path):
     return re.sub(r"[\\]+", "/", path)
 
 
+def normalize_path(path):
+    path = os.path.abspath(path)
+    if not IS_WINDOWS or not path.startswith("\\\\"):
+        return path
+    try:
+        result = proc.exec_command(["net", "use"])
+        if result["returncode"] != 0:
+            return path
+        share_re = re.compile(r"\s([A-Z]\:)\s+(\\\\[^\s]+)")
+        for line in result["out"].split("\n"):
+            share = share_re.search(line)
+            if not share:
+                continue
+            path = path.replace(share.group(2), share.group(1))
+    except OSError:
+        pass
+    return path
+
+
 def expanduser(path):
     """
     Be compatible with Python 3.8, on Windows skip HOME and check for USERPROFILE

platformio/package/lockfile.py

@@ -48,7 +48,7 @@ class LockFile(object):
     def __init__(self, path, timeout=LOCKFILE_TIMEOUT, delay=LOCKFILE_DELAY):
         self.timeout = timeout
         self.delay = delay
-        self._lock_path = os.path.realpath(path) + ".lock"
+        self._lock_path = os.path.abspath(path) + ".lock"
         self._fp = None
 
     def _lock(self):

platformio/package/manager/base.py

@@ -252,9 +252,9 @@ class BasePackageManager(  # pylint: disable=too-many-public-methods
         # external "URL" mismatch
         if spec.external:
             # local folder mismatch
-            if os.path.realpath(spec.url) == os.path.realpath(pkg.path) or (
+            if os.path.abspath(spec.url) == os.path.abspath(pkg.path) or (
                 spec.url.startswith("file://")
-                and os.path.realpath(pkg.path) == os.path.realpath(spec.url[7:])
+                and os.path.abspath(pkg.path) == os.path.abspath(spec.url[7:])
             ):
                 return True
             if spec.url != pkg.metadata.spec.url:

platformio/project/generator.py

@@ -82,7 +82,7 @@ class ProjectGenerator(object):
             "project_dir": self.project_dir,
             "original_env_name": self.original_env_name,
             "env_name": self.env_name,
-            "user_home_dir": os.path.realpath(fs.expanduser("~")),
+            "user_home_dir": os.path.abspath(fs.expanduser("~")),
             "platformio_path": sys.argv[0]
             if os.path.isfile(sys.argv[0])
             else where_is_program("platformio"),
@@ -125,7 +125,9 @@ class ProjectGenerator(object):
         with fs.cd(self.project_dir):
             for root, _, files in os.walk(self.config.get("platformio", "src_dir")):
                 for f in files:
-                    result.append(os.path.relpath(os.path.join(root, f)))
+                    result.append(
+                        os.path.relpath(os.path.join(os.path.realpath(root), f))
+                    )
         return result
 
     def get_tpls(self):

platformio/project/helpers.py

@@ -24,7 +24,7 @@ from platformio.project.config import ProjectConfig
 
 
 def get_project_dir():
-    return os.getcwd()
+    return fs.normalize_path(os.getcwd())
 
 
 def is_platformio_project(project_dir=None):

platformio/project/options.py

@@ -114,7 +114,7 @@ def validate_dir(path):
         path = fs.expanduser(path)
     if "$" in path:
         path = expand_dir_templates(path)
-    return os.path.realpath(path)
+    return fs.normalize_path(path)
 
 
 def validate_core_dir(path):