Refactor Unified Debugger to native Python Asynchronous I/O stack // Resolve #3793 , Resolve #3595

platformio/debug/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/debug/exception.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platformio.exception import PlatformioException, UserSideException
+
+
+class DebugError(PlatformioException):
+    pass
+
+
+class DebugSupportError(DebugError, UserSideException):
+
+    MESSAGE = (
+        "Currently, PlatformIO does not support debugging for `{0}`.\n"
+        "Please request support at https://github.com/platformio/"
+        "platformio-core/issues \nor visit -> https://docs.platformio.org"
+        "/page/plus/debugging.html"
+    )
+
+
+class DebugInvalidOptionsError(DebugError, UserSideException):
+    pass

platformio/debug/helpers.py

@@ -0,0 +1,303 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import sys
+import time
+from fnmatch import fnmatch
+from hashlib import sha1
+from io import BytesIO
+from os.path import isfile
+
+from platformio import fs, util
+from platformio.commands import PlatformioCLI
+from platformio.commands.run.command import cli as cmd_run
+from platformio.compat import is_bytes
+from platformio.debug.exception import DebugInvalidOptionsError
+from platformio.project.config import ProjectConfig
+from platformio.project.options import ProjectOptions
+
+
+class GDBMIConsoleStream(BytesIO):  # pylint: disable=too-few-public-methods
+
+    STDOUT = sys.stdout
+
+    def write(self, text):
+        self.STDOUT.write(escape_gdbmi_stream("~", text))
+        self.STDOUT.flush()
+
+
+def is_gdbmi_mode():
+    return "--interpreter" in " ".join(PlatformioCLI.leftover_args)
+
+
+def escape_gdbmi_stream(prefix, stream):
+    bytes_stream = False
+    if is_bytes(stream):
+        bytes_stream = True
+        stream = stream.decode()
+
+    if not stream:
+        return b"" if bytes_stream else ""
+
+    ends_nl = stream.endswith("\n")
+    stream = re.sub(r"\\+", "\\\\\\\\", stream)
+    stream = stream.replace('"', '\\"')
+    stream = stream.replace("\n", "\\n")
+    stream = '%s"%s"' % (prefix, stream)
+    if ends_nl:
+        stream += "\n"
+
+    return stream.encode() if bytes_stream else stream
+
+
+def get_default_debug_env(config):
+    default_envs = config.default_envs()
+    all_envs = config.envs()
+    for env in default_envs:
+        if config.get("env:" + env, "build_type") == "debug":
+            return env
+    for env in all_envs:
+        if config.get("env:" + env, "build_type") == "debug":
+            return env
+    return default_envs[0] if default_envs else all_envs[0]
+
+
+def predebug_project(ctx, project_dir, env_name, preload, verbose):
+    ctx.invoke(
+        cmd_run,
+        project_dir=project_dir,
+        environment=[env_name],
+        target=["debug"] + (["upload"] if preload else []),
+        verbose=verbose,
+    )
+    if preload:
+        time.sleep(5)
+
+
+def configure_initial_debug_options(platform, env_options):
+    def _cleanup_cmds(items):
+        items = ProjectConfig.parse_multi_values(items)
+        return ["$LOAD_CMDS" if item == "$LOAD_CMD" else item for item in items]
+
+    board_config = platform.board_config(env_options["board"])
+    tool_name = board_config.get_debug_tool_name(env_options.get("debug_tool"))
+    tool_settings = board_config.get("debug", {}).get("tools", {}).get(tool_name, {})
+    server_options = None
+
+    # specific server per a system
+    if isinstance(tool_settings.get("server", {}), list):
+        for item in tool_settings["server"][:]:
+            tool_settings["server"] = item
+            if util.get_systype() in item.get("system", []):
+                break
+
+    # user overwrites debug server
+    if env_options.get("debug_server"):
+        server_options = {
+            "cwd": None,
+            "executable": None,
+            "arguments": env_options.get("debug_server"),
+        }
+        server_options["executable"] = server_options["arguments"][0]
+        server_options["arguments"] = server_options["arguments"][1:]
+    elif "server" in tool_settings:
+        server_options = tool_settings["server"]
+        server_package = server_options.get("package")
+        server_package_dir = (
+            platform.get_package_dir(server_package) if server_package else None
+        )
+        if server_package and not server_package_dir:
+            platform.install_packages(
+                with_packages=[server_package], skip_default_package=True, silent=True
+            )
+            server_package_dir = platform.get_package_dir(server_package)
+        server_options.update(
+            dict(
+                cwd=server_package_dir if server_package else None,
+                executable=server_options.get("executable"),
+                arguments=[
+                    a.replace("$PACKAGE_DIR", server_package_dir)
+                    if server_package_dir
+                    else a
+                    for a in server_options.get("arguments", [])
+                ],
+            )
+        )
+
+    extra_cmds = _cleanup_cmds(env_options.get("debug_extra_cmds"))
+    extra_cmds.extend(_cleanup_cmds(tool_settings.get("extra_cmds")))
+    result = dict(
+        tool=tool_name,
+        upload_protocol=env_options.get(
+            "upload_protocol", board_config.get("upload", {}).get("protocol")
+        ),
+        load_cmds=_cleanup_cmds(
+            env_options.get(
+                "debug_load_cmds",
+                tool_settings.get(
+                    "load_cmds",
+                    tool_settings.get(
+                        "load_cmd", ProjectOptions["env.debug_load_cmds"].default
+                    ),
+                ),
+            )
+        ),
+        load_mode=env_options.get(
+            "debug_load_mode",
+            tool_settings.get(
+                "load_mode", ProjectOptions["env.debug_load_mode"].default
+            ),
+        ),
+        init_break=env_options.get(
+            "debug_init_break",
+            tool_settings.get(
+                "init_break", ProjectOptions["env.debug_init_break"].default
+            ),
+        ),
+        init_cmds=_cleanup_cmds(
+            env_options.get("debug_init_cmds", tool_settings.get("init_cmds"))
+        ),
+        extra_cmds=extra_cmds,
+        require_debug_port=tool_settings.get("require_debug_port", False),
+        port=reveal_debug_port(
+            env_options.get("debug_port", tool_settings.get("port")),
+            tool_name,
+            tool_settings,
+        ),
+        speed=env_options.get("debug_speed", tool_settings.get("speed")),
+        server=server_options,
+    )
+    return result
+
+
+def configure_esp32_load_cmds(debug_options, configuration):
+    """
+    DEPRECATED: Moved to ESP32 dev-platform
+    See platform.py::configure_debug_options
+    """
+    flash_images = configuration.get("extra", {}).get("flash_images")
+    ignore_conds = [
+        debug_options["load_cmds"] != ["load"],
+        "xtensa-esp32" not in configuration.get("cc_path", ""),
+        not flash_images,
+        not all(isfile(item["path"]) for item in flash_images),
+    ]
+    if any(ignore_conds):
+        return debug_options["load_cmds"]
+
+    mon_cmds = [
+        'monitor program_esp32 "{{{path}}}" {offset} verify'.format(
+            path=fs.to_unix_path(item["path"]), offset=item["offset"]
+        )
+        for item in flash_images
+    ]
+    mon_cmds.append(
+        'monitor program_esp32 "{%s.bin}" 0x10000 verify'
+        % fs.to_unix_path(configuration["prog_path"][:-4])
+    )
+    return mon_cmds
+
+
+def has_debug_symbols(prog_path):
+    if not isfile(prog_path):
+        return False
+    matched = {
+        b".debug_info": False,
+        b".debug_abbrev": False,
+        b" -Og": False,
+        b" -g": False,
+        b"__PLATFORMIO_BUILD_DEBUG__": False,
+    }
+    with open(prog_path, "rb") as fp:
+        last_data = b""
+        while True:
+            data = fp.read(1024)
+            if not data:
+                break
+            for pattern, found in matched.items():
+                if found:
+                    continue
+                if pattern in last_data + data:
+                    matched[pattern] = True
+            last_data = data
+    return all(matched.values())
+
+
+def is_prog_obsolete(prog_path):
+    prog_hash_path = prog_path + ".sha1"
+    if not isfile(prog_path):
+        return True
+    shasum = sha1()
+    with open(prog_path, "rb") as fp:
+        while True:
+            data = fp.read(1024)
+            if not data:
+                break
+            shasum.update(data)
+    new_digest = shasum.hexdigest()
+    old_digest = None
+    if isfile(prog_hash_path):
+        with open(prog_hash_path) as fp:
+            old_digest = fp.read()
+    if new_digest == old_digest:
+        return False
+    with open(prog_hash_path, "w") as fp:
+        fp.write(new_digest)
+    return True
+
+
+def reveal_debug_port(env_debug_port, tool_name, tool_settings):
+    def _get_pattern():
+        if not env_debug_port:
+            return None
+        if set(["*", "?", "[", "]"]) & set(env_debug_port):
+            return env_debug_port
+        return None
+
+    def _is_match_pattern(port):
+        pattern = _get_pattern()
+        if not pattern:
+            return True
+        return fnmatch(port, pattern)
+
+    def _look_for_serial_port(hwids):
+        for item in util.get_serialports(filter_hwid=True):
+            if not _is_match_pattern(item["port"]):
+                continue
+            port = item["port"]
+            if tool_name.startswith("blackmagic"):
+                if (
+                    "windows" in util.get_systype()
+                    and port.startswith("COM")
+                    and len(port) > 4
+                ):
+                    port = "\\\\.\\%s" % port
+                if "GDB" in item["description"]:
+                    return port
+            for hwid in hwids:
+                hwid_str = ("%s:%s" % (hwid[0], hwid[1])).replace("0x", "")
+                if hwid_str in item["hwid"]:
+                    return port
+        return None
+
+    if env_debug_port and not _get_pattern():
+        return env_debug_port
+    if not tool_settings.get("require_debug_port"):
+        return None
+
+    debug_port = _look_for_serial_port(tool_settings.get("hwids", []))
+    if not debug_port:
+        raise DebugInvalidOptionsError("Please specify `debug_port` for environment")
+    return debug_port

platformio/debug/initcfgs.py

@@ -0,0 +1,161 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GDB_DEFAULT_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor reset halt
+end
+
+define pio_reset_run_target
+    monitor reset
+end
+
+target extended-remote $DEBUG_PORT
+monitor init
+$LOAD_CMDS
+pio_reset_halt_target
+$INIT_BREAK
+"""
+
+GDB_STUTIL_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor reset
+    monitor halt
+end
+
+define pio_reset_run_target
+    monitor reset
+end
+
+target extended-remote $DEBUG_PORT
+$LOAD_CMDS
+pio_reset_halt_target
+$INIT_BREAK
+"""
+
+GDB_JLINK_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor reset
+    monitor halt
+end
+
+define pio_reset_run_target
+    monitor clrbp
+    monitor reset
+    monitor go
+end
+
+target extended-remote $DEBUG_PORT
+monitor clrbp
+monitor speed auto
+pio_reset_halt_target
+$LOAD_CMDS
+$INIT_BREAK
+"""
+
+GDB_BLACKMAGIC_INIT_CONFIG = """
+define pio_reset_halt_target
+    set language c
+    set *0xE000ED0C = 0x05FA0004
+    set $busy = (*0xE000ED0C & 0x4)
+    while ($busy)
+        set $busy = (*0xE000ED0C & 0x4)
+    end
+    set language auto
+end
+
+define pio_reset_run_target
+    pio_reset_halt_target
+end
+
+target extended-remote $DEBUG_PORT
+monitor swdp_scan
+attach 1
+set mem inaccessible-by-default off
+$LOAD_CMDS
+$INIT_BREAK
+
+set language c
+set *0xE000ED0C = 0x05FA0004
+set $busy = (*0xE000ED0C & 0x4)
+while ($busy)
+    set $busy = (*0xE000ED0C & 0x4)
+end
+set language auto
+"""
+
+GDB_MSPDEBUG_INIT_CONFIG = """
+define pio_reset_halt_target
+end
+
+define pio_reset_run_target
+end
+
+target extended-remote $DEBUG_PORT
+monitor erase
+$LOAD_CMDS
+pio_reset_halt_target
+$INIT_BREAK
+"""
+
+GDB_QEMU_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor system_reset
+end
+
+define pio_reset_run_target
+    monitor system_reset
+end
+
+target extended-remote $DEBUG_PORT
+$LOAD_CMDS
+pio_reset_halt_target
+$INIT_BREAK
+"""
+
+GDB_RENODE_INIT_CONFIG = """
+define pio_reset_halt_target
+    monitor machine Reset
+    $LOAD_CMDS
+    monitor start
+end
+
+define pio_reset_run_target
+    pio_reset_halt_target
+end
+
+target extended-remote $DEBUG_PORT
+$LOAD_CMDS
+$INIT_BREAK
+monitor start
+"""
+
+
+TOOL_TO_CONFIG = {
+    "jlink": GDB_JLINK_INIT_CONFIG,
+    "mspdebug": GDB_MSPDEBUG_INIT_CONFIG,
+    "qemu": GDB_QEMU_INIT_CONFIG,
+    "blackmagic": GDB_BLACKMAGIC_INIT_CONFIG,
+    "renode": GDB_RENODE_INIT_CONFIG,
+}
+
+
+def get_gdb_init_config(debug_options):
+    tool = debug_options.get("tool")
+    if tool and tool in TOOL_TO_CONFIG:
+        return TOOL_TO_CONFIG[tool]
+    server_exe = (debug_options.get("server") or {}).get("executable", "").lower()
+    if "st-util" in server_exe:
+        return GDB_STUTIL_INIT_CONFIG
+    return GDB_DEFAULT_INIT_CONFIG

platformio/debug/process/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

platformio/debug/process/base.py

@@ -0,0 +1,189 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import signal
+import subprocess
+import sys
+import time
+
+from platformio import fs
+from platformio.compat import (
+    create_task,
+    get_locale_encoding,
+    get_running_loop,
+    string_types,
+)
+from platformio.proc import get_pythonexe_path
+from platformio.project.helpers import get_project_core_dir
+
+
+class DebugSubprocessProtocol(asyncio.SubprocessProtocol):
+    def __init__(self, factory):
+        self.factory = factory
+        self._is_exited = False
+
+    def connection_made(self, transport):
+        self.factory.connection_made(transport)
+
+    def pipe_data_received(self, fd, data):
+        pipe_to_cb = [
+            self.factory.stdin_data_received,
+            self.factory.stdout_data_received,
+            self.factory.stderr_data_received,
+        ]
+        pipe_to_cb[fd](data)
+
+    def connection_lost(self, exc):
+        self.process_exited()
+
+    def process_exited(self):
+        if self._is_exited:
+            return
+        self.factory.process_exited()
+        self._is_exited = True
+
+
+class DebugBaseProcess:
+
+    STDOUT_CHUNK_SIZE = 2048
+    LOG_FILE = None
+
+    COMMON_PATTERNS = {
+        "PLATFORMIO_HOME_DIR": get_project_core_dir(),
+        "PLATFORMIO_CORE_DIR": get_project_core_dir(),
+        "PYTHONEXE": get_pythonexe_path(),
+    }
+
+    def __init__(self):
+        self.transport = None
+        self._is_running = False
+        self._last_activity = 0
+        self._exit_future = None
+        self._stdin_read_task = None
+        self._std_encoding = get_locale_encoding()
+
+    async def spawn(self, *args, **kwargs):
+        wait_until_exit = False
+        if "wait_until_exit" in kwargs:
+            wait_until_exit = kwargs["wait_until_exit"]
+            del kwargs["wait_until_exit"]
+        for pipe in ("stdin", "stdout", "stderr"):
+            if pipe not in kwargs:
+                kwargs[pipe] = subprocess.PIPE
+        loop = get_running_loop()
+        await loop.subprocess_exec(
+            lambda: DebugSubprocessProtocol(self), *args, **kwargs
+        )
+        if wait_until_exit:
+            self._exit_future = loop.create_future()
+            await self._exit_future
+
+    def is_running(self):
+        return self._is_running
+
+    def connection_made(self, transport):
+        self._is_running = True
+        self.transport = transport
+
+    def connect_stdin_pipe(self):
+        self._stdin_read_task = create_task(self._read_stdin_pipe())
+
+    async def _read_stdin_pipe(self):
+        loop = get_running_loop()
+        try:
+            loop.add_reader(
+                sys.stdin.fileno(),
+                lambda: self.stdin_data_received(sys.stdin.buffer.readline()),
+            )
+        except NotImplementedError:
+            while True:
+                self.stdin_data_received(
+                    await loop.run_in_executor(None, sys.stdin.buffer.readline)
+                )
+
+    def stdin_data_received(self, data):
+        self._last_activity = time.time()
+        if self.LOG_FILE:
+            with open(self.LOG_FILE, "ab") as fp:
+                fp.write(data)
+
+    def stdout_data_received(self, data):
+        self._last_activity = time.time()
+        if self.LOG_FILE:
+            with open(self.LOG_FILE, "ab") as fp:
+                fp.write(data)
+        while data:
+            chunk = data[: self.STDOUT_CHUNK_SIZE]
+            print(chunk.decode(self._std_encoding, "replace"), end="", flush=True)
+            data = data[self.STDOUT_CHUNK_SIZE :]
+
+    def stderr_data_received(self, data):
+        self._last_activity = time.time()
+        if self.LOG_FILE:
+            with open(self.LOG_FILE, "ab") as fp:
+                fp.write(data)
+        print(
+            data.decode(self._std_encoding, "replace"),
+            end="",
+            file=sys.stderr,
+            flush=True,
+        )
+
+    def process_exited(self):
+        self._is_running = False
+        self._last_activity = time.time()
+        # Allow terminating via SIGINT/CTRL+C
+        signal.signal(signal.SIGINT, signal.default_int_handler)
+        if self._stdin_read_task:
+            self._stdin_read_task.cancel()
+            self._stdin_read_task = None
+        if self._exit_future:
+            self._exit_future.set_result(True)
+            self._exit_future = None
+
+    def apply_patterns(self, source, patterns=None):
+        _patterns = self.COMMON_PATTERNS.copy()
+        _patterns.update(patterns or {})
+
+        for key, value in _patterns.items():
+            if key.endswith(("_DIR", "_PATH")):
+                _patterns[key] = fs.to_unix_path(value)
+
+        def _replace(text):
+            for key, value in _patterns.items():
+                pattern = "$%s" % key
+                text = text.replace(pattern, value or "")
+            return text
+
+        if isinstance(source, string_types):
+            source = _replace(source)
+        elif isinstance(source, (list, dict)):
+            items = enumerate(source) if isinstance(source, list) else source.items()
+            for key, value in items:
+                if isinstance(value, string_types):
+                    source[key] = _replace(value)
+                elif isinstance(value, (list, dict)):
+                    source[key] = self.apply_patterns(value, patterns)
+
+        return source
+
+    def terminate(self):
+        if not self.is_running() or not self.transport:
+            return
+        try:
+            self.transport.kill()
+            self.transport.close()
+        except:  # pylint: disable=bare-except
+            pass

platformio/debug/process/client.py

@@ -0,0 +1,270 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import os
+import re
+import signal
+import tempfile
+import time
+
+from platformio import fs, proc, telemetry, util
+from platformio.cache import ContentCache
+from platformio.compat import get_running_loop, hashlib_encode_data, is_bytes
+from platformio.debug import helpers
+from platformio.debug.exception import DebugInvalidOptionsError
+from platformio.debug.initcfgs import get_gdb_init_config
+from platformio.debug.process.base import DebugBaseProcess
+from platformio.debug.process.server import DebugServerProcess
+from platformio.project.helpers import get_project_cache_dir
+
+
+class DebugClientProcess(
+    DebugBaseProcess
+):  # pylint: disable=too-many-instance-attributes
+
+    PIO_SRC_NAME = ".pioinit"
+    INIT_COMPLETED_BANNER = "PlatformIO: Initialization completed"
+
+    def __init__(self, project_dir, args, debug_options, env_options):
+        super(DebugClientProcess, self).__init__()
+        self.project_dir = project_dir
+        self.args = list(args)
+        self.debug_options = debug_options
+        self.env_options = env_options
+
+        self._server_process = DebugServerProcess(debug_options, env_options)
+        self._session_id = None
+
+        if not os.path.isdir(get_project_cache_dir()):
+            os.makedirs(get_project_cache_dir())
+        self._gdbsrc_dir = tempfile.mkdtemp(
+            dir=get_project_cache_dir(), prefix=".piodebug-"
+        )
+
+        self._target_is_running = False
+        self._errors_buffer = b""
+
+    async def run(self, gdb_path, prog_path):
+        session_hash = gdb_path + prog_path
+        self._session_id = hashlib.sha1(hashlib_encode_data(session_hash)).hexdigest()
+        self._kill_previous_session()
+
+        patterns = {
+            "PROJECT_DIR": self.project_dir,
+            "PROG_PATH": prog_path,
+            "PROG_DIR": os.path.dirname(prog_path),
+            "PROG_NAME": os.path.basename(os.path.splitext(prog_path)[0]),
+            "DEBUG_PORT": self.debug_options["port"],
+            "UPLOAD_PROTOCOL": self.debug_options["upload_protocol"],
+            "INIT_BREAK": self.debug_options["init_break"] or "",
+            "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []),
+        }
+
+        await self._server_process.run(patterns)
+        if not patterns["DEBUG_PORT"]:
+            patterns["DEBUG_PORT"] = self._server_process.get_debug_port()
+
+        self.generate_pioinit(self._gdbsrc_dir, patterns)
+
+        # start GDB client
+        args = [
+            gdb_path,
+            "-q",
+            "--directory",
+            self._gdbsrc_dir,
+            "--directory",
+            self.project_dir,
+            "-l",
+            "10",
+        ]
+        args.extend(self.args)
+        if not gdb_path:
+            raise DebugInvalidOptionsError("GDB client is not configured")
+        gdb_data_dir = self._get_data_dir(gdb_path)
+        if gdb_data_dir:
+            args.extend(["--data-directory", gdb_data_dir])
+        args.append(patterns["PROG_PATH"])
+        await self.spawn(*args, cwd=self.project_dir, wait_until_exit=True)
+
+    @staticmethod
+    def _get_data_dir(gdb_path):
+        if "msp430" in gdb_path:
+            return None
+        gdb_data_dir = os.path.realpath(
+            os.path.join(os.path.dirname(gdb_path), "..", "share", "gdb")
+        )
+        return gdb_data_dir if os.path.isdir(gdb_data_dir) else None
+
+    def generate_pioinit(self, dst_dir, patterns):
+        # default GDB init commands depending on debug tool
+        commands = get_gdb_init_config(self.debug_options).split("\n")
+
+        if self.debug_options["init_cmds"]:
+            commands = self.debug_options["init_cmds"]
+        commands.extend(self.debug_options["extra_cmds"])
+
+        if not any("define pio_reset_run_target" in cmd for cmd in commands):
+            commands = [
+                "define pio_reset_run_target",
+                "   echo Warning! Undefined pio_reset_run_target command\\n",
+                "   monitor reset",
+                "end",
+            ] + commands
+        if not any("define pio_reset_halt_target" in cmd for cmd in commands):
+            commands = [
+                "define pio_reset_halt_target",
+                "   echo Warning! Undefined pio_reset_halt_target command\\n",
+                "   monitor reset halt",
+                "end",
+            ] + commands
+        if not any("define pio_restart_target" in cmd for cmd in commands):
+            commands += [
+                "define pio_restart_target",
+                "   pio_reset_halt_target",
+                "   $INIT_BREAK",
+                "   %s" % ("continue" if patterns["INIT_BREAK"] else "next"),
+                "end",
+            ]
+
+        banner = [
+            "echo PlatformIO Unified Debugger -> http://bit.ly/pio-debug\\n",
+            "echo PlatformIO: debug_tool = %s\\n" % self.debug_options["tool"],
+            "echo PlatformIO: Initializing remote target...\\n",
+        ]
+        footer = ["echo %s\\n" % self.INIT_COMPLETED_BANNER]
+        commands = banner + commands + footer
+
+        with open(os.path.join(dst_dir, self.PIO_SRC_NAME), "w") as fp:
+            fp.write("\n".join(self.apply_patterns(commands, patterns)))
+
+    def connection_made(self, transport):
+        super(DebugClientProcess, self).connection_made(transport)
+        self._lock_session(transport.get_pid())
+        # Disable SIGINT and allow GDB's Ctrl+C interrupt
+        signal.signal(signal.SIGINT, lambda *args, **kwargs: None)
+        self.connect_stdin_pipe()
+
+    def stdin_data_received(self, data):
+        super(DebugClientProcess, self).stdin_data_received(data)
+        if b"-exec-run" in data:
+            if self._target_is_running:
+                token, _ = data.split(b"-", 1)
+                self.stdout_data_received(token + b"^running\n")
+                return
+            data = data.replace(b"-exec-run", b"-exec-continue")
+
+        if b"-exec-continue" in data:
+            self._target_is_running = True
+        if b"-gdb-exit" in data or data.strip() in (b"q", b"quit"):
+            # Allow terminating via SIGINT/CTRL+C
+            signal.signal(signal.SIGINT, signal.default_int_handler)
+            self.transport.get_pipe_transport(0).write(b"pio_reset_run_target\n")
+        self.transport.get_pipe_transport(0).write(data)
+
+    def stdout_data_received(self, data):
+        super(DebugClientProcess, self).stdout_data_received(data)
+        self._handle_error(data)
+        # go to init break automatically
+        if self.INIT_COMPLETED_BANNER.encode() in data:
+            telemetry.send_event(
+                "Debug", "Started", telemetry.dump_run_environment(self.env_options)
+            )
+            self._auto_exec_continue()
+
+    def console_log(self, msg):
+        if helpers.is_gdbmi_mode():
+            msg = helpers.escape_gdbmi_stream("~", msg)
+        self.stdout_data_received(msg if is_bytes(msg) else msg.encode())
+
+    def _auto_exec_continue(self):
+        auto_exec_delay = 0.5  # in seconds
+        if self._last_activity > (time.time() - auto_exec_delay):
+            get_running_loop().call_later(0.1, self._auto_exec_continue)
+            return
+
+        if not self.debug_options["init_break"] or self._target_is_running:
+            return
+
+        self.console_log(
+            "PlatformIO: Resume the execution to `debug_init_break = %s`\n"
+            % self.debug_options["init_break"]
+        )
+        self.console_log(
+            "PlatformIO: More configuration options -> http://bit.ly/pio-debug\n"
+        )
+        self.transport.get_pipe_transport(0).write(
+            b"0-exec-continue\n" if helpers.is_gdbmi_mode() else b"continue\n"
+        )
+        self._target_is_running = True
+
+    def stderr_data_received(self, data):
+        super(DebugClientProcess, self).stderr_data_received(data)
+        self._handle_error(data)
+
+    def _handle_error(self, data):
+        self._errors_buffer = (self._errors_buffer + data)[-8192:]  # keep last 8 KBytes
+        if not (
+            self.PIO_SRC_NAME.encode() in self._errors_buffer
+            and b"Error in sourced" in self._errors_buffer
+        ):
+            return
+
+        last_erros = self._errors_buffer.decode()
+        last_erros = " ".join(reversed(last_erros.split("\n")))
+        last_erros = re.sub(r'((~|&)"|\\n\"|\\t)', " ", last_erros, flags=re.M)
+
+        err = "%s -> %s" % (
+            telemetry.dump_run_environment(self.env_options),
+            last_erros,
+        )
+        telemetry.send_exception("DebugInitError: %s" % err)
+        self.transport.close()
+
+    def process_exited(self):
+        self._unlock_session()
+        if self._gdbsrc_dir and os.path.isdir(self._gdbsrc_dir):
+            fs.rmtree(self._gdbsrc_dir)
+        if self._server_process:
+            self._server_process.terminate()
+        super(DebugClientProcess, self).process_exited()
+
+    def _kill_previous_session(self):
+        assert self._session_id
+        pid = None
+        with ContentCache() as cc:
+            pid = cc.get(self._session_id)
+            cc.delete(self._session_id)
+        if not pid:
+            return
+        if "windows" in util.get_systype():
+            kill = ["Taskkill", "/PID", pid, "/F"]
+        else:
+            kill = ["kill", pid]
+        try:
+            proc.exec_command(kill)
+        except:  # pylint: disable=bare-except
+            pass
+
+    def _lock_session(self, pid):
+        if not self._session_id:
+            return
+        with ContentCache() as cc:
+            cc.set(self._session_id, str(pid), "1h")
+
+    def _unlock_session(self):
+        if not self._session_id:
+            return
+        with ContentCache() as cc:
+            cc.delete(self._session_id)

platformio/debug/process/server.py

@@ -0,0 +1,148 @@
+# Copyright (c) 2014-present PlatformIO <contact@platformio.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import os
+import time
+
+from platformio import fs, util
+from platformio.debug.exception import DebugInvalidOptionsError
+from platformio.debug.helpers import escape_gdbmi_stream, is_gdbmi_mode
+from platformio.debug.process.base import DebugBaseProcess
+from platformio.proc import where_is_program
+
+
+class DebugServerProcess(DebugBaseProcess):
+    def __init__(self, debug_options, env_options):
+        super(DebugServerProcess, self).__init__()
+        self.debug_options = debug_options
+        self.env_options = env_options
+
+        self._debug_port = ":3333"
+        self._ready = False
+
+    async def run(self, patterns):  # pylint: disable=too-many-branches
+        systype = util.get_systype()
+        server = self.debug_options.get("server")
+        if not server:
+            return None
+        server = self.apply_patterns(server, patterns)
+        server_executable = server["executable"]
+        if not server_executable:
+            return None
+        if server["cwd"]:
+            server_executable = os.path.join(server["cwd"], server_executable)
+        if (
+            "windows" in systype
+            and not server_executable.endswith(".exe")
+            and os.path.isfile(server_executable + ".exe")
+        ):
+            server_executable = server_executable + ".exe"
+
+        if not os.path.isfile(server_executable):
+            server_executable = where_is_program(server_executable)
+        if not os.path.isfile(server_executable):
+            raise DebugInvalidOptionsError(
+                "\nCould not launch Debug Server '%s'. Please check that it "
+                "is installed and is included in a system PATH\n\n"
+                "See documentation or contact contact@platformio.org:\n"
+                "https://docs.platformio.org/page/plus/debugging.html\n"
+                % server_executable
+            )
+
+        openocd_pipe_allowed = all(
+            [not self.debug_options["port"], "openocd" in server_executable]
+        )
+        openocd_pipe_allowed = False
+        if openocd_pipe_allowed:
+            args = []
+            if server["cwd"]:
+                args.extend(["-s", server["cwd"]])
+            args.extend(
+                ["-c", "gdb_port pipe; tcl_port disabled; telnet_port disabled"]
+            )
+            args.extend(server["arguments"])
+            str_args = " ".join(
+                [arg if arg.startswith("-") else '"%s"' % arg for arg in args]
+            )
+            self._debug_port = '| "%s" %s' % (server_executable, str_args)
+            self._debug_port = fs.to_unix_path(self._debug_port)
+            return self._debug_port
+
+        env = os.environ.copy()
+        # prepend server "lib" folder to LD path
+        if (
+            "windows" not in systype
+            and server["cwd"]
+            and os.path.isdir(os.path.join(server["cwd"], "lib"))
+        ):
+            ld_key = "DYLD_LIBRARY_PATH" if "darwin" in systype else "LD_LIBRARY_PATH"
+            env[ld_key] = os.path.join(server["cwd"], "lib")
+            if os.environ.get(ld_key):
+                env[ld_key] = "%s:%s" % (env[ld_key], os.environ.get(ld_key))
+        # prepend BIN to PATH
+        if server["cwd"] and os.path.isdir(os.path.join(server["cwd"], "bin")):
+            env["PATH"] = "%s%s%s" % (
+                os.path.join(server["cwd"], "bin"),
+                os.pathsep,
+                os.environ.get("PATH", os.environ.get("Path", "")),
+            )
+
+        await self.spawn(
+            *([server_executable] + server["arguments"]), cwd=server["cwd"], env=env
+        )
+
+        if "mspdebug" in server_executable.lower():
+            self._debug_port = ":2000"
+        elif "jlink" in server_executable.lower():
+            self._debug_port = ":2331"
+        elif "qemu" in server_executable.lower():
+            self._debug_port = ":1234"
+
+        await self._wait_until_ready()
+
+        return self._debug_port
+
+    async def _wait_until_ready(self):
+        ready_pattern = self.debug_options.get("server", {}).get("ready_pattern")
+        timeout = 60 if ready_pattern else 10
+        elapsed = 0
+        delay = 0.5
+        auto_ready_delay = 0.5
+        while not self._ready and self.is_running() and elapsed < timeout:
+            await asyncio.sleep(delay)
+            if not ready_pattern:
+                self._ready = self._last_activity < (time.time() - auto_ready_delay)
+            elapsed += delay
+
+    def _check_ready_by_pattern(self, data):
+        if self._ready:
+            return self._ready
+        ready_pattern = self.debug_options.get("server", {}).get("ready_pattern")
+        if ready_pattern:
+            self._ready = ready_pattern.encode() in data
+        return self._ready
+
+    def get_debug_port(self):
+        return self._debug_port
+
+    def stdout_data_received(self, data):
+        super(DebugServerProcess, self).stdout_data_received(
+            escape_gdbmi_stream("@", data) if is_gdbmi_mode() else data
+        )
+        self._check_ready_by_pattern(data)
+
+    def stderr_data_received(self, data):
+        super(DebugServerProcess, self).stderr_data_received(data)
+        self._check_ready_by_pattern(data)