Merge branch 'bugfix/idf_monitor' into 'master'

idf_monitor: Small fixes (baud rate, EOL, /dev/tty.X on macOS, Ctrl-T on failure)

* "make monitor" now passed the configured baud rate.
  Closes #436 https://github.com/espressif/esp-idf/issues/436
* Pass toolchain prefix from sdkconfig into monitor tool
* Allow setting EOL in idf_monitor.py, use CRLF by default
* Detect if /dev/tty.X is used on macOS, warn and replace with /dev/cu.X
* If a build fails or gdb exits, ignore Ctrl-T (allowing Ctrl-T Ctrl-A/F to be same key sequence everywhere)
* Add a note about winpty on Windows.
  Ref 02fdf8271d (commitcomment-21369196)
* Fix problems with Console.cancel() not existing in older pyserial
* Print more user-friendly symbols for "start of iram" and "start of flash"

See merge request !594

components/esp32/ld/esp32.common.ld

@@ -71,6 +71,10 @@ SECTIONS
     *(.init.literal)
     *(.init)
     _init_end = ABSOLUTE(.);
+
+    /* This goes here, not at top of linker script, so addr2line finds it last,
+       and uses it in preference to the first symbol in IRAM */
+    _iram_start = ABSOLUTE(0);
   } > iram0_0_seg
 
   .iram0.text :
@@ -193,5 +197,11 @@ SECTIONS
     *(.gnu.version)
     _text_end = ABSOLUTE(.);
     _etext = .;
+
+    /* Similar to _iram_start, this symbol goes here so it is
+       resolved by addr2line in preference to the first symbol in
+       the flash.text segment.
+    */
+    _flash_cache_start = ABSOLUTE(0);
   } >iram0_2_seg
 }

components/esptool_py/Makefile.projbuild

@@ -83,12 +83,14 @@ endif
 simple_monitor: $(call prereq_if_explicit,%flash)
 	$(MONITOR_PYTHON) -m serial.tools.miniterm --rts 0 --dtr 0 --raw $(ESPPORT) $(MONITORBAUD)
 
+MONITOR_OPTS := --baud $(MONITORBAUD) --port $(ESPPORT) --toolchain-prefix $(CONFIG_TOOLPREFIX) --make "$(MAKE)"
+
 monitor: $(call prereq_if_explicit,%flash)
 	$(summary) MONITOR
 	[ -f $(APP_ELF) ] || echo "*** 'make monitor' target requires an app to be compiled and flashed first."
 	[ -f $(APP_ELF) ] || echo "*** Run 'make flash monitor' to build, flash and monitor"
 	[ -f $(APP_ELF) ] || echo "*** Or alternatively 'make simple_monitor' to view the serial port as-is."
 	[ -f $(APP_ELF) ] || exit 1
-	$(MONITOR_PYTHON) $(IDF_PATH)/tools/idf_monitor.py --port $(ESPPORT) --make "$(MAKE)" $(APP_ELF)
+	$(MONITOR_PYTHON) $(IDF_PATH)/tools/idf_monitor.py $(MONITOR_OPTS) $(APP_ELF)
 
 .PHONY: erase_flash

docs/idf-monitor.rst

@@ -102,6 +102,7 @@ Known Issues with idf_monitor
 Issues Observed on Windows
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+- If you are using the supported Windows environment and receive the error "winpty: command not found" then run ``pacman -S winpty`` to fix.
 - Arrow keys and some other special keys in gdb don't work, due to Windows Console limitations.
 - Occasionally when "make" exits, it may stall for up to 30 seconds before idf_monitor resumes.
 - Occasionally when "gdb" is run, it may stall for a short time before it begins communicating with the gdbstub.

make/component_wrapper.mk

@@ -165,7 +165,7 @@ $(1)/%.o: $$(COMPONENT_PATH)/$(1)/%.cpp $(COMMON_MAKEFILES) $(COMPONENT_MAKEFILE
 
 $(1)/%.o: $$(COMPONENT_PATH)/$(1)/%.S $(COMMON_MAKEFILES) $(COMPONENT_MAKEFILE) | $(1)
 	$$(summary) AS $$@
-	$$(CC) $$(CPPFLAGS) $$(addprefix -I ,$$(COMPONENT_INCLUDES)) $$(addprefix -I ,$$(COMPONENT_EXTRA_INCLUDES)) -I$(1) -c $$< -o $$@
+	$$(CC) $$(CPPFLAGS) $$(DEBUG_FLAGS) $$(addprefix -I ,$$(COMPONENT_INCLUDES)) $$(addprefix -I ,$$(COMPONENT_EXTRA_INCLUDES)) -I$(1) -c $$< -o $$@
 
 # CWD is build dir, create the build subdirectory if it doesn't exist
 $(1):

make/project.mk

@@ -241,13 +241,14 @@ OPTIMIZATION_FLAGS = -Og
 endif
 
 # Enable generation of debugging symbols
-OPTIMIZATION_FLAGS += -ggdb
+# (we generate even in Release mode, as this has no impact on final binary size.)
+DEBUG_FLAGS ?= -ggdb
 
 # List of flags to pass to C compiler
 # If any flags are defined in application Makefile, add them at the end.
 CFLAGS := $(strip \
 	-std=gnu99 \
-	$(OPTIMIZATION_FLAGS) \
+	$(OPTIMIZATION_FLAGS) $(DEBUG_FLAGS) \
 	$(COMMON_FLAGS) \
 	$(COMMON_WARNING_FLAGS) -Wno-old-style-declaration \
 	$(CFLAGS) \
@@ -259,7 +260,7 @@ CXXFLAGS := $(strip \
 	-std=gnu++11 \
 	-fno-exceptions \
 	-fno-rtti \
-	$(OPTIMIZATION_FLAGS) \
+	$(OPTIMIZATION_FLAGS) $(DEBUG_FLAGS) \
 	$(COMMON_FLAGS) \
 	$(COMMON_WARNING_FLAGS) \
 	$(CXXFLAGS) \

tools/idf_monitor.py

@@ -56,11 +56,20 @@ CTRL_T = '\x14'
 CTRL_RBRACKET = '\x1d'  # Ctrl+]
 
 # ANSI terminal codes
-ANSI_BLUE = '\033[0;34m'
 ANSI_RED = '\033[1;31m'
 ANSI_YELLOW = '\033[0;33m'
 ANSI_NORMAL = '\033[0m'
 
+def color_print(message, color):
+    """ Print a message to stderr with colored highlighting """
+    sys.stderr.write("%s%s%s\n" % (color, message,  ANSI_NORMAL))
+
+def yellow_print(message):
+    color_print(message, ANSI_YELLOW)
+
+def red_print(message):
+    color_print(message, ANSI_RED)
+
 __version__ = "1.0"
 
 # Tags for tuples in queues
@@ -70,6 +79,8 @@ TAG_SERIAL = 1
 # regex matches an potential PC value (0x4xxxxxxx)
 MATCH_PCADDR = re.compile(r'0x4[0-9a-f]{7}', re.IGNORECASE)
 
+DEFAULT_TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
+
 class StoppableThread(object):
     """
     Provide a Thread-like class which can be 'cancelled' via a subclass-provided
@@ -146,7 +157,15 @@ class ConsoleReader(StoppableThread):
             self.console.cleanup()
 
     def _cancel(self):
-        self.console.cancel()
+        if hasattr(self.console, "cancel"):
+            self.console.cancel()
+        elif os.name == 'posix':
+            # this is the way cancel() is implemented in pyserial 3.1 or newer,
+            # older pyserial doesn't have this method, hence this hack.
+            #
+            # on Windows there is a different (also hacky) fix, applied above.
+            import fcntl, termios
+            fcntl.ioctl(self.console.fd, termios.TIOCSTI, b'\0')
 
 class SerialReader(StoppableThread):
     """ Read serial data from the serial port and push to the
@@ -193,7 +212,7 @@ class Monitor(object):
 
     Main difference is that all event processing happens in the main thread, not the worker threads.
     """
-    def __init__(self, serial_instance, elf_file, make="make"):
+    def __init__(self, serial_instance, elf_file, make="make", toolchain_prefix=DEFAULT_TOOLCHAIN_PREFIX, eol="CRLF"):
         super(Monitor, self).__init__()
         self.event_queue = queue.Queue()
         self.console = miniterm.Console()
@@ -207,9 +226,16 @@ class Monitor(object):
         self.serial_reader = SerialReader(self.serial, self.event_queue)
         self.elf_file = elf_file
         self.make = make
+        self.toolchain_prefix = DEFAULT_TOOLCHAIN_PREFIX
         self.menu_key = CTRL_T
         self.exit_key = CTRL_RBRACKET
 
+        self.translate_eol = {
+            "CRLF": lambda c: c.replace(b"\n", b"\r\n"),
+            "CR":   lambda c: c.replace(b"\n", b"\r"),
+            "LF":   lambda c: c.replace(b"\r", b"\n"),
+        }[eol]
+
         # internal state
         self._pressed_menu_key = False
         self._read_line = b""
@@ -246,6 +272,7 @@ class Monitor(object):
             self.serial_reader.stop()
         else:
             try:
+                key = self.translate_eol(key)
                 self.serial.write(codecs.encode(key))
             except serial.SerialException:
                 pass # this shouldn't happen, but sometimes port has closed in serial thread
@@ -270,7 +297,7 @@ class Monitor(object):
         if c == self.exit_key or c == self.menu_key:  # send verbatim
             self.serial.write(codecs.encode(c))
         elif c in [ CTRL_H, 'h', 'H', '?' ]:
-            sys.stderr.write(self.get_help_text())
+            red_print(self.get_help_text())
         elif c == CTRL_R:  # Reset device via RTS
             self.serial.setRTS(True)
             time.sleep(0.2)
@@ -280,7 +307,7 @@ class Monitor(object):
         elif c == CTRL_A:  # Recompile & upload app only
             self.run_make("app-flash")
         else:
-            sys.stderr.write('--- unknown menu character {} --\n'.format(key_description(c)))
+            red_print('--- unknown menu character {} --'.format(key_description(c)))
 
     def get_help_text(self):
         return """
@@ -317,17 +344,18 @@ class Monitor(object):
     def prompt_next_action(self, reason):
         self.console.setup()  # set up console to trap input characters
         try:
-            sys.stderr.write(ANSI_RED)
+            red_print("""
-            sys.stderr.write("--- {}\n".format(reason))
+--- {}
-            sys.stderr.write("--- Press {} to exit monitor.\n"
+--- Press {} to exit monitor.
-                             .format(key_description(self.exit_key)))
+--- Press {} to run 'make flash'.
-            sys.stderr.write("--- Press {} to run 'make flash'.\n"
+--- Press {} to run 'make app-flash'.
-                             .format(key_description(CTRL_F)))
+--- Press any other key to resume monitor (resets target).""".format(reason,
-            sys.stderr.write("--- Press {} to run 'make app-flash'.\n"
+                                                                     key_description(self.exit_key),
-                             .format(key_description(CTRL_A)))
+                                                                     key_description(CTRL_F),
-            sys.stderr.write("--- Press any other key to resume monitor (resets target).\n")
+                                                                     key_description(CTRL_A)))
-            sys.stderr.write(ANSI_NORMAL)
+            k = CTRL_T  # ignore CTRL-T here, so people can muscle-memory Ctrl-T Ctrl-F, etc.
-            k = self.console.getkey()
+            while k == CTRL_T:
+                k = self.console.getkey()
         finally:
             self.console.cleanup()
         if k == self.exit_key:
@@ -338,7 +366,7 @@ class Monitor(object):
 
     def run_make(self, target):
         with self:
-            sys.stderr.write("%s--- Running make %s...\n" % (ANSI_NORMAL, target))
+            yellow_print("Running make %s..." % target)
             p = subprocess.Popen([self.make,
                                   target ])
             try:
@@ -350,11 +378,11 @@ class Monitor(object):
 
     def lookup_pc_address(self, pc_addr):
         translation = subprocess.check_output(
-            ["xtensa-esp32-elf-addr2line", "-pfia",
+            ["%saddr2line" % self.toolchain_prefix,
-             "-e", self.elf_file, pc_addr],
+             "-pfia", "-e", self.elf_file, pc_addr],
             cwd=".")
         if not "?? ??:0" in translation:
-            sys.stderr.write(ANSI_YELLOW + translation + ANSI_NORMAL)
+            yellow_print(translation)
 
     def check_gdbstub_trigger(self, c):
         self._gdb_buffer = self._gdb_buffer[-6:] + c  # keep the last 7 characters seen
@@ -368,14 +396,14 @@ class Monitor(object):
             if chsum == calc_chsum:
                 self.run_gdb()
             else:
-                sys.stderr.write("Malformed gdb message... calculated checksum %02x received %02x\n" % (chsum, calc_chsum))
+                red_print("Malformed gdb message... calculated checksum %02x received %02x" % (chsum, calc_chsum))
 
 
     def run_gdb(self):
         with self:  # disable console control
             sys.stderr.write(ANSI_NORMAL)
             try:
-                subprocess.call(["xtensa-esp32-elf-gdb",
+                subprocess.call(["%sgdb" % self.toolchain_prefix,
                                 "-ex", "set serial baud %d" % self.serial.baudrate,
                                 "-ex", "target remote %s" % self.serial.port,
                                 "-ex", "interrupt",  # monitor has already parsed the first 'reason' command, need a second
@@ -404,12 +432,29 @@ def main():
         help='Command to run make',
         type=str, default='make')
 
+    parser.add_argument(
+        '--toolchain-prefix',
+        help="Triplet prefix to add before cross-toolchain names",
+        default=DEFAULT_TOOLCHAIN_PREFIX)
+
+    parser.add_argument(
+        "--eol",
+        choices=['CR', 'LF', 'CRLF'],
+        type=lambda c: c.upper(),
+        help="End of line to use when sending to the serial port",
+        default='CRLF')
+
     parser.add_argument(
         'elf_file', help='ELF file of application',
         type=argparse.FileType('r'))
 
     args = parser.parse_args()
 
+    if args.port.startswith("/dev/tty."):
+        args.port = args.port.replace("/dev/tty.", "/dev/cu.")
+        yellow_print("--- WARNING: Serial ports accessed as /dev/tty.* will hang gdb if launched.")
+        yellow_print("--- Using %s instead..." % args.port)
+
     serial_instance = serial.serial_for_url(args.port, args.baud,
                                             do_not_open=True)
     serial_instance.dtr = False
@@ -428,11 +473,11 @@ def main():
     except KeyError:
         pass  # not running a make jobserver
 
-    monitor = Monitor(serial_instance, args.elf_file.name, args.make)
+    monitor = Monitor(serial_instance, args.elf_file.name, args.make, args.eol)
 
-    sys.stderr.write('--- idf_monitor on {p.name} {p.baudrate} ---\n'.format(
+    yellow_print('--- idf_monitor on {p.name} {p.baudrate} ---'.format(
         p=serial_instance))
-    sys.stderr.write('--- Quit: {} | Menu: {} | Help: {} followed by {} ---\n'.format(
+    yellow_print('--- Quit: {} | Menu: {} | Help: {} followed by {} ---'.format(
         key_description(monitor.exit_key),
         key_description(monitor.menu_key),
         key_description(monitor.menu_key),