Merge branch 'feature/pystyle' into 'master'

Check the Python coding style

See merge request idf/esp-idf!3748

.flake8

@@ -0,0 +1,163 @@
+[flake8]
+
+ignore =
+    E221, # Multiple spaces before operator
+    E231, # Missing whitespace after ',', ';', or ':'
+    E241, # Multiple spaces after ','
+    FI11, # __future__ import "absolute_import" missing
+    FI12, # __future__ import "with_statement" missing
+    FI14, # __future__ import "unicode_literals" missing
+    FI15, # __future__ import "generator_stop" missing
+    FI16, # __future__ import "nested_scopes" missing
+    FI17, # __future__ import "generators" missing
+    FI50, # __future__ import "division" present
+    FI53, # __future__ import "print_function" present
+    W503, # Line break occurred before a binary operator
+    W504  # Line break occurred after a binary operator
+
+max-line-length = 160
+
+show_source = True
+
+statistics = True
+
+exclude =
+    .git,
+    __pycache__,
+    # submodules
+        components/esptool_py/esptool,
+        components/micro-ecc/micro-ecc,
+        components/nghttp/nghttp2,
+        components/libsodium/libsodium,
+        components/json/cJSON,
+        components/mbedtls/mbedtls,
+        components/expat/expat,
+        components/unity/unity,
+    # temporary list (should be empty)
+        components/app_update/dump_otadata.py,
+        components/app_update/gen_empty_partition.py,
+        components/espcoredump/espcoredump.py,
+        components/espcoredump/test/test_espcoredump.py,
+        components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py,
+        components/partition_table/gen_esp32part.py,
+        components/partition_table/parttool.py,
+        components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py,
+        components/protocomm/python/constants_pb2.py,
+        components/protocomm/python/sec0_pb2.py,
+        components/protocomm/python/sec1_pb2.py,
+        components/protocomm/python/session_pb2.py,
+        components/ulp/esp32ulp_mapgen.py,
+        components/wifi_provisioning/python/wifi_config_pb2.py,
+        components/wifi_provisioning/python/wifi_constants_pb2.py,
+        docs/conf_common.py,
+        docs/en/conf.py,
+        docs/gen-dxd.py,
+        docs/gen-toolchain-links.py,
+        docs/gen-version-specific-includes.py,
+        docs/link-roles.py,
+        docs/local_util.py,
+        docs/zh_CN/conf.py,
+        examples/peripherals/can/can_alert_and_recovery/example_test.py,
+        examples/peripherals/can/can_network/example_test.py,
+        examples/peripherals/can/can_self_test/example_test.py,
+        examples/peripherals/i2s_adc_dac/tools/generate_audio_file.py,
+        examples/peripherals/sdio/sdio_test.py,
+        examples/protocols/asio/chat_client/asio_chat_client_test.py,
+        examples/protocols/asio/chat_server/asio_chat_server_test.py,
+        examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py,
+        examples/protocols/asio/udp_echo_server/asio_udp_server_test.py,
+        examples/protocols/esp_http_client/esp_http_client_test.py,
+        examples/protocols/http_server/advanced_tests/http_server_advanced_test.py,
+        examples/protocols/http_server/advanced_tests/scripts/test.py,
+        examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py,
+        examples/protocols/http_server/persistent_sockets/scripts/adder.py,
+        examples/protocols/http_server/simple/http_server_simple_test.py,
+        examples/protocols/http_server/simple/scripts/client.py,
+        examples/protocols/https_request/example_test.py,
+        examples/protocols/mdns/mdns_example_test.py,
+        examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py,
+        examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py,
+        examples/protocols/mqtt/ws/mqtt_ws_example_test.py,
+        examples/protocols/mqtt/wss/mqtt_wss_example_test.py,
+        examples/protocols/sockets/scripts/tcpclient.py,
+        examples/protocols/sockets/scripts/tcpserver.py,
+        examples/protocols/sockets/scripts/udpclient.py,
+        examples/protocols/sockets/scripts/udpserver.py,
+        examples/provisioning/ble_prov/ble_prov_test.py,
+        examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py,
+        examples/provisioning/softap_prov/softap_prov_test.py,
+        examples/provisioning/softap_prov/utils/wifi_tools.py,
+        examples/system/cpp_exceptions/example_test.py,
+        examples/system/esp_event/default_event_loop/example_test.py,
+        examples/system/esp_event/user_event_loops/example_test.py,
+        examples/system/esp_timer/example_test.py,
+        examples/system/light_sleep/example_test.py,
+        examples/wifi/iperf/iperf_test.py,
+        examples/wifi/iperf/test_report.py,
+        tools/check_python_dependencies.py,
+        tools/ci/apply_bot_filter.py,
+        tools/cmake/convert_to_cmake.py,
+        tools/esp_app_trace/apptrace_proc.py,
+        tools/esp_app_trace/logtrace_proc.py,
+        tools/esp_app_trace/pylibelf/__init__.py,
+        tools/esp_app_trace/pylibelf/constants/__init__.py,
+        tools/esp_app_trace/pylibelf/iterators/__init__.py,
+        tools/esp_app_trace/pylibelf/macros/__init__.py,
+        tools/esp_app_trace/pylibelf/types/__init__.py,
+        tools/esp_app_trace/pylibelf/util/__init__.py,
+        tools/esp_app_trace/pylibelf/util/syms/__init__.py,
+        tools/esp_prov/esp_prov.py,
+        tools/esp_prov/proto/__init__.py,
+        tools/esp_prov/prov/__init__.py,
+        tools/esp_prov/prov/custom_prov.py,
+        tools/esp_prov/prov/wifi_prov.py,
+        tools/esp_prov/security/__init__.py,
+        tools/esp_prov/security/security.py,
+        tools/esp_prov/security/security0.py,
+        tools/esp_prov/security/security1.py,
+        tools/esp_prov/transport/__init__.py,
+        tools/esp_prov/transport/ble_cli.py,
+        tools/esp_prov/transport/transport.py,
+        tools/esp_prov/transport/transport_ble.py,
+        tools/esp_prov/transport/transport_console.py,
+        tools/esp_prov/transport/transport_softap.py,
+        tools/esp_prov/utils/__init__.py,
+        tools/esp_prov/utils/convenience.py,
+        tools/gen_esp_err_to_name.py,
+        tools/idf.py,
+        tools/idf_size.py,
+        tools/kconfig_new/confgen.py,
+        tools/kconfig_new/confserver.py,
+        tools/kconfig_new/gen_kconfig_doc.py,
+        tools/kconfig_new/kconfiglib.py,
+        tools/kconfig_new/test/test_confserver.py,
+        tools/ldgen/fragments.py,
+        tools/ldgen/generation.py,
+        tools/ldgen/ldgen.py,
+        tools/ldgen/pyparsing.py,
+        tools/ldgen/sdkconfig.py,
+        tools/ldgen/test/test_fragments.py,
+        tools/ldgen/test/test_generation.py,
+        tools/mass_mfg/mfg_gen.py,
+        tools/test_idf_monitor/run_test_idf_monitor.py,
+        tools/test_idf_size/test_idf_size.py,
+        tools/tiny-test-fw/CIAssignExampleTest.py,
+        tools/tiny-test-fw/CIAssignUnitTest.py,
+        tools/tiny-test-fw/DUT.py,
+        tools/tiny-test-fw/EnvConfig.py,
+        tools/tiny-test-fw/IDF/IDFApp.py,
+        tools/tiny-test-fw/IDF/IDFDUT.py,
+        tools/tiny-test-fw/Runner.py,
+        tools/tiny-test-fw/TinyFW.py,
+        tools/tiny-test-fw/Utility/CaseConfig.py,
+        tools/tiny-test-fw/Utility/LineChart.py,
+        tools/tiny-test-fw/Utility/PowerControl.py,
+        tools/tiny-test-fw/Utility/SearchCases.py,
+        tools/tiny-test-fw/Utility/__init__.py,
+        tools/tiny-test-fw/docs/conf.py,
+        tools/tiny-test-fw/example.py,
+        tools/unit-test-app/idf_ext.py,
+        tools/unit-test-app/tools/CreateSectionTable.py,
+        tools/unit-test-app/tools/UnitTestParser.py,
+        tools/unit-test-app/unit_test.py,
+        tools/windows/eclipse_make.py,

.gitignore

@@ -68,3 +68,6 @@ test_multi_heap_host
 
 # VS Code Settings
 .vscode/
+
+# Results for the checking of the Python coding style
+flake8_output.txt

.gitlab-ci.yml

@@ -722,6 +722,18 @@ check_examples_cmake_make:
   script:
     - tools/ci/check_examples_cmake_make.sh
 
+check_python_style:
+  <<: *check_job_template
+  artifacts:
+    when: on_failure
+    paths:
+      - flake8_output.txt
+    expire_in: 1 week
+  before_script: *do_nothing_before
+  script:
+    # run it only under Python 3 (it is very slow under Python 2)
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
+
 check_ut_cmake_make:
   stage: check
   image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG

.travis.yml

@@ -0,0 +1,7 @@
+language: python
+sudo: false
+python:
+  - "3.4"
+script:
+  - pip install flake8
+  - travis_wait 20 python -m flake8 --config=.flake8 .

tools/idf_monitor.py

@@ -29,8 +29,6 @@
 #
 from __future__ import print_function, division
 from __future__ import unicode_literals
-from future import standard_library
-standard_library.install_aliases()
 from builtins import chr
 from builtins import object
 from builtins import bytes
@@ -71,16 +69,20 @@ ANSI_RED = '\033[1;31m'
 ANSI_YELLOW = '\033[0;33m'
 ANSI_NORMAL = '\033[0m'
 
+
 def color_print(message, color):
     """ Print a message to stderr with colored highlighting """
     sys.stderr.write("%s%s%s\n" % (color, message,  ANSI_NORMAL))
 
+
 def yellow_print(message):
     color_print(message, ANSI_YELLOW)
 
+
 def red_print(message):
     color_print(message, ANSI_RED)
 
+
 __version__ = "1.1"
 
 # Tags for tuples in queues
@@ -95,6 +97,7 @@ DEFAULT_TOOLCHAIN_PREFIX = "xtensa-esp32-elf-"
 
 DEFAULT_PRINT_FILTER = ""
 
+
 class StoppableThread(object):
     """
     Provide a Thread-like class which can be 'cancelled' via a subclass-provided
@@ -120,10 +123,10 @@ class StoppableThread(object):
             self._thread.start()
 
     def _cancel(self):
-        pass # override to provide cancellation functionality
+        pass  # override to provide cancellation functionality
 
     def run(self):
-        pass # override for the main thread behaviour
+        pass  # override for the main thread behaviour
 
     def _run_outer(self):
         try:
@@ -138,6 +141,7 @@ class StoppableThread(object):
             self._cancel()
             old_thread.join()
 
+
 class ConsoleReader(StoppableThread):
     """ Read input keys from the console and push them to the queue,
     until stopped.
@@ -192,9 +196,11 @@ class ConsoleReader(StoppableThread):
             # TODO: introduce some workaround to make it work there.
             #
             # Note: This would throw exception in testing mode when the stdin is connected to PTY.
-            import fcntl, termios
+            import fcntl
+            import termios
             fcntl.ioctl(self.console.fd, termios.TIOCSTI, b'\0')
 
+
 class SerialReader(StoppableThread):
     """ Read serial data from the serial port and push to the
     event queue, until stopped.
@@ -227,9 +233,10 @@ class SerialReader(StoppableThread):
         if hasattr(self.serial, 'cancel_read'):
             try:
                 self.serial.cancel_read()
-            except:
+            except Exception:
                 pass
 
+
 class LineMatcher(object):
     """
     Assembles a dictionary of filtering rules based on the --print_filter
@@ -244,14 +251,14 @@ class LineMatcher(object):
     LEVEL_V = 5
 
     level = {'N': LEVEL_N, 'E': LEVEL_E, 'W': LEVEL_W, 'I': LEVEL_I, 'D': LEVEL_D,
-            'V': LEVEL_V, '*': LEVEL_V, '': LEVEL_V}
+             'V': LEVEL_V, '*': LEVEL_V, '': LEVEL_V}
 
     def __init__(self, print_filter):
         self._dict = dict()
         self._re = re.compile(r'^(?:\033\[[01];?[0-9]+m?)?([EWIDV]) \([0-9]+\) ([^:]+): ')
         items = print_filter.split()
         if len(items) == 0:
-            self._dict["*"] = self.LEVEL_V # default is to print everything
+            self._dict["*"] = self.LEVEL_V  # default is to print everything
         for f in items:
             s = f.split(r':')
             if len(s) == 1:
@@ -267,6 +274,7 @@ class LineMatcher(object):
             else:
                 raise ValueError('Missing ":" in filter ' + f)
             self._dict[s[0]] = lev
+
     def match(self, line):
         try:
             m = self._re.search(line)
@@ -282,12 +290,14 @@ class LineMatcher(object):
         # We need something more than "*.N" for printing.
         return self._dict.get("*", self.LEVEL_N) > self.LEVEL_N
 
+
 class SerialStopException(Exception):
     """
     This exception is used for stopping the IDF monitor in testing mode.
     """
     pass
 
+
 class Monitor(object):
     """
     Monitor application main class.
@@ -316,7 +326,7 @@ class Monitor(object):
 
             self.console.getkey = types.MethodType(getkey_patched, self.console)
 
-        socket_mode = serial_instance.port.startswith("socket://") # testing hook - data from serial can make exit the monitor
+        socket_mode = serial_instance.port.startswith("socket://")  # testing hook - data from serial can make exit the monitor
         self.serial = serial_instance
         self.console_reader = ConsoleReader(self.console, self.event_queue, socket_mode)
         self.serial_reader = SerialReader(self.serial, self.event_queue)
@@ -331,8 +341,8 @@ class Monitor(object):
 
         self.translate_eol = {
             "CRLF": lambda c: c.replace("\n", "\r\n"),
-            "CR":   lambda c: c.replace("\n", "\r"),
+            "CR": lambda c: c.replace("\n", "\r"),
-            "LF":   lambda c: c.replace("\r", "\n"),
+            "LF": lambda c: c.replace("\r", "\n"),
         }[eol]
 
         # internal state
@@ -381,7 +391,7 @@ class Monitor(object):
                 # Cancelling _invoke_processing_last_line_timer is not
                 # important here because receiving empty data doesn't matter.
                 self._invoke_processing_last_line_timer = None
-            except:
+            except Exception:
                 pass
             sys.stderr.write(ANSI_NORMAL + "\n")
 
@@ -399,9 +409,9 @@ class Monitor(object):
                 key = self.translate_eol(key)
                 self.serial.write(codecs.encode(key))
             except serial.SerialException:
-                pass # this shouldn't happen, but sometimes port has closed in serial thread
+                pass  # this shouldn't happen, but sometimes port has closed in serial thread
             except UnicodeEncodeError:
-                pass # this can happen if a non-ascii character was passed, ignoring
+                pass  # this can happen if a non-ascii character was passed, ignoring
 
     def handle_serial_input(self, data, finalize_line=False):
         sp = data.split(b'\n')
@@ -427,7 +437,7 @@ class Monitor(object):
         # to make a decision.
         if self._last_line_part != b"":
             if self._force_line_print or (finalize_line and self._line_matcher.match(self._last_line_part.decode(errors="ignore"))):
-                self._force_line_print = True;
+                self._force_line_print = True
                 if self._output_enabled:
                     self.console.write_bytes(self._last_line_part)
                     self.handle_possible_pc_address_in_line(self._last_line_part)
@@ -454,7 +464,7 @@ class Monitor(object):
     def handle_menu_key(self, c):
         if c == self.exit_key or c == self.menu_key:  # send verbatim
             self.serial.write(codecs.encode(c))
-        elif c in [ CTRL_H, 'h', 'H', '?' ]:
+        elif c in [CTRL_H, 'h', 'H', '?']:
             red_print(self.get_help_text())
         elif c == CTRL_R:  # Reset device via RTS
             self.serial.setRTS(True)
@@ -472,10 +482,10 @@ class Monitor(object):
             # to fast trigger pause without press menu key
             self.serial.setDTR(False)  # IO0=HIGH
             self.serial.setRTS(True)   # EN=LOW, chip in reset
-            time.sleep(1.3) # timeouts taken from esptool.py, includes esp32r0 workaround. defaults: 0.1
+            time.sleep(1.3)  # timeouts taken from esptool.py, includes esp32r0 workaround. defaults: 0.1
             self.serial.setDTR(True)   # IO0=LOW
             self.serial.setRTS(False)  # EN=HIGH, chip out of reset
-            time.sleep(0.45) # timeouts taken from esptool.py, includes esp32r0 workaround. defaults: 0.05
+            time.sleep(0.45)  # timeouts taken from esptool.py, includes esp32r0 workaround. defaults: 0.05
             self.serial.setDTR(False)  # IO0=HIGH, done
         else:
             red_print('--- unknown menu character {} --'.format(key_description(c)))
@@ -502,7 +512,7 @@ class Monitor(object):
            makecmd=key_description(CTRL_F),
            appmake=key_description(CTRL_A),
            output=key_description(CTRL_Y),
-           pause=key_description(CTRL_P) )
+           pause=key_description(CTRL_P))
 
     def __enter__(self):
         """ Use 'with self' to temporarily disable monitoring behaviour """
@@ -525,7 +535,7 @@ class Monitor(object):
 --- Press any other key to resume monitor (resets target).""".format(reason,
                                                                      key_description(self.exit_key),
                                                                      key_description(CTRL_F),
-                                                                     key_description(CTRL_A) ))
+                                                                     key_description(CTRL_A)))
             k = CTRL_T  # ignore CTRL-T here, so people can muscle-memory Ctrl-T Ctrl-F, etc.
             while k == CTRL_T:
                 k = self.console.getkey()
@@ -533,16 +543,16 @@ class Monitor(object):
             self.console.cleanup()
         if k == self.exit_key:
             self.event_queue.put((TAG_KEY, k))
-        elif k in [ CTRL_F, CTRL_A ]:
+        elif k in [CTRL_F, CTRL_A]:
             self.event_queue.put((TAG_KEY, self.menu_key))
             self.event_queue.put((TAG_KEY, k))
 
     def run_make(self, target):
         with self:
             if isinstance(self.make, list):
-                popen_args = self.make + [ target ]
+                popen_args = self.make + [target]
             else:
-                popen_args = [ self.make, target ]
+                popen_args = [self.make, target]
             yellow_print("Running %s..." % " ".join(popen_args))
             p = subprocess.Popen(popen_args)
             try:
@@ -559,13 +569,13 @@ class Monitor(object):
             ["%saddr2line" % self.toolchain_prefix,
              "-pfiaC", "-e", self.elf_file, pc_addr],
             cwd=".")
-        if not b"?? ??:0" in translation:
+        if b"?? ??:0" not in translation:
             yellow_print(translation.decode())
 
     def check_gdbstub_trigger(self, line):
         line = self._gdb_buffer + line
         self._gdb_buffer = b""
-        m = re.search(b"\\$(T..)#(..)", line) # look for a gdb "reason" for a break
+        m = re.search(b"\\$(T..)#(..)", line)  # look for a gdb "reason" for a break
         if m is not None:
             try:
                 chsum = sum(ord(bytes([p])) for p in m.group(1)) & 0xFF
@@ -577,16 +587,15 @@ class Monitor(object):
             else:
                 red_print("Malformed gdb message... calculated checksum %02x received %02x" % (chsum, calc_chsum))
 
-
     def run_gdb(self):
         with self:  # disable console control
             sys.stderr.write(ANSI_NORMAL)
             try:
                 process = subprocess.Popen(["%sgdb" % self.toolchain_prefix,
-                                "-ex", "set serial baud %d" % self.serial.baudrate,
+                                            "-ex", "set serial baud %d" % self.serial.baudrate,
-                                "-ex", "target remote %s" % self.serial.port,
+                                            "-ex", "target remote %s" % self.serial.port,
-                                "-ex", "interrupt",  # monitor has already parsed the first 'reason' command, need a second
+                                            "-ex", "interrupt",  # monitor has already parsed the first 'reason' command, need a second
-                                self.elf_file], cwd=".")
+                                            self.elf_file], cwd=".")
                 process.wait()
             except KeyboardInterrupt:
                 pass  # happens on Windows, maybe other OSes
@@ -594,12 +603,12 @@ class Monitor(object):
                 try:
                     # on Linux, maybe other OSes, gdb sometimes seems to be alive even after wait() returns...
                     process.terminate()
-                except:
+                except Exception:
                     pass
                 try:
                     # also on Linux, maybe other OSes, gdb sometimes exits uncleanly and breaks the tty mode
                     subprocess.call(["stty", "sane"])
-                except:
+                except Exception:
                     pass  # don't care if there's no stty, we tried...
             self.prompt_next_action("gdb exited")
 
@@ -610,6 +619,7 @@ class Monitor(object):
         self._output_enabled = not self._output_enabled
         yellow_print("\nToggle output display: {}, Type Ctrl-T Ctrl-Y to show/disable output again.".format(self._output_enabled))
 
+
 def main():
     parser = argparse.ArgumentParser("idf_monitor - a serial output monitor for esp-idf")
 
@@ -690,6 +700,7 @@ def main():
 
     monitor.main_loop()
 
+
 if os.name == 'nt':
     # Windows console stuff
 
@@ -704,7 +715,7 @@ if os.name == 'nt':
     RE_ANSI_COLOR = re.compile(b'\033\\[([01]);3([0-7])m')
 
     # list mapping the 8 ANSI colors (the indexes) to Windows Console colors
-    ANSI_TO_WINDOWS_COLOR = [ 0, 4, 2, 6, 1, 5, 3, 7 ]
+    ANSI_TO_WINDOWS_COLOR = [0, 4, 2, 6, 1, 5, 3, 7]
 
     GetStdHandle = ctypes.windll.kernel32.GetStdHandle
     SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
@@ -745,10 +756,10 @@ if os.name == 'nt':
                 data = bytearray(data, 'utf-8')
             for b in data:
                 b = bytes([b])
-                l = len(self.matched)
+                length = len(self.matched)
                 if b == b'\033':  # ESC
                     self.matched = b
-                elif (l == 1 and b == b'[') or (1 < l < 7):
+                elif (length == 1 and b == b'[') or (1 < length < 7):
                     self.matched += b
                     if self.matched == ANSI_NORMAL.encode('latin-1'):  # reset console
                         # Flush is required only with Python3 - switching color before it is printed would mess up the console
@@ -765,7 +776,7 @@ if os.name == 'nt':
                             self.flush()
                             SetConsoleTextAttribute(self.handle, color)
                         else:
-                            self._output_write(self.matched) # not an ANSI color code, display verbatim
+                            self._output_write(self.matched)  # not an ANSI color code, display verbatim
                         self.matched = b''
                 else:
                     self._output_write(b)