docs: Start refactoring IDF-specific docs features into extensions

Run the actual IDF build system to determine what components are linked
for a particular target.

docs/conf_common.py

@@ -35,12 +35,16 @@ try:
 except KeyError:
     builddir = '_build'
 
+builddir = os.path.abspath(builddir)
+
 # Fill in a default IDF_PATH if it's missing (ie when Read The Docs is building the docs)
 try:
     idf_path = os.environ['IDF_PATH']
 except KeyError:
     idf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
 
+# Set the idf_target chip. This is a hack right now.
+idf_target = 'esp32s2'
 
 def call_with_python(cmd):
     # using sys.executable ensures that the scripts are called with the same Python interpreter
@@ -60,70 +64,6 @@ copy_if_modified('xml/', 'xml_in/')
 # Generate 'api_name.inc' files using the XML files by Doxygen
 call_with_python('../gen-dxd.py')
 
-
-def find_component_files(parent_dir, target_filename):
-    parent_dir = os.path.abspath(parent_dir)
-    result = []
-
-    component_files = dict()
-
-    for (dirpath, dirnames, filenames) in os.walk(parent_dir):
-        try:
-            # note: trimming "examples" dir as MQTT submodule
-            # has its own examples directory in the submodule, not part of IDF
-            dirnames.remove("examples")
-        except ValueError:
-            pass
-        if target_filename in filenames:
-            component_files[os.path.basename(dirpath)] = os.path.join(dirpath, target_filename)
-
-    components = sorted(component_files.keys())
-
-    for component in components:
-        result.append(component_files[component])
-
-    print("List of %s: %s" % (target_filename, ", ".join(components)))
-    return result
-
-
-# Generate 'kconfig.inc' file from components' Kconfig files
-print("Generating kconfig.inc from kconfig contents")
-kconfig_inc_path = '{}/inc/kconfig.inc'.format(builddir)
-temp_sdkconfig_path = '{}/sdkconfig.tmp'.format(builddir)
-
-kconfigs = find_component_files("../../components", "Kconfig")
-kconfig_projbuilds = find_component_files("../../components", "Kconfig.projbuild")
-sdkconfig_renames = find_component_files("../../components", "sdkconfig.rename")
-
-kconfigs_source_path = '{}/inc/kconfigs_source.in'.format(builddir)
-kconfig_projbuilds_source_path = '{}/inc/kconfig_projbuilds_source.in'.format(builddir)
-
-prepare_kconfig_files_args = [sys.executable,
-                              "../../tools/kconfig_new/prepare_kconfig_files.py",
-                              "--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)),
-                              "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)),
-                              "--env", "COMPONENT_KCONFIGS_SOURCE_FILE={}".format(kconfigs_source_path),
-                              "--env", "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE={}".format(kconfig_projbuilds_source_path),
-                              ]
-subprocess.check_call(prepare_kconfig_files_args)
-
-confgen_args = [sys.executable,
-                "../../tools/kconfig_new/confgen.py",
-                "--kconfig", "../../Kconfig",
-                "--sdkconfig-rename", "../../sdkconfig.rename",
-                "--config", temp_sdkconfig_path,
-                "--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)),
-                "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)),
-                "--env", "COMPONENT_SDKCONFIG_RENAMES={}".format(" ".join(sdkconfig_renames)),
-                "--env", "COMPONENT_KCONFIGS_SOURCE_FILE={}".format(kconfigs_source_path),
-                "--env", "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE={}".format(kconfig_projbuilds_source_path),
-                "--env", "IDF_PATH={}".format(idf_path),
-                "--env", "IDF_TARGET={}".format(os.environ.get('IDF_TARGET', 'esp32')),
-                "--output", "docs", kconfig_inc_path + '.in'
-                ]
-subprocess.check_call(confgen_args)
-copy_if_modified(kconfig_inc_path + '.in', kconfig_inc_path)
-
 # Generate 'esp_err_defs.inc' file with ESP_ERR_ error code definitions
 esp_err_inc_path = '{}/inc/esp_err_defs.inc'.format(builddir)
 call_with_python('../../tools/gen_esp_err_to_name.py --rst_output ' + esp_err_inc_path + '.in')
@@ -176,6 +116,8 @@ extensions = ['breathe',
               'sphinxcontrib.rackdiag',
               'sphinxcontrib.packetdiag',
               'html_redirects',
+              'idf_build_system',
+              'kconfig_reference',
               'sphinx.ext.todo',
               ]
 

docs/idf_build_system/CMakeLists.txt

@@ -0,0 +1,6 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(dummy_project)

docs/idf_build_system/__init__.py

@@ -0,0 +1,48 @@
+# Sphinx extension to integrate IDF build system information
+# into the Sphinx Build
+#
+# Runs early in the Sphinx process, runs CMake to generate the dummy IDF project
+# in this directory - including resolving paths, etc.
+#
+# Then emits the new 'idf-info' event which has information read from IDF
+# build system, that other extensions can use to generate relevant data.
+import os.path
+import sys
+import subprocess
+import json
+
+# this directory also contains the dummy IDF project
+project_path = os.path.abspath(os.path.dirname(__file__))
+project_build_dir = os.path.join(project_path, "build")
+
+def setup(app):
+    builddir = os.path.dirname(app.doctreedir.rstrip(os.sep))
+    app.add_config_value('idf_path', os.environ.get("IDF_PATH", ""), 'env')
+    app.add_config_value('idf_target', 'esp32', 'env')
+    app.add_event('idf-info')
+
+    # Attaching the generate event to env-get-outdated is a bit of a hack,
+    # we want this to run early in the docs build but unclear exactly when
+    app.connect('env-get-outdated', generate_idf_info)
+
+def generate_idf_info(app, env, added, changed, removed):
+    print("Running CMake on dummy project to get build info...")
+    idf_py_path = os.path.join(app.config.idf_path, "tools", "idf.py")
+    print("Running idf.py...")
+    subprocess.check_call([sys.executable,
+                           idf_py_path,
+                           "-C",
+                           project_path,
+                           "set-target",
+                           app.config.idf_target])
+    # TODO: can call these in one execution pass?
+    subprocess.check_call([sys.executable,
+                           idf_py_path,
+                           "-C",
+                           project_path,
+                           "reconfigure"])
+    with open(os.path.join(project_build_dir, "project_description.json")) as f:
+        project_description = json.load(f)
+    app.emit('idf-info', project_description)
+
+    return []

docs/kconfig_reference.py

@@ -0,0 +1,52 @@
+# Extension to generate the KConfig reference list
+import os.path
+import sys
+import subprocess
+
+from local_util import copy_if_modified
+
+def setup(app):
+    # The idf_build_system extension will emit this event once it
+    # has parsed the IDF project's information
+    app.connect('idf-info', generate_reference)
+
+def generate_reference(app, project_description):
+    build_dir = os.path.dirname(app.doctreedir.rstrip(os.sep))
+
+    # Generate 'kconfig.inc' file from components' Kconfig files
+    print("Generating kconfig.inc from kconfig contents")
+    kconfig_inc_path = '{}/inc/kconfig.inc'.format(build_dir)
+    temp_sdkconfig_path = '{}/sdkconfig.tmp'.format(build_dir)
+
+    kconfigs = project_description["config_environment"]["COMPONENT_KCONFIGS"].split(";")
+    kconfig_projbuilds = project_description["config_environment"]["COMPONENT_KCONFIGS_PROJBUILD"].split(";")
+
+    sdkconfig_renames = set()
+    # TODO: this should be generated in project description as well, if possible
+    for k in kconfigs + kconfig_projbuilds:
+        component_dir = os.path.dirname(k)
+        sdkconfig_rename = os.path.join(component_dir, "sdkconfig.rename")
+        if os.path.exists(sdkconfig_rename):
+            sdkconfig_renames.add(sdkconfig_rename)
+
+    kconfigs_source_path = '{}/inc/kconfigs_source.in'.format(build_dir)
+    kconfig_projbuilds_source_path = '{}/inc/kconfig_projbuilds_source.in'.format(build_dir)
+
+    confgen_args = [sys.executable,
+                    "../../tools/kconfig_new/confgen.py",
+                    "--kconfig", "../../Kconfig",
+                    "--sdkconfig-rename", "../../sdkconfig.rename",
+                    "--config", temp_sdkconfig_path,
+                    "--env", "COMPONENT_KCONFIGS={}".format(" ".join(kconfigs)),
+                    "--env", "COMPONENT_KCONFIGS_PROJBUILD={}".format(" ".join(kconfig_projbuilds)),
+                    "--env", "COMPONENT_SDKCONFIG_RENAMES={}".format(" ".join(sdkconfig_renames)),
+                    "--env", "COMPONENT_KCONFIGS_SOURCE_FILE={}".format(kconfigs_source_path),
+                    "--env", "COMPONENT_KCONFIGS_PROJBUILD_SOURCE_FILE={}".format(kconfig_projbuilds_source_path),
+                    "--env", "IDF_PATH={}".format(app.config.idf_path),
+                    "--env", "IDF_TARGET={}".format(app.config.idf_target),
+                    "--output", "docs", kconfig_inc_path + '.in'
+    ]
+    subprocess.check_call(confgen_args)
+    copy_if_modified(kconfig_inc_path + '.in', kconfig_inc_path)
+
+