docs(common): improving documentation

- update esp_modem to use esp_docs
- migrated docs from github pages to docs.espressif.com

.flake8

@@ -137,4 +137,4 @@ show_source = True
 statistics = True
 
 exclude =
-        components/asio/docs/conf_common.py
+        docs/asio/conf_common.py

.github/workflows/publish-docs-component.yml

@@ -5,13 +5,19 @@ on:
     branches:
       - master
 
+env:
+  DOCS_DEPLOY_URL_BASE: ${{ secrets.DOCS_DEPLOY_URL_BASE }}
+  DOCS_DEPLOY_SERVER: ${{ secrets.DOCS_DEPLOY_SERVER }}
+  DOCS_DEPLOY_SERVER_USER: ${{ secrets.DOCS_DEPLOY_SERVER_USER }}
+  DOCS_DEPLOY_KEY: ${{ secrets.DOCS_DEPLOY_PRIVATEKEY }}
+  DOCS_DEPLOY_PATH : ${{ secrets.DOCS_DEPLOY_PATH }}
+
 jobs:
   docs_build:
     name: Docs-Build-And-Upload
     runs-on: ubuntu-latest
     # Skip running on forks since it won't have access to secrets
     if: github.repository == 'espressif/esp-protocols'
-
     steps:
       - name: Checkout esp-protocols
         uses: actions/checkout@master
@@ -19,52 +25,27 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
           submodules: recursive
-
       - name: Generate docs
+        shell: bash
         run: |
-          sudo apt-get update
-          sudo apt-get -y install doxygen clang python3-pip
-          python -m pip install breathe recommonmark esp-docs
-          cd $GITHUB_WORKSPACE/components/esp_modem/docs
-          ./generate_docs
-          mkdir -p $GITHUB_WORKSPACE/docs/esp_modem
-          cp -r html/. $GITHUB_WORKSPACE/docs/esp_modem
-
-          cd $GITHUB_WORKSPACE/components/esp_websocket_client/docs
-          ./generate_docs
-          mkdir -p $GITHUB_WORKSPACE/docs/esp_websocket_client
-          cp -r html/. $GITHUB_WORKSPACE/docs/esp_websocket_client
-
-          cd $GITHUB_WORKSPACE/components/mdns/docs
-          ./generate_docs
-          mkdir -p $GITHUB_WORKSPACE/docs/mdns/en
-          mkdir -p $GITHUB_WORKSPACE/docs/mdns/zh_CN
-          cp -r html_en/. $GITHUB_WORKSPACE/docs/mdns/en
-          cp -r html_zh_CN/. $GITHUB_WORKSPACE/docs/mdns/zh_CN
-
-          cd $GITHUB_WORKSPACE/components/asio/docs
-          ./generate_docs
-          mkdir -p $GITHUB_WORKSPACE/docs/asio
-          cp -r html/. $GITHUB_WORKSPACE/docs/asio
-
-          cd $GITHUB_WORKSPACE/docs
-          touch .nojekyll
-          echo '<a href="esp_modem/index.html">esp-modem</a><br>' > index.html
-          echo '<a href="esp_websocket_client/index.html">esp-websocket-client</a><br>' >> index.html
-          echo '<a href="asio/index.html">ASIO</a><br>' >> index.html
-          echo '<a href="mdns/en/index.html">mDNS_en</a><br>' >> index.html
-          echo '<a href="mdns/zh_CN/index.html">mDNS_zh_CN</a><br>' >> index.html
-
-
+            sudo apt-get update
+            sudo apt-get -y install doxygen clang python3-pip
+            python -m pip install breathe recommonmark esp-docs==1.4.1
+            cd $GITHUB_WORKSPACE/docs
+            ./generate_docs
+      - name: Deploying generated docs
+        if: always()
+        shell: bash
+        run: |
+            source $GITHUB_WORKSPACE/docs/utils.sh
+            add_doc_server_ssh_keys $DOCS_DEPLOY_KEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
+            export GIT_VER=$(git describe --always)
+            export GITHUB_REF_NAME=latest
+            export DOCS_BUILD_DIR=$GITHUB_WORKSPACE/docs
+            deploy-docs
       - name: Upload components to component service
         uses: espressif/github-actions/upload_components@master
         with:
           directories: "components/esp_modem;components/esp_websocket_client;components/mdns;components/asio"
           namespace: "espressif"
           api_token: ${{ secrets.IDF_COMPONENT_API_TOKEN }}
-
-      - name: Deploy generated docs
-        uses: JamesIves/github-pages-deploy-action@4.1.5
-        with:
-          branch: gh-pages
-          folder: docs

README.md

@@ -1,26 +1,24 @@
 # Collection of protocol components for ESP-IDF
 
-[Documentation of esp-protocol](https://espressif.github.io/esp-protocols)
-
 ## Components
 
 ### esp_modem
 
 * Brief introduction [README](components/esp_modem/README.md)
-* Full html [documentation](https://espressif.github.io/esp-protocols/esp_modem/index.html)
+* Full html [documentation](https://docs.espressif.com/projects/esp-protocols/docs/latest/esp_modem/index.html)
 
 ### mDNS
 
 * Brief introduction [README](components/mdns/README.md)
-* Full html [documentation(English)](https://espressif.github.io/esp-protocols/mdns/en/index.html)
-* Full html [documentation(Chinese)](https://espressif.github.io/esp-protocols/mdns/zh_CN/index.html)
+* Full html [documentation(English)](https://docs.espressif.com/projects/esp-protocols/mdns/latest/en/index.html)
+* Full html [documentation(Chinese)](https://docs.espressif.com/projects/esp-protocols/mdns/latest/zh_CN/index.html)
 
 ### esp_websocket_client
 
 * Brief introduction [README](components/esp_websocket_client/README.md)
-* Full html [documentation](https://espressif.github.io/esp-protocols/esp_websocket_client/index.html)
+* Full html [documentation](https://docs.espressif.com/projects/esp-protocols/docs/latest/esp_websocket_client/index.html)
 
 ### ASIO port
 
 * Brief introduction [README](components/asio/README.md)
-* Full html [documentation](https://espressif.github.io/esp-protocols/asio/index.html)
+* Full html [documentation](https://docs.espressif.com/projects/esp-protocols/docs/latest/asio/index.html)

ci/check_copyright_config.yaml

@@ -57,7 +57,7 @@ slim_modem_examples:
 ignore:
   perform_check: no
   include:
-   - 'components/**/docs/**'
+   - '**/docs/**'
    - 'components/esp_modem/port/linux/**'
    - 'components/asio/examples/**'
    - 'components/mdns/**/esp_system_protocols_linux/**'

components/esp_modem/docs/generate_docs

@@ -1,23 +0,0 @@
-# Cleanup the generated html
-rm -rf html
-
-# Cleans example and test build dirs (to reduce the component size before upload)
-rm -rf ../examples/ap_to_pppos/build/ ../examples/simple_cmux_client/build/ ../examples/pppos_client/build/ ../examples/linux_modem/build/ ../examples/modem_console/build ../test/host_test/build/ ../test/target/build/
-
-# Generate C++ API header of the DCE
-cat ../include/generate/esp_modem_command_declare.inc | clang++ -E -P -CC  -xc++ -I../include -DGENERATE_DOCS  - | sed -n '1,/DCE command documentation/!p' > esp_modem_dce.hpp
-
-# Generate C API header of the modem_api.h
-cat ../include/generate/esp_modem_command_declare.inc | clang -E -P -CC  -xc -I../include -DGENERATE_DOCS  - | sed -n '1,/DCE command documentation/!p' > esp_modem_api_commands.h
-
-# RST with links to C++ API
-cat ../include/generate/esp_modem_command_declare.inc | clang -E -P  -xc -I../include -DGENERATE_DOCS -DGENERATE_RST_LINKS - | sed 's/NL/\n/g' > cxx_api_links.rst
-
-# Run doxygen
-doxygen
-
-# Generate the docs
-python -u -m sphinx.cmd.build -b html  . html
-
-# Cleanup the doxygen xml's and temporary headers
-rm -rf xml esp_modem_api_commands.h esp_modem_dce.hpp cxx_api_links.rst

docs/esp_modem/Doxyfile

@@ -93,14 +93,6 @@ ALLOW_UNICODE_NAMES    = NO
 
 OUTPUT_LANGUAGE        = English
 
-# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all
-# documentation generated by doxygen is written. Doxygen will use this
-# information to generate all generated output in the proper direction.
-# Possible values are: None, LTR, RTL and Context.
-# The default value is: None.
-
-OUTPUT_TEXT_DIRECTION  = None
-
 # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
 # descriptions after the members that are listed in the file and class
 # documentation (similar to Javadoc). Set to NO to disable this.
@@ -267,13 +259,6 @@ ALIASES                =
 # A mapping has the form "name=value". For example adding "class=itcl::class"
 # will allow you to use the command class in the itcl::class meaning.
 
-TCL_SUBST              =
-
-# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
-# only. Doxygen will then generate output that is more tailored for C. For
-# instance, some of the names that are used will be different. The list of all
-# members will be omitted, etc.
-# The default value is: NO.
 
 OPTIMIZE_OUTPUT_FOR_C  = NO
 
@@ -829,22 +814,23 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  =                 ../include/esp_modem_api.h \
-                                         ../include/esp_modem_c_api_types.h \
-                                         ../include/cxx_include/esp_modem_api.hpp \
-                                         ../include/cxx_include/esp_modem_dce.hpp \
-                                         ../include/esp_modem_config.h \
-                                         ../include/esp_modem_dce_config.h \
-                                         ../include/cxx_include/esp_modem_dce_factory.hpp \
-                                         ../include/cxx_include/esp_modem_command_library.hpp \
-                                         ../include/cxx_include/esp_modem_dce_module.hpp \
-                                         ../include/cxx_include/esp_modem_dte.hpp \
-                                         ../include/cxx_include/esp_modem_netif.hpp \
-                                         ../include/cxx_include/esp_modem_types.hpp \
-                                         ../include/cxx_include/esp_modem_terminal.hpp \
-                                         ../include/cxx_include/esp_modem_cmux.hpp \
-                                         esp_modem_api_commands.h \
-                                         esp_modem_dce.hpp
+INPUT = \
+    $(PROJECT_PATH)/../components/esp_modem/include/esp_modem_api.h \
+    $(PROJECT_PATH)/../components/esp_modem/include/esp_modem_c_api_types.h \
+    $(PROJECT_PATH)/../components/esp_modem/include/cxx_include/esp_modem_api.hpp \
+    $(PROJECT_PATH)/../components/esp_modem/include/esp_modem_config.h \
+    $(PROJECT_PATH)/../components/esp_modem/include/esp_modem_dce_config.h \
+    $(PROJECT_PATH)/../components/esp_modem/include/cxx_include/esp_modem_dce_factory.hpp \
+    $(PROJECT_PATH)/../components/esp_modem/include/cxx_include/esp_modem_command_library.hpp \
+    $(PROJECT_PATH)/../components/esp_modem/include/cxx_include/esp_modem_dce_module.hpp \
+    $(PROJECT_PATH)/../components/esp_modem/include/cxx_include/esp_modem_dte.hpp \
+    $(PROJECT_PATH)/../components/esp_modem/include/cxx_include/esp_modem_netif.hpp \
+    $(PROJECT_PATH)/../components/esp_modem/include/cxx_include/esp_modem_types.hpp \
+    $(PROJECT_PATH)/../components/esp_modem/include/cxx_include/esp_modem_terminal.hpp \
+    $(PROJECT_PATH)/../components/esp_modem/include/cxx_include/esp_modem_cmux.hpp \
+    $(PROJECT_PATH)/../components/esp_modem/include/cxx_include/esp_modem_dce.hpp
+
+
 # The last two are generated
 
 # This tag can be used to specify the character encoding of the source files
@@ -1094,12 +1080,6 @@ VERBATIM_HEADERS       = YES
 
 ALPHABETICAL_INDEX     = YES
 
-# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
-# which the alphabetical index list will be split.
-# Minimum value: 1, maximum value: 20, default value: 5.
-# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
-
-COLS_IN_ALPHA_INDEX    = 5
 
 # In case all classes in a project start with a common prefix, all classes will
 # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
@@ -1518,16 +1498,6 @@ EXT_LINKS_IN_WINDOW    = NO
 
 FORMULA_FONTSIZE       = 10
 
-# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
-# generated for formulas are transparent PNGs. Transparent PNGs are not
-# supported properly for IE 6.0, but are supported on all modern browsers.
-#
-# Note that when changing this option you need to delete any form_*.png files in
-# the HTML output directory before the changes have effect.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-FORMULA_TRANSPARENT    = YES
 
 # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
 # https://www.mathjax.org) which uses client side Javascript for the rendering
@@ -1821,15 +1791,6 @@ LATEX_BATCHMODE        = NO
 
 LATEX_HIDE_INDICES     = NO
 
-# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
-# code with syntax highlighting in the LaTeX output.
-#
-# Note that which sources are shown also depends on other settings such as
-# SOURCE_BROWSER.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_SOURCE_CODE      = NO
 
 # The LATEX_BIB_STYLE tag can be used to specify the style to use for the
 # bibliography, e.g. plainnat, or ieeetr. See
@@ -1911,15 +1872,6 @@ RTF_STYLESHEET_FILE    =
 
 RTF_EXTENSIONS_FILE    =
 
-# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
-# with syntax highlighting in the RTF output.
-#
-# Note that which sources are shown also depends on other settings such as
-# SOURCE_BROWSER.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_SOURCE_CODE        = NO
 
 #---------------------------------------------------------------------------
 # Configuration options related to the man page output
@@ -2017,14 +1969,6 @@ GENERATE_DOCBOOK       = NO
 
 DOCBOOK_OUTPUT         = docbook
 
-# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
-# program listings (including syntax highlighting and cross-referencing
-# information) to the DOCBOOK output. Note that enabling this will significantly
-# increase the size of the DOCBOOK output.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
-
-DOCBOOK_PROGRAMLISTING = NO
 
 #---------------------------------------------------------------------------
 # Configuration options for the AutoGen Definitions output
@@ -2204,14 +2148,6 @@ EXTERNAL_PAGES         = YES
 # Configuration options related to the dot tool
 #---------------------------------------------------------------------------
 
-# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
-# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
-# NO turns the diagrams off. Note that this option also works with HAVE_DOT
-# disabled, but it is recommended to install and use dot, since it yields more
-# powerful graphs.
-# The default value is: YES.
-
-CLASS_DIAGRAMS         = YES
 
 # You can include diagrams made with dia in doxygen documentation. Doxygen will
 # then run dia to produce the diagram and insert it in the documentation. The
@@ -2245,22 +2181,7 @@ HAVE_DOT               = YES
 
 DOT_NUM_THREADS        = 0
 
-# When you want a differently looking font in the dot files that doxygen
-# generates you can specify the font name using DOT_FONTNAME. You need to make
-# sure dot is able to find the font, which can be done by putting it in a
-# standard location or by setting the DOTFONTPATH environment variable or by
-# setting DOT_FONTPATH to the directory containing the font.
-# The default value is: Helvetica.
-# This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_FONTNAME           = Helvetica
-
-# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
-# dot graphs.
-# Minimum value: 4, maximum value: 24, default value: 10.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_FONTSIZE           = 10
 
 # By default doxygen will tell dot to use the default font as specified with
 # DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
@@ -2474,17 +2395,6 @@ DOT_GRAPH_MAX_NODES    = 250
 
 MAX_DOT_GRAPH_DEPTH    = 3
 
-# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
-# background. This is disabled by default, because dot on Windows does not seem
-# to support this out of the box.
-#
-# Warning: Depending on the platform used, enabling this option may lead to
-# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
-# read).
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_TRANSPARENT        = NO
 
 # Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
 # files in one run (i.e. multiple -o and -T options on the command line). This

docs/esp_mqtt_cxx/Doxyfile

@@ -21,8 +21,8 @@ PROJECT_NAME = "MQTT C++ client"
 ## and used to include in API reference documentation
 
 INPUT = \
-    $(PROJECT_PATH)/include/esp_mqtt.hpp \
-    $(PROJECT_PATH)/include/esp_mqtt_client_config.hpp
+    $(PROJECT_PATH)/../components/esp_mqtt_cxx/include/esp_mqtt.hpp \
+    $(PROJECT_PATH)/../components/esp_mqtt_cxx/include/esp_mqtt_client_config.hpp
 
 ## Get warnings for functions that have no documentation for their parameters or return value
 ##

docs/esp_websocket_client/Doxyfile

@@ -21,7 +21,7 @@ PROJECT_NAME = "ESP Protocols Programming Guide"
 ## and used to include in API reference documentation
 
 INPUT = \
-    $(PROJECT_PATH)/include/esp_websocket_client.h
+    $(PROJECT_PATH)/../components/esp_websocket_client/include/esp_websocket_client.h
 
 ## Get warnings for functions that have no documentation for their parameters or return value
 ##

docs/esp_websocket_client/conf_common.py

@@ -0,0 +1,23 @@
+# flake8: noqa
+from esp_docs.conf_docs import *
+
+extensions += [
+    'sphinx_copybutton',
+    # Needed as a trigger for running doxygen
+    'esp_docs.esp_extensions.dummy_build_system',
+    'esp_docs.esp_extensions.run_doxygen',
+]
+
+# link roles config
+github_repo = 'espressif/esp-protocols'
+
+# context used by sphinx_idf_theme
+html_context['github_user'] = 'espressif'
+html_context['github_repo'] = 'esp-docs'
+
+# Extra options required by sphinx_idf_theme
+project_slug = 'esp-idf'  # >=5.0
+versions_url = 'https://github.com/espressif/esp-protocols/docs/docs_versions.js'
+
+idf_targets = ['esp32']
+languages = ['en']

docs/esp_websocket_client/generate_docs

@@ -0,0 +1,26 @@
+build-docs --target esp32 --language en
+
+cp -rf _build/en/esp32/html .
+rm -rf _build __pycache__
+
+# Modifes some version and target fields of index.html
+echo "<script type="text/javascript">
+window.onload =(function() {
+    var myAnchor = document.getElementById('version-select');
+    var mySpan = document.createElement('input');
+    mySpan.setAttribute('type', 'text');
+    mySpan.setAttribute('maxLength', '10');
+    mySpan.value = 'latest';
+    mySpan.setAttribute('disabled', true);
+    myAnchor.parentNode.replaceChild(mySpan, myAnchor);
+
+    var myAnchor = document.getElementById('target-select');
+    var mySpan = document.createElement('input');
+    mySpan.setAttribute('type', 'text');
+    mySpan.setAttribute('maxLength', '10');
+    mySpan.value = 'all targets';
+    mySpan.setAttribute('disabled', true);
+    myAnchor.parentNode.replaceChild(mySpan, myAnchor);
+
+})();
+</script>" >> html/index.html

docs/generate_docs

@@ -0,0 +1,5 @@
+cd asio; ./generate_docs; cd ../
+cd esp_modem; ./generate_docs; cd ../
+cd esp_websocket_client; ./generate_docs; cd ../
+cd esp_mqtt_cxx; ./generate_docs; cd ../
+cd mdns; ./generate_docs; cd ../

docs/mdns/Doxyfile

@@ -21,7 +21,7 @@ PROJECT_NAME = "ESP Protocols Programming Guide"
 ## and used to include in API reference documentation
 
 INPUT = \
-   $(PROJECT_PATH)/include/mdns.h
+   $(PROJECT_PATH)/../components/mdns/include/mdns.h
 
 ## Get warnings for functions that have no documentation for their parameters or return value
 ##

docs/mdns/en/conf.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+#
+# English Language RTD & Sphinx config file
+#
+# Uses ../conf_common.py for most non-language-specific settings.
+
+# Importing conf_common adds all the non-language-specific
+# parts to this conf module
+
+try:
+    from conf_common import *  # noqa: F403,F401
+except ImportError:
+    import os
+    import sys
+    sys.path.insert(0, os.path.abspath('../'))
+    from conf_common import *  # noqa: F403,F401
+
+# General information about the project.
+project = u'ESP-Protocols'
+copyright = u'2016 - 2022, Espressif Systems (Shanghai) Co., Ltd'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+language = 'en'

docs/mdns/generate_docs

@@ -1,8 +1,8 @@
 build-docs --target esp32 --language en
 build-docs --target esp32 --language zh_CN
 
-cp -rf _build/en/esp32/html html_en
-cp -rf _build/zh_CN/esp32/html html_zh_CN
+cp -rf _build/en/esp32/html en
+cp -rf _build/zh_CN/esp32/html zh_CN
 rm -rf _build __pycache__ tee
 
 # Modifes some version and target fields of index.html

docs/utils.sh

@@ -0,0 +1,18 @@
+# Bash helper functions for adding SSH keys
+
+function add_ssh_keys() {
+  local key_string="${1}"
+  mkdir -p ~/.ssh
+  chmod 700 ~/.ssh
+  echo -n "${key_string}" >~/.ssh/id_rsa_base64
+  base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
+  chmod 600 ~/.ssh/id_rsa
+}
+
+function add_doc_server_ssh_keys() {
+  local key_string="${1}"
+  local server_url="${2}"
+  local server_user="${3}"
+  add_ssh_keys "${key_string}"
+  echo -e "Host ${server_url}\n\tStrictHostKeyChecking no\n\tUser ${server_user}\n" >>~/.ssh/config
+}