From 89179dc286439216ed01b6b915cbf9790bbda4b5 Mon Sep 17 00:00:00 2001
From: Marius Vikhammer <marius.vikhammer@espressif.com>
Date: Tue, 27 Apr 2021 12:24:22 +0800
Subject: [PATCH] docs: add fast build feature and activate it in CI

Adds feature for skipping include of doxygen headers into sphinx build
when env variable is set.

Builds incomplete docs (no function API documention)
but speeds up building (20min to 1<min).

Add this fast build as the default way of building docs in CI on
non protected branches.
---
 .gitlab/ci/dependencies/dependencies.yml | 16 -----
 .gitlab/ci/docs.yml                      | 90 ++++++++++++++++++++----
 .gitlab/ci/rules.yml                     | 39 ----------
 docs/build_docs.py                       |  6 ++
 docs/en/contribute/documenting-code.rst  | 10 +++
 docs/idf_extensions/run_doxygen.py       |  9 +++
 6 files changed, 103 insertions(+), 67 deletions(-)

diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml
index ac6ea4b825..d6fb3700af 100644
--- a/.gitlab/ci/dependencies/dependencies.yml
+++ b/.gitlab/ci/dependencies/dependencies.yml
@@ -49,22 +49,6 @@
     - build_system
     - macos
 
-"build:docs":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  patterns:
-    - docs
-  deploy:
-    - production
-
-"build:docs:label-only":
-  labels:
-    - docs
-    - build_docs  # for backward compatibility
-  deploy:
-    - preview
-
 # ---------------
 # Build Test Jobs
 # ---------------
diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml
index 9d53498a65..a0ce8e715e 100644
--- a/.gitlab/ci/docs.yml
+++ b/.gitlab/ci/docs.yml
@@ -1,8 +1,45 @@
+.patterns-docs: &patterns-docs
+  - ".gitlab/ci/docs.yml"
+  - "docs/**/*"
+  - "components/**/*.h"
+  - "components/**/Kconfig*"
+  - "components/**/CMakeList.txt"
+  - "components/**/sdkconfig*"
+  - "tools/kconfig_new/**/*"
+  - "CONTRIBUTING.rst"
+
+.if-protected: &if-protected
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
+
+.if-protected-no_label: &if-protected-no_label
+  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null'
+
+.if-label-build_docs: &if-label-build_docs
+  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs: &if-label-docs
+  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
+
+.if-label-docs_full: &if-label-docs_full
+  if: '$BOT_LABEL_DOCS_FULL || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs_full(?:,[^,\n\r]+)*$/i'
+
+.if-dev-push: &if-dev-push
+  if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
+
+.doc-rules:build:docs:
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-label-docs_full
+    - <<: *if-dev-push
+      changes: *patterns-docs
+
 # stage: pre_check
 check_readme_links:
   extends:
     - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   tags: ["build", "amd64", "internet"]
   allow_failure: true
   script:
@@ -11,7 +48,7 @@ check_readme_links:
 check_docs_lang_sync:
   extends:
     - .pre_check_job_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   script:
     - cd docs
     - ./check_lang_folder_sync.sh
@@ -34,7 +71,7 @@ check_docs_gh_links:
   extends:
     - .pre_check_job_template
     - .build_docs_template
-    - .rules:build:docs
+    - .doc-rules:build:docs
   script:
     - cd docs
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
@@ -45,7 +82,6 @@ check_docs_gh_links:
 .build_docs_build_stage_template:
   extends:
     - .build_docs_template
-    - .rules:build:docs
   stage: build_doc
   needs:
     - job: check_docs_lang_sync
@@ -53,9 +89,14 @@ check_docs_gh_links:
     - job: check_docs_gh_links
       artifacts: false
 
-build_docs_html:
+# Doc jobs have a lot of special cases, we specify rules here directly instead
+# in dependencies.yml to simplify things
+build_docs_html_full:
   extends:
     - .build_docs_build_stage_template
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
   artifacts:
     when: always
     paths:
@@ -65,10 +106,32 @@ build_docs_html:
   variables:
     DOC_BUILDERS: "html"
 
+build_docs_html_fast:
+  extends:
+    - .build_docs_build_stage_template
+  rules:
+    - <<: *if-label-docs_full
+      when: never
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "html"
+    DOCS_FAST_BUILD: "yes"
+
 build_docs_pdf:
   extends:
     - .build_docs_build_stage_template
-    - .rules:build:docs:label-only
+  rules:
+    - <<: *if-protected
+    - <<: *if-label-docs_full
   artifacts:
     when: always
     paths:
@@ -98,10 +161,12 @@ build_docs_pdf:
 deploy_docs_preview:
   extends:
     - .deploy_docs_template
-    - .rules:build:docs:label-only-preview
-  dependencies:  # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - build_docs_html
+  rules:
+    - <<: *if-label-build_docs
+    - <<: *if-label-docs
+  dependencies:
+    - build_docs_html_fast
+    - build_docs_html_full
     - build_docs_pdf
   variables:
     TYPE: "preview"
@@ -117,11 +182,12 @@ deploy_docs_production:
   # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
   extends:
     - .deploy_docs_template
-    - .rules:build:docs-production
+  rules:
+    - <<: *if-protected-no_label
   stage: post_deploy
   dependencies:  # set dependencies to null to avoid missing artifacts issue
   needs: # ensure runs after push_to_github succeeded
-    - build_docs_html
+    - build_docs_html_full
     - build_docs_pdf
     - job: push_to_github
       artifacts: false
diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml
index 3ae2eb0f27..8c2d26730e 100644
--- a/.gitlab/ci/rules.yml
+++ b/.gitlab/ci/rules.yml
@@ -1,16 +1,6 @@
 ############
 # Patterns #
 ############
-.patterns-docs: &patterns-docs
-  - ".gitlab/ci/docs.yml"
-  - "docs/**/*"
-  - "components/**/*.h"
-  - "components/**/Kconfig*"
-  - "components/**/CMakeList.txt"
-  - "components/**/sdkconfig*"
-  - "tools/kconfig_new/**/*"
-  - "CONTRIBUTING.rst"
-
 .patterns-c-files: &patterns-c-files
   - ".gitlab/ci/static-code-analysis.yml"
   - "tools/ci/static-analysis-rules.yml"
@@ -258,9 +248,6 @@
 .if-label-build: &if-label-build
   if: '$BOT_LABEL_BUILD || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i'
 
-.if-label-build_docs: &if-label-build_docs
-  if: '$BOT_LABEL_BUILD_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build_docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-component_ut: &if-label-component_ut
   if: '$BOT_LABEL_COMPONENT_UT || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*component_ut(?:,[^,\n\r]+)*$/i'
 
@@ -294,9 +281,6 @@
 .if-label-docker: &if-label-docker
   if: '$BOT_LABEL_DOCKER || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docker(?:,[^,\n\r]+)*$/i'
 
-.if-label-docs: &if-label-docs
-  if: '$BOT_LABEL_DOCS || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*docs(?:,[^,\n\r]+)*$/i'
-
 .if-label-example_test: &if-label-example_test
   if: '$BOT_LABEL_EXAMPLE_TEST || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*example_test(?:,[^,\n\r]+)*$/i'
 
@@ -486,29 +470,6 @@
     - <<: *if-dev-push
       changes: *patterns-docker
 
-.rules:build:docs:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-    - <<: *if-dev-push
-      changes: *patterns-docs
-
-.rules:build:docs-production:
-  rules:
-    - <<: *if-protected-no_label
-
-.rules:build:docs:label-only:
-  rules:
-    - <<: *if-protected
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
-.rules:build:docs:label-only-preview:
-  rules:
-    - <<: *if-label-build_docs
-    - <<: *if-label-docs
-
 .rules:build:example_test-esp32:
   rules:
     - <<: *if-protected
diff --git a/docs/build_docs.py b/docs/build_docs.py
index 5e980dbd3d..6841f0bb5c 100755
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@@ -100,6 +100,7 @@ def main():
 
     build_parser = action_parsers.add_parser('build', help='Build documentation')
     build_parser.add_argument('--check-warnings-only', '-w', action='store_true')
+    build_parser.add_argument('--fast-build', '-f', action='store_true', help='Skips including doxygen generated APIs into the Sphinx build')
 
     action_parsers.add_parser('linkcheck', help='Check links (a current IDF revision should be uploaded to GitHub)')
 
@@ -124,6 +125,11 @@ def main():
     if args.action == 'build' or args.action is None:
         if args.action is None:
             args.check_warnings_only = False
+            args.fast_build = False
+
+        if args.fast_build:
+            os.environ['DOCS_FAST_BUILD'] = 'y'
+
         sys.exit(action_build(args))
 
     if args.action == 'linkcheck':
diff --git a/docs/en/contribute/documenting-code.rst b/docs/en/contribute/documenting-code.rst
index 3f3d36d86f..4c496cca99 100644
--- a/docs/en/contribute/documenting-code.rst
+++ b/docs/en/contribute/documenting-code.rst
@@ -513,6 +513,16 @@ As well as wildcards::
 
 Note that this is a feature intended to simply testing and debugging during writing of documentation. The HTML output won't be perfect, i.e. it will not build a proper index that lists all the documents, and any references to documents that are not built will result in warnings.
 
+Fast build
+""""""""""
+Another trick to speed up building is to skip including doxygen generated API documention into the Sphinx build process, skipping this drastically reduces build times.
+
+This can be achieved by adding the fast-build argument::
+
+    ./build_docs.py build -f
+
+or by setting the environment variable `DOCS_FAST_BUILD`. Note that the the `-f` argument is a subargument to `build` and thus must be listed after `build`.
+
 Building PDF
 """"""""""""
 
diff --git a/docs/idf_extensions/run_doxygen.py b/docs/idf_extensions/run_doxygen.py
index 5e96b009b9..8c0b1a0f19 100644
--- a/docs/idf_extensions/run_doxygen.py
+++ b/docs/idf_extensions/run_doxygen.py
@@ -73,6 +73,8 @@ def convert_api_xml_to_inc(app, doxyfiles):
     xml_directory_path = '{}/xml'.format(build_dir)
     inc_directory_path = '{}/inc'.format(build_dir)
 
+    fast_build = os.environ.get('DOCS_FAST_BUILD', None)
+
     if not os.path.isdir(xml_directory_path):
         raise RuntimeError('Directory {} does not exist!'.format(xml_directory_path))
 
@@ -96,6 +98,13 @@ def convert_api_xml_to_inc(app, doxyfiles):
             with open(inc_file_path, 'w', encoding='utf-8') as inc_file:
                 inc_file.write(rst_output)
 
+        # For fast builds we wipe the doxygen api documention.
+        # Parsing this output during the sphinx build process is
+        # what takes 95% of the build time
+        if fast_build:
+            with open(inc_file_path, 'w', encoding='utf-8') as inc_file:
+                inc_file.write('')
+
 
 def get_doxyfile_input_paths(app, doxyfile_path):
     """Get contents of Doxyfile's INPUT statement.