ci(docs): Move doc related jobs to docs.yml

.gitlab-ci.yml

@@ -117,6 +117,7 @@ before_script:
 
 include:
   - '/tools/ci/config/rules.yml'
+  - '/tools/ci/config/docs.yml'
   - '/tools/ci/config/pre_check.yml'
   - '/tools/ci/config/build.yml'
   - '/tools/ci/config/assign-test.yml'
@@ -124,4 +125,3 @@ include:
   - '/tools/ci/config/target-test.yml'
   - '/tools/ci/config/post_check.yml'
   - '/tools/ci/config/deploy.yml'
-  - '/tools/ci/config/post_deploy.yml'

tools/ci/config/build.yml

@@ -50,16 +50,6 @@ fast_template_app:
   stage: pre_check
   variables:
     BUILD_COMMAND_ARGS: "-p"
-
-check_docs_gh_links:
-  extends: .build_docs_template
-  stage: pre_check
-  variables:
-    SUBMODULES_TO_FETCH: "none"
-  script:
-    - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
 #------------------------------------------------------------------------------
 
 .build_ssc_template:
@@ -285,45 +275,6 @@ build_component_ut_esp32c3:
   variables:
     IDF_TARGET: esp32c3
 
-.build_docs_template:
-  stage: build
-  image: $ESP_IDF_DOC_ENV_IMAGE
-  tags:
-    - build_docs
-  script:
-    - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs $DOC_BUILDERS -l $DOCLANG -t $DOCTGT build
-  parallel:
-    matrix:
-      - DOCLANG: [ "en", "zh_CN" ]
-        DOCTGT: [ "esp32", "esp32s2",  "esp32c3"]
-
-build_docs_html:
-  extends:
-    - .build_docs_template
-    - .rules:labels:build_docs
-  artifacts:
-    when: always
-    paths:
-      - docs/_build/*/*/*.txt
-      - docs/_build/*/*/html/*
-    expire_in: 4 days
-  variables:
-    DOC_BUILDERS: "html"
-
-build_docs_pdf:
-  extends:
-    - .build_docs_template
-    - .rules:labels:build_docs-slim
-  artifacts:
-    when: always
-    paths:
-      - docs/_build/*/*/latex/*
-    expire_in: 4 days
-  variables:
-    DOC_BUILDERS: "latex"
-
 .test_build_system_template:
   extends:
     - .build_template

tools/ci/config/deploy.yml

@@ -52,60 +52,6 @@ push_to_github:
     - git remote add github git@github.com:espressif/esp-idf.git
     - tools/ci/push_to_github.sh
 
-.deploy_docs_template:
-  extends:
-    - .deploy_job_template
-    - .before_script_lesser
-  image: $ESP_IDF_DOC_ENV_IMAGE
-  tags:
-    - deploy
-    - shiny
-  variables:
-    DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
-    PYTHONUNBUFFERED: 1
-  script:
-    - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
-    - export GIT_VER=$(git describe --always)
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py
-
-# deploys docs to CI_DOCKER_REGISTRY webserver, for internal review
-deploy_docs_preview:
-  extends:
-    - .deploy_docs_template
-    - .rules:labels:build_docs-preview
-  # Override default stage to happen before the post_check
-  stage: test_deploy
-  needs:
-    - build_docs_html
-    - build_docs_pdf
-  variables:
-    TYPE: "preview"
-    # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
-    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY"
-    DOCS_DEPLOY_SERVER: "$DOCS_SERVER"
-    DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER"
-    DOCS_DEPLOY_PATH: "$DOCS_PATH"
-    DOCS_DEPLOY_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
-
-# deploy docs to production webserver
-deploy_docs_production:
-  # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
-  extends:
-    - .deploy_docs_template
-    - .rules:protected-no_label
-  stage: post_deploy
-  needs: # ensure runs after push_to_github succeeded
-    - build_docs_html
-    - build_docs_pdf
-    - push_to_github
-  variables:
-    TYPE: "preview"
-    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
-    DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER"
-    DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
-    DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
-    DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
-
 deploy_test_result:
   extends:
     - .deploy_job_template

tools/ci/config/docs.yml

@@ -0,0 +1,151 @@
+.build_docs_template:
+  image: $ESP_IDF_DOC_ENV_IMAGE
+  tags:
+    - build_docs
+  script:
+    - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -bs $DOC_BUILDERS -l $DOCLANG -t $DOCTGT build
+  parallel:
+    matrix:
+      - DOCLANG: ["en", "zh_CN"]
+        DOCTGT: ["esp32", "esp32s2", "esp32c3"]
+
+# stage: pre_check
+check_readme_links:
+  extends:
+    - .pre_check_job_template
+    - .rules:patterns:docs
+  tags: ["internet"]
+  allow_failure: true
+  variables:
+    PYTHON_VER: 3
+  script:
+    - python ${IDF_PATH}/tools/ci/check_readme_links.py
+
+check_docs_lang_sync:
+  extends:
+    - .pre_check_job_template
+    - .rules:patterns:docs
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+  script:
+    - cd docs
+    - ./check_lang_folder_sync.sh
+
+check_docs_gh_links:
+  extends:
+    - .build_docs_template
+    - .rules:patterns:docs
+  stage: pre_check
+  variables:
+    SUBMODULES_TO_FETCH: "none"
+  script:
+    - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py gh-linkcheck
+
+# stage: build
+.build_docs_build_stage_template:
+  extends:
+    - .build_docs_template
+    - .rules:patterns:docs
+  stage: build
+  needs:
+    - check_docs_lang_sync
+    - check_docs_gh_links
+
+build_docs_html:
+  extends:
+    - .build_docs_build_stage_template
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/html/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "html"
+
+build_docs_pdf:
+  extends:
+    - .build_docs_build_stage_template
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/latex/*
+    expire_in: 4 days
+  variables:
+    DOC_BUILDERS: "latex"
+
+.deploy_docs_template:
+  extends:
+    - .before_script_lesser
+    - .rules:patterns:docs
+  image: $ESP_IDF_DOC_ENV_IMAGE
+  tags:
+    - deploy
+    - shiny
+  variables:
+    DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
+    PYTHONUNBUFFERED: 1
+  script:
+    - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
+    - export GIT_VER=$(git describe --always)
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ${IDF_PATH}/tools/ci/deploy_docs.py
+
+# stage: test_deploy
+deploy_docs_preview:
+  extends:
+    - .deploy_docs_template
+    - .rules:patterns:docs-preview
+  stage: test_deploy
+  needs:
+    - build_docs_html
+    - build_docs_pdf
+  variables:
+    TYPE: "preview"
+    # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY"
+    DOCS_DEPLOY_SERVER: "$DOCS_SERVER"
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER"
+    DOCS_DEPLOY_PATH: "$DOCS_PATH"
+    DOCS_DEPLOY_URL_BASE: "https://$CI_DOCKER_REGISTRY/docs/esp-idf"
+
+# stage: post_deploy
+deploy_docs_production:
+  # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
+  extends:
+    - .deploy_docs_template
+    - .rules:protected-no_label
+  stage: post_deploy
+  needs: # ensure runs after push_to_github succeeded
+    - build_docs_html
+    - build_docs_pdf
+    - push_to_github
+  variables:
+    TYPE: "preview"
+    DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
+    DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER"
+    DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
+    DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
+    DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
+
+check_doc_links:
+  extends:
+    - .build_docs_template
+    - .rules:protected
+  stage: post_deploy
+  tags: ["build", "amd64", "internet"]
+  artifacts:
+    when: always
+    paths:
+      - docs/_build/*/*/*.txt
+      - docs/_build/*/*/linkcheck/*.txt
+    expire_in: 1 week
+  allow_failure: true
+  dependencies: []
+  script:
+    - cd docs
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -l $DOCLANG -t $DOCTGT linkcheck

tools/ci/config/post_deploy.yml

@@ -1,45 +0,0 @@
-.check_doc_links_template:
-  extends: .rules:protected-no_label
-  stage: post_deploy
-  image: $ESP_IDF_DOC_ENV_IMAGE
-  tags: [ "build", "amd64", "internet" ]
-  needs: # ensure runs after push_to_github succeeded
-    - build_docs_html
-    - build_docs_pdf
-    - push_to_github
-  artifacts:
-    when: always
-    paths:
-      - docs/_build/*/*/*.txt
-      - docs/_build/*/*/linkcheck/*.txt
-    expire_in: 1 week
-  allow_failure: true
-  dependencies: []
-  script:
-    - cd docs
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./build_docs.py -l $DOCLANG -t $DOCTGT linkcheck
-
-check_doc_links_en_esp32:
-  extends: .check_doc_links_template
-  variables:
-    DOCLANG: "en"
-    DOCTGT: "esp32"
-
-check_doc_links_en_esp32s2:
-  extends: .check_doc_links_template
-  variables:
-    DOCLANG: "en"
-    DOCTGT: "esp32s2"
-
-check_doc_links_zh_CN_esp32:
-  extends: .check_doc_links_template
-  variables:
-    DOCLANG: "zh_CN"
-    DOCTGT: "esp32"
-
-check_doc_links_zh_CN_esp32s2:
-  extends: .check_doc_links_template
-  variables:
-    DOCLANG: "zh_CN"
-    DOCTGT: "esp32s2"

tools/ci/config/pre_check.yml

@@ -37,14 +37,6 @@ check_pre_commit_MR:
   script:
     - python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_COMMIT_BRANCH} | xargs pre-commit run --files
 
-check_docs_lang_sync:
-  extends: .pre_check_job_template
-  variables:
-    SUBMODULES_TO_FETCH: "none"
-  script:
-    - cd docs
-    - ./check_lang_folder_sync.sh
-
 check_version:
   # Don't run this for feature/bugfix branches, so that it is possible to modify
   # esp_idf_version.h in a branch before tagging the next version.
@@ -145,15 +137,6 @@ scan_tests:
     - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR
     - python $CI_SCAN_TESTS_PY component_ut $COMPONENT_UT_DIRS --exclude $COMPONENT_UT_EXCLUDES -c $TEST_CONFIG_FILE -o $COMPONENT_UT_OUTPUT_DIR
 
-check_readme_links:
-  extends: .pre_check_job_template
-  tags: [ "amd64", "deploy", "internet" ]
-  allow_failure: true
-  variables:
-    PYTHON_VER: 3
-  script:
-    - python ${IDF_PATH}/tools/ci/check_readme_links.py
-
 .clang_tidy_check_template:
   extends: .pre_check_base_template
   image: ${CI_DOCKER_REGISTRY}/clang-static-analysis

tools/ci/config/rules.yml

@@ -1,3 +1,13 @@
+# manually maintained patterns
+.patterns-docs: &patterns-docs
+  # - "tools/ci/config/docs.yml" # FIXME: after debug
+  - "docs/**/*"
+  - "components/**/*.h"
+  - "components/**/Kconfig"
+  - "components/**/CMakeList.txt"
+  - "components/**/sdkconfig*"
+  - "CONTRIBUTING.rst"
+
 # if anchors
 .if-ref-master: &if-ref-master
   if: '$CI_COMMIT_REF_NAME == "master"'
@@ -128,21 +138,18 @@
     - <<: *if-label-regular_test
     - <<: *if-label-build
 
-.rules:labels:build_docs:
+.rules:patterns:docs:
   rules:
-    - <<: *if-protected-no_label
+    - <<: *if-protected
-    - <<: *if-label-build
-    - <<: *if-label-regular_test
     - <<: *if-label-build_docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
 
-.rules:labels:build_docs-slim:
+.rules:patterns:docs-preview:
-  rules:
-    - <<: *if-protected-no_label
-    - <<: *if-label-build_docs
-
-.rules:labels:build_docs-preview:
   rules:
     - <<: *if-label-build_docs
+    - <<: *if-dev-push
+      changes: *patterns-docs
 
 .rules:labels:weekend_test-only:
   rules: