CI: support customize test with @bot

.gitlab-ci.yml

@@ -26,8 +26,16 @@ variables:
 # IDF environment
 
   IDF_PATH: "$CI_PROJECT_DIR"
+  APPLY_BOT_FILTER_SCRIPT: "$CI_PROJECT_DIR/tools/ci/apply_bot_filter.py"
+  CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
+
+# before each job, we need to check if this job is filtered by bot stage/job filter
+.apply_bot_filter: &apply_bot_filter
+  python $APPLY_BOT_FILTER_SCRIPT || exit 0
 
 before_script:
+  # apply bot filter in before script
+  - *apply_bot_filter
   # add gitlab ssh key
   - mkdir -p ~/.ssh
   - chmod 700 ~/.ssh
@@ -46,11 +54,15 @@ before_script:
 
 .do_nothing_before:
   before_script: &do_nothing_before
+    # apply bot filter in before script
+    - *apply_bot_filter
     - echo "Not setting up GitLab key, not fetching submodules"
     - source tools/ci/configure_ci_environment.sh
 
 .add_gitlab_key_before:
   before_script: &add_gitlab_key_before
+    # apply bot filter in before script
+    - *apply_bot_filter
     - echo "Not fetching submodules"
     - source tools/ci/configure_ci_environment.sh
     # add gitlab ssh key
@@ -75,7 +87,7 @@ build_template_app:
     # Try to use the same branch name for esp-idf-template that we're
     # using on esp-idf. If it doesn't exist then just stick to the default
     # branch
-    - git checkout ${CI_COMMIT_REF_NAME} || echo "Using esp-idf-template default branch..."
+    - python $CHECKOUT_REF_SCRIPT esp-idf-template
     # Test debug build (default)
     - make all V=1
     # Now test release build
@@ -106,7 +118,7 @@ build_ssc:
   script:
     - git clone $SSC_REPOSITORY
     - cd SSC
-    - git checkout ${CI_COMMIT_REF_NAME} || echo "Using SSC default branch..."
+    - python $CHECKOUT_REF_SCRIPT SSC
     - MAKEFLAGS= ./gen_misc_ng.sh
 
 build_esp_idf_tests:
@@ -281,6 +293,7 @@ test_report:
     # clone test bench
     - git clone $GITLAB_SSH_SERVER/yinling/auto_test_script.git
     - cd auto_test_script
+    - python $CHECKOUT_REF_SCRIPT auto_test_script
     # generate report
     - TEST_RESULT=Pass
     - python CITestReport.py -l $LOG_PATH -t $TEST_CASE_FILE_PATH -p $REPORT_PATH -r $RESULT_PATH -a $ARTIFACTS_PATH -m $MODULE_UPDATE_FILE || TEST_RESULT=Fail
@@ -438,7 +451,7 @@ assign_test:
     # clone test script to assign tests
     - git clone $TEST_SCRIPT_REPOSITORY
     - cd auto_test_script
-    - git checkout ${CI_COMMIT_REF_NAME} || echo "Using default branch..."
+    - python $CHECKOUT_REF_SCRIPT auto_test_script
     # assign unit test cases
     - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/unit_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/test_bins
     # assgin integration test cases
@@ -500,10 +513,12 @@ assign_test:
     - test -e $CONFIG_FILE || exit 0
     # clone local test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
+    - cd ci-test-runner-configs
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs
     # clone test bench
     - git clone $TEST_SCRIPT_REPOSITORY
     - cd auto_test_script
-    - git checkout ${CI_COMMIT_REF_NAME} || echo "Using default branch..."
+    - python $CHECKOUT_REF_SCRIPT auto_test_script
     # run test
     - python CIRunner.py -l "$LOG_PATH/$CI_JOB_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH -m $MODULE_UPDATE_FILE
 
@@ -533,6 +548,8 @@ nvs_compatible_test:
   script:
     # clone local test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
+    - cd ci-test-runner-configs
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs
     # clone test bench
     - git clone $TEST_SCRIPT_REPOSITORY
     - cd auto_test_script

tools/ci/apply_bot_filter.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+# internal use only
+# called by CI jobs to determine if it need to be executed
+
+import os
+import re
+import sys
+import json
+
+
+RE_FILTER_PATTERN = re.compile(r'^r"(.+)?"$')
+
+RE_TYPE = type(re.compile("", 0))
+
+
+def parse_filter(filter_name):
+    filter_raw = os.getenv(filter_name)
+    filters = []
+    if filter_raw:
+        filter_data = json.loads(filter_raw)
+        for _filter in filter_data:
+            match = RE_FILTER_PATTERN.search(_filter)
+            if match:
+                filters.append(re.compile(match.group(1)))
+            else:
+                filters.append(_filter)
+    return filters
+
+
+def process_filter(filter_name, ci_name):
+    execute = True
+    
+    # bot message is case insensitive (processed with lower case). so we also convert ci_name to lower case.
+    ci_name = ci_name.lower()
+
+    filter_list = parse_filter(filter_name)
+
+    for _filter in filter_list:
+        if isinstance(_filter, RE_TYPE):
+            match = _filter.search(ci_name) is not None
+        else:
+            match = _filter == ci_name
+
+        if match:
+            execute = True
+            break
+        else:
+            execute = False
+    return execute
+
+
+if __name__ == "__main__":
+    need_to_execute = process_filter("BOT_STAGE_FILTER", os.getenv("CI_JOB_STAGE")) \
+                      and process_filter("BOT_JOB_FILTER", os.getenv("CI_JOB_NAME"))
+    if need_to_execute:
+        sys.exit(0)
+    else:
+        print("Skip this job as it doesn't fit @bot's filter")
+        sys.exit(-1)

tools/ci/checkout_project_ref.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+# internal use only
+# called by CI jobs when it uses a project related to IDF
+
+import os
+import json
+import argparse
+import subprocess
+
+
+def checkout_branch(proj_name, customized_revision, default_ref_name):
+    try:
+        ref_to_use = customized_revision[proj_name.lower()]
+    except (KeyError, TypeError):
+        ref_to_use = default_ref_name
+
+    try:
+        subprocess.check_call(["git", "checkout", ref_to_use])
+        print("CI using ref {} for project {}".format(ref_to_use, proj_name))
+    except subprocess.CalledProcessError:
+        print("using default branch")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("project",
+                        help="the name of project")
+
+    args = parser.parse_args()
+    project_name = args.project
+
+    customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION")
+    if customized_project_revisions:
+        customized_project_revisions = json.loads(customized_project_revisions)
+    ci_ref_name = os.getenv("CI_COMMIT_REF_NAME")
+
+    checkout_branch(project_name, customized_project_revisions, ci_ref_name)