From d621d0e88ea75b338c104a3fc0b2c43989645a4a Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Wed, 27 Nov 2019 11:21:33 +0800 Subject: [PATCH 01/10] tiny-test-fw: move to tools/esp_python_packages: make `tiny_test_fw` as a package and move to root path of idf python packages --- .../python_packages/tiny_test_fw}/App.py | 0 .../python_packages/tiny_test_fw}/DUT.py | 0 .../python_packages/tiny_test_fw}/Env.py | 0 .../tiny_test_fw}/EnvConfig.py | 0 .../tiny_test_fw}/EnvConfigTemplate.yml | 0 .../python_packages/tiny_test_fw}/TinyFW.py | 0 .../tiny_test_fw}/Utility/CIAssignTest.py | 5 ++-- .../tiny_test_fw}/Utility/CaseConfig.py | 15 +++++------ .../tiny_test_fw}/Utility/GitlabCIJob.py | 0 .../tiny_test_fw}/Utility/SearchCases.py | 3 ++- .../tiny_test_fw/Utility}/TestCase.py | 0 .../tiny_test_fw}/Utility/__init__.py | 0 .../python_packages/tiny_test_fw/__init__.py} | 0 .../tiny_test_fw/bin}/Runner.py | 4 +-- .../tiny_test_fw/bin}/example.py | 0 .../tiny_test_fw}/docs/Makefile | 0 .../tiny_test_fw/docs/_static/.keep | 0 .../tiny_test_fw}/docs/conf.py | 0 .../tiny_test_fw}/docs/index.rst | 0 .../tiny_test_fw}/requirements.txt | 0 .../ttfw_idf}/CIAssignExampleTest.py | 26 +++++++++++-------- .../ttfw_idf}/CIAssignUnitTest.py | 12 ++------- .../python_packages/ttfw_idf}/IDFApp.py | 2 +- .../python_packages/ttfw_idf}/IDFDUT.py | 3 +-- .../python_packages/ttfw_idf}/__init__.py | 7 +++-- tools/ci/setup_python.sh | 3 +++ .../tiny_test_fw}/Utility/Attenuator.py | 0 .../tiny_test_fw}/Utility/LineChart.py | 0 .../tiny_test_fw}/Utility/PowerControl.py | 0 29 files changed, 37 insertions(+), 43 deletions(-) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/App.py (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/DUT.py (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/Env.py (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/EnvConfig.py (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/EnvConfigTemplate.yml (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/TinyFW.py (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/Utility/CIAssignTest.py (99%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/Utility/CaseConfig.py (97%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/Utility/GitlabCIJob.py (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/Utility/SearchCases.py (99%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw/Utility}/TestCase.py (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/Utility/__init__.py (100%) rename tools/{tiny-test-fw/docs/_static/.keep => ci/python_packages/tiny_test_fw/__init__.py} (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw/bin}/Runner.py (96%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw/bin}/example.py (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/docs/Makefile (100%) create mode 100644 tools/ci/python_packages/tiny_test_fw/docs/_static/.keep rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/docs/conf.py (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/docs/index.rst (100%) rename tools/{tiny-test-fw => ci/python_packages/tiny_test_fw}/requirements.txt (100%) rename tools/{tiny-test-fw => ci/python_packages/ttfw_idf}/CIAssignExampleTest.py (79%) rename tools/{tiny-test-fw => ci/python_packages/ttfw_idf}/CIAssignUnitTest.py (96%) rename tools/{tiny-test-fw/IDF => ci/python_packages/ttfw_idf}/IDFApp.py (99%) rename tools/{tiny-test-fw/IDF => ci/python_packages/ttfw_idf}/IDFDUT.py (99%) rename tools/{tiny-test-fw/IDF => ci/python_packages/ttfw_idf}/__init__.py (97%) rename tools/{tiny-test-fw => esp_python_packages/tiny_test_fw}/Utility/Attenuator.py (100%) rename tools/{tiny-test-fw => esp_python_packages/tiny_test_fw}/Utility/LineChart.py (100%) rename tools/{tiny-test-fw => esp_python_packages/tiny_test_fw}/Utility/PowerControl.py (100%) diff --git a/tools/tiny-test-fw/App.py b/tools/ci/python_packages/tiny_test_fw/App.py similarity index 100% rename from tools/tiny-test-fw/App.py rename to tools/ci/python_packages/tiny_test_fw/App.py diff --git a/tools/tiny-test-fw/DUT.py b/tools/ci/python_packages/tiny_test_fw/DUT.py similarity index 100% rename from tools/tiny-test-fw/DUT.py rename to tools/ci/python_packages/tiny_test_fw/DUT.py diff --git a/tools/tiny-test-fw/Env.py b/tools/ci/python_packages/tiny_test_fw/Env.py similarity index 100% rename from tools/tiny-test-fw/Env.py rename to tools/ci/python_packages/tiny_test_fw/Env.py diff --git a/tools/tiny-test-fw/EnvConfig.py b/tools/ci/python_packages/tiny_test_fw/EnvConfig.py similarity index 100% rename from tools/tiny-test-fw/EnvConfig.py rename to tools/ci/python_packages/tiny_test_fw/EnvConfig.py diff --git a/tools/tiny-test-fw/EnvConfigTemplate.yml b/tools/ci/python_packages/tiny_test_fw/EnvConfigTemplate.yml similarity index 100% rename from tools/tiny-test-fw/EnvConfigTemplate.yml rename to tools/ci/python_packages/tiny_test_fw/EnvConfigTemplate.yml diff --git a/tools/tiny-test-fw/TinyFW.py b/tools/ci/python_packages/tiny_test_fw/TinyFW.py similarity index 100% rename from tools/tiny-test-fw/TinyFW.py rename to tools/ci/python_packages/tiny_test_fw/TinyFW.py diff --git a/tools/tiny-test-fw/Utility/CIAssignTest.py b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py similarity index 99% rename from tools/tiny-test-fw/Utility/CIAssignTest.py rename to tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py index ddc0ac1a6b..056d75f1c9 100644 --- a/tools/tiny-test-fw/Utility/CIAssignTest.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py @@ -44,14 +44,13 @@ import re import json import yaml - -from Utility import (CaseConfig, SearchCases, GitlabCIJob, console_log) - try: from yaml import CLoader as Loader except ImportError: from yaml import Loader as Loader +from . import (CaseConfig, SearchCases, GitlabCIJob, console_log) + class Group(object): diff --git a/tools/tiny-test-fw/Utility/CaseConfig.py b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py similarity index 97% rename from tools/tiny-test-fw/Utility/CaseConfig.py rename to tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py index cf5d42f19c..8f373bce9f 100644 --- a/tools/tiny-test-fw/Utility/CaseConfig.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/CaseConfig.py @@ -20,7 +20,7 @@ Template Config File:: TestConfig: app: - path: Users/Test/TinyTestFW/IDF/IDFApp.py + package: ttfw_idf class: Example dut: path: @@ -38,22 +38,20 @@ Template Config File:: extra_data: some extra data passed to case with kwarg extra_data overwrite: # overwrite test configs app: - path: Users/Test/TinyTestFW/IDF/IDFApp.py + package: ttfw_idf class: Example - name: xxx """ +import importlib import yaml - -import TestCase - -from Utility import load_source - try: from yaml import CLoader as Loader except ImportError: from yaml import Loader as Loader +from . import TestCase + def _convert_to_lower_case_bytes(item): """ @@ -172,8 +170,7 @@ class Parser(object): """ output = dict() for key in overwrite: - path = overwrite[key]["path"] - module = load_source(path) + module = importlib.import_module(overwrite[key]["package"]) output[key] = module.__getattribute__(overwrite[key]["class"]) return output diff --git a/tools/tiny-test-fw/Utility/GitlabCIJob.py b/tools/ci/python_packages/tiny_test_fw/Utility/GitlabCIJob.py similarity index 100% rename from tools/tiny-test-fw/Utility/GitlabCIJob.py rename to tools/ci/python_packages/tiny_test_fw/Utility/GitlabCIJob.py diff --git a/tools/tiny-test-fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py similarity index 99% rename from tools/tiny-test-fw/Utility/SearchCases.py rename to tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py index 3ce98a07e1..b5e0762839 100644 --- a/tools/tiny-test-fw/Utility/SearchCases.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py @@ -17,7 +17,8 @@ import os import fnmatch import types import copy -from Utility import load_source + +from . import load_source class Search(object): diff --git a/tools/tiny-test-fw/TestCase.py b/tools/ci/python_packages/tiny_test_fw/Utility/TestCase.py similarity index 100% rename from tools/tiny-test-fw/TestCase.py rename to tools/ci/python_packages/tiny_test_fw/Utility/TestCase.py diff --git a/tools/tiny-test-fw/Utility/__init__.py b/tools/ci/python_packages/tiny_test_fw/Utility/__init__.py similarity index 100% rename from tools/tiny-test-fw/Utility/__init__.py rename to tools/ci/python_packages/tiny_test_fw/Utility/__init__.py diff --git a/tools/tiny-test-fw/docs/_static/.keep b/tools/ci/python_packages/tiny_test_fw/__init__.py similarity index 100% rename from tools/tiny-test-fw/docs/_static/.keep rename to tools/ci/python_packages/tiny_test_fw/__init__.py diff --git a/tools/tiny-test-fw/Runner.py b/tools/ci/python_packages/tiny_test_fw/bin/Runner.py similarity index 96% rename from tools/tiny-test-fw/Runner.py rename to tools/ci/python_packages/tiny_test_fw/bin/Runner.py index ea124c1489..a233de64f5 100644 --- a/tools/tiny-test-fw/Runner.py +++ b/tools/ci/python_packages/tiny_test_fw/bin/Runner.py @@ -26,8 +26,8 @@ import sys import argparse import threading -import TinyFW -from Utility import SearchCases, CaseConfig +from tiny_test_fw import TinyFW +from tiny_test_fw.Utility import SearchCases, CaseConfig class Runner(threading.Thread): diff --git a/tools/tiny-test-fw/example.py b/tools/ci/python_packages/tiny_test_fw/bin/example.py similarity index 100% rename from tools/tiny-test-fw/example.py rename to tools/ci/python_packages/tiny_test_fw/bin/example.py diff --git a/tools/tiny-test-fw/docs/Makefile b/tools/ci/python_packages/tiny_test_fw/docs/Makefile similarity index 100% rename from tools/tiny-test-fw/docs/Makefile rename to tools/ci/python_packages/tiny_test_fw/docs/Makefile diff --git a/tools/ci/python_packages/tiny_test_fw/docs/_static/.keep b/tools/ci/python_packages/tiny_test_fw/docs/_static/.keep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/tiny-test-fw/docs/conf.py b/tools/ci/python_packages/tiny_test_fw/docs/conf.py similarity index 100% rename from tools/tiny-test-fw/docs/conf.py rename to tools/ci/python_packages/tiny_test_fw/docs/conf.py diff --git a/tools/tiny-test-fw/docs/index.rst b/tools/ci/python_packages/tiny_test_fw/docs/index.rst similarity index 100% rename from tools/tiny-test-fw/docs/index.rst rename to tools/ci/python_packages/tiny_test_fw/docs/index.rst diff --git a/tools/tiny-test-fw/requirements.txt b/tools/ci/python_packages/tiny_test_fw/requirements.txt similarity index 100% rename from tools/tiny-test-fw/requirements.txt rename to tools/ci/python_packages/tiny_test_fw/requirements.txt diff --git a/tools/tiny-test-fw/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py similarity index 79% rename from tools/tiny-test-fw/CIAssignExampleTest.py rename to tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py index bd6bdaf3b8..85beb1f086 100644 --- a/tools/tiny-test-fw/CIAssignExampleTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py @@ -22,25 +22,29 @@ import sys import re import argparse -try: - from Utility.CIAssignTest import AssignTest -except ImportError: - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path: - sys.path.insert(0, test_fw_path) - from Utility.CIAssignTest import AssignTest - -from Utility.CIAssignTest import Group +import gitlab_api +from tiny_test_fw.Utility import CIAssignTest -class ExampleGroup(Group): +class ExampleGroup(CIAssignTest.Group): SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"] -class CIExampleAssignTest(AssignTest): +class CIExampleAssignTest(CIAssignTest.AssignTest): CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+") +class ArtifactFile(object): + def __init__(self, project_id, job_name, artifact_file_path): + self.gitlab_api = gitlab_api.Gitlab(project_id) + + def process(self): + pass + + def output(self): + pass + + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("test_case", diff --git a/tools/tiny-test-fw/CIAssignUnitTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py similarity index 96% rename from tools/tiny-test-fw/CIAssignUnitTest.py rename to tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py index e3c96b4f98..ef84103bdb 100644 --- a/tools/tiny-test-fw/CIAssignUnitTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py @@ -3,8 +3,6 @@ Command line tool to assign unit tests to CI test jobs. """ import re -import os -import sys import argparse import yaml @@ -14,13 +12,7 @@ try: except ImportError: from yaml import Loader as Loader -try: - from Utility import CIAssignTest -except ImportError: - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path: - sys.path.insert(0, test_fw_path) - from Utility import CIAssignTest +from tiny_test_fw.Utility import CIAssignTest class Group(CIAssignTest.Group): @@ -119,7 +111,7 @@ class Group(CIAssignTest.Group): if target: overwrite = { "dut": { - "path": "IDF/IDFDUT.py", + "package": "ttfw_idf", "class": self.DUT_CLS_NAME[target], } } diff --git a/tools/tiny-test-fw/IDF/IDFApp.py b/tools/ci/python_packages/ttfw_idf/IDFApp.py similarity index 99% rename from tools/tiny-test-fw/IDF/IDFApp.py rename to tools/ci/python_packages/ttfw_idf/IDFApp.py index 0018f7bef8..eff8c14584 100644 --- a/tools/tiny-test-fw/IDF/IDFApp.py +++ b/tools/ci/python_packages/ttfw_idf/IDFApp.py @@ -17,7 +17,7 @@ import subprocess import os import json -import App +from tiny_test_fw import App class IDFApp(App.BaseApp): diff --git a/tools/tiny-test-fw/IDF/IDFDUT.py b/tools/ci/python_packages/ttfw_idf/IDFDUT.py similarity index 99% rename from tools/tiny-test-fw/IDF/IDFDUT.py rename to tools/ci/python_packages/ttfw_idf/IDFDUT.py index 13838153f9..874d89273f 100644 --- a/tools/tiny-test-fw/IDF/IDFDUT.py +++ b/tools/ci/python_packages/ttfw_idf/IDFDUT.py @@ -30,8 +30,7 @@ except ImportError: from serial.tools import list_ports -import DUT -import Utility +from tiny_test_fw import DUT, Utility try: import esptool diff --git a/tools/tiny-test-fw/IDF/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py similarity index 97% rename from tools/tiny-test-fw/IDF/__init__.py rename to tools/ci/python_packages/ttfw_idf/__init__.py index 56db1b2872..d7aed38571 100644 --- a/tools/tiny-test-fw/IDF/__init__.py +++ b/tools/ci/python_packages/ttfw_idf/__init__.py @@ -14,10 +14,9 @@ import os import re -import TinyFW -import Utility -from IDF.IDFApp import IDFApp, Example, UT -from IDF.IDFDUT import IDFDUT +from tiny_test_fw import TinyFW, Utility +from IDFApp import IDFApp, Example, UT +from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT def format_case_id(chip, case_name): diff --git a/tools/ci/setup_python.sh b/tools/ci/setup_python.sh index c134ba816b..a0e1723be0 100644 --- a/tools/ci/setup_python.sh +++ b/tools/ci/setup_python.sh @@ -51,3 +51,6 @@ else echo 'No /opt/pyenv/activate exists and no Python interpreter is found!' exit 1 fi + +# add esp-idf local package path to PYTHONPATH so it can be imported directly +export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" diff --git a/tools/tiny-test-fw/Utility/Attenuator.py b/tools/esp_python_packages/tiny_test_fw/Utility/Attenuator.py similarity index 100% rename from tools/tiny-test-fw/Utility/Attenuator.py rename to tools/esp_python_packages/tiny_test_fw/Utility/Attenuator.py diff --git a/tools/tiny-test-fw/Utility/LineChart.py b/tools/esp_python_packages/tiny_test_fw/Utility/LineChart.py similarity index 100% rename from tools/tiny-test-fw/Utility/LineChart.py rename to tools/esp_python_packages/tiny_test_fw/Utility/LineChart.py diff --git a/tools/tiny-test-fw/Utility/PowerControl.py b/tools/esp_python_packages/tiny_test_fw/Utility/PowerControl.py similarity index 100% rename from tools/tiny-test-fw/Utility/PowerControl.py rename to tools/esp_python_packages/tiny_test_fw/Utility/PowerControl.py From 4d45932c5e3250141bab07481b721220925aec70 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Wed, 27 Nov 2019 11:22:14 +0800 Subject: [PATCH 02/10] CI: add utility `gitlab_api` --- tools/ci/python_packages/gitlab_api.py | 174 +++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 tools/ci/python_packages/gitlab_api.py diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py new file mode 100644 index 0000000000..d4fc4600fd --- /dev/null +++ b/tools/ci/python_packages/gitlab_api.py @@ -0,0 +1,174 @@ +import os +import re +import argparse +import tempfile +import tarfile +import zipfile + +import gitlab + + +class Gitlab(object): + JOB_NAME_PATTERN = re.compile(r"(\w+)(\s+(\d+)/(\d+))?") + + def __init__(self, project_id=None): + config_data_from_env = os.getenv("PYTHON_GITLAB_CONFIG") + if config_data_from_env: + # prefer to load config from env variable + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_file.write(config_data_from_env) + config_files = [temp_file.name] + else: + # otherwise try to use config file at local filesystem + config_files = None + self.gitlab_inst = gitlab.Gitlab.from_config(config_files=config_files) + self.gitlab_inst.auth() + if project_id: + self.project = self.gitlab_inst.projects.get(project_id) + else: + self.project = None + + def get_project_id(self, name, namespace=None): + """ + search project ID by name + + :param name: project name + :param namespace: namespace to match when we have multiple project with same name + :return: project ID + """ + projects = self.gitlab_inst.projects.list(search=name) + for project in projects: + if namespace is None: + if len(projects) == 1: + project_id = project.id + break + if project.namespace["path"] == namespace: + project_id = project.id + break + else: + raise ValueError("Can't find project") + return project_id + + def download_artifacts(self, job_id, destination): + """ + download full job artifacts and extract to destination. + + :param job_id: Gitlab CI job ID + :param destination: extract artifacts to path. + """ + job = self.project.jobs.get(job_id) + + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + job.artifacts(streamed=True, action=temp_file.write) + + with zipfile.ZipFile(temp_file.name, "r") as archive_file: + archive_file.extractall(destination) + + def download_artifact(self, job_id, artifact_path, destination=None): + """ + download specific path of job artifacts and extract to destination. + + :param job_id: Gitlab CI job ID + :param artifact_path: list of path in artifacts (relative path to artifact root path) + :param destination: destination of artifact. Do not save to file if destination is None + :return: A list of artifact file raw data. + """ + job = self.project.jobs.get(job_id) + + raw_data_list = [] + + for a_path in artifact_path: + try: + data = job.artifact(a_path) + except gitlab.GitlabGetError as e: + print("Failed to download '{}' form job {}".format(a_path, job_id)) + raise e + raw_data_list.append(data) + if destination: + file_path = os.path.join(destination, a_path) + try: + os.makedirs(os.path.dirname(file_path)) + except OSError: + # already exists + pass + with open(file_path, "wb") as f: + f.write(data) + + return raw_data_list + + def find_job_id(self, job_name, pipeline_id=None): + """ + Get Job ID from job name of specific pipeline + + :param job_name: job name + :param pipeline_id: If None, will get pipeline id from CI pre-defined variable. + :return: a list of job IDs (parallel job will generate multiple jobs) + """ + job_id_list = [] + if pipeline_id is None: + pipeline_id = os.getenv("CI_PIPELINE_ID") + pipeline = self.project.pipelines.get(pipeline_id) + jobs = pipeline.jobs.list(all=True) + for job in jobs: + match = self.JOB_NAME_PATTERN.match(job.name) + if match: + if match.group(1) == job_name: + job_id_list.append({"id": job.id, "parallel_num": match.group(3)}) + return job_id_list + + def download_archive(self, ref, destination, project_id=None): + """ + Download archive of certain commit of a repository and extract to destination path + + :param ref: commit or branch name + :param destination: destination path of extracted archive file + :param project_id: download project of current instance if project_id is None + :return: root path name of archive file + """ + if project_id is None: + project = self.project + else: + project = self.gitlab_inst.projects.get(project_id) + + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + try: + project.repository_archive(sha=ref, streamed=True, action=temp_file.write) + except gitlab.GitlabGetError as e: + print("Failed to archive from project {}".format(project_id)) + raise e + + print("archive size: {:.03f}MB".format(float(os.path.getsize(temp_file.name)) / (1024 * 1024))) + + with tarfile.open(temp_file.name, "r") as archive_file: + root_name = archive_file.getnames()[0] + archive_file.extractall(destination) + + return os.path.join(os.path.realpath(destination), root_name) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("action") + parser.add_argument("project_id", type=int) + parser.add_argument("--pipeline_id", "-i", type=int, default=None) + parser.add_argument("--ref", "-r", default="master") + parser.add_argument("--job_id", "-j", type=int, default=None) + parser.add_argument("--job_name", "-n", default=None) + parser.add_argument("--project_name", "-m", default=None) + parser.add_argument("--destination", "-d", default=None) + parser.add_argument("--artifact_path", "-a", nargs="*", default=None) + args = parser.parse_args() + + gitlab_inst = Gitlab(args.project_id) + if args.action == "download_artifacts": + gitlab_inst.download_artifacts(args.job_id, args.destination) + if args.action == "download_artifact": + gitlab_inst.download_artifact(args.job_id, args.artifact_path, args.destination) + elif args.action == "find_job_id": + job_ids = gitlab_inst.find_job_id(args.job_name, args.pipeline_id) + print(";".join([",".join([str(j["id"]), j["parallel_num"]]) for j in job_ids])) + elif args.action == "download_archive": + gitlab_inst.download_archive(args.ref, args.destination) + elif args.action == "get_project_id": + ret = gitlab_inst.get_project_id(args.project_name) + print("project id: {}".format(ret)) From c906e2afee9455bf55b7b163815dce69de766879 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Wed, 27 Nov 2019 11:58:07 +0800 Subject: [PATCH 03/10] test: update example and unit tests with new import roles: tiny_test_fw is a python package now. import it using normal way. --- .../bluetooth/nimble/blecent/blecent_test.py | 35 +- examples/bluetooth/nimble/blehr/blehr_test.py | 37 +- .../bluetooth/nimble/bleprph/bleprph_test.py | 35 +- examples/get-started/blink/example_test.py | 23 +- .../hello_world/loadable_elf_example_test.py | 19 +- .../can_alert_and_recovery/example_test.py | 18 +- .../can/can_network/example_test.py | 25 +- .../can/can_self_test/example_test.py | 17 +- .../peripherals/i2c/i2c_tools/example_test.py | 17 +- .../rmt/ir_protocols/example_test.py | 12 +- examples/peripherals/sdio/sdio_test.py | 26 +- .../asio/chat_client/asio_chat_client_test.py | 22 +- .../asio/chat_server/asio_chat_server_test.py | 23 +- .../tcp_echo_server/asio_tcp_server_test.py | 24 +- .../udp_echo_server/asio_udp_server_test.py | 24 +- .../esp_http_client/esp_http_client_test.py | 23 +- .../http_server_advanced_test.py | 31 +- .../http_server_persistence_test.py | 30 +- .../simple/http_server_simple_test.py | 28 +- .../protocols/https_request/example_test.py | 22 +- examples/protocols/mdns/mdns_example_test.py | 27 +- .../protocols/modbus/serial/example_test.py | 17 +- .../mqtt/ssl/mqtt_ssl_example_test.py | 27 +- .../mqtt/tcp/mqtt_tcp_example_test.py | 25 +- .../protocols/mqtt/ws/mqtt_ws_example_test.py | 25 +- .../mqtt/wss/mqtt_wss_example_test.py | 24 +- examples/protocols/websocket/example_test.py | 19 +- .../provisioning/ble_prov/ble_prov_test.py | 27 +- .../manager/wifi_prov_mgr_test.py | 27 +- .../softap_prov/softap_prov_test.py | 36 +- .../security/flash_encryption/example_test.py | 16 +- .../storage/ext_flash_fatfs/example_test.py | 16 +- examples/storage/parttool/example_test.py | 13 +- examples/storage/spiffsgen/example_test.py | 14 +- examples/system/console/example_test.py | 12 +- .../system/cpp_exceptions/example_test.py | 15 +- examples/system/cpp_rtti/example_test.py | 18 +- .../default_event_loop/example_test.py | 16 +- .../user_event_loops/example_test.py | 18 +- examples/system/esp_timer/example_test.py | 19 +- .../freertos/real_time_stats/example_test.py | 19 +- examples/system/light_sleep/example_test.py | 15 +- examples/system/ota/otatool/example_test.py | 17 +- .../ota/simple_ota_example/example_test.py | 25 +- examples/wifi/iperf/iperf_test.py | 62 +- tools/ble/__init__.py | 0 tools/ci/config/assign-test.yml | 7 +- tools/ci/config/target-test.yml | 10 +- .../idf_http_server_test/__init__.py | 0 .../idf_http_server_test}/adder.py | 3 +- .../idf_http_server_test}/client.py | 13 +- .../idf_http_server_test/test.py | 1025 +++++++++++++++++ .../idf_iperf_test_util}/Attenuator.py | 0 .../idf_iperf_test_util}/LineChart.py | 0 .../idf_iperf_test_util}/PowerControl.py | 0 .../idf_iperf_test_util/TestReport.py | 0 .../idf_iperf_test_util/__init__.py | 0 .../tiny_test_fw/requirements.txt | 1 - .../ci/python_packages}/wifi_tools.py | 0 tools/esp_prov/__init__.py | 1 + tools/unit-test-app/unit_test.py | 47 +- 61 files changed, 1283 insertions(+), 864 deletions(-) create mode 100644 tools/ble/__init__.py create mode 100644 tools/ci/python_packages/idf_http_server_test/__init__.py rename {examples/protocols/http_server/persistent_sockets/scripts => tools/ci/python_packages/idf_http_server_test}/adder.py (99%) rename {examples/protocols/http_server/simple/scripts => tools/ci/python_packages/idf_http_server_test}/client.py (96%) create mode 100644 tools/ci/python_packages/idf_http_server_test/test.py rename tools/{esp_python_packages/tiny_test_fw/Utility => ci/python_packages/idf_iperf_test_util}/Attenuator.py (100%) rename tools/{esp_python_packages/tiny_test_fw/Utility => ci/python_packages/idf_iperf_test_util}/LineChart.py (100%) rename tools/{esp_python_packages/tiny_test_fw/Utility => ci/python_packages/idf_iperf_test_util}/PowerControl.py (100%) rename examples/wifi/iperf/test_report.py => tools/ci/python_packages/idf_iperf_test_util/TestReport.py (100%) create mode 100644 tools/ci/python_packages/idf_iperf_test_util/__init__.py rename {examples/provisioning/softap_prov/utils => tools/ci/python_packages}/wifi_tools.py (100%) create mode 100644 tools/esp_prov/__init__.py diff --git a/examples/bluetooth/nimble/blecent/blecent_test.py b/examples/bluetooth/nimble/blecent/blecent_test.py index eabe8df261..9a85afc06a 100644 --- a/examples/bluetooth/nimble/blecent/blecent_test.py +++ b/examples/bluetooth/nimble/blecent/blecent_test.py @@ -16,43 +16,20 @@ from __future__ import print_function import os -import sys import re import uuid import subprocess -try: - # This environment variable is expected on the host machine - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError as e: - print(e) - print("\nCheck your IDF_PATH\nOR") - print("Try `export TEST_FW_PATH=$IDF_PATH/tools/tiny-test-fw` for resolving the issue\nOR") - print("Try `pip install -r $IDF_PATH/tools/tiny-test-fw/requirements.txt` for resolving the issue") - import IDF - -try: - import lib_ble_client -except ImportError: - lib_ble_client_path = os.getenv("IDF_PATH") + "/tools/ble" - if lib_ble_client_path and lib_ble_client_path not in sys.path: - sys.path.insert(0, lib_ble_client_path) - import lib_ble_client - - -import Utility +from tiny_test_fw import Utility +import ttfw_idf +from ble import lib_ble_client # When running on local machine execute the following before running this script # > make app bootloader # > make print_flash_cmd | tail -n 1 > build/download.config -# > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw -@IDF.idf_example_test(env_tag="Example_WIFI_BT") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT") def test_example_app_ble_central(env, extra_data): """ Steps: @@ -68,12 +45,12 @@ def test_example_app_ble_central(env, extra_data): subprocess.check_output(['rm','-rf','/var/lib/bluetooth/*']) subprocess.check_output(['hciconfig','hci0','reset']) # Acquire DUT - dut = env.get_dut("blecent", "examples/bluetooth/nimble/blecent", dut_class=ESP32DUT) + dut = env.get_dut("blecent", "examples/bluetooth/nimble/blecent", dut_class=ttfw_idf.ESP32DUT) # Get binary file binary_file = os.path.join(dut.app.binary_path, "blecent.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("blecent_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.log_performance("blecent_bin_size", "{}KB".format(bin_size // 1024)) # Upload binary and start testing Utility.console_log("Starting blecent example test app") diff --git a/examples/bluetooth/nimble/blehr/blehr_test.py b/examples/bluetooth/nimble/blehr/blehr_test.py index 432a160135..84de1eed5d 100644 --- a/examples/bluetooth/nimble/blehr/blehr_test.py +++ b/examples/bluetooth/nimble/blehr/blehr_test.py @@ -16,42 +16,19 @@ from __future__ import print_function import os -import sys import re import threading import traceback import Queue import subprocess -try: - # This environment variable is expected on the host machine - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError as e: - print(e) - print("\nCheck your IDF_PATH\nOR") - print("Try `export TEST_FW_PATH=$IDF_PATH/tools/tiny-test-fw` for resolving the issue\nOR") - print("Try `pip install -r $IDF_PATH/tools/tiny-test-fw/requirements.txt` for resolving the issue\n") - import IDF - -try: - import lib_ble_client -except ImportError: - lib_ble_client_path = os.getenv("IDF_PATH") + "/tools/ble" - if lib_ble_client_path and lib_ble_client_path not in sys.path: - sys.path.insert(0, lib_ble_client_path) - import lib_ble_client - - -import Utility +from tiny_test_fw import Utility +import ttfw_idf +from ble import lib_ble_client # When running on local machine execute the following before running this script # > make app bootloader # > make print_flash_cmd | tail -n 1 > build/download.config -# > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw def blehr_client_task(hr_obj, dut_addr): @@ -115,7 +92,7 @@ class BleHRThread(threading.Thread): self.exceptions_queue.put(traceback.format_exc(), block=False) -@IDF.idf_example_test(env_tag="Example_WIFI_BT") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT") def test_example_app_ble_hr(env, extra_data): """ Steps: @@ -129,13 +106,13 @@ def test_example_app_ble_hr(env, extra_data): subprocess.check_output(['hciconfig','hci0','reset']) # Acquire DUT - dut = env.get_dut("blehr", "examples/bluetooth/nimble/blehr", dut_class=ESP32DUT) + dut = env.get_dut("blehr", "examples/bluetooth/nimble/blehr", dut_class=ttfw_idf.ESP32DUT) # Get binary file binary_file = os.path.join(dut.app.binary_path, "blehr.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("blehr_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("blehr_bin_size", bin_size // 1024) + ttfw_idf.log_performance("blehr_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("blehr_bin_size", bin_size // 1024) # Upload binary and start testing Utility.console_log("Starting blehr simple example test app") diff --git a/examples/bluetooth/nimble/bleprph/bleprph_test.py b/examples/bluetooth/nimble/bleprph/bleprph_test.py index 70615c12bf..f557295716 100644 --- a/examples/bluetooth/nimble/bleprph/bleprph_test.py +++ b/examples/bluetooth/nimble/bleprph/bleprph_test.py @@ -16,36 +16,15 @@ from __future__ import print_function import os -import sys import re import Queue import traceback import threading import subprocess -try: - # This environment variable is expected on the host machine - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError as e: - print(e) - print("Try `export TEST_FW_PATH=$IDF_PATH/tools/tiny-test-fw` for resolving the issue") - print("Try `pip install -r $IDF_PATH/tools/tiny-test-fw/requirements.txt` for resolving the issue") - import IDF - -try: - import lib_ble_client -except ImportError: - lib_ble_client_path = os.getenv("IDF_PATH") + "/tools/ble" - if lib_ble_client_path and lib_ble_client_path not in sys.path: - sys.path.insert(0, lib_ble_client_path) - import lib_ble_client - - -import Utility +from tiny_test_fw import Utility +import ttfw_idf +from ble import lib_ble_client # When running on local machine execute the following before running this script # > make app bootloader @@ -136,7 +115,7 @@ class BlePrphThread(threading.Thread): self.exceptions_queue.put(traceback.format_exc(), block=False) -@IDF.idf_example_test(env_tag="Example_WIFI_BT") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT") def test_example_app_ble_peripheral(env, extra_data): """ Steps: @@ -150,13 +129,13 @@ def test_example_app_ble_peripheral(env, extra_data): subprocess.check_output(['hciconfig','hci0','reset']) # Acquire DUT - dut = env.get_dut("bleprph", "examples/bluetooth/nimble/bleprph", dut_class=ESP32DUT) + dut = env.get_dut("bleprph", "examples/bluetooth/nimble/bleprph", dut_class=ttfw_idf.ESP32DUT) # Get binary file binary_file = os.path.join(dut.app.binary_path, "bleprph.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("bleprph_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("bleprph_bin_size", bin_size // 1024) + ttfw_idf.log_performance("bleprph_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("bleprph_bin_size", bin_size // 1024) # Upload binary and start testing Utility.console_log("Starting bleprph simple example test app") diff --git a/examples/get-started/blink/example_test.py b/examples/get-started/blink/example_test.py index fbe274beab..10815cf081 100644 --- a/examples/get-started/blink/example_test.py +++ b/examples/get-started/blink/example_test.py @@ -5,21 +5,10 @@ from __future__ import print_function from __future__ import unicode_literals import re import os -import sys import hashlib -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # This environment variable is expected on the host machine - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - - import IDF - -import Utility +from tiny_test_fw import Utility +import ttfw_idf def verify_elf_sha256_embedding(dut): @@ -39,13 +28,13 @@ def verify_elf_sha256_embedding(dut): raise ValueError('ELF file SHA256 mismatch') -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_blink(env, extra_data): - dut = env.get_dut("blink", "examples/get-started/blink", dut_class=ESP32DUT) + dut = env.get_dut("blink", "examples/get-started/blink", dut_class=ttfw_idf.ESP32DUT) binary_file = os.path.join(dut.app.binary_path, "blink.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("blink_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("blink_bin_size", bin_size // 1024) + ttfw_idf.log_performance("blink_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("blink_bin_size", bin_size // 1024) dut.start_app() diff --git a/examples/get-started/hello_world/loadable_elf_example_test.py b/examples/get-started/hello_world/loadable_elf_example_test.py index c85dbd121b..3afaf270a2 100644 --- a/examples/get-started/hello_world/loadable_elf_example_test.py +++ b/examples/get-started/hello_world/loadable_elf_example_test.py @@ -1,19 +1,12 @@ import os -import pexpect -import serial -import sys import threading import time -try: - import IDF -except ImportError: - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import pexpect +import serial -import Utility +from tiny_test_fw import Utility +import ttfw_idf class CustomProcess(object): @@ -125,13 +118,13 @@ class SerialThread(object): Utility.console_log('The pyserial thread is still alive', 'O') -@IDF.idf_example_test(env_tag="test_jtag_arm") +@ttfw_idf.idf_example_test(env_tag="test_jtag_arm") def test_examples_loadable_elf(env, extra_data): idf_path = os.environ['IDF_PATH'] rel_project_path = os.path.join('examples', 'get-started', 'hello_world') proj_path = os.path.join(idf_path, rel_project_path) - example = IDF.Example(rel_project_path, target="esp32") + example = ttfw_idf.Example(rel_project_path, target="esp32") sdkconfig = example.get_sdkconfig() elf_path = os.path.join(example.get_binary_path(rel_project_path), 'hello-world.elf') esp_log_path = os.path.join(proj_path, 'esp.log') diff --git a/examples/peripherals/can/can_alert_and_recovery/example_test.py b/examples/peripherals/can/can_alert_and_recovery/example_test.py index 77110053d4..4c1a5d4611 100644 --- a/examples/peripherals/can/can_alert_and_recovery/example_test.py +++ b/examples/peripherals/can/can_alert_and_recovery/example_test.py @@ -1,28 +1,16 @@ # Need Python 3 string formatting functions from __future__ import print_function -import os -import sys - -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # The test cause is dependent on the Tiny Test Framework. Ensure the - # `TEST_FW_PATH` environment variable is set to `$IDF_PATH/tools/tiny-test-fw` - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf # CAN Self Test Example constants STR_EXPECT = ("CAN Alert and Recovery: Driver installed", "CAN Alert and Recovery: Driver uninstalled") EXPECT_TIMEOUT = 20 -@IDF.idf_example_test(env_tag='Example_CAN1') +@ttfw_idf.idf_example_test(env_tag='Example_CAN1') def test_can_alert_and_recovery_example(env, extra_data): - dut = env.get_dut('dut1', 'examples/peripherals/can/can_alert_and_recovery', dut_class=ESP32DUT) + dut = env.get_dut('dut1', 'examples/peripherals/can/can_alert_and_recovery', dut_class=ttfw_idf.ESP32DUT) dut.start_app() for string in STR_EXPECT: diff --git a/examples/peripherals/can/can_network/example_test.py b/examples/peripherals/can/can_network/example_test.py index 3183baa750..d59873aa73 100644 --- a/examples/peripherals/can/can_network/example_test.py +++ b/examples/peripherals/can/can_network/example_test.py @@ -1,19 +1,9 @@ # Need Python 3 string formatting functions from __future__ import print_function -import os -import sys from threading import Thread -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # The test cause is dependent on the Tiny Test Framework. Ensure the - # `TEST_FW_PATH` environment variable is set to `$IDF_PATH/tools/tiny-test-fw` - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF + +import ttfw_idf # Define tuple of strings to expect for each DUT. master_expect = ("CAN Master: Driver installed", "CAN Master: Driver uninstalled") @@ -37,13 +27,16 @@ def dut_thread_callback(**kwargs): result[0] = True -@IDF.idf_example_test(env_tag='Example_CAN2') +@ttfw_idf.idf_example_test(env_tag='Example_CAN2') def test_can_network_example(env, extra_data): # Get device under test. "dut1", "dut2", and "dut3" must be properly defined in EnvConfig - dut_master = env.get_dut("dut1", "examples/peripherals/can/can_network/can_network_master", dut_class=ESP32DUT) - dut_slave = env.get_dut("dut2", "examples/peripherals/can/can_network/can_network_slave", dut_class=ESP32DUT) - dut_listen_only = env.get_dut("dut3", "examples/peripherals/can/can_network/can_network_listen_only", dut_class=ESP32DUT) + dut_master = env.get_dut("dut1", "examples/peripherals/can/can_network/can_network_master", + dut_class=ttfw_idf.ESP32DUT) + dut_slave = env.get_dut("dut2", "examples/peripherals/can/can_network/can_network_slave", + dut_class=ttfw_idf.ESP32DUT) + dut_listen_only = env.get_dut("dut3", "examples/peripherals/can/can_network/can_network_listen_only", + dut_class=ttfw_idf.ESP32DUT) # Flash app onto each DUT, each DUT is reset again at the start of each thread dut_master.start_app() diff --git a/examples/peripherals/can/can_self_test/example_test.py b/examples/peripherals/can/can_self_test/example_test.py index 9a888643ca..a220f960f9 100644 --- a/examples/peripherals/can/can_self_test/example_test.py +++ b/examples/peripherals/can/can_self_test/example_test.py @@ -1,18 +1,7 @@ # Need Python 3 string formatting functions from __future__ import print_function -import os -import sys -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # The test cause is dependent on the Tiny Test Framework. Ensure the - # `TEST_FW_PATH` environment variable is set to `$IDF_PATH/tools/tiny-test-fw` - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf # CAN Self Test Example constants @@ -20,10 +9,10 @@ STR_EXPECT = ("CAN Self Test: Driver installed", "CAN Self Test: Driver uninstal EXPECT_TIMEOUT = 20 -@IDF.idf_example_test(env_tag='Example_CAN1') +@ttfw_idf.idf_example_test(env_tag='Example_CAN1') def test_can_self_test_example(env, extra_data): # Get device under test, flash and start example. "dut1" must be defined in EnvConfig - dut = env.get_dut('dut1', 'examples/peripherals/can/can_self_test', dut_class=ESP32DUT) + dut = env.get_dut('dut1', 'examples/peripherals/can/can_self_test', dut_class=ttfw_idf.ESP32DUT) dut.start_app() for string in STR_EXPECT: diff --git a/examples/peripherals/i2c/i2c_tools/example_test.py b/examples/peripherals/i2c/i2c_tools/example_test.py index 0f0c2271d5..3731c54d84 100644 --- a/examples/peripherals/i2c/i2c_tools/example_test.py +++ b/examples/peripherals/i2c/i2c_tools/example_test.py @@ -1,23 +1,14 @@ from __future__ import print_function -import os -import sys + +import ttfw_idf EXPECT_TIMEOUT = 20 -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -@IDF.idf_example_test(env_tag='Example_I2C_CCS811_SENSOR') +@ttfw_idf.idf_example_test(env_tag='Example_I2C_CCS811_SENSOR') def test_i2ctools_example(env, extra_data): # Get device under test, flash and start example. "i2ctool" must be defined in EnvConfig - dut = env.get_dut('i2ctools', 'examples/peripherals/i2c/i2c_tools', dut_class=ESP32DUT) + dut = env.get_dut('i2ctools', 'examples/peripherals/i2c/i2c_tools', dut_class=ttfw_idf.ESP32DUT) dut.start_app() dut.expect("esp32>", timeout=EXPECT_TIMEOUT) # Get i2c address diff --git a/examples/peripherals/rmt/ir_protocols/example_test.py b/examples/peripherals/rmt/ir_protocols/example_test.py index 9fd9fd8167..e61010e4f2 100644 --- a/examples/peripherals/rmt/ir_protocols/example_test.py +++ b/examples/peripherals/rmt/ir_protocols/example_test.py @@ -1,19 +1,11 @@ from __future__ import print_function -import os -import sys -try: - import IDF -except ImportError: - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf EXPECT_TIMEOUT = 20 -@IDF.idf_example_test(env_tag='Example_RMT_IR_PROTOCOLS') +@ttfw_idf.idf_example_test(env_tag='Example_RMT_IR_PROTOCOLS') def test_examples_rmt_ir_protocols(env, extra_data): dut = env.get_dut('ir_protocols_example', 'examples/peripherals/rmt/ir_protocols', app_config_name='nec') print("Using binary path: {}".format(dut.app.binary_path)) diff --git a/examples/peripherals/sdio/sdio_test.py b/examples/peripherals/sdio/sdio_test.py index 0fa8b2979f..4025d53946 100644 --- a/examples/peripherals/sdio/sdio_test.py +++ b/examples/peripherals/sdio/sdio_test.py @@ -12,25 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" example of writing test with TinyTestFW """ -import os -import sys - -try: - import TinyFW -except ImportError: - # if we want to run test case outside `tiny-test-fw` folder, - # we need to insert tiny-test-fw path into sys path - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import TinyFW - -import IDF -from IDF.IDFDUT import ESP32DUT +from tiny_test_fw import TinyFW +import ttfw_idf -@IDF.idf_example_test(env_tag="Example_SDIO", ignore=True) +@ttfw_idf.idf_example_test(env_tag="Example_SDIO", ignore=True) def test_example_sdio_communication(env, extra_data): """ Configurations @@ -50,8 +36,8 @@ def test_example_sdio_communication(env, extra_data): or use sdio test board, which has two wrover modules connect to a same FT3232 Assume that first dut is host and second is slave """ - dut1 = env.get_dut("sdio_host", "examples/peripherals/sdio/host", dut_class=ESP32DUT) - dut2 = env.get_dut("sdio_slave", "examples/peripherals/sdio/slave", dut_class=ESP32DUT) + dut1 = env.get_dut("sdio_host", "examples/peripherals/sdio/host", dut_class=ttfw_idf.ESP32DUT) + dut2 = env.get_dut("sdio_slave", "examples/peripherals/sdio/slave", dut_class=ttfw_idf.ESP32DUT) dut1.start_app() # wait until the master is ready to setup the slave dut1.expect("host ready, start initializing slave...") @@ -133,5 +119,5 @@ def test_example_sdio_communication(env, extra_data): if __name__ == '__main__': - TinyFW.set_default_config(env_config_file="EnvConfigTemplate.yml", dut=IDF.IDFDUT) + TinyFW.set_default_config(env_config_file="EnvConfigTemplate.yml", dut=ttfw_idf.IDFDUT) test_example_sdio_communication() diff --git a/examples/protocols/asio/chat_client/asio_chat_client_test.py b/examples/protocols/asio/chat_client/asio_chat_client_test.py index 4325806a36..d59b459c74 100644 --- a/examples/protocols/asio/chat_client/asio_chat_client_test.py +++ b/examples/protocols/asio/chat_client/asio_chat_client_test.py @@ -1,22 +1,10 @@ import re import os -import sys import socket from threading import Thread import time -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf global g_client_response global g_msg_to_client @@ -56,7 +44,7 @@ def chat_server_sketch(my_ip): print("server closed") -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_asio_chat_client(env, extra_data): """ steps: | @@ -70,12 +58,12 @@ def test_examples_protocol_asio_chat_client(env, extra_data): global g_client_response global g_msg_to_client test_msg = "ABC" - dut1 = env.get_dut("chat_client", "examples/protocols/asio/chat_client", dut_class=ESP32DUT) + dut1 = env.get_dut("chat_client", "examples/protocols/asio/chat_client", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "asio_chat_client.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("asio_chat_client_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("asio_chat_client_size", bin_size // 1024) + ttfw_idf.log_performance("asio_chat_client_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("asio_chat_client_size", bin_size // 1024) # 1. start a tcp server on the host host_ip = get_my_ip() thread1 = Thread(target=chat_server_sketch, args=(host_ip,)) diff --git a/examples/protocols/asio/chat_server/asio_chat_server_test.py b/examples/protocols/asio/chat_server/asio_chat_server_test.py index ccdd4a5570..2bfa026fc1 100644 --- a/examples/protocols/asio/chat_server/asio_chat_server_test.py +++ b/examples/protocols/asio/chat_server/asio_chat_server_test.py @@ -1,24 +1,11 @@ import re import os -import sys import socket - -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_asio_chat_server(env, extra_data): """ steps: | @@ -28,12 +15,12 @@ def test_examples_protocol_asio_chat_server(env, extra_data): 4. Test evaluates received test message from server """ test_msg = b" 4ABC\n" - dut1 = env.get_dut("chat_server", "examples/protocols/asio/chat_server", dut_class=ESP32DUT) + dut1 = env.get_dut("chat_server", "examples/protocols/asio/chat_server", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "asio_chat_server.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("asio_chat_server_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("asio_chat_server_size", bin_size // 1024) + ttfw_idf.log_performance("asio_chat_server_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("asio_chat_server_size", bin_size // 1024) # 1. start test dut1.start_app() # 2. get the server IP address diff --git a/examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py b/examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py index d5ee5cf077..b7dc16817c 100644 --- a/examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py +++ b/examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py @@ -1,25 +1,11 @@ import re import os -import sys import socket - -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_asio_tcp_server(env, extra_data): """ steps: | @@ -30,12 +16,12 @@ def test_examples_protocol_asio_tcp_server(env, extra_data): 5. Test evaluates received test message on server stdout """ test_msg = b"echo message from client to server" - dut1 = env.get_dut("tcp_echo_server", "examples/protocols/asio/tcp_echo_server", dut_class=ESP32DUT) + dut1 = env.get_dut("tcp_echo_server", "examples/protocols/asio/tcp_echo_server", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "asio_tcp_echo_server.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("asio_tcp_echo_server_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("asio_tcp_echo_server_size", bin_size // 1024) + ttfw_idf.log_performance("asio_tcp_echo_server_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("asio_tcp_echo_server_size", bin_size // 1024) # 1. start test dut1.start_app() # 2. get the server IP address diff --git a/examples/protocols/asio/udp_echo_server/asio_udp_server_test.py b/examples/protocols/asio/udp_echo_server/asio_udp_server_test.py index 726a39f4fa..3b6d0bff94 100644 --- a/examples/protocols/asio/udp_echo_server/asio_udp_server_test.py +++ b/examples/protocols/asio/udp_echo_server/asio_udp_server_test.py @@ -1,25 +1,11 @@ import re import os -import sys import socket - -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_asio_udp_server(env, extra_data): """ steps: | @@ -30,12 +16,12 @@ def test_examples_protocol_asio_udp_server(env, extra_data): 5. Test evaluates received test message on server stdout """ test_msg = b"echo message from client to server" - dut1 = env.get_dut("udp_echo_server", "examples/protocols/asio/udp_echo_server", dut_class=ESP32DUT) + dut1 = env.get_dut("udp_echo_server", "examples/protocols/asio/udp_echo_server", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "asio_udp_echo_server.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("asio_udp_echo_server_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("asio_udp_echo_server_size", bin_size // 1024) + ttfw_idf.log_performance("asio_udp_echo_server_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("asio_udp_echo_server_size", bin_size // 1024) # 1. start test dut1.start_app() # 2. get the server IP address diff --git a/examples/protocols/esp_http_client/esp_http_client_test.py b/examples/protocols/esp_http_client/esp_http_client_test.py index cdd0888c47..ce7072cdf1 100644 --- a/examples/protocols/esp_http_client/esp_http_client_test.py +++ b/examples/protocols/esp_http_client/esp_http_client_test.py @@ -1,35 +1,22 @@ import re import os -import sys -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag="Example_WIFI", ignore=True) +@ttfw_idf.idf_example_test(env_tag="Example_WIFI", ignore=True) def test_examples_protocol_esp_http_client(env, extra_data): """ steps: | 1. join AP 2. Send HTTP request to httpbin.org """ - dut1 = env.get_dut("esp_http_client", "examples/protocols/esp_http_client", dut_class=ESP32DUT) + dut1 = env.get_dut("esp_http_client", "examples/protocols/esp_http_client", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "esp-http-client-example.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("esp_http_client_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("esp_http_client_bin_size", bin_size // 1024) + ttfw_idf.log_performance("esp_http_client_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("esp_http_client_bin_size", bin_size // 1024) # start test dut1.start_app() dut1.expect("Connected to AP, begin http example", timeout=30) diff --git a/examples/protocols/http_server/advanced_tests/http_server_advanced_test.py b/examples/protocols/http_server/advanced_tests/http_server_advanced_test.py index 754eb5564e..c38bfbd537 100644 --- a/examples/protocols/http_server/advanced_tests/http_server_advanced_test.py +++ b/examples/protocols/http_server/advanced_tests/http_server_advanced_test.py @@ -19,31 +19,14 @@ from __future__ import print_function from __future__ import unicode_literals import re import os -import sys -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # This environment variable is expected on the host machine - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - - import IDF - -import Utility +from tiny_test_fw import Utility +import ttfw_idf +from idf_http_server_test import test as client # When running on local machine execute the following before running this script # > make app bootloader # > make print_flash_cmd | tail -n 1 > build/download.config -# > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw - -# Import client module -# TODO: replace with import -expath = os.path.dirname(os.path.realpath(__file__)) -client = Utility.load_source(expath + "/scripts/test.py") - # Due to connectivity issues (between runner host and DUT) in the runner environment, # some of the `advanced_tests` are ignored. These tests are intended for verifying @@ -51,16 +34,16 @@ client = Utility.load_source(expath + "/scripts/test.py") # of large HTTP packets and malformed requests, running multiple parallel sessions, etc. # It is advised that all these tests be run locally, when making changes or adding new # features to this component. -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_http_server_advanced(env, extra_data): # Acquire DUT - dut1 = env.get_dut("http_server", "examples/protocols/http_server/advanced_tests", dut_class=ESP32DUT) + dut1 = env.get_dut("http_server", "examples/protocols/http_server/advanced_tests", dut_class=ttfw_idf.ESP32DUT) # Get binary file binary_file = os.path.join(dut1.app.binary_path, "tests.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("http_server_bin_size", bin_size // 1024) + ttfw_idf.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024) # Upload binary and start testing Utility.console_log("Starting http_server advanced test app") diff --git a/examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py b/examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py index 71cf0db96d..1f635a13a3 100644 --- a/examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py +++ b/examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py @@ -21,42 +21,28 @@ from builtins import str from builtins import range import re import os -import sys import random -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # This environment variable is expected on the host machine - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -import Utility +from tiny_test_fw import Utility +import ttfw_idf +from idf_http_server_test import adder as client # When running on local machine execute the following before running this script # > make app bootloader # > make print_flash_cmd | tail -n 1 > build/download.config -# > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw - -# Import client module -# TODO: replace with import -expath = os.path.dirname(os.path.realpath(__file__)) -client = Utility.load_source(expath + "/scripts/adder.py") -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_http_server_persistence(env, extra_data): # Acquire DUT - dut1 = env.get_dut("http_server", "examples/protocols/http_server/persistent_sockets", dut_class=ESP32DUT) + dut1 = env.get_dut("http_server", "examples/protocols/http_server/persistent_sockets", + dut_class=ttfw_idf.ESP32DUT) # Get binary file binary_file = os.path.join(dut1.app.binary_path, "persistent_sockets.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("http_server_bin_size", bin_size // 1024) + ttfw_idf.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024) # Upload binary and start testing Utility.console_log("Starting http_server persistance test app") diff --git a/examples/protocols/http_server/simple/http_server_simple_test.py b/examples/protocols/http_server/simple/http_server_simple_test.py index bb209e2048..0e9242fc06 100644 --- a/examples/protocols/http_server/simple/http_server_simple_test.py +++ b/examples/protocols/http_server/simple/http_server_simple_test.py @@ -20,43 +20,29 @@ from __future__ import unicode_literals from builtins import range import re import os -import sys import string import random -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # This environment variable is expected on the host machine - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) +from tiny_test_fw import Utility +import ttfw_idf +from idf_http_server_test import client - import IDF - -import Utility # When running on local machine execute the following before running this script # > make app bootloader # > make print_flash_cmd | tail -n 1 > build/download.config -# > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw - -# Import client module -expath = os.path.dirname(os.path.realpath(__file__)) -client = Utility.load_source(expath + "/scripts/client.py") -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_http_server_simple(env, extra_data): # Acquire DUT - dut1 = env.get_dut("http_server", "examples/protocols/http_server/simple", dut_class=ESP32DUT) + dut1 = env.get_dut("http_server", "examples/protocols/http_server/simple", dut_class=ttfw_idf.ESP32DUT) # Get binary file binary_file = os.path.join(dut1.app.binary_path, "simple.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("http_server_bin_size", bin_size // 1024) + ttfw_idf.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024) # Upload binary and start testing Utility.console_log("Starting http_server simple test app") diff --git a/examples/protocols/https_request/example_test.py b/examples/protocols/https_request/example_test.py index 543edc3da2..c86213f89e 100644 --- a/examples/protocols/https_request/example_test.py +++ b/examples/protocols/https_request/example_test.py @@ -1,22 +1,10 @@ import re import os -import sys -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag="Example_WIFI", ignore=True) +@ttfw_idf.idf_example_test(env_tag="Example_WIFI", ignore=True) def test_examples_protocol_https_request(env, extra_data): """ steps: | @@ -24,12 +12,12 @@ def test_examples_protocol_https_request(env, extra_data): 2. connect to www.howsmyssl.com:443 3. send http request """ - dut1 = env.get_dut("https_request", "examples/protocols/https_request", dut_class=ESP32DUT) + dut1 = env.get_dut("https_request", "examples/protocols/https_request", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "https_request.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("https_request_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("https_request_bin_size", bin_size // 1024) + ttfw_idf.log_performance("https_request_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("https_request_bin_size", bin_size // 1024) # start test dut1.start_app() dut1.expect("Connection established...", timeout=30) diff --git a/examples/protocols/mdns/mdns_example_test.py b/examples/protocols/mdns/mdns_example_test.py index e86b84e42f..af897bb799 100644 --- a/examples/protocols/mdns/mdns_example_test.py +++ b/examples/protocols/mdns/mdns_example_test.py @@ -1,6 +1,5 @@ import re import os -import sys import socket import time import struct @@ -8,22 +7,8 @@ import dpkt import dpkt.dns from threading import Thread, Event - -# this is a test case write with tiny-test-fw. -# to run test cases outside tiny-test-fw, -# we need to set environment variable `TEST_FW_PATH`, -# then get and insert `TEST_FW_PATH` to sys path before import FW module - -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -import DUT +from tiny_test_fw import DUT +import ttfw_idf # g_run_server = True # g_done = False @@ -103,7 +88,7 @@ def mdns_server(esp_host): continue -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_mdns(env, extra_data): global stop_mdns_server """ @@ -113,12 +98,12 @@ def test_examples_protocol_mdns(env, extra_data): 3. check the mdns name is accessible 4. check DUT output if mdns advertized host is resolved """ - dut1 = env.get_dut("mdns-test", "examples/protocols/mdns", dut_class=ESP32DUT) + dut1 = env.get_dut("mdns-test", "examples/protocols/mdns", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "mdns-test.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("mdns-test_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("mdns-test_bin_size", bin_size // 1024) + ttfw_idf.log_performance("mdns-test_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("mdns-test_bin_size", bin_size // 1024) # 1. start mdns application dut1.start_app() # 2. get the dut host name (and IP address) diff --git a/examples/protocols/modbus/serial/example_test.py b/examples/protocols/modbus/serial/example_test.py index e1ffa9736f..0902e1cb8c 100644 --- a/examples/protocols/modbus/serial/example_test.py +++ b/examples/protocols/modbus/serial/example_test.py @@ -2,20 +2,11 @@ from __future__ import print_function import os -import sys import re import logging from threading import Thread -try: - import IDF -except ImportError: - # The test cause is dependent on the Tiny Test Framework. Ensure the - # `TEST_FW_PATH` environment variable is set to `$IDF_PATH/tools/tiny-test-fw` - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf LOG_LEVEL = logging.DEBUG LOGGER_NAME = "modbus_test" @@ -174,13 +165,13 @@ def test_check_mode(dut=None, mode_str=None, value=None): return False -@IDF.idf_example_test(env_tag='Example_T2_RS485') +@ttfw_idf.idf_example_test(env_tag='Example_T2_RS485') def test_modbus_communication(env, comm_mode): global logger # Get device under test. "dut1 - master", "dut2 - slave" must be properly connected through RS485 interface driver - dut_master = env.get_dut("modbus_master", "examples/protocols/modbus/serial/mb_master") - dut_slave = env.get_dut("modbus_slave", "examples/protocols/modbus/serial/mb_slave") + dut_master = env.get_dut("modbus_master", "examples/protocols/modbus/serial/mb_master", dut_class=ttfw_idf.ESP32DUT) + dut_slave = env.get_dut("modbus_slave", "examples/protocols/modbus/serial/mb_slave", dut_class=ttfw_idf.ESP32DUT) try: logger.debug("Environment vars: %s\r\n" % os.environ) diff --git a/examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py b/examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py index 43d4cec8bd..a392b2cae1 100644 --- a/examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py +++ b/examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py @@ -8,21 +8,8 @@ import ssl import paho.mqtt.client as mqtt from threading import Thread, Event - -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -import DUT +from tiny_test_fw import DUT +import ttfw_idf event_client_connected = Event() @@ -73,7 +60,7 @@ def on_message(client, userdata, msg): message_log += "Received data:" + msg.topic + " " + payload + "\n" -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_mqtt_ssl(env, extra_data): broker_url = "" broker_port = 0 @@ -85,13 +72,13 @@ def test_examples_protocol_mqtt_ssl(env, extra_data): 4. Test ESP32 client received correct qos0 message 5. Test python client receives binary data from running partition and compares it with the binary """ - dut1 = env.get_dut("mqtt_ssl", "examples/protocols/mqtt/ssl", dut_class=ESP32DUT) + dut1 = env.get_dut("mqtt_ssl", "examples/protocols/mqtt/ssl", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "mqtt_ssl.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("mqtt_ssl_bin_size", "{}KB" - .format(bin_size // 1024)) - IDF.check_performance("mqtt_ssl_size", bin_size // 1024) + ttfw_idf.log_performance("mqtt_ssl_bin_size", "{}KB" + .format(bin_size // 1024)) + ttfw_idf.check_performance("mqtt_ssl_size", bin_size // 1024) # Look for host:port in sdkconfig try: value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()["CONFIG_BROKER_URI"]) diff --git a/examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py b/examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py index 4e39705643..2d8dbe3419 100644 --- a/examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py +++ b/examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py @@ -6,21 +6,8 @@ from threading import Thread import struct import time - -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -import DUT +from tiny_test_fw import DUT +import ttfw_idf msgid = -1 @@ -66,7 +53,7 @@ def mqqt_server_sketch(my_ip, port): print("server closed") -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_mqtt_qos1(env, extra_data): global msgid """ @@ -76,12 +63,12 @@ def test_examples_protocol_mqtt_qos1(env, extra_data): 3. Test evaluates that qos1 message is queued and removed from queued after ACK received 4. Test the broker received the same message id evaluated in step 3 """ - dut1 = env.get_dut("mqtt_tcp", "examples/protocols/mqtt/tcp", dut_class=ESP32DUT) + dut1 = env.get_dut("mqtt_tcp", "examples/protocols/mqtt/tcp", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "mqtt_tcp.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("mqtt_tcp_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("mqtt_tcp_size", bin_size // 1024) + ttfw_idf.log_performance("mqtt_tcp_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("mqtt_tcp_size", bin_size // 1024) # 1. start mqtt broker sketch host_ip = get_my_ip() thread1 = Thread(target=mqqt_server_sketch, args=(host_ip,1883)) diff --git a/examples/protocols/mqtt/ws/mqtt_ws_example_test.py b/examples/protocols/mqtt/ws/mqtt_ws_example_test.py index 19ed395d65..cf40c9a3a6 100644 --- a/examples/protocols/mqtt/ws/mqtt_ws_example_test.py +++ b/examples/protocols/mqtt/ws/mqtt_ws_example_test.py @@ -7,21 +7,8 @@ import sys import paho.mqtt.client as mqtt from threading import Thread, Event - -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except Exception: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -import DUT +from tiny_test_fw import DUT +import ttfw_idf event_client_connected = Event() event_stop_client = Event() @@ -52,7 +39,7 @@ def on_message(client, userdata, msg): message_log += "Received data:" + msg.topic + " " + payload + "\n" -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_mqtt_ws(env, extra_data): broker_url = "" broker_port = 0 @@ -63,12 +50,12 @@ def test_examples_protocol_mqtt_ws(env, extra_data): 3. Test evaluates it received correct qos0 message 4. Test ESP32 client received correct qos0 message """ - dut1 = env.get_dut("mqtt_websocket", "examples/protocols/mqtt/ws", dut_class=ESP32DUT) + dut1 = env.get_dut("mqtt_websocket", "examples/protocols/mqtt/ws", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "mqtt_websocket.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("mqtt_websocket_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("mqtt_websocket_size", bin_size // 1024) + ttfw_idf.log_performance("mqtt_websocket_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("mqtt_websocket_size", bin_size // 1024) # Look for host:port in sdkconfig try: value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()["CONFIG_BROKER_URI"]) diff --git a/examples/protocols/mqtt/wss/mqtt_wss_example_test.py b/examples/protocols/mqtt/wss/mqtt_wss_example_test.py index 486d65d318..a4c4e6c023 100644 --- a/examples/protocols/mqtt/wss/mqtt_wss_example_test.py +++ b/examples/protocols/mqtt/wss/mqtt_wss_example_test.py @@ -8,21 +8,9 @@ import ssl import paho.mqtt.client as mqtt from threading import Thread, Event +from tiny_test_fw import DUT +import ttfw_idf -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -import DUT event_client_connected = Event() event_stop_client = Event() @@ -53,7 +41,7 @@ def on_message(client, userdata, msg): message_log += "Received data:" + msg.topic + " " + payload + "\n" -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_mqtt_wss(env, extra_data): broker_url = "" broker_port = 0 @@ -64,12 +52,12 @@ def test_examples_protocol_mqtt_wss(env, extra_data): 3. Test evaluates it received correct qos0 message 4. Test ESP32 client received correct qos0 message """ - dut1 = env.get_dut("mqtt_websocket_secure", "examples/protocols/mqtt/wss", dut_class=ESP32DUT) + dut1 = env.get_dut("mqtt_websocket_secure", "examples/protocols/mqtt/wss", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "mqtt_websocket_secure.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("mqtt_websocket_secure_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("mqtt_websocket_secure_size", bin_size // 1024) + ttfw_idf.log_performance("mqtt_websocket_secure_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("mqtt_websocket_secure_size", bin_size // 1024) # Look for host:port in sdkconfig try: value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()["CONFIG_BROKER_URI"]) diff --git a/examples/protocols/websocket/example_test.py b/examples/protocols/websocket/example_test.py index 4ca10b2043..8c21c6c03a 100644 --- a/examples/protocols/websocket/example_test.py +++ b/examples/protocols/websocket/example_test.py @@ -1,19 +1,10 @@ import re import os -import sys -import IDF -from IDF.IDFDUT import ESP32DUT -# this is a test case write with tiny-test-fw. -# to run test cases outside tiny-test-fw, -# we need to set environment variable `TEST_FW_PATH`, -# then get and insert `TEST_FW_PATH` to sys path before import FW module -test_fw_path = os.getenv("TEST_FW_PATH") -if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) +import ttfw_idf -@IDF.idf_example_test(env_tag="Example_WIFI", ignore=True) +@ttfw_idf.idf_example_test(env_tag="Example_WIFI", ignore=True) def test_examples_protocol_websocket(env, extra_data): """ steps: | @@ -21,12 +12,12 @@ def test_examples_protocol_websocket(env, extra_data): 2. connect to ws://echo.websocket.org 3. send and receive data """ - dut1 = env.get_dut("websocket", "examples/protocols/websocket", dut_class=ESP32DUT) + dut1 = env.get_dut("websocket", "examples/protocols/websocket", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "websocket-example.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("websocket_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("websocket_bin_size", bin_size // 1024) + ttfw_idf.log_performance("websocket_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("websocket_bin_size", bin_size // 1024) # start test dut1.start_app() dut1.expect("Waiting for wifi ...") diff --git a/examples/provisioning/ble_prov/ble_prov_test.py b/examples/provisioning/ble_prov/ble_prov_test.py index 760756a439..1e3e8508b6 100644 --- a/examples/provisioning/ble_prov/ble_prov_test.py +++ b/examples/provisioning/ble_prov/ble_prov_test.py @@ -17,40 +17,25 @@ from __future__ import print_function import re import os -import sys import time -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -try: - import esp_prov -except ImportError: - esp_prov_path = os.getenv("IDF_PATH") + "/tools/esp_prov" - if esp_prov_path and esp_prov_path not in sys.path: - sys.path.insert(0, esp_prov_path) - import esp_prov +import ttfw_idf +import esp_prov # Have esp_prov throw exception esp_prov.config_throw_except = True -@IDF.idf_example_test(env_tag="Example_WIFI_BT") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT") def test_examples_provisioning_ble(env, extra_data): # Acquire DUT - dut1 = env.get_dut("ble_prov", "examples/provisioning/ble_prov", dut_class=ESP32DUT) + dut1 = env.get_dut("ble_prov", "examples/provisioning/ble_prov", dut_class=ttfw_idf.ESP32DUT) # Get binary file binary_file = os.path.join(dut1.app.binary_path, "ble_prov.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("ble_prov_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("ble_prov_bin_size", bin_size // 1024) + ttfw_idf.log_performance("ble_prov_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("ble_prov_bin_size", bin_size // 1024) # Upload binary and start testing dut1.start_app() diff --git a/examples/provisioning/manager/wifi_prov_mgr_test.py b/examples/provisioning/manager/wifi_prov_mgr_test.py index 600bb66d0c..f305edd57d 100644 --- a/examples/provisioning/manager/wifi_prov_mgr_test.py +++ b/examples/provisioning/manager/wifi_prov_mgr_test.py @@ -17,40 +17,25 @@ from __future__ import print_function import re import os -import sys import time -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -try: - import esp_prov -except ImportError: - esp_prov_path = os.getenv("IDF_PATH") + "/tools/esp_prov" - if esp_prov_path and esp_prov_path not in sys.path: - sys.path.insert(0, esp_prov_path) - import esp_prov +import ttfw_idf +import esp_prov # Have esp_prov throw exception esp_prov.config_throw_except = True -@IDF.idf_example_test(env_tag="Example_WIFI_BT") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT") def test_examples_wifi_prov_mgr(env, extra_data): # Acquire DUT - dut1 = env.get_dut("wifi_prov_mgr", "examples/provisioning/manager", dut_class=ESP32DUT) + dut1 = env.get_dut("wifi_prov_mgr", "examples/provisioning/manager", dut_class=ttfw_idf.ESP32DUT) # Get binary file binary_file = os.path.join(dut1.app.binary_path, "wifi_prov_mgr.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("wifi_prov_mgr_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("wifi_prov_mgr_bin_size", bin_size // 1024) + ttfw_idf.log_performance("wifi_prov_mgr_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("wifi_prov_mgr_bin_size", bin_size // 1024) # Upload binary and start testing dut1.start_app() diff --git a/examples/provisioning/softap_prov/softap_prov_test.py b/examples/provisioning/softap_prov/softap_prov_test.py index 73d3a51815..9a6ee6858a 100644 --- a/examples/provisioning/softap_prov/softap_prov_test.py +++ b/examples/provisioning/softap_prov/softap_prov_test.py @@ -17,48 +17,26 @@ from __future__ import print_function import re import os -import sys import time -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -try: - import esp_prov -except ImportError: - esp_prov_path = os.getenv("IDF_PATH") + "/tools/esp_prov" - if esp_prov_path and esp_prov_path not in sys.path: - sys.path.insert(0, esp_prov_path) - import esp_prov - -try: - import wifi_tools -except ImportError: - wifi_tools_path = os.getenv("IDF_PATH") + "/examples/provisioning/softap_prov/utils" - if wifi_tools_path and wifi_tools_path not in sys.path: - sys.path.insert(0, wifi_tools_path) - import wifi_tools +import ttfw_idf +import esp_prov +import wifi_tools # Have esp_prov throw exception esp_prov.config_throw_except = True -@IDF.idf_example_test(env_tag="Example_WIFI_BT") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI_BT") def test_examples_provisioning_softap(env, extra_data): # Acquire DUT - dut1 = env.get_dut("softap_prov", "examples/provisioning/softap_prov", dut_class=ESP32DUT) + dut1 = env.get_dut("softap_prov", "examples/provisioning/softap_prov", dut_class=ttfw_idf.ESP32DUT) # Get binary file binary_file = os.path.join(dut1.app.binary_path, "softap_prov.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("softap_prov_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("softap_prov_bin_size", bin_size // 1024) + ttfw_idf.log_performance("softap_prov_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("softap_prov_bin_size", bin_size // 1024) # Upload binary and start testing dut1.start_app() diff --git a/examples/security/flash_encryption/example_test.py b/examples/security/flash_encryption/example_test.py index a48fac0875..5d99a37822 100644 --- a/examples/security/flash_encryption/example_test.py +++ b/examples/security/flash_encryption/example_test.py @@ -1,15 +1,5 @@ from __future__ import print_function -import os -import sys - -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf # To prepare a test runner for this example: @@ -19,9 +9,9 @@ except ImportError: # espefuse.py --do-not-confirm -p $ESPPORT burn_efuse FLASH_CRYPT_CONFIG 0xf # espefuse.py --do-not-confirm -p $ESPPORT burn_efuse FLASH_CRYPT_CNT 0x1 # espefuse.py --do-not-confirm -p $ESPPORT burn_key flash_encryption key.bin -@IDF.idf_example_test(env_tag='Example_Flash_Encryption') +@ttfw_idf.idf_example_test(env_tag='Example_Flash_Encryption') def test_examples_security_flash_encryption(env, extra_data): - dut = env.get_dut('flash_encryption', 'examples/security/flash_encryption', dut_class=ESP32DUT) + dut = env.get_dut('flash_encryption', 'examples/security/flash_encryption', dut_class=ttfw_idf.ESP32DUT) # start test dut.start_app() lines = [ diff --git a/examples/storage/ext_flash_fatfs/example_test.py b/examples/storage/ext_flash_fatfs/example_test.py index a0696d0573..3fa133bb51 100644 --- a/examples/storage/ext_flash_fatfs/example_test.py +++ b/examples/storage/ext_flash_fatfs/example_test.py @@ -1,21 +1,11 @@ from __future__ import print_function -import os -import sys -try: - import IDF -except ImportError: - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -from IDF.IDFDUT import ESP32DUT +import ttfw_idf -@IDF.idf_example_test(env_tag='Example_ExtFlash') +@ttfw_idf.idf_example_test(env_tag='Example_ExtFlash') def test_examples_storage_ext_flash_fatfs(env, extra_data): - dut = env.get_dut('ext_flash_fatfs', 'examples/storage/ext_flash_fatfs', dut_class=ESP32DUT) + dut = env.get_dut('ext_flash_fatfs', 'examples/storage/ext_flash_fatfs', dut_class=ttfw_idf.ESP32DUT) dut.start_app() dut.expect('Initialized external Flash') diff --git a/examples/storage/parttool/example_test.py b/examples/storage/parttool/example_test.py index 8b9d37775b..c75faf456a 100644 --- a/examples/storage/parttool/example_test.py +++ b/examples/storage/parttool/example_test.py @@ -3,19 +3,12 @@ import os import sys import subprocess -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_examples_parttool(env, extra_data): - dut = env.get_dut('parttool', 'examples/storage/parttool', dut_class=ESP32DUT) + dut = env.get_dut('parttool', 'examples/storage/parttool', dut_class=ttfw_idf.ESP32DUT) dut.start_app(False) # Verify factory firmware diff --git a/examples/storage/spiffsgen/example_test.py b/examples/storage/spiffsgen/example_test.py index a2ad4304a4..0fe0045681 100644 --- a/examples/storage/spiffsgen/example_test.py +++ b/examples/storage/spiffsgen/example_test.py @@ -1,22 +1,14 @@ from __future__ import print_function import os -import sys import hashlib -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_examples_spiffsgen(env, extra_data): # Test with default build configurations - dut = env.get_dut('spiffsgen', 'examples/storage/spiffsgen', dut_class=ESP32DUT) + dut = env.get_dut('spiffsgen', 'examples/storage/spiffsgen', dut_class=ttfw_idf.ESP32DUT) dut.start_app() base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'spiffs_image') diff --git a/examples/system/console/example_test.py b/examples/system/console/example_test.py index 23b25ef005..87cc8e3921 100644 --- a/examples/system/console/example_test.py +++ b/examples/system/console/example_test.py @@ -1,17 +1,9 @@ from __future__ import print_function -import os -import sys -try: - import IDF -except ImportError: - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_examples_system_console(env, extra_data): dut = env.get_dut('console_example', 'examples/system/console', app_config_name='history') print("Using binary path: {}".format(dut.app.binary_path)) diff --git a/examples/system/cpp_exceptions/example_test.py b/examples/system/cpp_exceptions/example_test.py index 2101458361..8340387c99 100644 --- a/examples/system/cpp_exceptions/example_test.py +++ b/examples/system/cpp_exceptions/example_test.py @@ -1,20 +1,11 @@ from __future__ import print_function -import os -import sys -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_examples_system_cpp_exceptions(env, extra_data): - dut = env.get_dut('cpp_exceptions_example', 'examples/system/cpp_exceptions', dut_class=ESP32DUT) + dut = env.get_dut('cpp_exceptions_example', 'examples/system/cpp_exceptions', dut_class=ttfw_idf.ESP32DUT) # start test dut.start_app() lines = ['app_main starting', diff --git a/examples/system/cpp_rtti/example_test.py b/examples/system/cpp_rtti/example_test.py index af63c98d49..d384d06c92 100644 --- a/examples/system/cpp_rtti/example_test.py +++ b/examples/system/cpp_rtti/example_test.py @@ -1,23 +1,11 @@ from __future__ import print_function -import os -import sys -try: - import IDF -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_cpp_rtti_example(env, extra_data): - dut = env.get_dut('cpp_rtti', 'examples/system/cpp_rtti') + dut = env.get_dut('cpp_rtti', 'examples/system/cpp_rtti', dut_class=ttfw_idf.ESP32DUT) dut.start_app() dut.expect('Type name of std::cout is: std::ostream') diff --git a/examples/system/esp_event/default_event_loop/example_test.py b/examples/system/esp_event/default_event_loop/example_test.py index 002c35eddf..43a5344515 100644 --- a/examples/system/esp_event/default_event_loop/example_test.py +++ b/examples/system/esp_event/default_event_loop/example_test.py @@ -1,18 +1,6 @@ from __future__ import print_function -import os -import sys -try: - import IDF -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf # Timer events TIMER_EVENT_LIMIT = 3 @@ -91,7 +79,7 @@ def _test_iteration_events(dut): print("Deleted task event source") -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_default_event_loop_example(env, extra_data): dut = env.get_dut('default_event_loop', 'examples/system/esp_event/default_event_loop') diff --git a/examples/system/esp_event/user_event_loops/example_test.py b/examples/system/esp_event/user_event_loops/example_test.py index ccd8eeab95..7bb3135bf3 100644 --- a/examples/system/esp_event/user_event_loops/example_test.py +++ b/examples/system/esp_event/user_event_loops/example_test.py @@ -1,18 +1,6 @@ from __future__ import print_function -import os -import sys -try: - import IDF -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf TASK_ITERATION_LIMIT = 10 @@ -20,9 +8,9 @@ TASK_ITERATION_POSTING = "posting TASK_EVENTS:TASK_ITERATION_EVENT to {}, iterat TASK_ITERATION_HANDLING = "handling TASK_EVENTS:TASK_ITERATION_EVENT from {}, iteration {}" -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_user_event_loops_example(env, extra_data): - dut = env.get_dut('user_event_loops', 'examples/system/esp_event/user_event_loops') + dut = env.get_dut('user_event_loops', 'examples/system/esp_event/user_event_loops', dut_class=ttfw_idf.ESP32DUT) dut.start_app() diff --git a/examples/system/esp_timer/example_test.py b/examples/system/esp_timer/example_test.py index 325088abea..9963bb768f 100644 --- a/examples/system/esp_timer/example_test.py +++ b/examples/system/esp_timer/example_test.py @@ -1,20 +1,7 @@ from __future__ import print_function import re -import os -import sys -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf STARTING_TIMERS_REGEX = re.compile(r'Started timers, time since boot: (\d+) us') @@ -38,9 +25,9 @@ LIGHT_SLEEP_TIME = 500000 ONE_SHOT_TIMER_PERIOD = 5000000 -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_examples_system_esp_timer(env, extra_data): - dut = env.get_dut('esp_timer_example', 'examples/system/esp_timer', dut_class=ESP32DUT) + dut = env.get_dut('esp_timer_example', 'examples/system/esp_timer', dut_class=ttfw_idf.ESP32DUT) # start test dut.start_app() groups = dut.expect(STARTING_TIMERS_REGEX, timeout=30) diff --git a/examples/system/freertos/real_time_stats/example_test.py b/examples/system/freertos/real_time_stats/example_test.py index 2d2d2cc0bc..d9de7b4b77 100644 --- a/examples/system/freertos/real_time_stats/example_test.py +++ b/examples/system/freertos/real_time_stats/example_test.py @@ -1,27 +1,14 @@ from __future__ import print_function -import os -import sys -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf STATS_TASK_ITERS = 3 STATS_TASK_EXPECT = "Real time stats obtained" -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_real_time_stats_example(env, extra_data): - dut = env.get_dut('real_time_stats', 'examples/system/freertos/real_time_stats', dut_class=ESP32DUT) + dut = env.get_dut('real_time_stats', 'examples/system/freertos/real_time_stats', dut_class=ttfw_idf.ESP32DUT) dut.start_app() for iteration in range(0, STATS_TASK_ITERS): diff --git a/examples/system/light_sleep/example_test.py b/examples/system/light_sleep/example_test.py index a29b5dcc8b..4c6e428339 100644 --- a/examples/system/light_sleep/example_test.py +++ b/examples/system/light_sleep/example_test.py @@ -1,17 +1,8 @@ from __future__ import print_function import re -import os -import sys import time -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf ENTERING_SLEEP_STR = 'Entering light sleep' EXIT_SLEEP_REGEX = re.compile(r'Returned from light sleep, reason: (\w+), t=(\d+) ms, slept for (\d+) ms') @@ -20,9 +11,9 @@ WAITING_FOR_GPIO_STR = 'Waiting for GPIO0 to go high...' WAKEUP_INTERVAL_MS = 2000 -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_examples_system_light_sleep(env, extra_data): - dut = env.get_dut('light_sleep_example', 'examples/system/light_sleep', dut_class=ESP32DUT) + dut = env.get_dut('light_sleep_example', 'examples/system/light_sleep', dut_class=ttfw_idf.ESP32DUT) dut.start_app() # Ensure DTR and RTS are de-asserted for proper control of GPIO0 diff --git a/examples/system/ota/otatool/example_test.py b/examples/system/ota/otatool/example_test.py index cfbf06450d..404f075780 100644 --- a/examples/system/ota/otatool/example_test.py +++ b/examples/system/ota/otatool/example_test.py @@ -3,23 +3,12 @@ import os import sys import subprocess -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv('TEST_FW_PATH') - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +import ttfw_idf -@IDF.idf_example_test(env_tag='Example_WIFI') +@ttfw_idf.idf_example_test(env_tag='Example_WIFI') def test_otatool_example(env, extra_data): - dut = env.get_dut('otatool', 'examples/system/ota/otatool', dut_class=ESP32DUT) + dut = env.get_dut('otatool', 'examples/system/ota/otatool', dut_class=ttfw_idf.ESP32DUT) # Verify factory firmware dut.start_app() diff --git a/examples/system/ota/simple_ota_example/example_test.py b/examples/system/ota/simple_ota_example/example_test.py index 0b79b68b59..4ea3045ad0 100644 --- a/examples/system/ota/simple_ota_example/example_test.py +++ b/examples/system/ota/simple_ota_example/example_test.py @@ -1,26 +1,13 @@ import re import os -import sys import socket import BaseHTTPServer import SimpleHTTPServer from threading import Thread import ssl -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -import DUT +from tiny_test_fw import DUT +import ttfw_idf server_cert = "-----BEGIN CERTIFICATE-----\n" \ "MIIDXTCCAkWgAwIBAgIJAP4LF7E72HakMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV\n"\ @@ -108,7 +95,7 @@ def start_https_server(ota_image_dir, server_ip, server_port): httpd.serve_forever() -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_protocol_simple_ota_example(env, extra_data): """ steps: | @@ -116,12 +103,12 @@ def test_examples_protocol_simple_ota_example(env, extra_data): 2. Fetch OTA image over HTTPS 3. Reboot with the new OTA image """ - dut1 = env.get_dut("simple_ota_example", "examples/system/ota/simple_ota_example", dut_class=ESP32DUT) + dut1 = env.get_dut("simple_ota_example", "examples/system/ota/simple_ota_example", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "simple_ota.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("simple_ota_bin_size", "{}KB".format(bin_size // 1024)) - IDF.check_performance("simple_ota_bin_size", bin_size // 1024) + ttfw_idf.log_performance("simple_ota_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("simple_ota_bin_size", bin_size // 1024) # start test host_ip = get_my_ip() thread1 = Thread(target=start_https_server, args=(dut1.app.binary_path, host_ip, 8000)) diff --git a/examples/wifi/iperf/iperf_test.py b/examples/wifi/iperf/iperf_test.py index b6af1a62a3..074cc7a5f7 100644 --- a/examples/wifi/iperf/iperf_test.py +++ b/examples/wifi/iperf/iperf_test.py @@ -26,34 +26,13 @@ from builtins import range from builtins import object import re import os -import sys import time import subprocess -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF +from tiny_test_fw import TinyFW, DUT, Utility +import ttfw_idf +from idf_iperf_test_util import (Attenuator, PowerControl, LineChart, TestReport) -import DUT -import TinyFW -import Utility -from Utility import (Attenuator, PowerControl, LineChart) - -try: - from test_report import (ThroughputForConfigsReport, ThroughputVsRssiReport) -except ImportError: - # add current folder to system path for importing test_report - sys.path.append(os.path.dirname(__file__)) - from test_report import (ThroughputForConfigsReport, ThroughputVsRssiReport) # configurations TEST_TIME = TEST_TIMEOUT = 60 @@ -166,8 +145,8 @@ class TestResult(object): throughput = 0.0 if throughput == 0 and rssi > self.ZERO_THROUGHPUT_THRESHOLD: - self.error_list.append("[Error][Fatal][{}][att: {}][rssi: {}]: No throughput data found" - .format(ap_ssid, att, rssi)) + self.error_list.append("[Error][Fatal][{}][att: {}][rssi: {}]: No throughput data found" + .format(ap_ssid, att, rssi)) self._save_result(throughput, ap_ssid, att, rssi, heap_size) @@ -467,7 +446,7 @@ class IperfTestUtility(object): return ret -@IDF.idf_example_test(env_tag="Example_ShieldBox_Basic", category="stress") +@ttfw_idf.idf_example_test(env_tag="Example_ShieldBox_Basic", category="stress") def test_wifi_throughput_with_different_configs(env, extra_data): """ steps: | @@ -492,7 +471,8 @@ def test_wifi_throughput_with_different_configs(env, extra_data): "sdkconfig.ci.{}".format(config_name)) # 2. get DUT and download - dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT, app_config_name=config_name) + dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT, + app_config_name=config_name) dut.start_app() dut.expect("esp32>") @@ -519,12 +499,12 @@ def test_wifi_throughput_with_different_configs(env, extra_data): env.close_dut("iperf") # 5. generate report - report = ThroughputForConfigsReport(os.path.join(env.log_path, "ThroughputForConfigsReport"), - ap_info["ssid"], test_result, sdkconfig_files) + report = TestReport.ThroughputForConfigsReport(os.path.join(env.log_path, "ThroughputForConfigsReport"), + ap_info["ssid"], test_result, sdkconfig_files) report.generate_report() -@IDF.idf_example_test(env_tag="Example_ShieldBox", category="stress") +@ttfw_idf.idf_example_test(env_tag="Example_ShieldBox", category="stress") def test_wifi_throughput_vs_rssi(env, extra_data): """ steps: | @@ -547,7 +527,8 @@ def test_wifi_throughput_vs_rssi(env, extra_data): } # 1. get DUT and download - dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT, app_config_name=BEST_PERFORMANCE_CONFIG) + dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT, + app_config_name=BEST_PERFORMANCE_CONFIG) dut.start_app() dut.expect("esp32>") @@ -573,12 +554,12 @@ def test_wifi_throughput_vs_rssi(env, extra_data): env.close_dut("iperf") # 4. generate report - report = ThroughputVsRssiReport(os.path.join(env.log_path, "ThroughputVsRssiReport"), - test_result) + report = TestReport.ThroughputVsRssiReport(os.path.join(env.log_path, "ThroughputVsRssiReport"), + test_result) report.generate_report() -@IDF.idf_example_test(env_tag="Example_ShieldBox_Basic") +@ttfw_idf.idf_example_test(env_tag="Example_ShieldBox_Basic") def test_wifi_throughput_basic(env, extra_data): """ steps: | @@ -593,7 +574,8 @@ def test_wifi_throughput_basic(env, extra_data): } # 1. get DUT - dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ESP32DUT, app_config_name=BEST_PERFORMANCE_CONFIG) + dut = env.get_dut("iperf", "examples/wifi/iperf", dut_class=ttfw_idf.ESP32DUT, + app_config_name=BEST_PERFORMANCE_CONFIG) dut.start_app() dut.expect("esp32>") @@ -615,8 +597,8 @@ def test_wifi_throughput_basic(env, extra_data): # 4. log performance and compare with pass standard performance_items = [] for throughput_type in test_result: - IDF.log_performance("{}_throughput".format(throughput_type), - "{:.02f} Mbps".format(test_result[throughput_type].get_best_throughput())) + ttfw_idf.log_performance("{}_throughput".format(throughput_type), + "{:.02f} Mbps".format(test_result[throughput_type].get_best_throughput())) performance_items.append(["{}_throughput".format(throughput_type), "{:.02f} Mbps".format(test_result[throughput_type].get_best_throughput())]) @@ -624,8 +606,8 @@ def test_wifi_throughput_basic(env, extra_data): TinyFW.JunitReport.update_performance(performance_items) # do check after logging, otherwise test will exit immediately if check fail, some performance can't be logged. for throughput_type in test_result: - IDF.check_performance("{}_throughput".format(throughput_type), - test_result[throughput_type].get_best_throughput()) + ttfw_idf.check_performance("{}_throughput".format(throughput_type), + test_result[throughput_type].get_best_throughput()) env.close_dut("iperf") diff --git a/tools/ble/__init__.py b/tools/ble/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index ffc7c9f27e..9846b8d00e 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -10,7 +10,6 @@ assign_test: - build_ssc - build_esp_idf_tests_cmake variables: - TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs" UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml" artifacts: @@ -18,6 +17,7 @@ assign_test: - components/idf_test/*/CIConfigs - components/idf_test/*/TC.sqlite - $EXAMPLE_CONFIG_OUTPUT_PATH + - build_examples/artifact_index.json expire_in: 1 week only: variables: @@ -27,9 +27,9 @@ assign_test: - $BOT_LABEL_EXAMPLE_TEST script: # assign example tests - - python $TEST_FW_PATH/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH + - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH # assign unit test cases - - python $TEST_FW_PATH/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs + - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs # clone test script to assign tests - git clone $TEST_SCRIPT_REPOSITORY - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script @@ -57,7 +57,6 @@ update_test_cases: variables: UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml" BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml" - TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script" PYTHON_VER: 3 script: diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 47044a1c54..baa30916f1 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -30,7 +30,6 @@ reports: junit: $LOG_PATH/*/XUNIT_RESULT.xml variables: - TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" TEST_CASE_PATH: "$CI_PROJECT_DIR/examples" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" @@ -46,7 +45,7 @@ # Get esptool - git submodule sync - git submodule update --init components/esptool_py/esptool - - cd $TEST_FW_PATH + - cd tools/ci/python_packages/tiny_test_fw/bin # run test - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE @@ -75,7 +74,6 @@ reports: junit: $LOG_PATH/*/XUNIT_RESULT.xml variables: - TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" TEST_CASE_PATH: "$CI_PROJECT_DIR/examples" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" @@ -87,7 +85,7 @@ # clone test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - - cd $TEST_FW_PATH + - cd tools/ci/python_packages/tiny_test_fw/bin # run test - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE @@ -170,7 +168,6 @@ test_weekend_mqtt: - $BOT_LABEL_WEEKEND_TEST variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test" - TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml" CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test" @@ -187,7 +184,6 @@ test_weekend_network: - $BOT_LABEL_WEEKEND_TEST variables: TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test" - TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml" CONFIG_FILE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test" @@ -214,7 +210,7 @@ example_test_002: # clone test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - - cd $TEST_FW_PATH + - cd tools/ci/python_packages/tiny_test_fw/bin # run test - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE diff --git a/tools/ci/python_packages/idf_http_server_test/__init__.py b/tools/ci/python_packages/idf_http_server_test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/protocols/http_server/persistent_sockets/scripts/adder.py b/tools/ci/python_packages/idf_http_server_test/adder.py similarity index 99% rename from examples/protocols/http_server/persistent_sockets/scripts/adder.py rename to tools/ci/python_packages/idf_http_server_test/adder.py index a1c9c990f7..46eb75a557 100644 --- a/examples/protocols/http_server/persistent_sockets/scripts/adder.py +++ b/tools/ci/python_packages/idf_http_server_test/adder.py @@ -20,7 +20,8 @@ from builtins import str from builtins import range import http.client import argparse -import Utility + +from tiny_test_fw import Utility def start_session(ip, port): diff --git a/examples/protocols/http_server/simple/scripts/client.py b/tools/ci/python_packages/idf_http_server_test/client.py similarity index 96% rename from examples/protocols/http_server/simple/scripts/client.py rename to tools/ci/python_packages/idf_http_server_test/client.py index 5e070b98b8..95d195d824 100644 --- a/examples/protocols/http_server/simple/scripts/client.py +++ b/tools/ci/python_packages/idf_http_server_test/client.py @@ -20,19 +20,8 @@ from builtins import str import http.client import argparse -try: - import Utility -except ImportError: - import sys - import os - # This environment variable is expected on the host machine - # > export TEST_FW_PATH=~/esp/esp-idf/tools/tiny-test-fw - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - - import Utility +from tiny_test_fw import Utility def verbose_print(verbosity, *args): diff --git a/tools/ci/python_packages/idf_http_server_test/test.py b/tools/ci/python_packages/idf_http_server_test/test.py new file mode 100644 index 0000000000..3ad5bcc9c7 --- /dev/null +++ b/tools/ci/python_packages/idf_http_server_test/test.py @@ -0,0 +1,1025 @@ +#!/usr/bin/env python +# +# Copyright 2018 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Utility for testing the web server. Test cases: +# Assume the device supports 'n' simultaneous open sockets +# +# HTTP Server Tests +# +# 0. Firmware Settings: +# - Create a dormant thread whose sole job is to call httpd_stop() when instructed +# - Measure the following before httpd_start() is called: +# - current free memory +# - current free sockets +# - Measure the same whenever httpd_stop is called +# - Register maximum possible URI handlers: should be successful +# - Register one more URI handler: should fail +# - Deregister on URI handler: should be successful +# - Register on more URI handler: should succeed +# - Register separate handlers for /hello, /hello/type_html. Also +# ensure that /hello/type_html is registered BEFORE /hello. (tests +# that largest matching URI is picked properly) +# - Create URI handler /adder. Make sure it uses a custom free_ctx +# structure to free it up + +# 1. Using Standard Python HTTP Client +# - simple GET on /hello (returns Hello World. Ensures that basic +# firmware tests are complete, or returns error) +# - POST on /hello (should fail) +# - PUT on /hello (should fail) +# - simple POST on /echo (returns whatever the POST data) +# - simple PUT on /echo (returns whatever the PUT data) +# - GET on /echo (should fail) +# - simple GET on /hello/type_html (returns Content type as text/html) +# - simple GET on /hello/status_500 (returns HTTP status 500) +# - simple GET on /false_uri (returns HTTP status 404) +# - largest matching URI handler is picked is already verified because +# of /hello and /hello/type_html tests +# +# +# 2. Session Tests +# - Sessions + Pipelining basics: +# - Create max supported sessions +# - On session i, +# - send 3 back-to-back POST requests with data i on /adder +# - read back 3 responses. They should be i, 2i and 3i +# - Tests that +# - pipelining works +# - per-session context is maintained for all supported +# sessions +# - Close all sessions +# +# - Cleanup leftover data: Tests that the web server properly cleans +# up leftover data +# - Create a session +# - POST on /leftover_data with 52 bytes of data (data includes +# \r\n)(the handler only +# reads first 10 bytes and returns them, leaving the rest of the +# bytes unread) +# - GET on /hello (should return 'Hello World') +# - POST on /false_uri with 52 bytes of data (data includes \r\n) +# (should return HTTP 404) +# - GET on /hello (should return 'Hello World') +# +# - Test HTTPd Asynchronous response +# - Create a session +# - GET on /async_data +# - returns 'Hello World!' as a response +# - the handler schedules an async response, which generates a second +# response 'Hello Double World!' +# +# - Spillover test +# - Create max supported sessions with the web server +# - GET /hello on all the sessions (should return Hello World) +# - Create one more session, this should fail +# - GET /hello on all the sessions (should return Hello World) +# +# - Timeout test +# - Create a session and only Send 'GE' on the same (simulates a +# client that left the network halfway through a request) +# - Wait for recv-wait-timeout +# - Server should automatically close the socket + + +# ############ TODO TESTS ############# + +# 3. Stress Tests +# +# - httperf +# - Run the following httperf command: +# httperf --server=10.31.130.126 --wsess=8,50,0.5 --rate 8 --burst-length 2 +# +# - The above implies that the test suite will open +# - 8 simultaneous connections with the server +# - the rate of opening the sessions will be 8 per sec. So in our +# case, a new connection will be opened every 0.2 seconds for 1 second +# - The burst length 2 indicates that 2 requests will be sent +# simultaneously on the same connection in a single go +# - 0.5 seconds is the time between sending out 2 bursts +# - 50 is the total number of requests that will be sent out +# +# - So in the above example, the test suite will open 8 +# connections, each separated by 0.2 seconds. On each connection +# it will send 2 requests in a single burst. The bursts on a +# single connection will be separated by 0.5 seconds. A total of +# 25 bursts (25 x 2 = 50) will be sent out + +# 4. Leak Tests +# - Simple Leak test +# - Simple GET on /hello/restart (returns success, stop web server, measures leaks, restarts webserver) +# - Simple GET on /hello/restart_results (returns the leak results) +# - Leak test with open sockets +# - Open 8 sessions +# - Simple GET on /hello/restart (returns success, stop web server, +# measures leaks, restarts webserver) +# - All sockets should get closed +# - Simple GET on /hello/restart_results (returns the leak results) + + +from __future__ import division +from __future__ import print_function +from builtins import str +from builtins import range +from builtins import object +import threading +import socket +import time +import argparse +import http.client +import sys +import string +import random + +from tiny_test_fw import Utility + +_verbose_ = False + + +class Session(object): + def __init__(self, addr, port, timeout=15): + self.client = socket.create_connection((addr, int(port)), timeout=timeout) + self.target = addr + self.status = 0 + self.encoding = '' + self.content_type = '' + self.content_len = 0 + + def send_err_check(self, request, data=None): + rval = True + try: + self.client.sendall(request.encode()) + if data: + self.client.sendall(data.encode()) + except socket.error as err: + self.client.close() + Utility.console_log("Socket Error in send :", err) + rval = False + return rval + + def send_get(self, path, headers=None): + request = "GET " + path + " HTTP/1.1\r\nHost: " + self.target + if headers: + for field, value in headers.items(): + request += "\r\n" + field + ": " + value + request += "\r\n\r\n" + return self.send_err_check(request) + + def send_put(self, path, data, headers=None): + request = "PUT " + path + " HTTP/1.1\r\nHost: " + self.target + if headers: + for field, value in headers.items(): + request += "\r\n" + field + ": " + value + request += "\r\nContent-Length: " + str(len(data)) + "\r\n\r\n" + return self.send_err_check(request, data) + + def send_post(self, path, data, headers=None): + request = "POST " + path + " HTTP/1.1\r\nHost: " + self.target + if headers: + for field, value in headers.items(): + request += "\r\n" + field + ": " + value + request += "\r\nContent-Length: " + str(len(data)) + "\r\n\r\n" + return self.send_err_check(request, data) + + def read_resp_hdrs(self): + try: + state = 'nothing' + resp_read = '' + while True: + char = self.client.recv(1).decode() + if char == '\r' and state == 'nothing': + state = 'first_cr' + elif char == '\n' and state == 'first_cr': + state = 'first_lf' + elif char == '\r' and state == 'first_lf': + state = 'second_cr' + elif char == '\n' and state == 'second_cr': + state = 'second_lf' + else: + state = 'nothing' + resp_read += char + if state == 'second_lf': + break + # Handle first line + line_hdrs = resp_read.splitlines() + line_comp = line_hdrs[0].split() + self.status = line_comp[1] + del line_hdrs[0] + self.encoding = '' + self.content_type = '' + headers = dict() + # Process other headers + for h in range(len(line_hdrs)): + line_comp = line_hdrs[h].split(':') + if line_comp[0] == 'Content-Length': + self.content_len = int(line_comp[1]) + if line_comp[0] == 'Content-Type': + self.content_type = line_comp[1].lstrip() + if line_comp[0] == 'Transfer-Encoding': + self.encoding = line_comp[1].lstrip() + if len(line_comp) == 2: + headers[line_comp[0]] = line_comp[1].lstrip() + return headers + except socket.error as err: + self.client.close() + Utility.console_log("Socket Error in recv :", err) + return None + + def read_resp_data(self): + try: + read_data = '' + if self.encoding != 'chunked': + while len(read_data) != self.content_len: + read_data += self.client.recv(self.content_len).decode() + else: + chunk_data_buf = '' + while (True): + # Read one character into temp buffer + read_ch = self.client.recv(1) + # Check CRLF + if (read_ch == '\r'): + read_ch = self.client.recv(1).decode() + if (read_ch == '\n'): + # If CRLF decode length of chunk + chunk_len = int(chunk_data_buf, 16) + # Keep adding to contents + self.content_len += chunk_len + rem_len = chunk_len + while (rem_len): + new_data = self.client.recv(rem_len) + read_data += new_data + rem_len -= len(new_data) + chunk_data_buf = '' + # Fetch remaining CRLF + if self.client.recv(2) != "\r\n": + # Error in packet + Utility.console_log("Error in chunked data") + return None + if not chunk_len: + # If last chunk + break + continue + chunk_data_buf += '\r' + # If not CRLF continue appending + # character to chunked data buffer + chunk_data_buf += read_ch + return read_data + except socket.error as err: + self.client.close() + Utility.console_log("Socket Error in recv :", err) + return None + + def close(self): + self.client.close() + + +def test_val(text, expected, received): + if expected != received: + Utility.console_log(" Fail!") + Utility.console_log(" [reason] " + text + ":") + Utility.console_log(" expected: " + str(expected)) + Utility.console_log(" received: " + str(received)) + return False + return True + + +class adder_thread (threading.Thread): + def __init__(self, id, dut, port): + threading.Thread.__init__(self) + self.id = id + self.dut = dut + self.depth = 3 + self.session = Session(dut, port) + + def run(self): + self.response = [] + + # Pipeline 3 requests + if (_verbose_): + Utility.console_log(" Thread: Using adder start " + str(self.id)) + + for _ in range(self.depth): + self.session.send_post('/adder', str(self.id)) + time.sleep(2) + + for _ in range(self.depth): + self.session.read_resp_hdrs() + self.response.append(self.session.read_resp_data()) + + def adder_result(self): + if len(self.response) != self.depth: + Utility.console_log("Error : missing response packets") + return False + for i in range(len(self.response)): + if not test_val("Thread" + str(self.id) + " response[" + str(i) + "]", + str(self.id * (i + 1)), str(self.response[i])): + return False + return True + + def close(self): + self.session.close() + + +def get_hello(dut, port): + # GET /hello should return 'Hello World!' + Utility.console_log("[test] GET /hello returns 'Hello World!' =>", end=' ') + conn = http.client.HTTPConnection(dut, int(port), timeout=15) + conn.request("GET", "/hello") + resp = conn.getresponse() + if not test_val("status_code", 200, resp.status): + conn.close() + return False + if not test_val("data", "Hello World!", resp.read().decode()): + conn.close() + return False + if not test_val("data", "text/html", resp.getheader('Content-Type')): + conn.close() + return False + Utility.console_log("Success") + conn.close() + return True + + +def put_hello(dut, port): + # PUT /hello returns 405' + Utility.console_log("[test] PUT /hello returns 405 =>", end=' ') + conn = http.client.HTTPConnection(dut, int(port), timeout=15) + conn.request("PUT", "/hello", "Hello") + resp = conn.getresponse() + if not test_val("status_code", 405, resp.status): + conn.close() + return False + Utility.console_log("Success") + conn.close() + return True + + +def post_hello(dut, port): + # POST /hello returns 405' + Utility.console_log("[test] POST /hello returns 405 =>", end=' ') + conn = http.client.HTTPConnection(dut, int(port), timeout=15) + conn.request("POST", "/hello", "Hello") + resp = conn.getresponse() + if not test_val("status_code", 405, resp.status): + conn.close() + return False + Utility.console_log("Success") + conn.close() + return True + + +def post_echo(dut, port): + # POST /echo echoes data' + Utility.console_log("[test] POST /echo echoes data =>", end=' ') + conn = http.client.HTTPConnection(dut, int(port), timeout=15) + conn.request("POST", "/echo", "Hello") + resp = conn.getresponse() + if not test_val("status_code", 200, resp.status): + conn.close() + return False + if not test_val("data", "Hello", resp.read().decode()): + conn.close() + return False + Utility.console_log("Success") + conn.close() + return True + + +def put_echo(dut, port): + # PUT /echo echoes data' + Utility.console_log("[test] PUT /echo echoes data =>", end=' ') + conn = http.client.HTTPConnection(dut, int(port), timeout=15) + conn.request("PUT", "/echo", "Hello") + resp = conn.getresponse() + if not test_val("status_code", 200, resp.status): + conn.close() + return False + if not test_val("data", "Hello", resp.read().decode()): + conn.close() + return False + Utility.console_log("Success") + conn.close() + return True + + +def get_echo(dut, port): + # GET /echo returns 404' + Utility.console_log("[test] GET /echo returns 405 =>", end=' ') + conn = http.client.HTTPConnection(dut, int(port), timeout=15) + conn.request("GET", "/echo") + resp = conn.getresponse() + if not test_val("status_code", 405, resp.status): + conn.close() + return False + Utility.console_log("Success") + conn.close() + return True + + +def get_test_headers(dut, port): + # GET /test_header returns data of Header2' + Utility.console_log("[test] GET /test_header =>", end=' ') + conn = http.client.HTTPConnection(dut, int(port), timeout=15) + custom_header = {"Header1": "Value1", "Header3": "Value3"} + header2_values = ["", " ", "Value2", " Value2", "Value2 ", " Value2 "] + for val in header2_values: + custom_header["Header2"] = val + conn.request("GET", "/test_header", headers=custom_header) + resp = conn.getresponse() + if not test_val("status_code", 200, resp.status): + conn.close() + return False + hdr_val_start_idx = val.find("Value2") + if hdr_val_start_idx == -1: + if not test_val("header: Header2", "", resp.read().decode()): + conn.close() + return False + else: + if not test_val("header: Header2", val[hdr_val_start_idx:], resp.read().decode()): + conn.close() + return False + resp.read() + Utility.console_log("Success") + conn.close() + return True + + +def get_hello_type(dut, port): + # GET /hello/type_html returns text/html as Content-Type' + Utility.console_log("[test] GET /hello/type_html has Content-Type of text/html =>", end=' ') + conn = http.client.HTTPConnection(dut, int(port), timeout=15) + conn.request("GET", "/hello/type_html") + resp = conn.getresponse() + if not test_val("status_code", 200, resp.status): + conn.close() + return False + if not test_val("data", "Hello World!", resp.read().decode()): + conn.close() + return False + if not test_val("data", "text/html", resp.getheader('Content-Type')): + conn.close() + return False + Utility.console_log("Success") + conn.close() + return True + + +def get_hello_status(dut, port): + # GET /hello/status_500 returns status 500' + Utility.console_log("[test] GET /hello/status_500 returns status 500 =>", end=' ') + conn = http.client.HTTPConnection(dut, int(port), timeout=15) + conn.request("GET", "/hello/status_500") + resp = conn.getresponse() + if not test_val("status_code", 500, resp.status): + conn.close() + return False + Utility.console_log("Success") + conn.close() + return True + + +def get_false_uri(dut, port): + # GET /false_uri returns status 404' + Utility.console_log("[test] GET /false_uri returns status 404 =>", end=' ') + conn = http.client.HTTPConnection(dut, int(port), timeout=15) + conn.request("GET", "/false_uri") + resp = conn.getresponse() + if not test_val("status_code", 404, resp.status): + conn.close() + return False + Utility.console_log("Success") + conn.close() + return True + + +def parallel_sessions_adder(dut, port, max_sessions): + # POSTs on /adder in parallel sessions + Utility.console_log("[test] POST {pipelined} on /adder in " + str(max_sessions) + " sessions =>", end=' ') + t = [] + # Create all sessions + for i in range(max_sessions): + t.append(adder_thread(i, dut, port)) + + for i in range(len(t)): + t[i].start() + + for i in range(len(t)): + t[i].join() + + res = True + for i in range(len(t)): + if not test_val("Thread" + str(i) + " Failed", t[i].adder_result(), True): + res = False + t[i].close() + if (res): + Utility.console_log("Success") + return res + + +def async_response_test(dut, port): + # Test that an asynchronous work is executed in the HTTPD's context + # This is tested by reading two responses over the same session + Utility.console_log("[test] Test HTTPD Work Queue (Async response) =>", end=' ') + s = Session(dut, port) + + s.send_get('/async_data') + s.read_resp_hdrs() + if not test_val("First Response", "Hello World!", s.read_resp_data()): + s.close() + return False + s.read_resp_hdrs() + if not test_val("Second Response", "Hello Double World!", s.read_resp_data()): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +def leftover_data_test(dut, port): + # Leftover data in POST is purged (valid and invalid URIs) + Utility.console_log("[test] Leftover data in POST is purged (valid and invalid URIs) =>", end=' ') + s = http.client.HTTPConnection(dut + ":" + port, timeout=15) + + s.request("POST", url='/leftover_data', body="abcdefghijklmnopqrstuvwxyz\r\nabcdefghijklmnopqrstuvwxyz") + resp = s.getresponse() + if not test_val("Partial data", "abcdefghij", resp.read().decode()): + s.close() + return False + + s.request("GET", url='/hello') + resp = s.getresponse() + if not test_val("Hello World Data", "Hello World!", resp.read().decode()): + s.close() + return False + + s.request("POST", url='/false_uri', body="abcdefghijklmnopqrstuvwxyz\r\nabcdefghijklmnopqrstuvwxyz") + resp = s.getresponse() + if not test_val("False URI Status", str(404), str(resp.status)): + s.close() + return False + # socket would have been closed by server due to error + s.close() + + s = http.client.HTTPConnection(dut + ":" + port, timeout=15) + s.request("GET", url='/hello') + resp = s.getresponse() + if not test_val("Hello World Data", "Hello World!", resp.read().decode()): + s.close() + return False + + s.close() + Utility.console_log("Success") + return True + + +def spillover_session(dut, port, max_sess): + # Session max_sess_sessions + 1 is rejected + Utility.console_log("[test] Session max_sess_sessions (" + str(max_sess) + ") + 1 is rejected =>", end=' ') + s = [] + _verbose_ = True + for i in range(max_sess + 1): + if (_verbose_): + Utility.console_log("Executing " + str(i)) + try: + a = http.client.HTTPConnection(dut + ":" + port, timeout=15) + a.request("GET", url='/hello') + resp = a.getresponse() + if not test_val("Connection " + str(i), "Hello World!", resp.read().decode()): + a.close() + break + s.append(a) + except Exception: + if (_verbose_): + Utility.console_log("Connection " + str(i) + " rejected") + a.close() + break + + # Close open connections + for a in s: + a.close() + + # Check if number of connections is equal to max_sess + Utility.console_log(["Fail","Success"][len(s) == max_sess]) + return (len(s) == max_sess) + + +def recv_timeout_test(dut, port): + Utility.console_log("[test] Timeout occurs if partial packet sent =>", end=' ') + s = Session(dut, port) + s.client.sendall(b"GE") + s.read_resp_hdrs() + resp = s.read_resp_data() + if not test_val("Request Timeout", "Server closed this connection", resp): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +def packet_size_limit_test(dut, port, test_size): + Utility.console_log("[test] send size limit test =>", end=' ') + retry = 5 + while (retry): + retry -= 1 + Utility.console_log("data size = ", test_size) + s = http.client.HTTPConnection(dut + ":" + port, timeout=15) + random_data = ''.join(string.printable[random.randint(0,len(string.printable)) - 1] for _ in list(range(test_size))) + path = "/echo" + s.request("POST", url=path, body=random_data) + resp = s.getresponse() + if not test_val("Error", "200", str(resp.status)): + if test_val("Error", "500", str(resp.status)): + Utility.console_log("Data too large to be allocated") + test_size = test_size // 10 + else: + Utility.console_log("Unexpected error") + s.close() + Utility.console_log("Retry...") + continue + resp = resp.read().decode() + result = (resp == random_data) + if not result: + test_val("Data size", str(len(random_data)), str(len(resp))) + s.close() + Utility.console_log("Retry...") + continue + s.close() + Utility.console_log("Success") + return True + Utility.console_log("Failed") + return False + + +def arbitrary_termination_test(dut, port): + Utility.console_log("[test] Arbitrary termination test =>", end=' ') + cases = [ + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: SomeValue\r\n\r\n", + "code": "200", + "header": "SomeValue" + }, + { + "request": "POST /echo HTTP/1.1\nHost: " + dut + "\r\nCustom: SomeValue\r\n\r\n", + "code": "200", + "header": "SomeValue" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\nCustom: SomeValue\r\n\r\n", + "code": "200", + "header": "SomeValue" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: SomeValue\n\r\n", + "code": "200", + "header": "SomeValue" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: SomeValue\r\n\n", + "code": "200", + "header": "SomeValue" + }, + { + "request": "POST /echo HTTP/1.1\nHost: " + dut + "\nCustom: SomeValue\n\n", + "code": "200", + "header": "SomeValue" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\n\r\nABCDE", + "code": "200", + "body": "ABCDE" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\r\n\nABCDE", + "code": "200", + "body": "ABCDE" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\n\nABCDE", + "code": "200", + "body": "ABCDE" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 5\n\n\rABCD", + "code": "200", + "body": "\rABCD" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\r\nCustom: SomeValue\r\r\n\r\r\n", + "code": "400" + }, + { + "request": "POST /echo HTTP/1.1\r\r\nHost: " + dut + "\r\n\r\n", + "code": "400" + }, + { + "request": "POST /echo HTTP/1.1\r\n\rHost: " + dut + "\r\n\r\n", + "code": "400" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\rCustom: SomeValue\r\n", + "code": "400" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom: Some\rValue\r\n", + "code": "400" + }, + { + "request": "POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nCustom- SomeValue\r\n\r\n", + "code": "400" + } + ] + for case in cases: + s = Session(dut, port) + s.client.sendall((case['request']).encode()) + resp_hdrs = s.read_resp_hdrs() + resp_body = s.read_resp_data() + s.close() + if not test_val("Response Code", case["code"], s.status): + return False + if "header" in case.keys(): + resp_hdr_val = None + if "Custom" in resp_hdrs.keys(): + resp_hdr_val = resp_hdrs["Custom"] + if not test_val("Response Header", case["header"], resp_hdr_val): + return False + if "body" in case.keys(): + if not test_val("Response Body", case["body"], resp_body): + return False + Utility.console_log("Success") + return True + + +def code_500_server_error_test(dut, port): + Utility.console_log("[test] 500 Server Error test =>", end=' ') + s = Session(dut, port) + # Sending a very large content length will cause malloc to fail + content_len = 2**30 + s.client.sendall(("POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: " + str(content_len) + "\r\n\r\nABCD").encode()) + s.read_resp_hdrs() + s.read_resp_data() + if not test_val("Server Error", "500", s.status): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +def code_501_method_not_impl(dut, port): + Utility.console_log("[test] 501 Method Not Implemented =>", end=' ') + s = Session(dut, port) + path = "/hello" + s.client.sendall(("ABC " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode()) + s.read_resp_hdrs() + s.read_resp_data() + # Presently server sends back 400 Bad Request + # if not test_val("Server Error", "501", s.status): + # s.close() + # return False + if not test_val("Server Error", "400", s.status): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +def code_505_version_not_supported(dut, port): + Utility.console_log("[test] 505 Version Not Supported =>", end=' ') + s = Session(dut, port) + path = "/hello" + s.client.sendall(("GET " + path + " HTTP/2.0\r\nHost: " + dut + "\r\n\r\n").encode()) + s.read_resp_hdrs() + s.read_resp_data() + if not test_val("Server Error", "505", s.status): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +def code_400_bad_request(dut, port): + Utility.console_log("[test] 400 Bad Request =>", end=' ') + s = Session(dut, port) + path = "/hello" + s.client.sendall(("XYZ " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode()) + s.read_resp_hdrs() + s.read_resp_data() + if not test_val("Client Error", "400", s.status): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +def code_404_not_found(dut, port): + Utility.console_log("[test] 404 Not Found =>", end=' ') + s = Session(dut, port) + path = "/dummy" + s.client.sendall(("GET " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode()) + s.read_resp_hdrs() + s.read_resp_data() + if not test_val("Client Error", "404", s.status): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +def code_405_method_not_allowed(dut, port): + Utility.console_log("[test] 405 Method Not Allowed =>", end=' ') + s = Session(dut, port) + path = "/hello" + s.client.sendall(("POST " + path + " HTTP/1.1\r\nHost: " + dut + "\r\n\r\n").encode()) + s.read_resp_hdrs() + s.read_resp_data() + if not test_val("Client Error", "405", s.status): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +def code_408_req_timeout(dut, port): + Utility.console_log("[test] 408 Request Timeout =>", end=' ') + s = Session(dut, port) + s.client.sendall(("POST /echo HTTP/1.1\r\nHost: " + dut + "\r\nContent-Length: 10\r\n\r\nABCD").encode()) + s.read_resp_hdrs() + s.read_resp_data() + if not test_val("Client Error", "408", s.status): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +def code_411_length_required(dut, port): + Utility.console_log("[test] 411 Length Required =>", end=' ') + s = Session(dut, port) + path = "/echo" + s.client.sendall(("POST " + path + " HTTP/1.1\r\nHost: " + dut + "\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n").encode()) + s.read_resp_hdrs() + s.read_resp_data() + # Presently server sends back 400 Bad Request + # if not test_val("Client Error", "411", s.status): + # s.close() + # return False + if not test_val("Client Error", "400", s.status): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +def send_getx_uri_len(dut, port, length): + s = Session(dut, port) + method = "GET " + version = " HTTP/1.1\r\n" + path = "/" + "x" * (length - len(method) - len(version) - len("/")) + s.client.sendall(method.encode()) + time.sleep(1) + s.client.sendall(path.encode()) + time.sleep(1) + s.client.sendall((version + "Host: " + dut + "\r\n\r\n").encode()) + s.read_resp_hdrs() + s.read_resp_data() + s.close() + return s.status + + +def code_414_uri_too_long(dut, port, max_uri_len): + Utility.console_log("[test] 414 URI Too Long =>", end=' ') + status = send_getx_uri_len(dut, port, max_uri_len) + if not test_val("Client Error", "404", status): + return False + status = send_getx_uri_len(dut, port, max_uri_len + 1) + if not test_val("Client Error", "414", status): + return False + Utility.console_log("Success") + return True + + +def send_postx_hdr_len(dut, port, length): + s = Session(dut, port) + path = "/echo" + host = "Host: " + dut + custom_hdr_field = "\r\nCustom: " + custom_hdr_val = "x" * (length - len(host) - len(custom_hdr_field) - len("\r\n\r\n") + len("0")) + request = ("POST " + path + " HTTP/1.1\r\n" + host + custom_hdr_field + custom_hdr_val + "\r\n\r\n").encode() + s.client.sendall(request[:length // 2]) + time.sleep(1) + s.client.sendall(request[length // 2:]) + hdr = s.read_resp_hdrs() + resp = s.read_resp_data() + s.close() + if hdr and ("Custom" in hdr): + return (hdr["Custom"] == custom_hdr_val), resp + return False, s.status + + +def code_431_hdr_too_long(dut, port, max_hdr_len): + Utility.console_log("[test] 431 Header Too Long =>", end=' ') + res, status = send_postx_hdr_len(dut, port, max_hdr_len) + if not res: + return False + res, status = send_postx_hdr_len(dut, port, max_hdr_len + 1) + if not test_val("Client Error", "431", status): + return False + Utility.console_log("Success") + return True + + +def test_upgrade_not_supported(dut, port): + Utility.console_log("[test] Upgrade Not Supported =>", end=' ') + s = Session(dut, port) + # path = "/hello" + s.client.sendall(("OPTIONS * HTTP/1.1\r\nHost:" + dut + "\r\nUpgrade: TLS/1.0\r\nConnection: Upgrade\r\n\r\n").encode()) + s.read_resp_hdrs() + s.read_resp_data() + if not test_val("Client Error", "400", s.status): + s.close() + return False + s.close() + Utility.console_log("Success") + return True + + +if __name__ == '__main__': + # Execution begins here... + # Configuration + # Max number of threads/sessions + max_sessions = 7 + max_uri_len = 512 + max_hdr_len = 512 + + parser = argparse.ArgumentParser(description='Run HTTPD Test') + parser.add_argument('-4','--ipv4', help='IPv4 address') + parser.add_argument('-6','--ipv6', help='IPv6 address') + parser.add_argument('-p','--port', help='Port') + args = vars(parser.parse_args()) + + dut4 = args['ipv4'] + dut6 = args['ipv6'] + port = args['port'] + dut = dut4 + + _verbose_ = True + + Utility.console_log("### Basic HTTP Client Tests") + get_hello(dut, port) + post_hello(dut, port) + put_hello(dut, port) + post_echo(dut, port) + get_echo(dut, port) + put_echo(dut, port) + get_hello_type(dut, port) + get_hello_status(dut, port) + get_false_uri(dut, port) + get_test_headers(dut, port) + + Utility.console_log("### Error code tests") + code_500_server_error_test(dut, port) + code_501_method_not_impl(dut, port) + code_505_version_not_supported(dut, port) + code_400_bad_request(dut, port) + code_404_not_found(dut, port) + code_405_method_not_allowed(dut, port) + code_408_req_timeout(dut, port) + code_414_uri_too_long(dut, port, max_uri_len) + code_431_hdr_too_long(dut, port, max_hdr_len) + test_upgrade_not_supported(dut, port) + + # Not supported yet (Error on chunked request) + # code_411_length_required(dut, port) + + Utility.console_log("### Sessions and Context Tests") + parallel_sessions_adder(dut, port, max_sessions) + leftover_data_test(dut, port) + async_response_test(dut, port) + spillover_session(dut, port, max_sessions) + recv_timeout_test(dut, port) + packet_size_limit_test(dut, port, 50 * 1024) + arbitrary_termination_test(dut, port) + get_hello(dut, port) + + sys.exit() diff --git a/tools/esp_python_packages/tiny_test_fw/Utility/Attenuator.py b/tools/ci/python_packages/idf_iperf_test_util/Attenuator.py similarity index 100% rename from tools/esp_python_packages/tiny_test_fw/Utility/Attenuator.py rename to tools/ci/python_packages/idf_iperf_test_util/Attenuator.py diff --git a/tools/esp_python_packages/tiny_test_fw/Utility/LineChart.py b/tools/ci/python_packages/idf_iperf_test_util/LineChart.py similarity index 100% rename from tools/esp_python_packages/tiny_test_fw/Utility/LineChart.py rename to tools/ci/python_packages/idf_iperf_test_util/LineChart.py diff --git a/tools/esp_python_packages/tiny_test_fw/Utility/PowerControl.py b/tools/ci/python_packages/idf_iperf_test_util/PowerControl.py similarity index 100% rename from tools/esp_python_packages/tiny_test_fw/Utility/PowerControl.py rename to tools/ci/python_packages/idf_iperf_test_util/PowerControl.py diff --git a/examples/wifi/iperf/test_report.py b/tools/ci/python_packages/idf_iperf_test_util/TestReport.py similarity index 100% rename from examples/wifi/iperf/test_report.py rename to tools/ci/python_packages/idf_iperf_test_util/TestReport.py diff --git a/tools/ci/python_packages/idf_iperf_test_util/__init__.py b/tools/ci/python_packages/idf_iperf_test_util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/ci/python_packages/tiny_test_fw/requirements.txt b/tools/ci/python_packages/tiny_test_fw/requirements.txt index aa6b53b4b6..2a742540e2 100644 --- a/tools/ci/python_packages/tiny_test_fw/requirements.txt +++ b/tools/ci/python_packages/tiny_test_fw/requirements.txt @@ -2,4 +2,3 @@ pyserial pyyaml junit_xml netifaces -matplotlib diff --git a/examples/provisioning/softap_prov/utils/wifi_tools.py b/tools/ci/python_packages/wifi_tools.py similarity index 100% rename from examples/provisioning/softap_prov/utils/wifi_tools.py rename to tools/ci/python_packages/wifi_tools.py diff --git a/tools/esp_prov/__init__.py b/tools/esp_prov/__init__.py new file mode 100644 index 0000000000..cbd5e5b60f --- /dev/null +++ b/tools/esp_prov/__init__.py @@ -0,0 +1 @@ +from esp_prov import * # noqa: export esp_prov module to users diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index b30cead1a3..c6ed79b747 100755 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -19,33 +19,12 @@ Test script for unit test case. """ import re -import os -import sys import time import argparse import threading -try: - import TinyFW -except ImportError: - # if we want to run test case outside `tiny-test-fw` folder, - # we need to insert tiny-test-fw path into sys path - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - else: - # or try the copy in IDF - idf_path = os.getenv("IDF_PATH") - tiny_test_path = idf_path + "/tools/tiny-test-fw" - if os.path.exists(tiny_test_path): - sys.path.insert(0, tiny_test_path) - import TinyFW - -import IDF -import Utility -import Env -from DUT import ExpectTimeout -from IDF.IDFApp import UT +from tiny_test_fw import TinyFW, Utility, Env, DUT +import ttfw_idf UT_APP_BOOT_UP_DONE = "Press ENTER to see the list of tests." @@ -180,7 +159,7 @@ def reset_dut(dut): try: dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT) break - except ExpectTimeout: + except DUT.ExpectTimeout: pass else: raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port)) @@ -255,14 +234,14 @@ def run_one_normal_case(dut, one_case, junit_test_case): (FINISH_PATTERN, handle_test_finish), (UT_APP_BOOT_UP_DONE, handle_reset_finish), timeout=one_case["timeout"]) - except ExpectTimeout: + except DUT.ExpectTimeout: Utility.console_log("Timeout in expect", color="orange") junit_test_case.add_failure_info("timeout") one_case_finish(False) break -@IDF.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True) +@ttfw_idf.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True) def run_unit_test_cases(env, extra_data): """ extra_data can be three types of value @@ -401,7 +380,7 @@ class Handler(threading.Thread): time.sleep(1) self.dut.write("\"{}\"".format(self.parent_case_name)) self.dut.expect("Running " + self.parent_case_name + "...") - except ExpectTimeout: + except DUT.ExpectTimeout: Utility.console_log("No case detected!", color="orange") while not self.finish and not self.force_stop.isSet(): try: @@ -411,7 +390,7 @@ class Handler(threading.Thread): (self.SEND_SIGNAL_PATTERN, device_send_action), # send signal pattern (self.FINISH_PATTERN, handle_device_test_finish), # test finish pattern timeout=self.timeout) - except ExpectTimeout: + except DUT.ExpectTimeout: Utility.console_log("Timeout in expect", color="orange") one_device_case_finish(False) break @@ -471,7 +450,7 @@ def run_one_multiple_devices_case(duts, ut_config, env, one_case, app_bin, junit return result -@IDF.idf_unit_test(env_tag="UT_T2_1", junit_report_by_case=True) +@ttfw_idf.idf_unit_test(env_tag="UT_T2_1", junit_report_by_case=True) def run_multiple_devices_cases(env, extra_data): """ extra_data can be two types of value @@ -618,7 +597,7 @@ def run_one_multiple_stage_case(dut, one_case, junit_test_case): (FINISH_PATTERN, handle_test_finish), (UT_APP_BOOT_UP_DONE, handle_next_stage), timeout=one_case["timeout"]) - except ExpectTimeout: + except DUT.ExpectTimeout: Utility.console_log("Timeout in expect", color="orange") one_case_finish(False) break @@ -627,7 +606,7 @@ def run_one_multiple_stage_case(dut, one_case, junit_test_case): break -@IDF.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True) +@ttfw_idf.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True) def run_multiple_stage_cases(env, extra_data): """ extra_data can be 2 types of value @@ -734,7 +713,7 @@ def detect_update_unit_test_info(env, extra_data, app_bin): for _dic in extra_data: if 'type' not in _dic: raise ValueError("Unit test \"{}\" doesn't exist in the flashed device!".format(_dic.get('name'))) - except ExpectTimeout: + except DUT.ExpectTimeout: Utility.console_log("Timeout during getting the test list", color="red") finally: dut.close() @@ -789,8 +768,8 @@ if __name__ == '__main__': TinyFW.set_default_config(env_config_file=args.env_config_file) env_config = TinyFW.get_default_config() - env_config['app'] = UT - env_config['dut'] = IDF.IDFDUT + env_config['app'] = ttfw_idf.UT + env_config['dut'] = ttfw_idf.IDFDUT env_config['test_suite_name'] = 'unit_test_parsing' test_env = Env.Env(**env_config) detect_update_unit_test_info(test_env, extra_data=list_of_dicts, app_bin=args.app_bin) From 89f8e198503e664efd3a49b18d00735d7861622d Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Wed, 27 Nov 2019 11:14:29 +0800 Subject: [PATCH 04/10] CI: support only download artifacts by request: use CI dependencies could waste a lot bandwidth for target test jobs, as example binary artifacts are very large. Now we will parse required artifacts first, then use API to download required files in artifacts. --- tools/ci/config/target-test.yml | 4 - .../ttfw_idf/CIAssignExampleTest.py | 44 ++++- tools/ci/python_packages/ttfw_idf/IDFApp.py | 150 ++++++++++++++---- tools/ci/python_packages/ttfw_idf/__init__.py | 2 +- 4 files changed, 157 insertions(+), 43 deletions(-) diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index baa30916f1..0ec079ee4d 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -21,7 +21,6 @@ - $BOT_LABEL_EXAMPLE_TEST dependencies: - assign_test - - build_examples_cmake_esp32 artifacts: when: always paths: @@ -64,8 +63,6 @@ - $BOT_LABEL_EXAMPLE_TEST dependencies: - assign_test - - build_examples_make - - build_examples_cmake_esp32 artifacts: when: always paths: @@ -282,7 +279,6 @@ example_test_010: example_test_011: extends: .example_debug_template - parallel: 4 tags: - ESP32 - Example_T2_RS485 diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py index 85beb1f086..ed27c58eb3 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py @@ -18,14 +18,23 @@ Command line tool to assign example tests to CI test jobs. # TODO: Need to handle running examples on different chips import os -import sys import re import argparse +import json import gitlab_api from tiny_test_fw.Utility import CIAssignTest +EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"] +IDF_PATH_FROM_ENV = os.getenv("IDF_PATH") +if IDF_PATH_FROM_ENV: + ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV, + "build_examples", "artifact_index.json") +else: + ARTIFACT_INDEX_FILE = "artifact_index.json" + + class ExampleGroup(CIAssignTest.Group): SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"] @@ -34,15 +43,33 @@ class CIExampleAssignTest(CIAssignTest.AssignTest): CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+") -class ArtifactFile(object): - def __init__(self, project_id, job_name, artifact_file_path): - self.gitlab_api = gitlab_api.Gitlab(project_id) +def create_artifact_index_file(project_id=None, pipeline_id=None): + if project_id is None: + project_id = os.getenv("CI_PROJECT_ID") + if pipeline_id is None: + pipeline_id = os.getenv("CI_PIPELINE_ID") + gitlab_inst = gitlab_api.Gitlab(project_id) + artifact_index_list = [] - def process(self): + def format_build_log_path(): + return "build_examples/list_job_{}.json".format(job_info["parallel_num"]) + + for build_job_name in EXAMPLE_BUILD_JOB_NAMES: + job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) + for job_info in job_info_list: + raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0] + build_info_list = [json.loads(line) for line in raw_data.splitlines()] + for build_info in build_info_list: + build_info["ci_job_id"] = job_info["id"] + artifact_index_list.append(build_info) + try: + os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE)) + except OSError: + # already created pass - def output(self): - pass + with open(ARTIFACT_INDEX_FILE, "w") as f: + json.dump(artifact_index_list, f) if __name__ == '__main__': @@ -53,8 +80,11 @@ if __name__ == '__main__': help="gitlab ci config file") parser.add_argument("output_path", help="output path of config files") + parser.add_argument("--pipeline_id", "-p", type=int, default=None, + help="pipeline_id") args = parser.parse_args() assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup) assign_test.assign_cases() assign_test.output_configs(args.output_path) + create_artifact_index_file() diff --git a/tools/ci/python_packages/ttfw_idf/IDFApp.py b/tools/ci/python_packages/ttfw_idf/IDFApp.py index eff8c14584..e90418caa0 100644 --- a/tools/ci/python_packages/ttfw_idf/IDFApp.py +++ b/tools/ci/python_packages/ttfw_idf/IDFApp.py @@ -17,7 +17,102 @@ import subprocess import os import json + from tiny_test_fw import App +from . import CIAssignExampleTest + +try: + import gitlab_api +except ImportError: + gitlab_api = None + + +def parse_flash_settings(path): + file_name = os.path.basename(path) + if file_name == "flasher_args.json": + # CMake version using build metadata file + with open(path, "r") as f: + args = json.load(f) + flash_files = [(offs, binary) for (offs, binary) in args["flash_files"].items() if offs != ""] + flash_settings = args["flash_settings"] + else: + # GNU Make version uses download.config arguments file + with open(path, "r") as f: + args = f.readlines()[-1].split(" ") + flash_files = [] + flash_settings = {} + for idx in range(0, len(args), 2): # process arguments in pairs + if args[idx].startswith("--"): + # strip the -- from the command line argument + flash_settings[args[idx][2:]] = args[idx + 1] + else: + # offs, filename + flash_files.append((args[idx], args[idx + 1])) + return flash_files, flash_settings + + +class Artifacts(object): + def __init__(self, dest_root_path): + assert gitlab_api + self.gitlab_inst = gitlab_api.Gitlab(os.getenv("CI_PROJECT_ID")) + self.dest_root_path = dest_root_path + + @staticmethod + def _find_artifact(artifact_index, app_path, config_name, target): + for artifact_info in artifact_index: + match_result = True + if app_path: + match_result = app_path in artifact_info["app_dir"] + if config_name: + match_result = match_result and config_name == artifact_info["config"] + if target: + match_result = match_result and target == artifact_info["target"] + if match_result: + ret = artifact_info + break + else: + ret = None + return ret + + def download_artifact(self, artifact_index_file, app_path, config_name, target): + # at least one of app_path or config_name is not None. otherwise we can't match artifact + assert app_path or config_name + assert os.path.exists(artifact_index_file) + with open(artifact_index_file, "r") as f: + artifact_index = json.load(f) + + artifact_info = self._find_artifact(artifact_index, app_path, config_name, target) + + if artifact_info: + base_path = os.path.join(artifact_info["work_dir"], artifact_info["build_dir"]) + job_id = artifact_info["ci_job_id"] + + # 1. download flash args file + if artifact_info["build_system"] == "cmake": + flash_arg_file = os.path.join(base_path, "flasher_args.json") + else: + flash_arg_file = os.path.join(base_path, "download.config") + + self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path) + + # 2. download all binary files + flash_files, flash_settings = parse_flash_settings(os.path.join(self.dest_root_path, flash_arg_file)) + artifact_files = [] + for p in flash_files: + artifact_files.append(os.path.join(base_path, p[1])) + if not os.path.dirname(p[1]): + # find app bin and also download elf + elf_file = os.path.splitext(p[1])[0] + ".elf" + artifact_files.append(os.path.join(base_path, elf_file)) + + self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path) + + # 3. download sdkconfig file + self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")], + self.dest_root_path) + else: + base_path = None + return base_path class IDFApp(App.BaseApp): @@ -34,7 +129,7 @@ class IDFApp(App.BaseApp): self.config_name = config_name self.target = target self.idf_path = self.get_sdk_path() - self.binary_path = self.get_binary_path(app_path, config_name) + self.binary_path = self.get_binary_path(app_path, config_name, target) self.elf_file = self._get_elf_file_path(self.binary_path) assert os.path.exists(self.binary_path) sdkconfig_dict = self.get_sdkconfig() @@ -72,7 +167,6 @@ class IDFApp(App.BaseApp): """ reads sdkconfig and returns a dictionary with all configuredvariables - :param sdkconfig_file: location of sdkconfig :raise: AssertionError: if sdkconfig file does not exist in defined paths """ d = {} @@ -89,14 +183,15 @@ class IDFApp(App.BaseApp): d[configs[0]] = configs[1].rstrip() return d - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): """ get binary path according to input app_path. subclass must overwrite this method. :param app_path: path of application - :param config_name: name of the application build config + :param config_name: name of the application build config. Will match any config if None + :param target: target name. Will match for target if None :return: abs app binary path """ pass @@ -123,24 +218,12 @@ class IDFApp(App.BaseApp): if self.IDF_FLASH_ARGS_FILE in os.listdir(self.binary_path): # CMake version using build metadata file - with open(os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE), "r") as f: - args = json.load(f) - flash_files = [(offs,file) for (offs,file) in args["flash_files"].items() if offs != ""] - flash_settings = args["flash_settings"] + path = os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE) else: # GNU Make version uses download.config arguments file - with open(os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE), "r") as f: - args = f.readlines()[-1].split(" ") - flash_files = [] - flash_settings = {} - for idx in range(0, len(args), 2): # process arguments in pairs - if args[idx].startswith("--"): - # strip the -- from the command line argument - flash_settings[args[idx][2:]] = args[idx + 1] - else: - # offs, filename - flash_files.append((args[idx], args[idx + 1])) + path = os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE) + flash_files, flash_settings = parse_flash_settings(path) # The build metadata file does not currently have details, which files should be encrypted and which not. # Assume that all files should be encrypted if flash encryption is enabled in development mode. sdkconfig_dict = self.get_sdkconfig() @@ -149,7 +232,7 @@ class IDFApp(App.BaseApp): # make file offsets into integers, make paths absolute flash_files = [(int(offs, 0), os.path.join(self.binary_path, path.strip())) for (offs, path) in flash_files] - return (flash_files, flash_settings) + return flash_files, flash_settings def _parse_partition_table(self): """ @@ -209,7 +292,7 @@ class Example(IDFApp): """ return [os.path.join(self.binary_path, "..", "sdkconfig")] - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): # build folder of example path path = os.path.join(self.idf_path, app_path, "build") if os.path.exists(path): @@ -227,18 +310,23 @@ class Example(IDFApp): for dirpath in os.listdir(example_path): if os.path.basename(dirpath) == app_path_underscored: path = os.path.join(example_path, dirpath, config_name, self.target, "build") - return path + if os.path.exists(path): + return path + else: + # app path exists, but config name not exists. try to download artifacts. + break - raise OSError("Failed to find example binary") + artifacts = Artifacts(self.idf_path) + path = artifacts.download_artifact(CIAssignExampleTest.ARTIFACT_INDEX_FILE, + app_path, config_name, target) + if path: + return os.path.join(self.idf_path, path) + else: + raise OSError("Failed to find example binary") class UT(IDFApp): - def get_binary_path(self, app_path, config_name=None): - """ - :param app_path: app path - :param config_name: config name - :return: binary path - """ + def get_binary_path(self, app_path, config_name=None, target=None): if not config_name: config_name = "default" @@ -262,12 +350,12 @@ class UT(IDFApp): class SSC(IDFApp): - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): # TODO: to implement SSC get binary path return app_path class AT(IDFApp): - def get_binary_path(self, app_path, config_name=None): + def get_binary_path(self, app_path, config_name=None, target=None): # TODO: to implement AT get binary path return app_path diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py index d7aed38571..4262a26434 100644 --- a/tools/ci/python_packages/ttfw_idf/__init__.py +++ b/tools/ci/python_packages/ttfw_idf/__init__.py @@ -16,7 +16,7 @@ import re from tiny_test_fw import TinyFW, Utility from IDFApp import IDFApp, Example, UT -from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT +from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT # noqa: export DUTs for users def format_case_id(chip, case_name): From 5d1be3bb8bd6832fb4126f0ad73ed35b1782358a Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Wed, 27 Nov 2019 20:30:49 +0800 Subject: [PATCH 05/10] CI: raise import error instead of exit for ble tools --- tools/ble/lib_ble_client.py | 2 +- tools/ble/lib_gap.py | 2 +- tools/ble/lib_gatt.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/ble/lib_ble_client.py b/tools/ble/lib_ble_client.py index 6fda14288c..a17cf0a3fc 100644 --- a/tools/ble/lib_ble_client.py +++ b/tools/ble/lib_ble_client.py @@ -29,7 +29,7 @@ try: from gi.repository import GLib except ImportError as e: if 'linux' not in sys.platform: - sys.exit("Error: Only supported on Linux platform") + raise e print(e) print("Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue") print("Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue") diff --git a/tools/ble/lib_gap.py b/tools/ble/lib_gap.py index 2033dc4ab8..02466c7aa1 100644 --- a/tools/ble/lib_gap.py +++ b/tools/ble/lib_gap.py @@ -25,7 +25,7 @@ try: import dbus.service except ImportError as e: if 'linux' not in sys.platform: - sys.exit("Error: Only supported on Linux platform") + raise e print(e) print("Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue") print("Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue") diff --git a/tools/ble/lib_gatt.py b/tools/ble/lib_gatt.py index c666d8e7e8..4710e1549c 100644 --- a/tools/ble/lib_gatt.py +++ b/tools/ble/lib_gatt.py @@ -25,7 +25,7 @@ try: import dbus.service except ImportError as e: if 'linux' not in sys.platform: - sys.exit("Error: Only supported on Linux platform") + raise e print(e) print("Install packages `libgirepository1.0-dev gir1.2-gtk-3.0 libcairo2-dev libdbus-1-dev libdbus-glib-1-dev` for resolving the issue") print("Run `pip install -r $IDF_PATH/tools/ble/requirements.txt` for resolving the issue") From b4863551a9a66b7aa35ae62359538ed2b25b7a84 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Thu, 28 Nov 2019 17:08:25 +0800 Subject: [PATCH 06/10] CI: modify fetch submodule method: download archive for submodules instead of clone --- .gitlab-ci.yml | 20 +++--- tools/ci/ci_fetch_submodule.py | 106 ++++++++++++++++++++++++++++++++ tools/ci/config/assign-test.yml | 2 + tools/ci/config/post_check.yml | 2 +- tools/ci/config/target-test.yml | 9 +-- 5 files changed, 124 insertions(+), 15 deletions(-) create mode 100644 tools/ci/ci_fetch_submodule.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3897135ce8..2b5e8cf790 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,9 +23,14 @@ variables: # GIT_STRATEGY is not defined here. # Use an option from "CI / CD Settings" - "General pipelines". - # "normal" strategy for fetching only top-level submodules since nothing requires the sub-submodules code for building IDF. - # If the "recursive" strategy is used we have a problem with using relative URLs for sub-submodules. - GIT_SUBMODULE_STRATEGY: normal + # we will download archive for each submodule instead of clone. + # we don't do "recursive" when fetch submodule as they're not used in CI now. + GIT_SUBMODULE_STRATEGY: none + SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py" + # by default we will fetch all submodules + # jobs can overwrite this variable to only fetch submodules they required + # set to "none" if don't need to fetch submodules + SUBMODULES_TO_FETCH: "all" UNIT_TEST_BUILD_SYSTEM: cmake EXAMPLE_TEST_BUILD_SYSTEM: cmake @@ -44,6 +49,7 @@ variables: CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml" + # before each job, we need to check if this job is filtered by bot stage/job filter .apply_bot_filter: &apply_bot_filter python $APPLY_BOT_FILTER_SCRIPT || exit 0 @@ -72,12 +78,10 @@ variables: tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1 fi -.show_submodule_urls: &show_submodule_urls | - git config --get-regexp '^submodule\..*\.url$' || true +.fetch_submodules: &fetch_submodules | + python $SUBMODULE_FETCH_TOOL -s $SUBMODULES_TO_FETCH before_script: - - echo "Running common script" - - *show_submodule_urls - source tools/ci/setup_python.sh # apply bot filter in before script - *apply_bot_filter @@ -93,6 +97,8 @@ before_script: - *setup_tools_unless_target_test + - *fetch_submodules + - *setup_custom_toolchain # used for check scripts which we want to run unconditionally diff --git a/tools/ci/ci_fetch_submodule.py b/tools/ci/ci_fetch_submodule.py new file mode 100644 index 0000000000..89dce82ea6 --- /dev/null +++ b/tools/ci/ci_fetch_submodule.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python + +# internal use only for CI +# download archive of one commit instead of cloning entire submodule repo + +import re +import os +import subprocess +import argparse +import shutil +import time + +import gitlab_api + +SUBMODULE_PATTERN = re.compile(r"\[submodule \"([^\"]+)\"]") +PATH_PATTERN = re.compile(r"path\s+=\s+(\S+)") +URL_PATTERN = re.compile(r"url\s+=\s+(\S+)") + +SUBMODULE_ARCHIVE_TEMP_FOLDER = "submodule_archive" + + +class SubModule(object): + # We don't need to support recursive submodule clone now + + GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r"\d+\s+commit\s+([0-9a-f]+)\s+") + + def __init__(self, gitlab_inst, path, url): + self.path = path + self.gitlab_inst = gitlab_inst + self.project_id = self._get_project_id(url) + self.commit_id = self._get_commit_id(path) + + def _get_commit_id(self, path): + output = subprocess.check_output(["git", "ls-tree", "HEAD", path]) + # example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0 components/esp_wifi/lib + match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output) + return match.group(1) + + def _get_project_id(self, url): + base_name = os.path.basename(url) + project_id = self.gitlab_inst.get_project_id(os.path.splitext(base_name)[0], # remove .git + namespace="espressif") + return project_id + + def download_archive(self): + print("Update submodule: {}: {}".format(self.path, self.commit_id)) + path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER, + self.project_id) + renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path)) + os.rename(path_name, renamed_path) + shutil.rmtree(self.path, ignore_errors=True) + shutil.move(renamed_path, os.path.dirname(self.path)) + + +def update_submodule(git_module_file, submodules_to_update): + gitlab_inst = gitlab_api.Gitlab() + submodules = [] + with open(git_module_file, "r") as f: + data = f.read() + match = SUBMODULE_PATTERN.search(data) + while True: + next_match = SUBMODULE_PATTERN.search(data, pos=match.end()) + if next_match: + end_pos = next_match.start() + else: + end_pos = len(data) + path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos) + url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos) + path = path_match.group(1) + url = url_match.group(1) + + filter_result = True + if submodules_to_update: + if path not in submodules_to_update: + filter_result = False + if filter_result: + submodules.append(SubModule(gitlab_inst, path, url)) + + match = next_match + if not match: + break + + shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True) + + for submodule in submodules: + submodule.download_archive() + + +if __name__ == '__main__': + start_time = time.time() + parser = argparse.ArgumentParser() + parser.add_argument("--repo_path", "-p", default=".", help="repo path") + parser.add_argument("--submodule", "-s", default="all", + help="Submodules to update. By default update all submodules. " + "For multiple submodules, separate them with `;`. " + "`all` and `none` are special values that indicates we fetch all / none submodules") + args = parser.parse_args() + if args.submodule == "none": + print("don't need to update submodules") + exit(0) + if args.submodule == "all": + _submodules = [] + else: + _submodules = args.submodule.split(";") + update_submodule(os.path.join(args.repo_path, ".gitmodules"), _submodules) + print("total time spent on update submodule: {:.02f}s".format(time.time() - start_time)) diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index 9846b8d00e..e997062e36 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -10,6 +10,7 @@ assign_test: - build_ssc - build_esp_idf_tests_cmake variables: + SUBMODULES_TO_FETCH: "components/esptool_py/esptool" EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs" UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml" artifacts: @@ -55,6 +56,7 @@ update_test_cases: - ${CI_PROJECT_DIR}/test-management/*.log expire_in: 1 week variables: + SUBMODULES_TO_FETCH: "components/esptool_py/esptool" UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml" BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml" AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script" diff --git a/tools/ci/config/post_check.yml b/tools/ci/config/post_check.yml index 2f9dacc125..d3d577892d 100644 --- a/tools/ci/config/post_check.yml +++ b/tools/ci/config/post_check.yml @@ -10,7 +10,7 @@ check_submodule_sync: retry: 2 variables: GIT_STRATEGY: clone - GIT_SUBMODULE_STRATEGY: none + SUBMODULES_TO_FETCH: "none" PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" before_script: [] after_script: [] diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 0ec079ee4d..63bffea83f 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -33,7 +33,7 @@ CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml" - GIT_SUBMODULE_STRATEGY: none + SUBMODULES_TO_FETCH: "components/esptool_py/esptool" script: - *define_config_file_name # first test if config file exists, if not exist, exit 0 @@ -41,9 +41,6 @@ # clone test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # Get esptool - - git submodule sync - - git submodule update --init components/esptool_py/esptool - cd tools/ci/python_packages/tiny_test_fw/bin # run test - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE @@ -134,7 +131,7 @@ - $LOG_PATH expire_in: 1 week variables: - GIT_SUBMODULE_STRATEGY: none + SUBMODULES_TO_FETCH: "components/esptool_py/esptool" LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test" @@ -195,8 +192,6 @@ example_test_001: example_test_002: extends: .example_test_template image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG - variables: - GIT_SUBMODULE_STRATEGY: normal tags: - ESP32 - Example_ShieldBox_Basic From a11b15b92b1bc7386bbeb904e8ccebad3deab860 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Sat, 30 Nov 2019 13:36:31 +0800 Subject: [PATCH 07/10] CI: build system do not check submodule for CI --- .gitlab-ci.yml | 2 ++ make/project.mk | 13 +++++++++---- tools/cmake/git_submodules.cmake | 8 ++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b5e8cf790..d20088dad1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,8 @@ variables: # jobs can overwrite this variable to only fetch submodules they required # set to "none" if don't need to fetch submodules SUBMODULES_TO_FETCH: "all" + # tell build system do not check submodule update as we download archive instead of clone + IDF_SKIP_CHECK_SUBMODULES: 1 UNIT_TEST_BUILD_SYSTEM: cmake EXAMPLE_TEST_BUILD_SYSTEM: cmake diff --git a/make/project.mk b/make/project.mk index 796f2e39cc..af30fe6097 100644 --- a/make/project.mk +++ b/make/project.mk @@ -144,7 +144,7 @@ EXTRA_COMPONENT_DIRS ?= COMPONENT_DIRS := $(PROJECT_PATH)/components $(EXTRA_COMPONENT_DIRS) $(IDF_PATH)/components $(PROJECT_PATH)/main endif # Make sure that every directory in the list is an absolute path without trailing slash. -# This is necessary to split COMPONENT_DIRS into SINGLE_COMPONENT_DIRS and MULTI_COMPONENT_DIRS below. +# This is necessary to split COMPONENT_DIRS into SINGLE_COMPONENT_DIRS and MULTI_COMPONENT_DIRS below. COMPONENT_DIRS := $(foreach cd,$(COMPONENT_DIRS),$(abspath $(cd))) export COMPONENT_DIRS @@ -153,11 +153,11 @@ $(warning SRCDIRS variable is deprecated. These paths can be added to EXTRA_COMP COMPONENT_DIRS += $(abspath $(SRCDIRS)) endif -# List of component directories, i.e. directories which contain a component.mk file +# List of component directories, i.e. directories which contain a component.mk file SINGLE_COMPONENT_DIRS := $(abspath $(dir $(dir $(foreach cd,$(COMPONENT_DIRS),\ $(wildcard $(cd)/component.mk))))) -# List of components directories, i.e. directories which may contain components +# List of components directories, i.e. directories which may contain components MULTI_COMPONENT_DIRS := $(filter-out $(SINGLE_COMPONENT_DIRS),$(COMPONENT_DIRS)) # The project Makefile can define a list of components, but if it does not do this @@ -635,6 +635,11 @@ clean: app-clean bootloader-clean config-clean ldgen-clean # # This only works for components inside IDF_PATH check-submodules: +# for internal use: +# skip submodule check if running on Gitlab CI and job is configured as not clone submodules +ifeq ($(IDF_SKIP_CHECK_SUBMODULES),1) + @echo "skip submodule check on internal CI" +else # Check if .gitmodules exists, otherwise skip submodule check, assuming flattened structure ifneq ("$(wildcard ${IDF_PATH}/.gitmodules)","") @@ -662,7 +667,7 @@ endef # so the argument is suitable for use with 'git submodule' commands $(foreach submodule,$(subst $(IDF_PATH)/,,$(filter $(IDF_PATH)/%,$(COMPONENT_SUBMODULES))),$(eval $(call GenerateSubmoduleCheckTarget,$(submodule)))) endif # End check for .gitmodules existence - +endif # PHONY target to list components in the build and their paths list-components: diff --git a/tools/cmake/git_submodules.cmake b/tools/cmake/git_submodules.cmake index 09098b24af..e36ee919f7 100644 --- a/tools/cmake/git_submodules.cmake +++ b/tools/cmake/git_submodules.cmake @@ -11,6 +11,14 @@ if(NOT GIT_FOUND) else() function(git_submodule_check root_path) + # for internal use: + # skip submodule check if running on Gitlab CI and job is configured as not clone submodules + if($ENV{IDF_SKIP_CHECK_SUBMODULES}) + if($ENV{IDF_SKIP_CHECK_SUBMODULES} EQUAL 1) + message("skip submodule check on internal CI") + return() + endif() + endif() execute_process( COMMAND ${GIT_EXECUTABLE} submodule status From cca08b3d2bf9842aeae0bdad817f41e3db9cfc33 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Sun, 1 Dec 2019 08:39:53 +0800 Subject: [PATCH 08/10] CI: remove not exist file from executable-list.txt --- tools/ci/executable-list.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 53795a4899..c1ced73f5c 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -6,7 +6,6 @@ components/espcoredump/espcoredump.py components/espcoredump/test/test_espcoredump.py components/espcoredump/test/test_espcoredump.sh components/heap/test_multi_heap_host/test_all_configs.sh -components/idf_test/unit_test/TestCaseScript/IDFUnitTest/__init__.py components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py components/partition_table/gen_empty_partition.py components/partition_table/gen_esp32part.py From 164e8ba21fcbdb570fed185c1ce538ba2e9f2de0 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Fri, 6 Dec 2019 17:14:15 +0800 Subject: [PATCH 09/10] test: support loadable elf test cases: loadable elf example is different for 2 reasons: 1. loadable elf binary don't have flasher_args.json, so we can't use normal to get from CI artifacts 2. it don't have binary and don't need to downloaded binary to DUT. Some related functions can be ignored --- .../hello_world/loadable_elf_example_test.py | 7 +- tools/ci/python_packages/ttfw_idf/IDFApp.py | 148 ++++++++++++------ tools/ci/python_packages/ttfw_idf/__init__.py | 2 +- 3 files changed, 104 insertions(+), 53 deletions(-) diff --git a/examples/get-started/hello_world/loadable_elf_example_test.py b/examples/get-started/hello_world/loadable_elf_example_test.py index 3afaf270a2..bd1cbe05b4 100644 --- a/examples/get-started/hello_world/loadable_elf_example_test.py +++ b/examples/get-started/hello_world/loadable_elf_example_test.py @@ -121,12 +121,13 @@ class SerialThread(object): @ttfw_idf.idf_example_test(env_tag="test_jtag_arm") def test_examples_loadable_elf(env, extra_data): - idf_path = os.environ['IDF_PATH'] rel_project_path = os.path.join('examples', 'get-started', 'hello_world') + app_files = ['hello-world.elf', 'partition_table/partition-table.bin'] + example = ttfw_idf.LoadableElfExample(rel_project_path, app_files, target="esp32") + idf_path = example.get_sdk_path() proj_path = os.path.join(idf_path, rel_project_path) - example = ttfw_idf.Example(rel_project_path, target="esp32") sdkconfig = example.get_sdkconfig() - elf_path = os.path.join(example.get_binary_path(rel_project_path), 'hello-world.elf') + elf_path = os.path.join(example.binary_path, 'hello-world.elf') esp_log_path = os.path.join(proj_path, 'esp.log') assert(sdkconfig['CONFIG_IDF_TARGET_ESP32'] == 'y'), "Only ESP32 target is supported" diff --git a/tools/ci/python_packages/ttfw_idf/IDFApp.py b/tools/ci/python_packages/ttfw_idf/IDFApp.py index e90418caa0..295577670f 100644 --- a/tools/ci/python_packages/ttfw_idf/IDFApp.py +++ b/tools/ci/python_packages/ttfw_idf/IDFApp.py @@ -33,8 +33,9 @@ def parse_flash_settings(path): # CMake version using build metadata file with open(path, "r") as f: args = json.load(f) - flash_files = [(offs, binary) for (offs, binary) in args["flash_files"].items() if offs != ""] - flash_settings = args["flash_settings"] + flash_files = [(offs, binary) for (offs, binary) in args["flash_files"].items() if offs != ""] + flash_settings = args["flash_settings"] + app_name = os.path.splitext(args["app"]["file"])[0] else: # GNU Make version uses download.config arguments file with open(path, "r") as f: @@ -48,14 +49,28 @@ def parse_flash_settings(path): else: # offs, filename flash_files.append((args[idx], args[idx + 1])) - return flash_files, flash_settings + # we can only guess app name in download.config. + for p in flash_files: + if not os.path.dirname(p[1]) and "partition" not in p[1]: + # app bin usually in the same dir with download.config and it's not partition table + app_name = os.path.splitext(p[1])[0] + break + else: + app_name = None + return flash_files, flash_settings, app_name class Artifacts(object): - def __init__(self, dest_root_path): + def __init__(self, dest_root_path, artifact_index_file, app_path, config_name, target): assert gitlab_api + # at least one of app_path or config_name is not None. otherwise we can't match artifact + assert app_path or config_name + assert os.path.exists(artifact_index_file) self.gitlab_inst = gitlab_api.Gitlab(os.getenv("CI_PROJECT_ID")) self.dest_root_path = dest_root_path + with open(artifact_index_file, "r") as f: + artifact_index = json.load(f) + self.artifact_info = self._find_artifact(artifact_index, app_path, config_name, target) @staticmethod def _find_artifact(artifact_index, app_path, config_name, target): @@ -74,21 +89,13 @@ class Artifacts(object): ret = None return ret - def download_artifact(self, artifact_index_file, app_path, config_name, target): - # at least one of app_path or config_name is not None. otherwise we can't match artifact - assert app_path or config_name - assert os.path.exists(artifact_index_file) - with open(artifact_index_file, "r") as f: - artifact_index = json.load(f) - - artifact_info = self._find_artifact(artifact_index, app_path, config_name, target) - - if artifact_info: - base_path = os.path.join(artifact_info["work_dir"], artifact_info["build_dir"]) - job_id = artifact_info["ci_job_id"] + def download_artifacts(self): + if self.artifact_info: + base_path = os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"]) + job_id = self.artifact_info["ci_job_id"] # 1. download flash args file - if artifact_info["build_system"] == "cmake": + if self.artifact_info["build_system"] == "cmake": flash_arg_file = os.path.join(base_path, "flasher_args.json") else: flash_arg_file = os.path.join(base_path, "download.config") @@ -96,14 +103,10 @@ class Artifacts(object): self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path) # 2. download all binary files - flash_files, flash_settings = parse_flash_settings(os.path.join(self.dest_root_path, flash_arg_file)) - artifact_files = [] - for p in flash_files: - artifact_files.append(os.path.join(base_path, p[1])) - if not os.path.dirname(p[1]): - # find app bin and also download elf - elf_file = os.path.splitext(p[1])[0] + ".elf" - artifact_files.append(os.path.join(base_path, elf_file)) + flash_files, flash_settings, app_name = parse_flash_settings(os.path.join(self.dest_root_path, + flash_arg_file)) + artifact_files = [os.path.join(base_path, p[1]) for p in flash_files] + artifact_files.append(os.path.join(base_path, app_name + ".elf")) self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path) @@ -114,6 +117,22 @@ class Artifacts(object): base_path = None return base_path + def download_artifact_files(self, file_names): + if self.artifact_info: + base_path = os.path.join(self.artifact_info["work_dir"], self.artifact_info["build_dir"]) + job_id = self.artifact_info["ci_job_id"] + + # download all binary files + artifact_files = [os.path.join(base_path, fn) for fn in file_names] + self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path) + + # download sdkconfig file + self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")], + self.dest_root_path) + else: + base_path = None + return base_path + class IDFApp(App.BaseApp): """ @@ -132,24 +151,22 @@ class IDFApp(App.BaseApp): self.binary_path = self.get_binary_path(app_path, config_name, target) self.elf_file = self._get_elf_file_path(self.binary_path) assert os.path.exists(self.binary_path) - sdkconfig_dict = self.get_sdkconfig() - if "CONFIG_APP_BUILD_GENERATE_BINARIES" in sdkconfig_dict: - # There are no flashing targets available when no binaries where generated. - if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path): - if self.IDF_FLASH_ARGS_FILE not in os.listdir(self.binary_path): - msg = ("Neither {} nor {} exists. " - "Try to run 'make print_flash_cmd | tail -n 1 > {}/{}' " - "or 'idf.py build' " - "for resolving the issue." - "").format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE, - self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE) - raise AssertionError(msg) + if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path): + if self.IDF_FLASH_ARGS_FILE not in os.listdir(self.binary_path): + msg = ("Neither {} nor {} exists. " + "Try to run 'make print_flash_cmd | tail -n 1 > {}/{}' " + "or 'idf.py build' " + "for resolving the issue." + "").format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE, + self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE) + raise AssertionError(msg) - self.flash_files, self.flash_settings = self._parse_flash_download_config() - self.partition_table = self._parse_partition_table() + self.flash_files, self.flash_settings = self._parse_flash_download_config() + self.partition_table = self._parse_partition_table() @classmethod def get_sdk_path(cls): + # type: () -> str idf_path = os.getenv("IDF_PATH") assert idf_path assert os.path.exists(idf_path) @@ -184,6 +201,7 @@ class IDFApp(App.BaseApp): return d def get_binary_path(self, app_path, config_name=None, target=None): + # type: (str, str, str) -> str """ get binary path according to input app_path. @@ -223,7 +241,7 @@ class IDFApp(App.BaseApp): # GNU Make version uses download.config arguments file path = os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE) - flash_files, flash_settings = parse_flash_settings(path) + flash_files, flash_settings, app_name = parse_flash_settings(path) # The build metadata file does not currently have details, which files should be encrypted and which not. # Assume that all files should be encrypted if flash encryption is enabled in development mode. sdkconfig_dict = self.get_sdkconfig() @@ -292,7 +310,7 @@ class Example(IDFApp): """ return [os.path.join(self.binary_path, "..", "sdkconfig")] - def get_binary_path(self, app_path, config_name=None, target=None): + def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None): # build folder of example path path = os.path.join(self.idf_path, app_path, "build") if os.path.exists(path): @@ -309,20 +327,52 @@ class Example(IDFApp): example_path = os.path.join(self.idf_path, "build_examples") for dirpath in os.listdir(example_path): if os.path.basename(dirpath) == app_path_underscored: - path = os.path.join(example_path, dirpath, config_name, self.target, "build") + path = os.path.join(example_path, dirpath, config_name, target, "build") if os.path.exists(path): return path else: - # app path exists, but config name not exists. try to download artifacts. - break + return None - artifacts = Artifacts(self.idf_path) - path = artifacts.download_artifact(CIAssignExampleTest.ARTIFACT_INDEX_FILE, - app_path, config_name, target) + def get_binary_path(self, app_path, config_name=None, target=None): + path = self._try_get_binary_from_local_fs(app_path, config_name, target) if path: - return os.path.join(self.idf_path, path) + return path else: - raise OSError("Failed to find example binary") + artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE, + app_path, config_name, target) + path = artifacts.download_artifacts() + if path: + return os.path.join(self.idf_path, path) + else: + raise OSError("Failed to find example binary") + + +class LoadableElfExample(Example): + def __init__(self, app_path, app_files, config_name=None, target=None): + # add arg `app_files` for loadable elf example. + # Such examples only build elf files, so it doesn't generate flasher_args.json. + # So we can't get app files from config file. Test case should pass it to application. + super(IDFApp, self).__init__(app_path) + self.app_files = app_files + self.config_name = config_name + self.target = target + self.idf_path = self.get_sdk_path() + self.binary_path = self.get_binary_path(app_path, config_name, target) + self.elf_file = self._get_elf_file_path(self.binary_path) + assert os.path.exists(self.binary_path) + + def get_binary_path(self, app_path, config_name=None, target=None): + path = self._try_get_binary_from_local_fs(app_path, config_name, target) + if path: + return path + else: + artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE, + app_path, config_name, target) + path = artifacts.download_artifact_files(self.app_files) + if path: + return os.path.join(self.idf_path, path) + else: + raise OSError("Failed to find example binary") class UT(IDFApp): diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py index 4262a26434..8c72047d5f 100644 --- a/tools/ci/python_packages/ttfw_idf/__init__.py +++ b/tools/ci/python_packages/ttfw_idf/__init__.py @@ -15,7 +15,7 @@ import os import re from tiny_test_fw import TinyFW, Utility -from IDFApp import IDFApp, Example, UT +from IDFApp import IDFApp, Example, LoadableElfExample, UT # noqa: export all Apps for users from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT # noqa: export DUTs for users From ca0ce7171db88373a2dc298e6923039e513c74a5 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Sat, 7 Dec 2019 10:26:51 +0800 Subject: [PATCH 10/10] CI: reduce number of test jobs --- tools/ci/config/target-test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 63bffea83f..82657c6389 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -303,7 +303,7 @@ UT_001: UT_002: extends: .unit_test_template - parallel: 11 + parallel: 10 tags: - ESP32_IDF - UT_T1_1 @@ -453,7 +453,7 @@ UT_034: UT_035: extends: .unit_test_template - parallel: 20 + parallel: 19 tags: - ESP32S2BETA_IDF - UT_T1_1 @@ -487,7 +487,6 @@ UT_043: UT_044: extends: .unit_test_template - parallel: 4 tags: - ESP32_IDF - UT_SDIO