From f8bd9dbe3e2619fd889f5d6c9e85c5b959c51125 Mon Sep 17 00:00:00 2001 From: Pascal Vizeli Date: Sun, 4 Aug 2019 14:32:35 +0200 Subject: [PATCH] Azure ci templates (#25688) * Use templates for Azure-CI steps * Migrate ci * fix endpoint --- azure-pipelines-ci.yml | 85 ++++++++----------------- azure-pipelines-wheels.yml | 127 +++++++++++++------------------------ 2 files changed, 70 insertions(+), 142 deletions(-) diff --git a/azure-pipelines-ci.yml b/azure-pipelines-ci.yml index 1e3d914aaf0..5297fd80231 100644 --- a/azure-pipelines-ci.yml +++ b/azure-pipelines-ci.yml @@ -18,9 +18,12 @@ resources: image: homeassistant/ci-azure:3.6 - container: 37 image: homeassistant/ci-azure:3.7 + repositories: + - repository: azure + type: github + name: 'home-assistant/ci-azure' + endpoint: 'home-assistant' variables: - - name: ArtifactFeed - value: '2df3ae11-3bf6-49bc-a809-ba0d340d6a6d' - name: PythonMain value: '36' - group: codecov @@ -95,39 +98,19 @@ stages: python.container: '37' container: $[ variables['python.container'] ] steps: - - script: | - python --version > .cache - displayName: 'Set python $(python.container) for requirement cache' - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 - displayName: 'Restore artifacts based on Requirements' - inputs: + - template: templates/azp-step-cache.yaml@azure + parameters: keyfile: 'requirements_test_all.txt, .cache, homeassistant/package_constraints.txt' - targetfolder: './venv' - vstsFeed: '$(ArtifactFeed)' - - script: | - set -e - python -m venv venv + build: | + set -e + python -m venv venv - . venv/bin/activate - pip install -U pip setuptools pytest-azurepipelines -c homeassistant/package_constraints.txt - pip install -r requirements_test_all.txt -c homeassistant/package_constraints.txt - # This is a TEMP. Eventually we should make sure our 4 dependencies drop typing. - # Find offending deps with `pipdeptree -r -p typing` - pip uninstall -y typing - displayName: 'Create Virtual Environment & Install Requirements' - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) - # Explicit Cache Save (instead of using RestoreAndSaveCache) - # Dont wait with cache save for all the other task in this job to complete (±30 minutes), other parallel jobs might utilize this - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - displayName: 'Save artifacts based on Requirements' - inputs: - keyfile: 'requirements_test_all.txt, .cache, homeassistant/package_constraints.txt' - targetfolder: './venv' - vstsFeed: '$(ArtifactFeed)' - - script: | - . venv/bin/activate - pip install -e . - displayName: 'Install Home Assistant for python $(python.container)' + . venv/bin/activate + pip install -U pip setuptools pytest-azurepipelines -c homeassistant/package_constraints.txt + pip install -r requirements_test_all.txt -c homeassistant/package_constraints.txt + # This is a TEMP. Eventually we should make sure our 4 dependencies drop typing. + # Find offending deps with `pipdeptree -r -p typing` + pip uninstall -y typing - script: | . venv/bin/activate pytest --timeout=9 --durations=10 --junitxml=test-results.xml -qq -o console_output_style=count -p no:sugar tests @@ -162,35 +145,17 @@ stages: vmImage: 'ubuntu-latest' container: $[ variables['PythonMain'] ] steps: - - script: | - python --version > .cache - displayName: 'Set python $(PythonMain) for requirement cache' - - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 - displayName: 'Restore artifacts based on Requirements' - inputs: + - template: templates/azp-step-cache.yaml@azure + parameters: keyfile: 'requirements_all.txt, requirements_test.txt, .cache, homeassistant/package_constraints.txt' - targetfolder: './venv' - vstsFeed: '$(ArtifactFeed)' - - script: | - set -e - python -m venv venv + build: | + set -e + python -m venv venv - . venv/bin/activate - pip install -U pip setuptools - pip install -r requirements_all.txt -c homeassistant/package_constraints.txt - pip install -r requirements_test.txt -c homeassistant/package_constraints.txt - displayName: 'Create Virtual Environment & Install Requirements' - condition: and(succeeded(), ne(variables['CacheRestored'], 'true')) - - task: 1ESLighthouseEng.PipelineArtifactCaching.SaveCacheV1.SaveCache@1 - displayName: 'Save artifacts based on Requirements' - inputs: - keyfile: 'requirements_all.txt, requirements_test.txt, .cache, homeassistant/package_constraints.txt' - targetfolder: './venv' - vstsFeed: '$(ArtifactFeed)' - - script: | - . venv/bin/activate - pip install -e . - displayName: 'Install Home Assistant for python $(PythonMain)' + . venv/bin/activate + pip install -U pip setuptools + pip install -r requirements_all.txt -c homeassistant/package_constraints.txt + pip install -r requirements_test.txt -c homeassistant/package_constraints.txt - script: | . venv/bin/activate pylint homeassistant diff --git a/azure-pipelines-wheels.yml b/azure-pipelines-wheels.yml index 89e45fc31da..7e1c9991d57 100644 --- a/azure-pipelines-wheels.yml +++ b/azure-pipelines-wheels.yml @@ -12,88 +12,51 @@ pr: none variables: - name: versionWheels value: '1.0-3.7-alpine3.10' - - group: wheels - +resources: + repositories: + - repository: azure + type: github + name: 'home-assistant/ci-azure' + endpoint: 'home-assistant' jobs: +- template: templates/azp-job-wheels.yaml@azure + parameters: + builderVersion: '$(versionWheels)' + builderApk: 'build-base;cmake;git;linux-headers;bluez-dev;libffi-dev;openssl-dev;glib-dev;eudev-dev;libxml2-dev;libxslt-dev;libpng-dev;libjpeg-turbo-dev;tiff-dev;autoconf;automake;cups-dev;linux-headers;gmp-dev;mpfr-dev;mpc1-dev;ffmpeg-dev' + wheelsRequirement: 'requirements_wheels.txt' + wheelsRequirementDiff: 'requirements_diff.txt' + preBuild: + - script: | + cp requirements_all.txt requirements_wheels.txt + if [[ "$(Build.Reason)" =~ (Schedule|Manual) ]]; then + touch requirements_diff.txt + else + curl -s -o requirements_diff.txt https://raw.githubusercontent.com/home-assistant/home-assistant/master/requirements_all.txt + fi -- job: 'Wheels' - timeoutInMinutes: 360 - pool: - vmImage: 'ubuntu-latest' - strategy: - maxParallel: 5 - matrix: - amd64: - buildArch: 'amd64' - i386: - buildArch: 'i386' - armhf: - buildArch: 'armhf' - armv7: - buildArch: 'armv7' - aarch64: - buildArch: 'aarch64' - steps: - - script: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - qemu-user-static \ - binfmt-support \ - curl - - sudo mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc - sudo update-binfmts --enable qemu-arm - sudo update-binfmts --enable qemu-aarch64 - displayName: 'Initial cross build' - - script: | - mkdir -p .ssh - echo -e "-----BEGIN RSA PRIVATE KEY-----\n$(wheelsSSH)\n-----END RSA PRIVATE KEY-----" >> .ssh/id_rsa - ssh-keyscan -H $(wheelsHost) >> .ssh/known_hosts - chmod 600 .ssh/* - displayName: 'Install ssh key' - - script: sudo docker pull homeassistant/$(buildArch)-wheels:$(versionWheels) - displayName: 'Install wheels builder' - - script: | - cp requirements_all.txt requirements_wheels.txt - if [[ "$(Build.Reason)" =~ (Schedule|Manual) ]]; then - touch requirements_diff.txt - else - curl -s -o requirements_diff.txt https://raw.githubusercontent.com/home-assistant/home-assistant/master/requirements_all.txt - fi - - requirement_files="requirements_wheels.txt requirements_diff.txt" - for requirement_file in ${requirement_files}; do - sed -i "s|# pytradfri|pytradfri|g" ${requirement_file} - sed -i "s|# pybluez|pybluez|g" ${requirement_file} - sed -i "s|# bluepy|bluepy|g" ${requirement_file} - sed -i "s|# beacontools|beacontools|g" ${requirement_file} - sed -i "s|# RPi.GPIO|RPi.GPIO|g" ${requirement_file} - sed -i "s|# raspihats|raspihats|g" ${requirement_file} - sed -i "s|# rpi-rf|rpi-rf|g" ${requirement_file} - sed -i "s|# blinkt|blinkt|g" ${requirement_file} - sed -i "s|# fritzconnection|fritzconnection|g" ${requirement_file} - sed -i "s|# pyuserinput|pyuserinput|g" ${requirement_file} - sed -i "s|# evdev|evdev|g" ${requirement_file} - sed -i "s|# smbus-cffi|smbus-cffi|g" ${requirement_file} - sed -i "s|# i2csense|i2csense|g" ${requirement_file} - sed -i "s|# python-eq3bt|python-eq3bt|g" ${requirement_file} - sed -i "s|# pycups|pycups|g" ${requirement_file} - sed -i "s|# homekit|homekit|g" ${requirement_file} - sed -i "s|# decora_wifi|decora_wifi|g" ${requirement_file} - sed -i "s|# decora|decora|g" ${requirement_file} - sed -i "s|# PySwitchbot|PySwitchbot|g" ${requirement_file} - sed -i "s|# pySwitchmate|pySwitchmate|g" ${requirement_file} - sed -i "s|# face_recognition|face_recognition|g" ${requirement_file} - done - displayName: 'Prepare requirements files for Hass.io' - - script: | - sudo docker run --rm -v $(pwd):/data:ro -v $(pwd)/.ssh:/root/.ssh:rw \ - homeassistant/$(buildArch)-wheels:$(versionWheels) \ - --apk "build-base;cmake;git;linux-headers;bluez-dev;libffi-dev;openssl-dev;glib-dev;eudev-dev;libxml2-dev;libxslt-dev;libpng-dev;libjpeg-turbo-dev;tiff-dev;autoconf;automake;cups-dev;linux-headers;gmp-dev;mpfr-dev;mpc1-dev;ffmpeg-dev" \ - --index $(wheelsIndex) \ - --requirement requirements_wheels.txt \ - --requirement-diff requirements_diff.txt \ - --upload rsync \ - --remote wheels@$(wheelsHost):/opt/wheels - displayName: 'Run wheels build' + requirement_files="requirements_wheels.txt requirements_diff.txt" + for requirement_file in ${requirement_files}; do + sed -i "s|# pytradfri|pytradfri|g" ${requirement_file} + sed -i "s|# pybluez|pybluez|g" ${requirement_file} + sed -i "s|# bluepy|bluepy|g" ${requirement_file} + sed -i "s|# beacontools|beacontools|g" ${requirement_file} + sed -i "s|# RPi.GPIO|RPi.GPIO|g" ${requirement_file} + sed -i "s|# raspihats|raspihats|g" ${requirement_file} + sed -i "s|# rpi-rf|rpi-rf|g" ${requirement_file} + sed -i "s|# blinkt|blinkt|g" ${requirement_file} + sed -i "s|# fritzconnection|fritzconnection|g" ${requirement_file} + sed -i "s|# pyuserinput|pyuserinput|g" ${requirement_file} + sed -i "s|# evdev|evdev|g" ${requirement_file} + sed -i "s|# smbus-cffi|smbus-cffi|g" ${requirement_file} + sed -i "s|# i2csense|i2csense|g" ${requirement_file} + sed -i "s|# python-eq3bt|python-eq3bt|g" ${requirement_file} + sed -i "s|# pycups|pycups|g" ${requirement_file} + sed -i "s|# homekit|homekit|g" ${requirement_file} + sed -i "s|# decora_wifi|decora_wifi|g" ${requirement_file} + sed -i "s|# decora|decora|g" ${requirement_file} + sed -i "s|# PySwitchbot|PySwitchbot|g" ${requirement_file} + sed -i "s|# pySwitchmate|pySwitchmate|g" ${requirement_file} + sed -i "s|# face_recognition|face_recognition|g" ${requirement_file} + done + displayName: 'Prepare requirements files for Hass.io'