From a77b82de989a95d4c8e5cb751b4e6833edd298db Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Thu, 25 Jul 2019 11:34:43 +0200 Subject: [PATCH] Freeze LOCI images with a date In the last patch to freeze images with a date, the LOCI images were not added, do to the length of the patch [1]. This patch fixes the absence of LOCI based frozen image by changing the CI appropriately, in the same way as the previous patch ([1]). However, LOCI configuration here was most of the time relying on inheritance, which made bugs hard to find. This cleans up the LOCI building by: 1) Having a common abstract profile for which all the main zuul variables are inherited 2) Making sure each branch has content without inheritance, but with only limited repetitions. This uses anchors to reduce the information to type about _projects, as the project details are most likely the same as master. [1]: https://review.opendev.org/#/c/663744 Depends-On: https://review.opendev.org/676398 Change-Id: I6f33d5b7a33904cdd1c5ec352046ca9a9bca2aba --- zuul.d/openstack-loci.yaml | 365 ++++++++++++++++++++++--------------- 1 file changed, 220 insertions(+), 145 deletions(-) diff --git a/zuul.d/openstack-loci.yaml b/zuul.d/openstack-loci.yaml index c55182a..42cce29 100644 --- a/zuul.d/openstack-loci.yaml +++ b/zuul.d/openstack-loci.yaml @@ -16,8 +16,6 @@ - project: check: jobs: - - openstack-helm-images-build-openstack-loci-master-ubuntu_xenial - - openstack-helm-images-build-openstack-loci-master-ubuntu_bionic - openstack-helm-images-build-openstack-loci-ocata-ubuntu_xenial - openstack-helm-images-build-openstack-loci-pike-ubuntu_xenial - openstack-helm-images-build-openstack-loci-queens-ubuntu_xenial @@ -25,10 +23,9 @@ - openstack-helm-images-build-openstack-loci-rocky-ubuntu_xenial - openstack-helm-images-build-openstack-loci-rocky-ubuntu_bionic - openstack-helm-images-build-openstack-loci-stein-ubuntu_bionic + - openstack-helm-images-build-openstack-loci-master-ubuntu_bionic gate: jobs: - - openstack-helm-images-upload-openstack-loci-master-ubuntu_xenial - - openstack-helm-images-upload-openstack-loci-master-ubuntu_bionic - openstack-helm-images-upload-openstack-loci-ocata-ubuntu_xenial - openstack-helm-images-upload-openstack-loci-pike-ubuntu_xenial - openstack-helm-images-upload-openstack-loci-queens-ubuntu_xenial @@ -36,10 +33,9 @@ - openstack-helm-images-upload-openstack-loci-rocky-ubuntu_xenial - openstack-helm-images-upload-openstack-loci-rocky-ubuntu_bionic - openstack-helm-images-upload-openstack-loci-stein-ubuntu_bionic + - openstack-helm-images-upload-openstack-loci-master-ubuntu_bionic promote: jobs: - - openstack-helm-images-promote-openstack-loci-master-ubuntu_xenial - - openstack-helm-images-promote-openstack-loci-master-ubuntu_bionic - openstack-helm-images-promote-openstack-loci-ocata-ubuntu_xenial - openstack-helm-images-promote-openstack-loci-pike-ubuntu_xenial - openstack-helm-images-promote-openstack-loci-queens-ubuntu_xenial @@ -47,10 +43,9 @@ - openstack-helm-images-promote-openstack-loci-rocky-ubuntu_xenial - openstack-helm-images-promote-openstack-loci-rocky-ubuntu_bionic - openstack-helm-images-promote-openstack-loci-stein-ubuntu_bionic + - openstack-helm-images-promote-openstack-loci-master-ubuntu_bionic periodic: jobs: - - openstack-helm-images-build-openstack-loci-master-ubuntu_xenial - - openstack-helm-images-build-openstack-loci-master-ubuntu_bionic - openstack-helm-images-build-openstack-loci-ocata-ubuntu_xenial - openstack-helm-images-build-openstack-loci-pike-ubuntu_xenial - openstack-helm-images-build-openstack-loci-queens-ubuntu_xenial @@ -58,29 +53,43 @@ - openstack-helm-images-build-openstack-loci-rocky-ubuntu_xenial - openstack-helm-images-build-openstack-loci-rocky-ubuntu_bionic - openstack-helm-images-build-openstack-loci-stein-ubuntu_bionic + - openstack-helm-images-build-openstack-loci-master-ubuntu_bionic +################ +# Abstract job # +################ + +# This is a sample job, for which it is easy to inherit. +# It gives an example structure, should something be edited/copy pasted. +# At each release, copy the whole blob, and edit the variables appropriately. - job: - name: openstack-helm-images-build-openstack-loci-master-ubuntu_xenial - # Inherits secrets and vars + name: openstack-helm-images-build-openstack-loci-master + # Inherits secrets, descriptions, and details for the building parent: openstack-helm-images-build - description: Build master LOCI images for xenial - # Increase timeout due to many images to build + abstract: true timeout: 5400 # This enables depends-on to work by using local repos required-projects: - openstack/loci + # TODO(evrardjp): Now that infra has buildset registry + # We should move to it. + # In the meantime, keep our pseudo "buildset" registry in here + # through prep_docker_images. # Fork of standard playbook to build prep images first (reusing # role) with a different zuul_role_dir, publish wheels on http server. # zuul_work_dir: "src/opendev.org/openstack/loci" run: zuul.d/playbooks/build-loci.yml - vars: &build_master_xenial_images + vars: &build_master_images + currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" branchname: "master" shortbranchname: "master" - distroname: "ubuntu_xenial" - from: "gcr.io/google_containers/ubuntu-slim:0.14" - python3: "no" + distroname: "" + from: "" + python3: "yes" #172.17.0.1 is default docker0 ip. wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" + # the macros named *_project apply project wide. Be careful + # if editing those. prep_docker_images: - &requirements_project context: "." @@ -104,6 +113,7 @@ - "WHEELS='{{ wheels_location }}'" tags: &imagetag - "{{ shortbranchname }}-{{ distroname }}" + - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag - &heat_project context: "." repository: openstackhelm/heat @@ -176,6 +186,7 @@ - "WHEELS='{{ wheels_location }}'" tags: - "{{ shortbranchname }}-18.04-sriov" + - "{{ shortbranchname }}-18.04-sriov-{{ currentdate }}" - &nova_project context: "." repository: openstackhelm/nova @@ -252,43 +263,35 @@ # so we can inherit from this. The post run is not caring about context, # only repository, and tags. - job: - name: openstack-helm-images-upload-openstack-loci-master-ubuntu_xenial - description: Build and upload master LOCI images for xenial - # Inherits secrets and vars + name: openstack-helm-images-upload-openstack-loci-master parent: openstack-helm-images-upload + abstract: true # Increase timeout due to many images to build timeout: 5400 - # This enables depends-on to work by using local repos required-projects: - openstack/loci run: zuul.d/playbooks/build-loci.yml - vars: *build_master_xenial_images + vars: *build_master_images files: *loci-files # Promote is a run job. Let's not change it. - job: - name: openstack-helm-images-promote-openstack-loci-master-ubuntu_xenial + name: openstack-helm-images-promote-openstack-loci-master parent: openstack-helm-images-promote - description: Promote previously published loci xenial master images - vars: *build_master_xenial_images + abstract: true + vars: *build_master_images files: *loci-files -# Bionic +############# +# Real jobs # +############# + - job: name: openstack-helm-images-build-openstack-loci-master-ubuntu_bionic - # Inherits secrets and vars - parent: openstack-helm-images-build - description: Build master LOCI images for bionic - # Increase timeout due to many images to build - timeout: 5400 - # This enables depends-on to work by using local repos - required-projects: - - openstack/loci - # Fork of standard playbook to build prep images first (reusing - # role) with a different zuul_role_dir, publish wheels on http server. - # zuul_work_dir: "src/opendev.org/openstack/loci" - run: zuul.d/playbooks/build-loci.yml + description: Build LOCI master bionic images + parent: openstack-helm-images-build-openstack-loci-master vars: &build_master_bionic_images + currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" branchname: "master" shortbranchname: "master" distroname: "ubuntu_bionic" @@ -297,27 +300,9 @@ #172.17.0.1 is default docker0 ip. wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" prep_docker_images: - - context: "." - repository: openstackhelm/requirements - build_args: - - "FROM='{{ from }}'" - - "PYTHON3={{ python3 }}" - - "PROJECT='requirements'" - - "PROJECT_REF={{ branchname }}" - <<: *requirements_project - # For keystone replace ldap by python-ldap docker_images: - - context: "." - repository: openstackhelm/keystone - build_args: - - "FROM='{{ from }}'" - - "PYTHON3={{ python3 }}" - - "PROJECT='keystone'" - - "PROJECT_REF={{ branchname }}" - - "PROFILES='fluent apache python-ldap {{ shortbranchname }}'" - - "PIP_PACKAGES='python-openstackclient'" - - "WHEELS='{{ wheels_location }}'" - tags: *imagetag + - <<: *keystone_project - <<: *glance_project - <<: *cinder_project - <<: *nova_project @@ -328,91 +313,152 @@ - job: name: openstack-helm-images-upload-openstack-loci-master-ubuntu_bionic - description: Build and upload master LOCI images for bionic - # Inherits secrets and vars - parent: openstack-helm-images-upload - # Increase timeout due to many images to build - timeout: 5400 - # This enables depends-on to work by using local repos - required-projects: - - openstack/loci - run: zuul.d/playbooks/build-loci.yml + description: Build and upload LOCI master bionic images + parent: openstack-helm-images-upload-openstack-loci-master vars: *build_master_bionic_images files: *loci-files - job: name: openstack-helm-images-promote-openstack-loci-master-ubuntu_bionic - parent: openstack-helm-images-promote - description: Promote previously published loci bionic master images + parent: openstack-helm-images-promote-openstack-loci-master + description: Promote previously published LOCI master bionic images vars: *build_master_bionic_images files: *loci-files -# Rocky-bionic -- job: - name: openstack-helm-images-build-openstack-loci-rocky-ubuntu_bionic - parent: openstack-helm-images-build-openstack-loci-master-ubuntu_bionic - description: Build rocky LOCI images for bionic - vars: &build_rocky_bionic_images - <<: *build_master_bionic_images - branchname: "stable/rocky" - shortbranchname: "rocky" +######### +# STEIN # +######### -- job: - name: openstack-helm-images-upload-openstack-loci-rocky-ubuntu_bionic - parent: openstack-helm-images-upload-openstack-loci-master-ubuntu_bionic - description: Build and upload rocky LOCI images for bionic - vars: *build_rocky_bionic_images - -- job: - name: openstack-helm-images-promote-openstack-loci-rocky-ubuntu_bionic - parent: openstack-helm-images-promote-openstack-loci-master-ubuntu_bionic - description: Promote previously published loci bionic rocky images - vars: *build_rocky_bionic_images - -# Stein-bionic - job: name: openstack-helm-images-build-openstack-loci-stein-ubuntu_bionic - parent: openstack-helm-images-build-openstack-loci-master-ubuntu_bionic - description: Build stein LOCI images for bionic + description: Build LOCI stein bionic images + parent: openstack-helm-images-build-openstack-loci-master vars: &build_stein_bionic_images - <<: *build_master_bionic_images + currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" branchname: "stable/stein" shortbranchname: "stein" + distroname: "ubuntu_bionic" + from: "docker.io/ubuntu:18.04" + python3: "yes" + wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" + prep_docker_images: + - <<: *requirements_project + docker_images: + - <<: *keystone_project + - <<: *glance_project + - <<: *cinder_project + - <<: *nova_project + - <<: *neutron_project + - <<: *horizon_project + - <<: *heat_project + - <<: *barbican_project - job: name: openstack-helm-images-upload-openstack-loci-stein-ubuntu_bionic - parent: openstack-helm-images-upload-openstack-loci-master-ubuntu_bionic - description: Build and upload stein LOCI images for bionic + parent: openstack-helm-images-upload-openstack-loci-master + description: Build and upload LOCI stein bionic images vars: *build_stein_bionic_images - job: name: openstack-helm-images-promote-openstack-loci-stein-ubuntu_bionic - parent: openstack-helm-images-promote-openstack-loci-master-ubuntu_bionic - description: Promote previously published loci bionic stein images + parent: openstack-helm-images-promote-openstack-loci-master + description: Promote previously published LOCI stein bionic images vars: *build_stein_bionic_images -# Rocky-xenial +######### +# ROCKY # +######### + +# Rocky is a stop gap for images. +# It is built using both Xenial (python2) and Bionic (python3) +# Bionic - job: - name: openstack-helm-images-build-openstack-loci-rocky-ubuntu_xenial - parent: openstack-helm-images-build-openstack-loci-master-ubuntu_xenial - description: Build rocky LOCI images for xenial - vars: &build_rocky_xenial_images - <<: *build_master_xenial_images + name: openstack-helm-images-build-openstack-loci-rocky-ubuntu_bionic + parent: openstack-helm-images-build-openstack-loci-master + description: Build LOCI rocky bionic images + vars: &build_rocky_bionic_images + currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" branchname: "stable/rocky" shortbranchname: "rocky" + distroname: "ubuntu_bionic" + from: "docker.io/ubuntu:18.04" + python3: "yes" + wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" + prep_docker_images: + - <<: *requirements_project + docker_images: + - <<: *keystone_project + - <<: *glance_project + - <<: *cinder_project + - <<: *nova_project + - <<: *neutron_project + - <<: *horizon_project + - <<: *heat_project + - <<: *barbican_project +- job: + name: openstack-helm-images-upload-openstack-loci-rocky-ubuntu_bionic + parent: openstack-helm-images-upload-openstack-loci-master + description: Build and upload LOCI rocky bionic images + vars: *build_rocky_bionic_images + +- job: + name: openstack-helm-images-promote-openstack-loci-rocky-ubuntu_bionic + parent: openstack-helm-images-promote-openstack-loci-master + description: Promote previously published LOCI rocky bionic images + vars: *build_rocky_bionic_images + +# Xenial +- job: + name: openstack-helm-images-build-openstack-loci-rocky-ubuntu_xenial + parent: openstack-helm-images-build-openstack-loci-master + description: Build LOCI rocky xenial images + vars: &build_rocky_xenial_images + currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" + branchname: "stable/rocky" + shortbranchname: "rocky" + distroname: "ubuntu_xenial" + from: "gcr.io/google_containers/ubuntu-slim:0.14" + python3: "no" + wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" + prep_docker_images: + - <<: *requirements_project + docker_images: + - <<: *keystone_project + - <<: *glance_project + - <<: *cinder_project + - <<: *nova_project + - <<: *neutron_project + - <<: *horizon_project + - <<: *heat_project + - <<: *barbican_project + +- job: + name: openstack-helm-images-upload-openstack-loci-rocky-ubuntu_xenial + parent: openstack-helm-images-upload-openstack-loci-master + description: Build and upload LOCI rocky xenial images + vars: *build_rocky_xenial_images + +- job: + name: openstack-helm-images-promote-openstack-loci-rocky-ubuntu_xenial + parent: openstack-helm-images-promote-openstack-loci-master + description: Promote previously published LOCI xenial rocky images + vars: *build_rocky_xenial_images + +# openSUSE - job: name: openstack-helm-images-build-openstack-loci-rocky-opensuse_15 - parent: openstack-helm-images-build-openstack-loci-master-ubuntu_xenial - description: "Build rocky LOCI images for leap 15" + parent: openstack-helm-images-build-openstack-loci-master + description: "Build LOCI rocky Leap15.0 images" timeout: 7200 # need to build base image too vars: &build_rocky_leap15_images + currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" branchname: "stable/rocky" shortbranchname: "rocky" distroname: "opensuse_15" - # This from will be used in all the docker_images section. - from: "openstackhelm/base:{{ distroname }}" + from: "openstackhelm/base:{{ distroname }}" # This 'from' will be used in all the docker_images section. python3: "yes" + wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" # We override the base image's FROM to 15.0 (instead of 15, which can # shift over time). That is because of the fact that upper-constraints # for Rocky contains a libvirt-python version too old for 15.1 @@ -438,83 +484,111 @@ - <<: *horizon_project - <<: *heat_project - <<: *barbican_project -- job: - name: openstack-helm-images-upload-openstack-loci-rocky-ubuntu_xenial - parent: openstack-helm-images-upload-openstack-loci-master-ubuntu_xenial - description: Build and upload rocky LOCI images for xenial - vars: *build_rocky_xenial_images - job: name: openstack-helm-images-upload-openstack-loci-rocky-opensuse_15 - parent: openstack-helm-images-upload-openstack-loci-master-ubuntu_xenial - description: "Build and upload rocky LOCI images for leap 15" + parent: openstack-helm-images-upload-openstack-loci-master + description: "Build and upload LOCI rocky Leap15.0 images" timeout: 7200 # need to build base image too vars: *build_rocky_leap15_images -- job: - name: openstack-helm-images-promote-openstack-loci-rocky-ubuntu_xenial - parent: openstack-helm-images-promote-openstack-loci-master-ubuntu_xenial - description: Promote previously published loci xenial rocky images - vars: *build_rocky_xenial_images - - job: name: openstack-helm-images-promote-openstack-loci-rocky-opensuse_15 - parent: openstack-helm-images-promote-openstack-loci-master-ubuntu_xenial - description: "Promote previously published loci leap 15 rocky images" + parent: openstack-helm-images-promote-openstack-loci-master + description: "Promote previously published LOCI rocky Leap 15.0 images" vars: *build_rocky_leap15_images -# Queens-xenial +########## +# Queens # +########## + - job: name: openstack-helm-images-build-openstack-loci-queens-ubuntu_xenial - parent: openstack-helm-images-build-openstack-loci-master-ubuntu_xenial - description: Build queens LOCI images for xenial + parent: openstack-helm-images-build-openstack-loci-master + description: Build LOCI queens xenial images vars: &build_queens_xenial_images - <<: *build_master_xenial_images + currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" branchname: "stable/queens" shortbranchname: "queens" + distroname: "ubuntu_xenial" + from: "gcr.io/google_containers/ubuntu-slim:0.14" + python3: "no" + wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" + prep_docker_images: + - <<: *requirements_project + docker_images: + - <<: *keystone_project + - <<: *glance_project + - <<: *cinder_project + - <<: *nova_project + - <<: *neutron_project + - <<: *horizon_project + - <<: *heat_project + - <<: *barbican_project - job: name: openstack-helm-images-upload-openstack-loci-queens-ubuntu_xenial - parent: openstack-helm-images-upload-openstack-loci-master-ubuntu_xenial - description: Build and upload queens LOCI images for xenial + parent: openstack-helm-images-upload-openstack-loci-master + description: Build and upload LOCI queens xenial images vars: *build_queens_xenial_images - job: name: openstack-helm-images-promote-openstack-loci-queens-ubuntu_xenial - parent: openstack-helm-images-promote-openstack-loci-master-ubuntu_xenial - description: Promote a previously published loci xenial queens images + parent: openstack-helm-images-promote-openstack-loci-master + description: Promote a previously published LOCI queens xenial images vars: *build_queens_xenial_images -# Pike-xenial +######## +# Pike # +######## + - job: name: openstack-helm-images-build-openstack-loci-pike-ubuntu_xenial - parent: openstack-helm-images-build-openstack-loci-master-ubuntu_xenial - description: Build pike LOCI images for xenial + parent: openstack-helm-images-build-openstack-loci-master + description: Build LOCI pike xenial images vars: &build_pike_xenial_images - <<: *build_master_xenial_images + currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" branchname: "stable/pike" shortbranchname: "pike" + distroname: "ubuntu_xenial" + from: "gcr.io/google_containers/ubuntu-slim:0.14" + python3: "no" + wheels_location: "http://172.17.0.1:8080/wheels.tar.gz" + prep_docker_images: + - <<: *requirements_project + docker_images: + - <<: *keystone_project + - <<: *glance_project + - <<: *cinder_project + - <<: *nova_project + - <<: *neutron_project + - <<: *horizon_project + - <<: *heat_project + - <<: *barbican_project - job: name: openstack-helm-images-upload-openstack-loci-pike-ubuntu_xenial - parent: openstack-helm-images-upload-openstack-loci-master-ubuntu_xenial - description: Build and upload pike LOCI images for xenial + parent: openstack-helm-images-upload-openstack-loci-master + description: Build and upload LOCI pike xenial images vars: *build_pike_xenial_images - job: name: openstack-helm-images-promote-openstack-loci-pike-ubuntu_xenial - parent: openstack-helm-images-promote-openstack-loci-master-ubuntu_xenial - description: Promote a previously published loci xenial pike images + parent: openstack-helm-images-promote-openstack-loci-master + description: Promote a previously published LOCI pike xenial images vars: *build_pike_xenial_images -# Ocata-xenial -# Ocata requires pycrypto installed for keystonemiddleware memcache support, -# so we can't reuse Master profile. +######### +# Ocata # +######### + +# Ocata requires pycrypto installed for keystonemiddleware memcache support. - job: name: openstack-helm-images-build-openstack-loci-ocata-ubuntu_xenial - parent: openstack-helm-images-build-openstack-loci-master-ubuntu_xenial + parent: openstack-helm-images-build-openstack-loci-master description: Build ocata LOCI images for xenial vars: &build_ocata_xenial_images + currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}" branchname: "stable/ocata" shortbranchname: "ocata" distroname: "ubuntu_xenial" @@ -611,6 +685,7 @@ - "WHEELS='{{ wheels_location }}'" tags: - "{{ shortbranchname }}-18.04-sriov" + - "{{ shortbranchname }}-18.04-sriov-{{ currentdate }}" - context: "." repository: openstackhelm/nova build_args: @@ -681,12 +756,12 @@ - job: name: openstack-helm-images-upload-openstack-loci-ocata-ubuntu_xenial - parent: openstack-helm-images-upload-openstack-loci-master-ubuntu_xenial - description: Build and upload ocata LOCI images for xenial + parent: openstack-helm-images-upload-openstack-loci-master + description: Build and upload LOCI ocata xenial images vars: *build_ocata_xenial_images - job: name: openstack-helm-images-promote-openstack-loci-ocata-ubuntu_xenial - parent: openstack-helm-images-promote-openstack-loci-master-ubuntu_xenial - description: Promote a previously published loci xenial ocata images + parent: openstack-helm-images-promote-openstack-loci-master + description: Promote a previously published LOCI ocata xenial images vars: *build_ocata_xenial_images