owned this note
owned this note
Published
Linked with GitHub
OSP Internal RHEL8 Container builds
===================================
###### tags: `RHEL8 Containers Internal`
###### tags: `Design`
centos-8 notes: [folco hackmd.io](https://hackmd.io/dSagCbocQ4KSVEZR1uf8Tw) ----- [weshay hackmd.io](https://hackmd.io/cv1483MkQlS8UsnteNyXqg)
* * *
#### Configure certs on RHEL-8 box
```
sudo yum install -y http://hdn.corp.redhat.com/rhel7-csb-stage/RPMS/noarch/redhat-internal-cert-install-0.1-9.el7.csb.noarch.rpm
```
#### Install rhos-release rpm
```
sudo yum -y install http://rhos-release.virt.bos.redhat.com/repos/rhos-release/rhos-release-latest.noarch.rpm
```
#### Setup repos with rhos-release
Taking latest from [download-node-02.eng.bos.redhat.com/rcm-guest/puddles/OpenStack/17.0-RHEL-8](http://download-node-02.eng.bos.redhat.com/rcm-guest/puddles/OpenStack/17.0-RHEL-8/) for the -p:
```
sudo rhos-release -P 17 -p RHOS_TRUNK-17.0-RHEL-8-20191224.n.1
```
* * *
#### git clone python-tripleoclient, kolla, tripleo-common and checkout rhos-17.0-trunk-patches
```
git clone git://git.host.prod.eng.bos.redhat.com/python-tripleoclient.git
git clone git://git.host.prod.eng.bos.redhat.com/openstack-tripleo-common.git
git clone git://git.host.prod.eng.bos.redhat.com/kolla.git
for i in python-tripleoclient openstack-tripleo-common kolla; do
pushd $i
git checkout rhos-17.0-trunk-patches
popd
done
```
***
#### Patch & install tripleo-common and kolla
```
# https://code.engineering.redhat.com/gerrit/#/c/190338
pushd kolla
git fetch https://code.engineering.redhat.com/gerrit/kolla refs/changes/38/190338/2 && git checkout FETCH_HEAD
sudo python3 setup.py install
popd
pushd openstack-tripleo-common
# MERGED https://code.engineering.redhat.com/gerrit/#/c/190538 https://code.engineering.redhat.com/gerrit/openstack-tripleo-common refs/changes/38/190538/3 && git checkout FETCH_HEAD
sudo python3 setup.py install
popd
# create symlink - tripleo-common files installed to /usr/local/share..
cd /usr/share
ln -s /usr/local/share/tripleo-common/ tripleo-common
```
* * *
#### dnf and pip install requirements
```
sudo dnf install -y git python3 buildah* podman* "@Development Tools" python3-devel
pip3 install -e python-tripleoclient/ --user
```
* * *
#### pull ubi image & build rhel:8
```
podman pull registry-proxy.engineering.redhat.com/rh-osbs/ubi8:8.2-ondeck
```
* Make sure you have the RH key on the box: http://git.app.eng.bos.redhat.com/git/openstack/tripleo-ci-internal-jobs.git/tree/playbooks/configure-rhel.yaml#n58
* Add internal reg to insecure registries in /etc/containers/registries.conf (marios: i didn't have to do that one)
```
mkdir ~/build && cd ~/build
cat <<"EOF" > Dockerfile
> FROM registry-proxy.engineering.redhat.com/rh-osbs/ubi8:8.2-ondeck
> COPY *.repo /etc/yum.repos.d/.
> RUN dnf clean all
> EOF
# get repos into build directory
pushd /etc/yum.repos.d/
cp rhos-release-rhel-8.1.repo ~/build/
cp rhos-release-17.repo ~/build/
cp osptrunk-deps.repo
popd
curl -O http://osp-trunk.hosted.upshift.rdu2.redhat.com/rhel8-osp17/osptrunk-deps.repo
curl -O http://osp-trunk.hosted.upshift.rdu2.redhat.com/rhel8-osp17/consistent/delorean.repo
# BUILD !
sudo buildah bud -t rhel:8 .
```
#### Prepare kolla-build.conf
RDO
```
cat <<"EOF" > kolla-build.conf
[DEFAULT]
base=rhel
base_image=rhel
base_tag=8
base_distro_tag=8
type=binary
registry=registry-proxy.engineering.redhat.com
tag=5678
template_override=/usr/share/tripleo-common/container-images/tripleo_kolla_template_overrides.j2
# we use the repos off the main host as they will already have the correct mirror info in them
rpm_setup_config=""
push=False
pull=False
namespace=rhosp17
name_prefix=openstack-
debug=True
EOF
```
#### Clone the downstream containers to /usr/local/share/kolla/docker
Eg. I was trying to build nova* only. We always need dependencies.
```
pushd /usr/local/share/kolla/docker
for i in dependencies nova-base nova-api nova-compute-ironic nova-compute nova-conductor nova-libvirt nova-novncproxy nova-scheduler novajoin-notifier novajoin-server ; do
git clone git://pkgs.devel.redhat.com/containers/openstack-$i;
pushd openstack-$i;
git checkout rhos-17.0-rhel-8;
popd;
done
popd
```
#### Prepare containers.yaml.j2 file for build
Again, I was trying to build nova only for now.
```
(as su or add sudos below):
pushd /usr/local/share/tripleo-common/container-images
mv overcloud_containers.yaml.j2 overcloud_containers.yaml.j2.ORIG
cat <<"EOF" > overcloud_containers.yaml.j2
container_images_template:
- imagename: "{{namespace}}/openstack-dependencies:{{tag}}"
image_source: kolla
- imagename: "{{namespace}}/{{name_prefix}}nova-base{{name_suffix}}:{{tag}}"
image_source: kolla
- imagename: "{{namespace}}/{{name_prefix}}nova-api{{name_suffix}}:{{tag}}"
image_source: kolla
params:
- ContainerNovaApiImage
- ContainerNovaConfigImage
- ContainerNovaMetadataImage
- ContainerNovaMetadataConfigImage
services:
- OS::TripleO::Services::NovaApi
- OS::TripleO::Services::NovaConductor
- OS::TripleO::Services::NovaIronic
- OS::TripleO::Services::NovaScheduler
- OS::TripleO::Services::NovaVncProxy
- OS::TripleO::Services::NovaMetadata
- imagename: "{{namespace}}/{{name_prefix}}nova-compute-ironic{{name_suffix}}:{{tag}}"
image_source: kolla
params:
- ContainerNovaComputeIronicImage
services:
- OS::TripleO::Services::NovaIronic
- imagename: "{{namespace}}/{{name_prefix}}nova-compute{{name_suffix}}:{{tag}}"
image_source: kolla
params:
- ContainerNovaComputeImage
- ContainerNovaLibvirtConfigImage
services:
- OS::TripleO::Services::NovaCompute
- OS::TripleO::Services::NovaLibvirt
- OS::TripleO::Services::NovaMigrationTarget
- imagename: "{{namespace}}/{{name_prefix}}nova-conductor{{name_suffix}}:{{tag}}"
image_source: kolla
params:
- ContainerNovaConductorImage
services:
- OS::TripleO::Services::NovaConductor
- imagename: "{{namespace}}/{{name_prefix}}nova-libvirt{{name_suffix}}:{{tag}}"
image_source: kolla
params:
- ContainerNovaLibvirtImage
services:
- OS::TripleO::Services::NovaLibvirt
- imagename: "{{namespace}}/{{name_prefix}}nova-novncproxy{{name_suffix}}:{{tag}}"
image_source: kolla
params:
- ContainerNovaVncProxyImage
services:
- OS::TripleO::Services::NovaVncProxy
- imagename: "{{namespace}}/{{name_prefix}}nova-scheduler{{name_suffix}}:{{tag}}"
image_source: kolla
params:
- ContainerNovaSchedulerImage
services:
- OS::TripleO::Services::NovaScheduler
- imagename: "{{namespace}}/{{name_prefix}}novajoin-notifier{{name_suffix}}:{{tag}}"
image_source: kolla
params:
- ContainerNovajoinNotifierImage
services:
- OS::TripleO::Services::Novajoin
- imagename: "{{namespace}}/{{name_prefix}}novajoin-server{{name_suffix}}:{{tag}}"
image_source: kolla
params:
- ContainerNovajoinServerImage
- ContainerNovajoinConfigImage
services:
- OS::TripleO::Services::Novajoin
EOF
popd
```
#### 'patch' tripleo_kolla_template_overrides.j2
Manual for now... vim find/replace. Looks like http://paste.openstack.org/raw/791587/ (diff from original). Adding in the package removals mainly for openstack_dependencies and the name prefixes... nova_api_packages becomes openstack_nova_api_packages. Part of our 'naming' problem. TBD.
#### Build
```
mkdir -p ~/build/temp
openstack overcloud container image build \
--kolla-config-file ~/build/kolla-build.conf \
--list-images --use-buildah --work-dir ~/build/temp/
```
Fails for nova dependencies like:
```
Stderr: 'error building at STEP "RUN dnf -y module disable virt:rhel && dnf -y module enable virt:8.2 && dnf -y install genisoimage nvme-cli openstack-nova-compute python3-novajoin && dnf clean all && rm -rf /var/cache/dnf": error while running runtime: exit status 1\n'
* Error: Problems in request:
missing groups or modules: virt:8.2
```
But nova-base nova-api build fine so process wise looks ok:
```
[marios@mariossprint24task1597 docker]$ sudo podman images
REPOSITORY TAG IMAGE ID CREATED SIZE
registry-proxy.engineering.redhat.com/rhosp17/rhel-binary-openstack-nova-api 5678 d02a0bb49976 About a minute ago 1.26 GB
registry-proxy.engineering.redhat.com/rhosp17/rhel-binary-openstack-nova-base 5678 facd7c901989 7 minutes ago 1.12 GB
registry-proxy.engineering.redhat.com/rhosp17/rhel-binary-nova-base 5678 c0b44bb9b4f1 7 minutes ago 1.12 GB
registry-proxy.engineering.redhat.com/rhosp17/rhel-binary-openstack-base 5678 85c7f74f4ef9 13 minutes ago 783 MB
registry-proxy.engineering.redhat.com/rhosp17/rhel-binary-openstack-dependencies 5678 2d81cf4218eb 13 minutes ago 783 MB
registry-proxy.engineering.redhat.com/rhosp17/rhel-binary-base 5678 851ea2b649e7 22 minutes ago 523 MB
<none> <none> ae0b6b0625f7 45 minutes ago 783 MB
<none> <none> 0265d7401de2 54 minutes ago 523 MB
<none> <none> 172c4629577e 21 hours ago 783 MB
<none> <none> 0092d003936a 21 hours ago 523 MB
<none> <none> 89a0af7ce009 21 hours ago 783 MB
<none> <none> f19c646c67a7 22 hours ago 523 MB
localhost/rhel 8 75198ffb9379 22 hours ago 212 MB
registry-proxy.engineering.redhat.com/rh-osbs/ubi8 8.2-ondeck ff18d576179d 2 days ago 211 MB
```
[ **Notes rlandy - 04/01**]
Managed to build containers:
* localhost/rhel - from uib (**builder bud**)
* rhel-binary-base - from (uib or rhel) with openstack-base rendered Dockerfile (**builder bud**)
* **openstack overcloud container build:**
* rhel-binary-openstack-dependencies - from rhel-binary-openstack-dependencies
* rhel-binary-openstack-glance-base - from rhel-binary-openstack-glance-base
* rhel-binary-openstack-glance-api - from rhel-binary-openstack-glance-api
```
[cloud-user@rlandy17containers build]$ sudo podman images
REPOSITORY TAG IMAGE ID CREATED SIZE
localhost/tripleo-ci-rhosp17/rhel-binary-openstack-glance-api 1234 f4adcb85d331 3 hours ago 1.02 GB
localhost/tripleo-ci-rhosp17/rhel-binary-openstack-glance-base 1234 a991533919f7 3 hours ago 976 MB
localhost/tripleo-ci-rhosp17/rhel-binary-openstack-dependencies 1234 6a83930fdefd 3 hours ago 714 MB
registry-proxy.engineering.redhat.com/rh-osbs/ubi8 8.2-ondeck ff18d576179d 26 hours ago 211 MB
localhost/tripleo-ci-rhosp17/rhel-binary-base 1234 8cd382cbc4b8 29 hours ago 522 MB
localhost/rhel 8 ee2b4f20bdc2 31 hours ago 212 MB
```
Hacks used:
* Containers are built from dirs in `/usr/local/share/kolla/docker/`. Copy files fron kolla/docker/<> to ^^
* openstack overcloud container image build --kolla-config-file ~/build/kolla-build.conf --list-images --use-buildah --work-dir ~/build/temp
* Note use of `--list-images` - w/o that you get: `docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))` No idea why --list-images changes that outcome
* kolla-build.conf
```
[DEFAULT]
base=rhel
base_image=rhel
base_tag=8
base_distro_tag=8
type=binary
registry=localhost
image_prefix=openstack-
tag=1234
template_override=/usr/share/tripleo-common/container-images/tripleo_kolla_template_overrides.j2
rpm_setup_config=""
push=False
pull=False
namespace=tripleo-ci-rhosp17
debug=True
```
* Add localhost to insecure registries list
* `/usr/share/tripleo-common/container-images/tripleo_kolla_template_overrides.j2`
needs to be edited to change the right variables for the containers used. For example: to remove python3-cloudkittyclient etc., you need `set openstack_dependencies_packages_remove` . Note: if we name the containers w/o openstack-, we will save some edits here
* `/usr/local/share/tripleo-common/container-images/overcloud_containers.yaml.j2`
Contaners to be built need to match an entry in ^^. Edited the file to add:
```
container_images_template:
- imagename: "{{namespace}}/openstack-dependencies:{{tag}}"
image_source: kolla
- imagename: "{{namespace}}/openstack-glance-base:{{tag}}"
image_source: kolla
- imagename: "{{namespace}}/openstack-glance-api:{{tag}}"
image_source: kolla
params:
- ContainerGlanceApiConfigImage
- ContainerGlanceApiImage
services:
- OS::TripleO::Services::GlanceApi
- OS::TripleO::Services::GlanceApiEdge
```
Again, if we refine the naming to rhel-binary and remove openstack-, we will save some edits here
**Next steps:**
* We need to agree on a naming scheme
* Working on a pre-role to include the above steps and hacks so the process is repeatable in CI
#### Try to build containers
```
openstack overcloud container image build \
--kolla-config-file ~/build/kolla-build.conf \
--exclude haproxy --exclude fluentd \
--exclude skydive-agent --exclude zookeeper \
--exclude zun-api --exclude zun-base \
--exclude zun-compute --exclude zun-wsproxy \
--use-buildah
```
Working with Dockerfiles from Release Eng, manage to build:
```
REPOSITORY TAG IMAGE ID CREATED SIZE
localhost/rhosp16/openstack-glance-base 20200130.1 ebd2838f7aa3 39 seconds ago 967 MB
localhost/rhosp16/openstack-keystone 20200130.1 17d816102031 2 minutes ago 763 MB
localhost/rhosp16/openstack-keystone-base 20200130.1 aa5f6dafc640 11 minutes ago 728 MB
localhost/rhosp16/openstack-dependencies 20200130.1 aec944912929 17 minutes ago 648 MB
localhost/rhosp16/openstack-base 20200130.1 5d11357a647c 39 minutes ago 388 MB
registry.access.redhat.com/ubi8 8.1-397 fd73e6738a95 11 days ago 239 MB
```
TODO: write a script to copy the Rel Eng Dockerfiles to correct locations in kolla.
afaict, kolla expects the dir structure - unless there is a way to override that?
## Links
* Base image: http://pkgs.devel.redhat.com/cgit/containers/openstack-base/plain/Dockerfile?h=rhos-17.0-rhel-8
* All containers: http://pkgs.devel.redhat.com/cgit/containers/
* Example: http://pkgs.devel.redhat.com/cgit/containers/openstack-ironic-pxe/tree/?h=rhos-17.0-rhel-8
* List: https://code.engineering.redhat.com/gerrit/gitweb?p=dist-git-manifests.git;a=blob_plain;f=combined-images.xml;h=4a45a1ff00f92fe233d5f071bbf67b707b05c826;hb=refs/heads/rhos-17.0-rhel-8
* Rel Del build scipts https://code.engineering.redhat.com/gerrit/gitweb?p=rhos-release-standup-scripts.git;a=blob;f=build_container_images.py;h=066769f641eb328132e2dd9c80cc5721e1d7bdac;hb=HEAD
* Rel Del build order https://code.engineering.redhat.com/gerrit/gitweb?p=rhos-release-standup-scripts.git;a=blob;f=buildorder.py;h=fe1b5222248fb0f4ba6a7d238dc935547abd5210;hb=HEAD
## Design decisions to automate changes required for OSP container builds with Kolla
Now that we have managed to go through the process of building containers manually, we have a list of tasks/changes that are required to automate the process in a Zuul job. The points below lay out the changes required, where they need to be made and what options we have to get the needed outcome.
### pkgs.devel.../containers
* Container repo cloning and naming
* Clone is done by zuul. Thanks to tristanC https://code.engineering.redhat.com/gerrit/#/c/196441 to add repos to the tenant. `scripts/get-projects.py` returns the list of containers and we can use that in *Copy to kolla/docker*
* OR we clone within the zuul job. The advantage of this approach is that it is more dynamic. We run the script and determine the directories in real time.
* Copy to kolla/docker
* [**ASK RELDEL**] *Copy to kolla/docker* is done in the zuul job - unless reldel does not care about that dir. The default image dir is /usr/local/share/kolla/docker
* Note that this ^^ directory requires *a dir called "base"*. Should that be empty? We build dependencies as base here?? see question below.
* Suggestion - if we *change the container dir names to remove openstack-* when we copy them to kolla/docker/, we will require fewer changes to overrides.
* *List of containers to clone* ... list of containers to build ... reldel produces a list of containers in https://code.engineering.redhat.com/gerrit/gitweb?p=dist-git-manifests.git;a=blob_plain;f=combined-images.xml;h=4a45a1ff00f92fe233d5f071bbf67b707b05c826;hb=refs/heads/rhos-17.0-rhel-8. We could use and manipulate this to generate a list of conatiners in CI OR we could check a static list into code.eng/openstack-tripleo-common
* What is THE base container?
* What is the first container built by *'buildah bud'*
* What is the first container built by *'openstack overcloud conatiner build'*?
* If *'dependencies'* considered a separate container OR do we take the rhos-release-rhel-8.1, delorean repos, AND what is in dependencies and add that to be built into what is currently openstack-base and use that as the first container built by builder bud? ie: rhel-binary-base? This could be templated in the role.
* <wes> are we saying ^ build openstack-base and openstack-dependencies as one container called openstack-base?</wes>
* Changes to Dockerfile.j2
* [**ASK RELDEL**] Does reldel use the .j2 file at all? If not, we could make the changes directly in the repo. We need to change the FROM line to match the containers we build
* If we can't change the .j2 files in the repos directly, this is a simple `sed` line, so we could carry it in the CI.
### code.engineering
* We need to make edits to:
* /usr/share/tripleo-common/container-images/tripleo_kolla_template_overrides.j2 (may be minial depending on how we name the base container)
* /usr/local/share/tripleo-common/container-images/overcloud_containers.yaml.j2 (we need to add the container-base images here)
[**ASK RELDEL**] If they use these files and if we can make the changes directly in the repos.
* We are still carrying patches to kolla:
git fetch https://code.engineering.redhat.com/gerrit/kolla refs/changes/38/190338/2 && git checkout FETCH_HEAD
### registry
* registry.access.redhat.com
* agree on a namespace
* image_prefix, image_suffix
### /openstack/tripleo-ci/ roles/build-containers
* We own this code and should be able to make required changes to support OSP container builds
## Reviews
* https://code.engineering.redhat.com/gerrit/#/c/196441 tripleo-ci-internal: add generated osp-containers project list (still in progress - tristanC wants to generalize this more)
* https://code.engineering.redhat.com/gerrit/#/c/196447/ Reorg the tripleo-ci-internal tenant resources and add kolla
## notes 04/03/2020 combining dockerfiles w/ rlandy
* clean all local images, except for ubi image
```
[cloud-user@rlandy17containers build]$ mv Dockerfile Dockerfile-openstack-base
[cloud-user@rlandy17containers build]$ mv Dockerfile-template Dockerfile
[cloud-user@rlandy17containers build]$ vi Dockerfile
[cloud-user@rlandy17containers build]$ ls *.repo
delorean.repo osptrunk-deps.repo rhos-release-17.repo rhos-release-rhel-8.1.repo
[cloud-user@rlandy17containers build]$
```
1. smush base and openstack-depencies into base
1. line 25 from openstack-dependencies add to BASE
2. build rhel8 as BASE
4. openstack container build
```
sudo buildah bud -t rhel:8 .
# this builds container named localhost/rhel w/ deps
```
#### this resulted in missing grafana repo
#### next attempt
1. smush openstack-base and openstack-depencies into openstack-base
1. take dnf install from openstack-dependencies and add to the bottom of openstack-base prior to kolla-start
2. build rhel8 as BASE
4. openstack container build
If we follow the above steps, we do not need to make any
source change to docker jinja files.
#### script for local clone and copy
Following https://hackmd.io/cv1483MkQlS8UsnteNyXqg directions for local build
https://gitlab.cee.redhat.com/snippets/1949
https://gitlab.cee.redhat.com/snippets/1948
## Local job run
Containers built:
```
REPOSITORY TAG IMAGE ID CREATED SIZE
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-nova-scheduler abcdef b1200d376d8e 7 minutes ago 1.12 GB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-zaqar-wsgi abcdef 61e9aede1b26 7 minutes ago 592 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-zaqar-base abcdef 3a577d32191d 7 minutes ago 592 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-tempest abcdef d53c9b038dea 7 minutes ago 577 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-swift-base abcdef c5fe827dc8c0 9 minutes ago 567 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-placement-base abcdef 9cf7be8b1c6b 10 minutes ago 639 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-nova-base abcdef c168a8e56c79 10 minutes ago 897 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-novajoin-server abcdef e3df2c460c12 10 minutes ago 653 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-novajoin-notifier abcdef 2c31ae8cbfb2 10 minutes ago 653 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-novajoin-base abcdef e6d7be0c76f2 10 minutes ago 653 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-octavia-base abcdef 676c4a1cfab9 10 minutes ago 775 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-manila-share abcdef 759ffd24d94b 11 minutes ago 1.01 GB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-manila-api abcdef a6f3fac66e16 12 minutes ago 704 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-heat-all abcdef a08a324754fa 12 minutes ago 744 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-manila-base abcdef 9055844afc41 13 minutes ago 656 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-ironic-base abcdef 0e6dccb316d4 14 minutes ago 610 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-keystone-base abcdef e892666ccb58 14 minutes ago 606 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-heat-base abcdef 65e538416ac2 15 minutes ago 713 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-glance-base abcdef 5162cdaeba8b 17 minutes ago 901 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-designate-producer abcdef f5559ee3a989 18 minutes ago 605 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-designate-sink abcdef 2989a80bf991 18 minutes ago 605 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-designate-worker abcdef 03fbd9f01b2b 18 minutes ago 610 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-gnocchi-base abcdef 8a12e1323a78 18 minutes ago 792 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-designate-backend-bind9 abcdef b5fc40a259e1 18 minutes ago 610 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-designate-central abcdef 29c365c36653 19 minutes ago 605 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-designate-api abcdef 10e49b5bc73d 19 minutes ago 605 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-designate-mdns abcdef f3bbea1e9db3 19 minutes ago 605 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-designate-base abcdef b93298f30fd1 19 minutes ago 574 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-cinder-base abcdef 28f7359bc02e 20 minutes ago 1.03 GB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-aodh-base abcdef baf8b1333b77 22 minutes ago 614 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-ceilometer-base abcdef 10b5dec7c5f3 22 minutes ago 598 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-barbican-base abcdef 155f1ab57125 22 minutes ago 601 MB
registry-proxy.engineering.redhat.com/tripleo-ci-rhosp17/rhel-binary-openstack-base abcdef c582f88e5453 23 minutes ago 529 MB
localhost/rhel 8 e169bb655b2a 28 minutes ago 212 MB
registry-proxy.engineering.redhat.com/rh-osbs/ubi8 8.2-ondeck 793f523ecdaf 2 days ago 211 MB
```
* https://review.opendev.org/#/c/717895/ - tripleo-ci role changes
* will need vars set on internal job http://pastebin.test.redhat.com/852586
* pre changes: https://code.engineering.redhat.com/gerrit/196758 WIP: Modify container pre to set up fpr osp builds
* See snippet for git clone: https://gitlab.cee.redhat.com/snippets/1948