# Setup Harbor Registry
###### tags: `harbor`, `image registry`
This documents steps to setup a Harbor Registry and store OKD4 images for "Deploy Tiny OKD4 VM Cluster" (https://hackmd.io/LDi2Tb9oTQe1qjqXJKFhTA)
Check "Deploy Tiny OKD4 VM Cluster with Harbor registry" (https://hackmd.io/Srw39wwaQrqwgoIn4zxUNA?edit) which put everything together.
##### 1 install Harbor for image registry
Install latest docker (Remove docker if exist)
```
[root@ssd-okd4-services ~]# rpm -qa | grep docker
docker-client-1.13.1-209.git7d71120.el7.centos.x86_64
docker-compose-plugin-2.10.2-3.el7.x86_64
docker-common-1.13.1-209.git7d71120.el7.centos.x86_64
docker-1.13.1-209.git7d71120.el7.centos.x86_64
[root@ssd-okd4-services ~]# yum remove docker-client-1.13.1-209.git7d71120.el7.centos.x86_64 docker-common-1.13.1-209.git7d71120.el7.centos.x86_64 docker-1.13.1-209.git7d71120.el7.centos.x86_64 docker-compose-plugin-2.10.2-3.el7.x86_64 -y
```
```
[root@ssd-okd4-services ~]# curl -fsSL https://get.docker.com/ | sh
[root@ssd-okd4-services ~]# systemctl restart docker
[root@ssd-okd4-services ~]# systemctl enable docker
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /usr/lib/systemd/system/docker.service.
[root@ssd-okd4-services ~]# docker -v
Docker version 20.10.18, build b40c2f6
[root@ssd-okd4-services ~]# docker version
Client: Docker Engine - Community
Version: 20.10.18
API version: 1.41
Go version: go1.18.6
Git commit: b40c2f6
Built: Thu Sep 8 23:14:08 2022
OS/Arch: linux/amd64
Context: default
Experimental: true
Server: Docker Engine - Community
Engine:
Version: 20.10.18
API version: 1.41 (minimum version 1.12)
Go version: go1.18.6
Git commit: e42327a
Built: Thu Sep 8 23:12:21 2022
OS/Arch: linux/amd64
Experimental: false
containerd:
Version: 1.6.8
GitCommit: 9cd3357b7fd7218e4aec3eae239db1f68a5a6ec6
runc:
Version: 1.1.4
GitCommit: v1.1.4-0-g5fd4c4d
docker-init:
Version: 0.19.0
GitCommit: de40ad0
```
Install docker compose
```
[root@ssd-okd4-services ~]# curl -L https://github.com/docker/compose/releases/download/v2.11.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 42.4M 100 42.4M 0 0 5929k 0 0:00:07 0:00:07 --:--:-- 6512k
[root@ssd-okd4-services ~]# chmod +x /usr/local/bin/docker-compose
[root@ssd-okd4-services ~]# docker-compose version
Docker Compose version v2.11.0
```
Download and install Harbor
```
[root@ssd-okd4-services ~]# curl -L https://github.com/goharbor/harbor/releases/download/v2.6.0/harbor-offline-installer-v2.6.0.tgz -o harbor-offline-installer-v2.6.0.tgz
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 721M 100 721M 0 0 3405k 0 0:03:36 0:03:36 --:--:-- 4024k
[root@ssd-okd4-services ~]# tar -zxf ./harbor-offline-installer-v2.6.0.tgz -C /opt/
```
Generate HTTPS certificate for Harbor
```
[root@ssd-okd4-services ~]# mkdir /opt/harbor/cert
[root@ssd-okd4-services ~]# cd /opt/harbor/cert/
```
a. Generate CA Certificate
a-1 generate private key
```
[root@ssd-okd4-services cert]# openssl genrsa -out homelab.com.key 4096
Generating RSA private key, 4096 bit long modulus
.................................++
....................................++
e is 65537 (0x10001)
```
a-2 generate certificate
```
[root@ssd-okd4-services cert]# openssl req -x509 -new -nodes -sha512 -days 3650 -subj "/C=AU/ST=NSW/L=Sydney/O=homelab/OU=okd/CN=okd.homelab.com" -key homelab.com.key -out homelab.com.crt
[root@ssd-okd4-services cert]# ls -l
total 8
-rw-r--r--. 1 root root 2009 Sep 17 17:03 homelab.com.crt
-rw-r--r--. 1 root root 3243 Sep 17 17:03 homelab.com.key
```
b. Generate Server Certificate
b-1 generate private key
```
[root@ssd-okd4-services cert]# openssl genrsa -out registry.okd4.okd.homelab.com.key 4096
Generating RSA private key, 4096 bit long modulus
......................++
......................++
e is 65537 (0x10001)
```
b-2 create CSR
```
[root@ssd-okd4-services cert]# openssl req -sha512 -new -subj "/C=CN/ST=NSW/L=Sydney/O=homelab.com/OU=okd/CN=registery.okd4.okd.homelab.com" -key registry.okd4.okd.homelab.com.key -out registry.okd4.okd.homelab.com.csr
```
b-3 create x509 v3 extension file, registry.okd4.okd.homelab.com.x509-v3.conf
```
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1=registry.okd4.okd.homelab.com
DNS.2=harbor.okd4.okd.homelab.com
DNS.3=registry
DNS.4=harbor
```
```
[root@ssd-okd4-services cert]# ls -l
total 20
-rw-r--r--. 1 root root 2009 Sep 17 17:03 homelab.com.crt
-rw-r--r--. 1 root root 3243 Sep 17 17:03 homelab.com.key
-rw-r--r--. 1 root root 1720 Sep 17 17:04 registry.okd4.okd.homelab.com.csr
-rw-r--r--. 1 root root 3243 Sep 17 17:04 registry.okd4.okd.homelab.com.key
-rw-r--r--. 1 root root 310 Sep 17 17:05 registry.okd4.okd.homelab.com.x509-v3.conf
```
b-4 generate certificate with the x509 v3 file for the registry
```
[root@ssd-okd4-services cert]# openssl x509 -req -sha512 -days 30650 -extfile registry.okd4.okd.homelab.com.x509-v3.conf -CA homelab.com.crt -CAkey homelab.com.key -CAcreateserial -in registry.okd4.okd.homelab.com.csr -out registry.okd4.okd.homelab.com.crt
Signature ok
subject=/C=CN/ST=NSW/L=Sydney/O=homelab.com/OU=okd/CN=registery.okd4.okd.homelab.com
Getting CA Private Key
[root@ssd-okd4-services cert]# ls -l
total 28
-rw-r--r--. 1 root root 2009 Sep 17 17:03 homelab.com.crt
-rw-r--r--. 1 root root 3243 Sep 17 17:03 homelab.com.key
-rw-r--r--. 1 root root 17 Sep 17 17:06 homelab.srl
-rw-r--r--. 1 root root 2163 Sep 17 17:06 registry.okd4.okd.homelab.com.crt
-rw-r--r--. 1 root root 1720 Sep 17 17:04 registry.okd4.okd.homelab.com.csr
-rw-r--r--. 1 root root 3243 Sep 17 17:04 registry.okd4.okd.homelab.com.key
-rw-r--r--. 1 root root 310 Sep 17 17:05 registry.okd4.okd.homelab.com.x509-v3.conf
```
Copy certificate to System and update ca trust.
__*Note*__: ``/etc/pki/ca-trust/source/anchors/`` for CentOS, ``/usr/local/share/ca-certificates/`` for Ubuntu
```
[root@ssd-okd4-services cert]# cp ./homelab.com.crt registry.okd4.okd.homelab.com.crt /etc/pki/ca-trust/source/anchors/
[root@ssd-okd4-services cert]# ls -l /etc/pki/ca-trust/source/anchors/
total 8
-rw-r--r--. 1 root root 2009 Sep 18 22:49 homelab.com.crt
-rw-r--r--. 1 root root 2163 Sep 17 17:16 registry.okd4.okd.homelab.com.crt
[root@ssd-okd4-services cert]# update-ca-trust
```
Copy certificate to Harbor running directory
```
[root@ssd-okd4-services cert]# mkdir -p /data/cert
[root@ssd-okd4-services cert]# cp registry.okd4.okd.homelab.com.crt registry.okd4.okd.homelab.com.key /data/cert/
```
Provide certificate to Docker
```
[root@ssd-okd4-services cert]# openssl x509 -inform PEM -in registry.okd4.okd.homelab.com.crt -out registry.okd4.okd.homelab.com.cert
[root@ssd-okd4-services cert]# cp ./registry.okd4.okd.homelab.com.cert ./registry.okd4.okd.homelab.com.key homelab.com.crt /etc/docker/certs.d/registry.okd4.okd.homelab.com\:9443/
[root@ssd-okd4-services cert]# ls -l /etc/docker/certs.d/registry.okd4.okd.homelab.com\:9443/
total 12
-rw-r--r--. 1 root root 2009 Sep 17 22:23 homelab.com.crt
-rw-r--r--. 1 root root 2163 Sep 17 22:23 registry.okd4.okd.homelab.com.cert
-rw-r--r--. 1 root root 3243 Sep 17 22:23 registry.okd4.okd.homelab.com.key
[root@ssd-okd4-services cert]# systemctl restart docker
```
Update harbor config `/opt/harbor/harbor.yml`
```
[root@ssd-okd4-services cert]# cd /opt/harbor/
[root@ssd-okd4-services harbor]# cp ./harbor.yml.tmpl harbor.yml
[root@ssd-okd4-services harbor]# vim harbor.yml
```
```
3 # The IP address or hostname to access admin UI and registry service.
4 # DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
5 hostname: registry.okd4.okd.homelab.com
8 http:
9 # port for http, default is 80. If https enabled, this port will redirect to https port
10 port: 9090
12 # https related config
13 https:
14 # https port for harbor, default is 443
15 port: 9443
16 # The path of cert and key files for nginx
17 certificate: /data/cert/registry.okd4.okd.homelab.com.crt
18 private_key: /data/cert/registry.okd4.okd.homelab.com.key
```
Install and start harbor
```
[root@ssd-okd4-services harbor]# /opt/harbor/install.sh
[Step 0]: checking if docker is installed ...
Note: docker version: 20.10.18
[Step 1]: checking docker-compose is installed ...
Note: Docker Compose version v2.10.2
[Step 2]: loading Harbor images ...
output omit...
[Step 5]: starting Harbor ...
[+] Running 10/10
⠿ Network harbor_harbor Created 0.2s
⠿ Container harbor-log Started 0.6s
⠿ Container registryctl Started 1.6s
⠿ Container registry Started 2.0s
⠿ Container harbor-db Started 2.0s
⠿ Container harbor-portal Started 1.2s
⠿ Container redis Started 1.9s
⠿ Container harbor-core Started 2.3s
⠿ Container harbor-jobservice Started 3.3s
⠿ Container nginx Started 3.3s
✔ ----Harbor has been installed and started successfully.----
```
Setup Harbor service auto start. `/etc/systemd/system/harbor.service`
```
[Unit]
Description=Harbor
After=docker.service systemd-networkd.service systemd-resolved.service
Requires=docker.service
Documentation=http://github.com/goharbor/harbor
[Service]
Type=simple
Restart=on-failure
RestartSec=5
ExecStart=/usr/local/bin/docker-compose -f /opt/harbor/docker-compose.yml up
ExecStop=/usr/local/bin/docker-compose -f /opt/harbor/docker-compose.yml down
[Install]
WantedBy=multi-user.target
```
```
[root@ssd-okd4-services harbor]# systemctl enable harbor
Created symlink from /etc/systemd/system/multi-user.target.wants/harbor.service to /etc/systemd/system/harbor.service.
```
Check Harbor process status
```
[root@ssd-okd4-services harbor]# docker-compose ps
NAME COMMAND SERVICE STATUS PORTS
harbor-core "/harbor/entrypoint.…" core running (healthy)
harbor-db "/docker-entrypoint.…" postgresql running (healthy)
harbor-jobservice "/harbor/entrypoint.…" jobservice running (healthy)
harbor-log "/bin/sh -c /usr/loc…" log running (healthy) 127.0.0.1:1514->10514/tcp
harbor-portal "nginx -g 'daemon of…" portal running (healthy)
nginx "nginx -g 'daemon of…" proxy running (healthy) 0.0.0.0:9090->8080/tcp, :::9090->8080/tcp, 0.0.0.0:9443->8443/tcp, :::9443->8443/tcp
redis "redis-server /etc/r…" redis running (healthy)
registry "/home/harbor/entryp…" registry running (healthy)
registryctl "/home/harbor/start.…" registryctl running (healthy)
```
Update /etc/hosts with harbor host
```
192.168.122.50 registry.okd4.okd.homelab.com
```
Verify login to Harbor, default username/password = admin/Harbor12345
```
[root@ssd-okd4-services harbor]# docker login registry.okd4.okd.homelab.com:9443
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
```
Allow tcp port 9090, 9443
```
[root@ssd-okd4-services harbor]# firewall-cmd --permanent --add-port=9443/tcp
success
[root@ssd-okd4-services harbor]# firewall-cmd --permanent --add-port=9090/tcp
success
[root@ssd-okd4-services harbor]# firewall-cmd --reload
success
```
Check iptables on KVM Server, remove rules which prevent traffic to "default" network (virbr0)
```
$ iptables -D LIBVIRT_FWI 2
$ iptables -D LIBVIRT_FWO 2
```
Update /etc/hosts on client
```
# on client host, add route if not exist
$ sudo ip route add 192.168.122.0/24 via 192.168.1.15 dev eno1
```
Open web browser to access Harbor from client
Login with admin/Harbor12345
Create project "openshift"
##### 2 Sync OKD image to Harbor registry
Get pull secret from "https://console.redhat.com/openshift/install/pull-secret"
```
[root@ssd-okd4-services ~]# mkdir -p /opt/okd-install/4.10.0
[root@ssd-okd4-services ~]# cd /opt/okd-install/4.10.0/
[root@ssd-okd4-services 4.10.0]# cat pull-secret .txt
{"auths":{"cloud.openshift.com":{"auth":"b3BlbnNoaWZ0LXJlbGVhc2UtZGV2K29jbV9hY2Nlc3NfZDMxZjk2NWQwMmI2NDhhNmI0NWFlYzliNzk3MzhjZmQ6RUhEWjNZUk5EVzVPRTI1RklBMV
output omit...
```
Install jq if not available
```
$ yum install epel-release -y
$ yum update -y
$ yum install jq -y
```
Convert pull-secret.txt to json format
```
[root@ssd-okd4-services 4.10.0]# cat ./pull-secret.txt | jq . > pull-secret.json
[root@ssd-okd4-services 4.10.0]# ll
total 8
-rw-r--r--. 1 root root 2967 Sep 18 14:33 pull-secret.json
-rw-r--r--. 1 root root 2851 Sep 18 14:25 pull-secret.txt
```
Generate base64 credential for Harbor image registry
```
[root@ssd-okd4-services 4.10.0]# echo -n 'admin:Harbor12345' | base64 -w0
YWRtaW46SGFyYm9yMTIzNDU=
```
Create Harbor image registry authentication file
```
[root@ssd-okd4-services 4.10.0]# cat >pull-secret-local.json<<EOF
{
"auths":{
"registry.okd4.okd.homelab.com:9443": {
"auth": "YWRtaW46SGFyYm9yMTIzNDU=",
"email": ""
}
}
}
EOF
```
Append Harbor image registry authentication file with the pull-secret.json
```
[root@ssd-okd4-services 4.10.0]# jq -s '.[0] * .[1]' pull-secret.json pull-secret-local.json > pull-secret-all.json
```
Check oc client version
```
[root@ssd-okd4-services 4.10.0]# oc version
Client Version: 4.10.0-0.okd-2022-07-09-073606
```
Export following environment variables
```
export OKD_RELEASE="4.10.0-0.okd-2022-07-09-073606"
export LOCAL_REGISTRY="registry.okd4.okd.homelab.com:9443"
export LOCAL_REPOSITORY="openshift/okd"
export PRODUCT_REPO="openshift"
export LOCAL_SECRET_JSON="/opt/okd-install/4.10.0/pull-secret-all.json"
export RELEASE_NAME="okd"
```
```
[root@ssd-okd4-services 4.10.0]# export OKD_RELEASE="4.10.0-0.okd-2022-07-09-073606"
[root@ssd-okd4-services 4.10.0]# export LOCAL_REGISTRY="registry.okd4.okd.homelab.com:9443"
[root@ssd-okd4-services 4.10.0]# export LOCAL_REPOSITORY="openshift/okd"
[root@ssd-okd4-services 4.10.0]# export PRODUCT_REPO="openshift"
[root@ssd-okd4-services 4.10.0]# export LOCAL_SECRET_JSON="/opt/okd-install/4.10.0/pull-secret-all.json"
[root@ssd-okd4-services 4.10.0]# export RELEASE_NAME="okd"
```
Pull OKD image from quay.io to local Harbor registry
```
oc adm release mirror -a ${LOCAL_SECRET_JSON} \
--from=quay.io/${PRODUCT_REPO}/${RELEASE_NAME}:${OKD_RELEASE} \
--to=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY} \
--to-release-image=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OKD_RELEASE}
```
```
[root@ssd-okd4-services 4.10.0]# oc adm release mirror -a ${LOCAL_SECRET_JSON} --from=quay.io/${PRODUCT_REPO}/${RELEASE_NAME}:${OKD_RELEASE} --to=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY} --to-release-image=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OKD_RELEASE}
info: Mirroring 170 images to registry.okd4.okd.homelab.com:9443/openshift/okd ...
omit output...
Success
Update image: registry.okd4.okd.homelab.com:9443/openshift/okd:4.10.0-0.okd-2022-07-09-073606
Mirror prefix: registry.okd4.okd.homelab.com:9443/openshift/okd
Mirror prefix: registry.okd4.okd.homelab.com:9443/openshift/okd:4.10.0-0.okd-2022-07-09-073606
To use the new mirrored repository to install, add the following section to the install-config.yaml:
imageContentSources:
- mirrors:
- registry.okd4.okd.homelab.com:9443/openshift/okd
source: quay.io/openshift/okd
- mirrors:
- registry.okd4.okd.homelab.com:9443/openshift/okd
source: quay.io/openshift/okd-content
To use the new mirrored repository for upgrades, use the following to create an ImageContentSourcePolicy:
apiVersion: operator.openshift.io/v1alpha1
kind: ImageContentSourcePolicy
metadata:
name: example
spec:
repositoryDigestMirrors:
- mirrors:
- registry.okd4.okd.homelab.com:9443/openshift/okd
source: quay.io/openshift/okd
- mirrors:
- registry.okd4.okd.homelab.com:9443/openshift/okd
source: quay.io/openshift/okd-content
```
Check repositories on web console
#### Reference
- https://goharbor.io
- https://goharbor.io/docs/2.4.0/install-config/configure-https/
- https://goharbor.io/docs/2.4.0/install-config/troubleshoot-installation/#https
- https://www.digitalocean.com/community/tutorials/how-to-set-up-and-configure-a-certificate-authority-ca-on-ubuntu-20-04
- https://computingforgeeks.com/add-harbor-image-registry-pull-secret-to-kubernetes-openshift/
Sign in with Wallet
Connect another wallet