# Note for deploy private registry

## Deploy privated registry on existing microk8s registry
### 1. Make certificate for tls
- Prepare config file

# ssl.conf
[ req ]
distinguished_name = req_distinguished_name
req_extensions     = req_ext

[ req_distinguished_name ]
commonName                  = container-registry

[ req_ext ]
subjectAltName = @alt_names

[alt_names]
IP.1    = 127.0.0.1
DNS.1   = localhost

[ v3_ca ]
subjectAltName = @alt_names

- Make certificate for tls

export SNAP_DATA="/var/snap/microk8s/current"
export OPENSSL_CONF="/snap/microk8s/current/etc/ssl/openssl.cnf"

openssl genrsa -out ${SNAP_DATA}/certs/registry.key 2048
openssl req -new -sha256 -key ${SNAP_DATA}/certs/registry.key -out ${SNAP_DATA}/certs/registry.csr -config ssl.conf -subj "/CN=container-registry"
openssl x509 -req -sha256 -in ${SNAP_DATA}/certs/registry.csr -CA ${SNAP_DATA}/certs/ca.crt -CAkey ${SNAP_DATA}/certs/ca.key -CAcreateserial -out ${SNAP_DATA}/certs/registry.crt -days 365 -extensions v3_ext -extfile ssl.conf

### 2. Create auth for private registry

$USER=registry-username
$PASSWD=registry-password

docker run --rm --entrypoint htpasswd registry:2.7.0 -Bbn $USER $PASSWD > htpasswd

- Notice that the images version `registry:2.7.0` should be the version that contains htpasswd

### 3. Configure certificate and auth as a Secret for Kubernetes

NAMESPACE=container-registry
microk8s kubectl create secret tls certs-secret -n $NAMESPACE --cert=${SNAP_DATA}/certs/registry.crt --key=${SNAP_DATA}/certs/registry.key
microk8s kubectl create secret generic auth-secret -n $NAMESPACE --from-file=htpasswd

### 4. Configure the registry deployment with those secrets and restart it
- Create yaml for new configuration

# registry-auth.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: registry
  name: registry
  namespace: container-registry
spec:
  replicas: 1
  selector:
    matchLabels:
      app: registry
  template:
    metadata:
      labels:
        app: registry
    spec:
      containers:
      - name: registry
        image: cdkbot/registry-amd64:2.6
        env:
        - name: REGISTRY_HTTP_ADDR
          value: :5000
        - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
          value: /var/lib/registry
        - name: REGISTRY_STORAGE_DELETE_ENABLED
          value: "yes"
        - name: REGISTRY_AUTH
          value: "htpasswd"
        - name: REGISTRY_AUTH_HTPASSWD_REALM
          value: "Registry Realm"
        - name: REGISTRY_AUTH_HTPASSWD_PATH
          value: "/auth/htpasswd"
        - name: REGISTRY_HTTP_TLS_CERTIFICATE
          value: "/certs/tls.crt"
        - name: REGISTRY_HTTP_TLS_KEY
          value: "/certs/tls.key"
        ports:
        - containerPort: 5000
          name: registry
          protocol: TCP
        volumeMounts:
          - mountPath: /var/lib/registry
            name: registry-data
          - name: certs-vol
            mountPath: "/certs"
            readOnly: true
          - name: auth-vol
            mountPath: "/auth"
            readOnly: true
      volumes:
        - name: registry-data
          persistentVolumeClaim:
            claimName: registry-claim
        - name: certs-vol
          secret:
            secretName: certs-secret
        - name: auth-vol
          secret:
            secretName: auth-secret

- restart the registry deployment

microk8s kubectl rollout restart deployment/registry -n $NAMESPACE

## Configure the private registry for containerd
### 1. Generate docker login auth
- Generate auth using docker

docker login localhost:32000 -u registry-username -p registry-password
cat ~/.docker/config.json

- Output will like that:

{
	"auths": {
		"localhost:32000": {
			"auth": "YWljO......"
		}
	},
	"HttpHeaders": {
		"User-Agent": "Docker-Client/19.03.8 (linux)"
	}
}

### 2. Add auth to containerd configuration

    [plugins."io.containerd.grpc.v1.cri".registry.configs]
      [plugins."io.containerd.grpc.v1.cri".registry.configs."localhost:32000".tls]
        ca_file = "${SNAP_DATA}/certs/ca.crt"
      [plugins."io.containerd.grpc.v1.cri".registry.configs."localhost:32000".auth]
        username = "registry-username"
        password = "registry-password"
        auth = "YWljO......"

### 3. Restart containerd daemon to load configuration by restart microk8s

microk8s stop
microk8s start

## Trouble shooting
### certificate signed by unknown authority

Failed to pull image "localhost:32000/nginx": rpc error: code = Unknown desc = failed to pull and unpack image "localhost:32000/nginx:latest": failed to resolve reference "localhost:32000/nginx:latest": failed to do request: Head "https://localhost:32000/v2/nginx/manifests/latest": x509: certificate signed by unknown authority

- Need to config the ca file that signed for registry
- in `${SNAP_DATA}/args/containerd-template.toml`

          [plugins."io.containerd.grpc.v1.cri".registry.configs."localhost:32000".tls]
            ca_file = "${SNAP_DATA}/certs/ca.crt"

### 401 Unauthorized

Failed to pull image "localhost:32000/nginx": rpc error: code = Unknown desc = failed to pull and unpack image "localhost:32000/nginx:latest": failed to resolve reference "localhost:32000/nginx:latest": unexpected status code [manifests latest]: 401 Unauthorized

- Need to config correct auth for registry setting
- in `${SNAP_DATA}/args/containerd-template.toml`

     [plugins."io.containerd.grpc.v1.cri".registry.configs."localhost:32000".auth]
            username = "registry-username"
            password = "registry-password"
            auth = "YWljO......"

### certificate is not valid for any names, but wanted to match localhost

Failed to pull image "localhost:32000/nginx": rpc error: code = Unknown desc = failed to pull and unpack image "localhost:32000/nginx:latest": failed to resolve reference "localhost:32000/nginx:latest": failed to do request: Head "https://localhost:32000/v2/nginx/manifests/latest": x509: certificate is not valid for any names, but wanted to match localhost

- Need to sign certificate with `subjectAltName` and the DNS is your registry doamin name.
- See [this step](https://hackmd.io/2wkiLQuZQBOj69oUUwNz4g?both#1-Make-certificate-for-tls).
## Reference
https://medium.com/swlh/deploy-your-private-docker-registry-as-a-pod-in-kubernetes-f6a489bf0180
https://github.com/containerd/cri/blob/master/docs/registry.md