# Set up OpenShift Service Mesh + Distributed Tracing (Tempo)

## ！！！此文件目的為驗證服務架構，非生產環境安裝指引！！！

## The whole picture of OSSM + Tempo


## Environment and important pre-set

- Resource Hub OCP 4.16 cluster

- minio is installed in ns/minio
- Minio Access Key : `tempo`
- Minio Secret Key : `supersecret`
- Minio's svc is named `minio`
- TempoStack is installed in ns/tracing-system
- TempoStack is named `sample`
- A OpenTelemetryCollector named `otel` is installed in ns/otel-collector
- otel collector's svc is named `otel-collector`

## Install NFS Provisioner Operator (You can use other Storage class)

### Install Operator

# Create a new namespace
oc new-project nfsprovisioner-operator

# Deploy NFS Provisioner operator in the terminal (You can also use OpenShift Console
cat << EOF | oc apply -f -  
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  name: nfs-provisioner-operator
  namespace: openshift-operators
spec:
  channel: alpha
  installPlanApproval: Automatic
  name: nfs-provisioner-operator
  source: community-operators
  sourceNamespace: openshift-marketplace
EOF

### Create NFS server mount path on a worker node

# Check nodes
oc get nodes
NAME                                      STATUS   ROLES                  AGE   VERSION
master-0.smtest.lab.psi.pnq2.redhat.com   Ready    control-plane,master   44m   v1.29.6+aba1e8d
master-1.smtest.lab.psi.pnq2.redhat.com   Ready    control-plane,master   44m   v1.29.6+aba1e8d
master-2.smtest.lab.psi.pnq2.redhat.com   Ready    control-plane,master   44m   v1.29.6+aba1e8d
worker-0.smtest.lab.psi.pnq2.redhat.com   Ready    worker                 22m   v1.29.6+aba1e8d
worker-1.smtest.lab.psi.pnq2.redhat.com   Ready    worker                 22m   v1.29.6+aba1e8d

# Set Env variable for the target node name
export target_node=worker-0.smtest.lab.psi.pnq2.redhat.com
oc label node/${target_node} app=nfs-provisioner

# ssh to the node
oc debug node/${target_node}

# Create a directory and set up the Selinux label.
chroot /host
mkdir -p /home/core/nfs
chcon -Rvt svirt_sandbox_file_t /home/core/nfs
exit 
exit

### Create NFS Provisioner CRD

# Create NFSProvisioner Custom Resource
cat << EOF | oc apply -f -  
apiVersion: cache.jhouse.com/v1alpha1
kind: NFSProvisioner
metadata:
  name: nfsprovisioner-sample
  namespace: nfsprovisioner-operator
spec:
  hostPathDir: /home/core/nfs
  nodeSelector:
    app: nfs-provisioner
EOF

# Check if NFS Server is running
oc get pod
NAME                               READY   STATUS    RESTARTS   AGE
nfs-provisioner-77bc99bd9c-57jf2   1/1     Running   0          2m32s

oc adm policy add-scc-to-user hostmount-anyuid -z nfs-provisioner -n nfsprovisioner-operator

# Update annotation of the NFS StorageClass
oc patch storageclass nfs -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

# Check the default next to nfs StorageClass
oc get sc
NAME            PROVISIONER       RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs (default)   example.com/nfs   Delete          Immediate           false                  4m29s

## Install Minio to provide Object storage

# Create NS for minio
oc new-project minio

# Create SS for minio

cat << EOF | oc apply -f -
kind: StatefulSet
apiVersion: apps/v1
metadata:
  name: minio
  namespace: minio
  labels:
    app: minio
spec:
  serviceName: minio
  revisionHistoryLimit: 10
  persistentVolumeClaimRetentionPolicy:
    whenDeleted: Retain
    whenScaled: Retain
  volumeClaimTemplates:
    - kind: PersistentVolumeClaim
      apiVersion: v1
      metadata:
        name: data
        creationTimestamp: null
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: 1Gi
        storageClassName: nfs # << Should fit your default SC
        volumeMode: Filesystem
      status:
        phase: Pending
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: minio
    spec:
      containers:
        - resources: {}
          terminationMessagePath: /dev/termination-log
          name: minio
          env:
            - name: MINIO_ACCESS_KEY
              value: tempo
            - name: MINIO_SECRET_KEY
              value: supersecret
            - name: MINIO_KMS_SECRET_KEY
              value: 'my-minio-key:oyArl7zlPECEduNbB1KXgdzDn2Bdpvvw0l8VO51HQnY='
          ports:
            - containerPort: 9000
              protocol: TCP
          imagePullPolicy: Always
          volumeMounts:
            - name: data
              mountPath: /data
          terminationMessagePolicy: File
          image: 'quay.io/rhn_support_jaliang/minio/minio:RELEASE.2023-05-27T05-56-19Z'
          args:
            - server
            - /data
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      dnsPolicy: ClusterFirst
      securityContext: {}
      schedulerName: default-scheduler
  podManagementPolicy: OrderedReady
  replicas: 1
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      partition: 0
  selector:
    matchLabels:
      app: minio
EOF

Create Service for Minio

cat << EOF | oc apply -f -
apiVersion: v1
kind: Service
metadata:
  name: minio
  labels:
    app: minio
spec:
  type: NodePort
  ports:
    - port: 9000
      name: minio
      nodePort: 30080
  selector:
    app: minio
EOF

Create route for minio client

oc expose svc/minio -n minio

Using minio client to create bucket for Tempo on Bastion

curl https://dl.min.io/client/mc/release/linux-amd64/mc \
  --create-dirs \
  -o $HOME/minio-binaries/mc

chmod +x $HOME/minio-binaries/mc
export PATH=$PATH:$HOME/minio-binaries/


[quickcluster@upi-0 ~]$ oc get route
NAME    HOST/PORT                                         PATH   SERVICES   PORT    TERMINATION   WILDCARD
minio   minio-minio.apps.smtest.lab.psi.pnq2.redhat.com          minio      minio                 None
[quickcluster@upi-0 ~]$
[quickcluster@upi-0 ~]$ mc alias set myminio http://minio-minio.apps.smtest.lab.psi.pnq2.redhat.com --insecure
mc: Configuration written to `/home/quickcluster/.mc/config.json`. Please update your access credentials.
mc: Successfully created `/home/quickcluster/.mc/share`.
mc: Initialized share uploads `/home/quickcluster/.mc/share/uploads.json` file.
mc: Initialized share downloads `/home/quickcluster/.mc/share/downloads.json` file.
Enter Access Key: tempo
Enter Secret Key:
Added `myminio` successfully.


[quickcluster@upi-0 ~]$ mc ready play
The cluster is ready

[quickcluster@upi-0 ~]$ mc mb myminio/tempo
Bucket created successfully `myminio/tempo`.

## Install Tempo Operator and Create TempoStack

### Install Tempo Operator with default setting
https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/html/distributed_tracing/distributed-tracing-platform-tempo#dist-tracing-tempo-installing

### Create TempoStack

oc new-project tracing-system

Check minio's svc under ns/minio

[quickcluster@upi-0 ~]$ oc get svc -n minio
NAME    TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
minio   NodePort   172.30.63.66   <none>        9000:30080/TCP   8m9s

>> minio.minio.svc:9000

Create Secret with S3 endpoint info for Tempo

cat << EOF | oc apply -f -
apiVersion: v1
kind: Secret
metadata:
  name: minio
  namespace: tracing-system
stringData:
  endpoint: http://minio.minio.svc:9000
  bucket: tempo
  access_key_id: tempo
  access_key_secret: supersecret
type: Opaque
EOF

Create a TempoStack named `sample` under ns/tracing-system

cat << EOF | oc apply -f -
apiVersion: tempo.grafana.com/v1alpha1
kind: TempoStack
metadata:
  name: sample
  namespace: tracing-system
spec:
  storageSize: 1Gi
  storage: 
    secret:
      name: minio
      type: s3
  resources:
    total:
      limits:
        memory: 2Gi
        cpu: 2000m
  template:
    queryFrontend:
      jaegerQuery: 
        enabled: true
        ingress:
          route:
            termination: edge
          type: route
EOF

Once TempoStack is ready, You will be able to access Jaeger console in ns/tracing-system.

[quickcluster@upi-0 ~]$ oc get route
NAME                          HOST/PORT                                                                        PATH   SERVICES                      PORT        TERMINATION   WILDCARD
tempo-sample-query-frontend   tempo-sample-query-frontend-tracing-system.apps.smtest.lab.psi.pnq2.redhat.com          tempo-sample-query-frontend   jaeger-ui   edge          None

## Install Red Hat build of OpenTelemetry Operator

Install RH build of OpenTelemetry Operator with default setting.
https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/html/red_hat_build_of_opentelemetry/install-otel


### Create NS for otel-collector and create a OpenTelemetryCollector in this NS

oc new-project otel-collector

cat << EOF | oc apply -f -
apiVersion: opentelemetry.io/v1alpha1
kind: OpenTelemetryCollector
metadata:
  name: otel
  namespace: otel-collector  
  annotations:
    sidecar.istio.io/inject: 'true' 
spec:
  mode: deployment
  config: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
    exporters:
      otlp:
        endpoint: "tempo-sample-distributor.tracing-system.svc.cluster.local:4317" 
        tls:
          insecure: true
    service:
      pipelines:
        traces:
          receivers: [otlp]
          processors: []
          exporters: [otlp]
EOF

Check otel-collector pod and svc

[quickcluster@upi-0 ~]$ oc get po,svc -n otel-collector
NAME                                  READY   STATUS    RESTARTS   AGE
pod/otel-collector-7659b9d696-ktsgk   2/2     Running   0          112s

NAME                                TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE
service/otel-collector              ClusterIP   172.30.215.76   <none>        4317/TCP   112s
service/otel-collector-headless     ClusterIP   None            <none>        4317/TCP   112s
service/otel-collector-monitoring   ClusterIP   172.30.185.70   <none>        8888/TCP   112s

## Install ServiceMesh Operator 2.6

### Install required Operators
https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/html/service_mesh/service-mesh-2-x#installing-ossm

https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/html/service_mesh/service-mesh-2-x#ossm-viewing-metrics_observability

Install Servcie Mesh Operator 2.6 with default setting.

Install Kiali Operator with default setting.

### Configure SMCP

# Create NS for SM control Plane
oc new-project istio-system

# Create SMCP in ns/istio-system

cat << EOF | oc apply -f -
kind: ServiceMeshControlPlane
apiVersion: maistra.io/v2
metadata:
  name: basic
  namespace: istio-system
spec:
  addons:
    grafana:
      enabled: true
    kiali:
      enabled: true
    prometheus:
      enabled: true
  meshConfig:
    extensionProviders:
      - name: otel
        opentelemetry:
          port: 4317
          service: otel-collector.otel-collector.svc.cluster.local
  policy:
    type: Istiod
  telemetry:
    type: Istiod
  version: v2.6
EOF

# Create a Telemetry resource in istio-system
cat << EOF | oc apply -f -
apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: mesh-default
  namespace: istio-system
spec:
  tracing:
  - providers:
    - name: otel
    randomSamplingPercentage: 100
EOF

### Configure the Kiali resource specification to enable a Kiali workload traces dashboard. You can use the dashboard to view tracing query results.

oc edit kiali kiali -n istio-system

apiVersion: kiali.io/v1alpha1
kind: Kiali
# ...
spec:
  external_services:
    tracing:
      query_timeout: 30
      enabled: true
      in_cluster_url: 'http://tempo-sample-query-frontend.tracing-system.svc.cluster.local:16685'
      url: 'http://tempo-sample-query-frontend-tracing-system.apps.smtest.lab.psi.pnq2.redhat.com'
      use_grpc: true

### Create SMMR for bookinfo and otel-collector NS

cat << EOF | oc apply -f -
kind: ServiceMeshMemberRoll
apiVersion: maistra.io/v1
metadata:
  name: default
  namespace: istio-system
spec:
  members:
    - bookinfo
    - otel-collector
EOF

### Install bookinfo example in ns/bookinfo

oc new-project bookinfo

oc apply -n bookinfo -f https://raw.githubusercontent.com/Maistra/istio/maistra-2.6/samples/bookinfo/platform/kube/bookinfo.yaml


oc apply -n bookinfo -f https://raw.githubusercontent.com/Maistra/istio/maistra-2.6/samples/bookinfo/networking/bookinfo-gateway.yaml

oc apply -n bookinfo -f https://raw.githubusercontent.com/Maistra/istio/maistra-2.6/samples/bookinfo/networking/destination-rule-all.yaml


export GATEWAY_URL=$(oc -n istio-system get route istio-ingressgateway -o jsonpath='{.spec.host}')

echo "http://$GATEWAY_URL/productpage"

curl -k -I "http://$GATEWAY_URL/productpage"
HTTP/1.1 200 OK
server: istio-envoy
date: Fri, 23 Aug 2024 06:46:13 GMT
content-type: text/html; charset=utf-8
content-length: 5290
vary: Cookie
x-envoy-upstream-service-time: 37
set-cookie: cd10b69e39387eb7ec9ac241201ab1ab=479a0937580de7ceadd29a6d0ad0e86b; path=/; HttpOnly

## You can now access Jaeger in tracing-system NS and see bookinfo's tracing data

[quickcluster@upi-0 ~]$ oc get route -n tracing-system
NAME                          HOST/PORT                                                                        PATH   SERVICES                      PORT        TERMINATION   WILDCARD
tempo-sample-query-frontend   tempo-sample-query-frontend-tracing-system.apps.smtest.lab.psi.pnq2.redhat.com          tempo-sample-query-frontend   jaeger-ui   edge          None


# References

Install NFS Operator - https://developers.redhat.com/articles/2022/04/20/create-and-manage-local-persistent-volumes-codeready-containers

Install Minio in OCP CRC - https://blog.min.io/develop-on-openshift-with-minio/