Telegraf
===

###### tags: `QCT` `Data Center`

This note is for developing QCT Telegraf.

# Dev Env
Telegraf requires golang version 1.10 or newer, the Makefile requires GNU make.

1. [Install Go](https://golang.org/doc/install) >=1.10 (1.12 recommended)
2. [Install dep](https://golang.github.io/dep/docs/installation.html) ==v0.5.0
3. Download Telegraf source:

    mkdir $HOME/go/src/github.com/influxdata
    cd $HOME/go/src/github.com/influxdata
    git clone https://git.rd2.qct.io/telemetry/telegraf.git

4. Run [make](https://hackmd.io/@derailment/make) from the source directory

    cd $HOME/go/src/github.com/influxdata/telegraf
    make

5. Update Docker image

    docker build -t qctrd2/qct-telegraf . --no-cache
    docker login
    docker push qctrd2/qct-telegraf:latest

::: danger
Before running ```./telegraf --config CONFIG_FILE```, make sure ```./telegraf --version``` is on the branch with the commit hash you want.
:::

---

# Output Env

## Collectd Exporter
:::danger
Do not add command option ```-collectd.listen-address=":25826"``` or ```-web.collectd-push-path``` for entrypoint **/bin/collectd_exporter** of the image.
:::

# docker-compose.yml
version: '3'
services:
   Collectd_Exporter:
     container_name: collectd_exporter
     image: prom/collectd-exporter
     volumes:
       - /etc/localtime:/etc/localtime:ro
     ports:
       - "9103:9103"
     logging:
       driver: "json-file"
       options:
         max-size: "500m"
         max-file: "3"
     tty: true
     restart: always

# in the same directory
docker-compose up -d

## Prometheus

# docker-compose.yml
version: '3'
services:
   Prometheus:
     container_name: prometheus
     image: prom/prometheus
     volumes:
       - /etc/localtime:/etc/localtime:ro
       - /etc/telegraf/prometheus.yml:/etc/prometheus/prometheus.yml
     ports:
       - "9090:9090"
     logging:
       driver: "json-file"
       options:
         max-size: "500m"
         max-file: "3"
     tty: true
     restart: always

# prometheus.yml
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # Override the global default and scrape targets from this job every 5 seconds.
    scrape_interval: 5s

    static_configs:
      - targets: ['10.103.11.100:9095']

# in the same directory
docker-compose up -d

---

# Input Env

## K8s
* Install K8s
* [~/.ssh/id_rsa](https://blog.gtwang.org/linux/linux-ssh-public-key-authentication/)
* [Rancher Kubernetes Engine (RKE)](https://onap.readthedocs.io/en/latest/submodules/oom.git/docs/oom_setup_kubernetes_rancher.html)
* Get bearer token

apiVersion: v1
kind: Namespace
metadata:
  name: miq

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: miq-sa
  namespace: miq

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: miq-cr
rules:
- apiGroups: [""]
  resources: ["*"]
  verbs: ["*"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: miq-crb
  namespace: miq
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: miq-cr
subjects:
- kind: ServiceAccount
  name: miq-sa
  namespace: miq

SECRET=$(kubectl get secrets -n <NAMESPACE> | grep <SA> | awk '{print $1}')
TOKEN=$(kubectl describe secret ${SECRET} -n <NAMESPACE> | grep -E '^token' | awk '{print $2}')

## HWs
- S.M.A.R.T. Input Plugin

smartctl --info --attributes --health -n standby --format=brief /dev/sda

- IPMI Sensor Input Plugin

ipmitool -I lan -H SERVER -U USERID -P PASSW0RD sdr

- Net Plugin

lspci | grep Ethernet

---

# Running Env
有3種方式執行Telegraf：
1. 在OS執行binary

git clone -b qct-addon --single-branch http://git.rd2.qct.io/telemetry/telegraf.git
cd telegraf
./telegraf --config qct/deploy/telegraf.conf

2. 在Docker執行container

# docker-compose.yml
version: '3'
services:
   QCT_Telegraf:
     container_name: telegraf
     network_mode: host
     privileged: true
     image: nfvdtmp0/telegraf:9dcec011
     volumes:
       - /etc/localtime:/etc/localtime:ro
       - /etc/telegraf/telegraf.conf:/etc/telegraf/telegraf.conf
       - /etc/telegraf/hardware.miq:/etc/telegraf/hardware.miq
       - /etc/telegraf/MIQ_bearer_token:/etc/telegraf/MIQ_bearer_token
     tty: true
     restart: always
     command: "/telegraf --config /etc/telegraf/telegraf.conf"

# in the same directory
docker-compose up -d

3. 在K8s執行DaemonSet

# telegraf-daemonset.yml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: telegraf-sa
  namespace: miq

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: influx:cluster:viewer
  labels:
    rbac.authorization.k8s.io/aggregate-view-telegraf: "true"
rules:
- apiGroups: [""]
  resources: ["*"]
  verbs: ["*"]

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: influx:telegraf
aggregationRule:
  clusterRoleSelectors:
  - matchLabels:
      rbac.authorization.k8s.io/aggregate-view-telegraf: "true"
  - matchLabels:
      rbac.authorization.k8s.io/aggregate-to-view: "true"
rules: [] # Rules are automatically filled in by the controller manager.

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: influx:telegraf:viewer
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: influx:telegraf
subjects:
- kind: ServiceAccount
  name: telegraf-sa
  namespace: miq

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: telegraf-cm
  namespace: miq
  labels:
    k8s-app: telegraf
data:
  telegraf.conf: |+
    [agent]
      interval = "10s"
      round_interval = true
      metric_batch_size = 1000
      metric_buffer_limit = 10000
      collection_jitter = "0s"
      flush_interval = "10s"
      flush_jitter = "0s"
      precision = ""
      debug = true
      quiet = false
      hostname = "S5BQ"
      omit_hostname = false
    [[inputs.qct_kube_inventory]]
      url = "https://10.103.11.72:6443"
      bearer_token = "/var/run/secrets/kubernetes.io/serviceaccount/token"
      namespace = ""
      qct_miq_k8s_ems_id = "1234"
      insecure_skip_verify = true
    [[inputs.qct_kubernetes]]
      url = "https://10.103.11.72:10250"
      qct_miq_k8s_ems_id = "1234"
      bearer_token = "/var/run/secrets/kubernetes.io/serviceaccount/token"
      insecure_skip_verify = true
    [[outputs.http]]
      url = "http://10.103.3.89:9103/collectd-post"
      data_format = "qct_prometheus"
      method = "POST"
      [outputs.http.headers]
        Content-Type = "application/json; charset=utf-8"

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: telegraf-ds
  namespace: miq
spec:
  selector:
    matchLabels:
      app: telegraf
  template:
    metadata:
      labels:
        app: telegraf
    spec:
      serviceAccountName: telegraf-sa
      containers:
      - name: qct-telegraf
        image: qctrd2/qct-telegraf:1.2.0
        imagePullPolicy: Always
        volumeMounts:
        - name: telegraf-config
          mountPath: /etc/telegraf
      terminationGracePeriodSeconds: 30
      volumes:
      - name: telegraf-config
        configMap:
          name: telegraf-cm

kubectl apply -f telegraf-daemonset.yml