[TOC]

## 簡介
EFK是Elasticsearch、Fluentd、 Kibana的縮寫
* Elasticsearch是一個分散式搜索和分析引擎，它用於儲存和索引日誌數據。
* Fluentd是一個數據收集和日誌處理系統，用於將日誌從各種搜集的來源轉到Elasticsearch。
* Kibana是一個圖形介面工具，用於在Elasticsearch中查找和圖形化數據。

## 目錄結構&專案
[專案連結](https://github.com/Wibur/EFK/tree/es-cluster)

## 步驟
1. 先執行service.sh 建立共用network
2. 個別到資料夾下執行docker-compose up -d
3. 開啟portainer(localhost:9000) 查看容器logs是否正常

## elasticsearch single

version: "3"
services:

  es:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
    restart: unless-stopped
    container_name: es01
    environment:
      # 單一節點
      - discovery.type=single-node
      # 不信任的ssl憑證
      - xpack.security.enabled=false
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - efk_service

networks:
  efk_service:
    external: true

## fluented

version: "3"
services:

  fluentd:
      build: ./plugins
      volumes:
        - ./fluentd/conf/:/fluentd/etc/
      ports:
        - "24224:24224"
        - "24224:24224/udp"
      networks:
        - efk_service

networks:
  efk_service:
    external: true

FROM --platform=arm64 fluent/fluentd:edge
USER root
RUN fluent-gem install fluent-plugin-elasticsearch \
    # 要裝此插件 才可以將log json parser
    && gem install fluent-plugin-docker
USER fluent

### fluented json logformat

# 預設的一定要
<source>
  @type forward
  port 24224
  bind 0.0.0.0
</source>

# filter 負責轉譯 docker裡的stdout log
<filter *.**>
  @type docker
  key_name log
  format json
  reserve_data true
</filter>

# 將log 傳送至es
<match *.**>
  @type elasticsearch
  host es01
  port 9200
  logstash_format true
</match>

## kibana

version: '3'
services:

  kibana:
      image: docker.elastic.co/kibana/kibana:8.8.0
      container_name: kibana
      restart: unless-stopped
      environment:
        - ELASTICSEARCH_HOSTS=http://es01:9200
        # - I18N_LOCALE=zh-CN
      ports:
        - 5601:5601
      networks:
        - efk_service

networks:
  efk_service:
    external: true

## web-server
[docker image 上傳](https://hackmd.io/@MJo5IwuWSFKmPE076E5P2g/HkBoVV853)

version: '3'
services:

    golang-http:
      image: vancer9527/go-http:v1.0.0
      logging:
        driver: "fluentd"
        options:
          "fluentd-address": "172.21.0.4:24224"
      container_name: golang-http
      restart: always
      ports:
        - 9527:9527
      networks:
        - efk_service

networks:
  efk_service:
    external: true

## elasticsearch cluster
* cluster版本需要8.8.0以上, 否則fluented會報錯物提示
* 要使用cluster 直接將此yaml替換掉single的

version: "3"
services:

  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
    restart: unless-stopped
    container_name: es01
    environment:
      - node.name=es01
      - cluster.name=es-cluster
      # 首次啟動時 會自動選出一個適合當master的節點,因此須設定適合當master的節點
      - cluster.initial_master_nodes=es01
      # 集群中的節點可以彼此發現並選出一個master,因此不能寫自己
      - discovery.seed_hosts=es02,es03
      # 不信任的ssl憑證
      - xpack.security.enabled=false
    # 一定要加 不然進程數過大 會導致node拉不起來
    ulimits:
      memlock:
        soft: -1
        hard: -1
    ports:
      - 9200:9200
    networks:
      - efk_service

  es02:
    # es01須先拉起來 才拉es02
    depends_on:
      - es01
    image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
    restart: unless-stopped
    container_name: es02
    environment:
      - node.name=es02
      - cluster.name=es-cluster
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es03
      # 不信任的ssl憑證
      - xpack.security.enabled=false
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks:
      - efk_service

  es03:
    depends_on:
      - es01
    image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
    restart: unless-stopped
    container_name: es03
    environment:
      - node.name=es03
      - cluster.name=es-cluster
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es02
      # 不信任的ssl憑證
      - xpack.security.enabled=false
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks:
      - efk_service

networks:
  efk_service:
    external: true

## 參考文件
[fluent-plugin-docker](https://github.com/edsiper/fluent-plugin-docker)
[fluented](https://docs.fluentd.org/)
[elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html)