DevOpsDays Taipei 2023 工作坊 - 如何在 Elasticsearch 實現敏捷的資料建模與管理
===
# 事先定義好的 Data Model
## 明確的欄位定義
### Create Index or Update Mapping
```
PUT my-index
{
"mappings": {
"properties": {
"name": { "type": "text" },
"age": { "type": "integer" },
"birthday": { "type": "date" },
"email": { "type": "keyword" },
"salary": { "type": "double" }
}
}
}
```
```
PUT my-index/_mapping
{
"properties": {
"name": { "type": "text" },
"age": { "type": "integer" },
"birthday": { "type": "date" },
"email": { "type": "keyword" },
"salary": { "type": "double" }
}
}
```
### Sprint 1: 收集電商訂單 Logs
配合的後端工程師告訴你，這個 Sprint 會完成一個訂單系統，右方是訂單的資料結構，每筆訂單完成都會產生一筆 Log 記錄到 Elasticsearch，未來 Data Team 會要用這些訂單資料做數據分析。
```
{
"currency": "EUR",
"customer_first_name": "Eddie",
"customer_gender": "MALE",
"customer_id": 38,
"customer_last_name": "Underwood",
"order_date": "2023-10-09T09:28:48+00:00",
"order_id": 584677,
"products": [
{
"base_price": 11.99,
"discount_percentage": 0,
"quantity": 1,
"manufacturer": "Elitelligence",
"tax_amount": 0,
"product_id": 6283,
"category": "Men's Clothing",
"sku": "ZO0549605496",
"taxless_price": 11.99,
"unit_discount_amount": 0,
"min_price": 6.35,
"_id": "sold_product_584677_6283",
"discount_amount": 0,
"created_on": "2016-12-26T09:28:48+00:00",
"product_name": "Basic T-shirt - dark blue/white",
"price": 11.99,
"taxful_price": 11.99,
"base_unit_price": 11.99
}
],
"total_quantity": 2,
"total_unique_products": 2,
"type": "order",
"user": "eddie"
}
```
溝通好會將訂單 Logs 資料寫入到 devopsdays-taipei-2023-ec-order 的 index 之中。
```
# 我們要先將這個 Order 的 Data Model 進行定義。
PUT devopsdays-taipei-2023-ec-order
{
"mappings": {
"properties": {
"currency": { "type": "keyword" },
"customer_first_name": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 }} },
"customer_gender": { "type": "keyword" },
"customer_id": { "type": "keyword" },
"customer_last_name": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 }} },
"order_date": { "type": "date" },
"order_id": { "type": "keyword" },
"products": {
"properties": {
"base_price": { "type": "double" },
"discount_percentage": { "type": "float" },
"quantity": { "type": "float" },
"manufacturer": { "type": "keyword" },
"tax_amount": { "type": "float" },
"product_id": { "type": "keyword" },
"category": { "type": "keyword" },
"sku": { "type": "keyword" },
"taxless_price": { "type": "double" },
"unit_discount_amount": { "type": "float" },
"min_price": { "type": "double" },
"_id": { "type": "keyword" },
"discount_amount": { "type": "float" },
"created_on": { "type": "date" },
"product_name": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 }} },
"price": { "type": "double" },
"taxful_price": { "type": "double" },
"base_unit_price": { "type": "double" }
}
},
"total_quantity": { "type": "float" },
"total_unique_products": { "type": "float" },
"type": { "type": "keyword" },
"user": { "type": "keyword" }
}
}
}
# 查看 mapping
GET devopsdays-taipei-2023-ec-order/_mapping
```
#### Testing Data for Sprint 1
```
# 第一批資料進來了
PUT devopsdays-taipei-2023-ec-order/_bulk
{ "index" : { "_id" : "584677" } }
{"currency":"EUR","customer_first_name":"Eddie","customer_gender":"MALE","customer_id":38,"customer_last_name":"Underwood","customer_phone":"","order_date":"2023-10-09T09:28:48+00:00","order_id":584677,"products":[{"base_price":11.99,"discount_percentage":0,"quantity":1,"manufacturer":"Elitelligence","tax_amount":0,"product_id":6283,"category":"Men's Clothing","sku":"ZO0549605496","taxless_price":11.99,"unit_discount_amount":0,"min_price":6.35,"_id":"sold_product_584677_6283","discount_amount":0,"created_on":"2016-12-26T09:28:48+00:00","product_name":"Basic T-shirt - dark blue/white","price":12,"taxful_price":12,"base_unit_price":12},{"base_price":24.99,"discount_percentage":0,"quantity":1,"manufacturer":"Oceanavigations","tax_amount":0,"product_id":19400,"category":"Men's Clothing","sku":"ZO0299602996","taxless_price":25,"unit_discount_amount":0,"min_price":11.75,"_id":"sold_product_584677_19400","discount_amount":0,"created_on":"2016-12-26T09:28:48+00:00","product_name":"Sweatshirt - grey multicolor","price":25,"taxful_price":25,"base_unit_price":25}],"taxful_total_price":37,"taxless_total_price":37,"total_quantity":2,"total_unique_products":2,"type":"order","user":"eddie"}
{ "index" : { "_id" : "584021" } }
{"currency":"EUR","customer_first_name":"Mary","customer_gender":"FEMALE","customer_id":20,"customer_last_name":"Bailey","customer_phone":"","order_date":"2023-10-08T21:59:02+00:00","order_id":584021,"products":[{"base_price":24.99,"discount_percentage":0,"quantity":1,"manufacturer":"Champion Arts","tax_amount":0,"product_id":11238,"category":"Women's Clothing","sku":"ZO0489604896","taxless_price":24.99,"unit_discount_amount":0,"min_price":11.75,"_id":"sold_product_584021_11238","discount_amount":0,"created_on":"2016-12-25T21:59:02+00:00","product_name":"Denim dress - black denim","price":24.99,"taxful_price":24.99,"base_unit_price":24.99},{"base_price":28.99,"discount_percentage":0,"quantity":1,"manufacturer":"Pyramidustries","tax_amount":0,"product_id":20149,"category":"Women's Clothing","sku":"ZO0185501855","taxless_price":28.99,"unit_discount_amount":0,"min_price":15.65,"_id":"sold_product_584021_20149","discount_amount":0,"created_on":"2016-12-25T21:59:02+00:00","product_name":"Shorts - black","price":28.99,"taxful_price":28.99,"base_unit_price":28.99}],"taxful_total_price":53.98,"taxless_total_price":53.98,"total_quantity":2,"total_unique_products":2,"type":"order","user":"mary"}
{ "index" : { "_id" : "584058" } }
{"currency":"EUR","customer_first_name":"Gwen","customer_gender":"FEMALE","customer_id":26,"customer_last_name":"Butler","customer_phone":"","order_date":"2023-10-04T22:32:10+00:00","order_id":584058,"products":[{"base_price":99.99,"discount_percentage":0,"quantity":1,"manufacturer":"Low Tide Media","tax_amount":0,"product_id":22794,"category":"Women's Shoes","sku":"ZO0374603746","taxless_price":99.99,"unit_discount_amount":0,"min_price":46.01,"_id":"sold_product_584058_22794","discount_amount":0,"created_on":"2016-12-25T22:32:10+00:00","product_name":"Boots - Midnight Blue","price":99.99,"taxful_price":99.99,"base_unit_price":99.99},{"base_price":99.99,"discount_percentage":0,"quantity":1,"manufacturer":"Oceanavigations","tax_amount":0,"product_id":23386,"category":"Women's Clothing","sku":"ZO0272202722","taxless_price":99.99,"unit_discount_amount":0,"min_price":53.99,"_id":"sold_product_584058_23386","discount_amount":0,"created_on":"2016-12-25T22:32:10+00:00","product_name":"Short coat - white/black","price":99.99,"taxful_price":99.99,"base_unit_price":99.99}],"taxful_total_price":199.98,"taxless_total_price":199.98,"total_quantity":2,"total_unique_products":2,"type":"order","user":"gwen"}
# 工程師反應，查看資料有誤？
GET devopsdays-taipei-2023-ec-order/_search
{
"query": {
"range": {
"taxful_total_price": {
"lte": 53
}
}
}
}
# 查看 mapping
GET devopsdays-taipei-2023-ec-order/_mapping
# 嘗試修復
PUT devopsdays-taipei-2023-ec-order/_mapping
{
"properties": {
"taxful_total_price": { "type": "double" },
"taxless_total_price": { "type": "double" }
}
}
```
### Sprint 2: Index Mapping 的版本选代
```
# 刪除 index
DELETE devopsdays-taipei-2023-ec-order
# 建立 Index Template
# 包含 `_meta` 資料，以及 `aliases`。
PUT _index_template/devopsdays-taipei-2023-ec-order
{
"index_patterns": [
"devopsdays-taipei-2023-ec-order*"
],
"template": {
"aliases": {
"devopsdays-taipei-2023-ec-order": {}
},
"mappings": {
"_meta": {
"version": 1,
"version_creation_date": "2023-09-26"
},
"properties": {
"currency": { "type": "keyword" },
"customer_first_name": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 }} },
"customer_gender": { "type": "keyword" },
"customer_id": { "type": "keyword" },
"customer_last_name": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 }} },
"order_date": { "type": "date" },
"order_id": { "type": "keyword" },
"products": {
"properties": {
"base_price": { "type": "double" },
"discount_percentage": { "type": "float" },
"quantity": { "type": "float" },
"manufacturer": { "type": "keyword" },
"tax_amount": { "type": "float" },
"product_id": { "type": "keyword" },
"category": { "type": "keyword" },
"sku": { "type": "keyword" },
"taxless_price": { "type": "double" },
"unit_discount_amount": { "type": "float" },
"min_price": { "type": "double" },
"_id": { "type": "keyword" },
"discount_amount": { "type": "float" },
"created_on": { "type": "date" },
"product_name": { "type": "text", "fields": { "keyword": { "type": "keyword", "ignore_above": 256 }} },
"price": { "type": "double" },
"taxful_price": { "type": "double" },
"base_unit_price": { "type": "double" }
}
},
"taxful_total_price": { "type": "double" },
"taxless_total_price": { "type": "double" },
"total_quantity": { "type": "float" },
"total_unique_products": { "type": "float" },
"type": { "type": "keyword" },
"user": { "type": "keyword" }
}
}
},
"priority": 100,
"version": 1,
"_meta": {
"description": "devopsdays workshop",
"version_creation_date": "2023-09-26"
}
}
# 這次我們將 Index 名字後面加上 `_v1`，用來區別這是第一版，重新將資料匯入。
PUT devopsdays-taipei-2023-ec-order_v1/_bulk
{ "index" : { "_id" : "584677" } }
{"currency":"EUR","customer_first_name":"Eddie","customer_gender":"MALE","customer_id":38,"customer_last_name":"Underwood","customer_phone":"","order_date":"2023-10-09T09:28:48+00:00","order_id":584677,"products":[{"base_price":11.99,"discount_percentage":0,"quantity":1,"manufacturer":"Elitelligence","tax_amount":0,"product_id":6283,"category":"Men's Clothing","sku":"ZO0549605496","taxless_price":11.99,"unit_discount_amount":0,"min_price":6.35,"_id":"sold_product_584677_6283","discount_amount":0,"created_on":"2016-12-26T09:28:48+00:00","product_name":"Basic T-shirt - dark blue/white","price":12,"taxful_price":12,"base_unit_price":12},{"base_price":24.99,"discount_percentage":0,"quantity":1,"manufacturer":"Oceanavigations","tax_amount":0,"product_id":19400,"category":"Men's Clothing","sku":"ZO0299602996","taxless_price":25,"unit_discount_amount":0,"min_price":11.75,"_id":"sold_product_584677_19400","discount_amount":0,"created_on":"2016-12-26T09:28:48+00:00","product_name":"Sweatshirt - grey multicolor","price":25,"taxful_price":25,"base_unit_price":25}],"taxful_total_price":37,"taxless_total_price":37,"total_quantity":2,"total_unique_products":2,"type":"order","user":"eddie"}
{ "index" : { "_id" : "584021" } }
{"currency":"EUR","customer_first_name":"Mary","customer_gender":"FEMALE","customer_id":20,"customer_last_name":"Bailey","customer_phone":"","order_date":"2023-10-08T21:59:02+00:00","order_id":584021,"products":[{"base_price":24.99,"discount_percentage":0,"quantity":1,"manufacturer":"Champion Arts","tax_amount":0,"product_id":11238,"category":"Women's Clothing","sku":"ZO0489604896","taxless_price":24.99,"unit_discount_amount":0,"min_price":11.75,"_id":"sold_product_584021_11238","discount_amount":0,"created_on":"2016-12-25T21:59:02+00:00","product_name":"Denim dress - black denim","price":24.99,"taxful_price":24.99,"base_unit_price":24.99},{"base_price":28.99,"discount_percentage":0,"quantity":1,"manufacturer":"Pyramidustries","tax_amount":0,"product_id":20149,"category":"Women's Clothing","sku":"ZO0185501855","taxless_price":28.99,"unit_discount_amount":0,"min_price":15.65,"_id":"sold_product_584021_20149","discount_amount":0,"created_on":"2016-12-25T21:59:02+00:00","product_name":"Shorts - black","price":28.99,"taxful_price":28.99,"base_unit_price":28.99}],"taxful_total_price":53.98,"taxless_total_price":53.98,"total_quantity":2,"total_unique_products":2,"type":"order","user":"mary"}
{ "index" : { "_id" : "584058" } }
{"currency":"EUR","customer_first_name":"Gwen","customer_gender":"FEMALE","customer_id":26,"customer_last_name":"Butler","customer_phone":"","order_date":"2023-10-04T22:32:10+00:00","order_id":584058,"products":[{"base_price":99.99,"discount_percentage":0,"quantity":1,"manufacturer":"Low Tide Media","tax_amount":0,"product_id":22794,"category":"Women's Shoes","sku":"ZO0374603746","taxless_price":99.99,"unit_discount_amount":0,"min_price":46.01,"_id":"sold_product_584058_22794","discount_amount":0,"created_on":"2016-12-25T22:32:10+00:00","product_name":"Boots - Midnight Blue","price":99.99,"taxful_price":99.99,"base_unit_price":99.99},{"base_price":99.99,"discount_percentage":0,"quantity":1,"manufacturer":"Oceanavigations","tax_amount":0,"product_id":23386,"category":"Women's Clothing","sku":"ZO0272202722","taxless_price":99.99,"unit_discount_amount":0,"min_price":53.99,"_id":"sold_product_584058_23386","discount_amount":0,"created_on":"2016-12-25T22:32:10+00:00","product_name":"Short coat - white/black","price":99.99,"taxful_price":99.99,"base_unit_price":99.99}],"taxful_total_price":199.98,"taxless_total_price":199.98,"total_quantity":2,"total_unique_products":2,"type":"order","user":"gwen"}
# 檢查 `taxful_total_price` 與 `taxless_total_price` 的型態
GET devopsdays-taipei-2023-ec-order/_mapping
# 重新查詢，現在已正常
GET devopsdays-taipei-2023-ec-order/_search
{
"query": {
"range": {
"taxful_total_price": {
"lte": 53
}
}
}
}
```
## 動態新增欄位 Dynamic Mapping
### Dynamic Template 範例
- 將 String 型態的欄位，定義成 `keyword`，而不是預設的 `text` + `keyword`。
```
PUT my-index
{
"mappings": {
"dynamic_templates": [
{
"keyword_strings": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
]
}
}
```
- 將 String 型態的欄位，且欄位名字是 `long_` 開頭，同時不是 `_text` 結尾，就定義成 `long` 型態。
```
PUT my-index/
{
"mappings": {
"dynamic_templates": [
{
"string_as_long": {
"match_mapping_type": "string",
"match": "long_*",
"unmatch": "*_text",
"mapping": {
"type": "long"
}
}
}
]
}
}
```
- 將 `name` 物件裡除了 `middle` 之外的所有欄位，都 copy 到 `full_name` 的欄位，並指定為 `text` 型態。
```
PUT my-index
{
"mappings": {
"dynamic_templates": [
{
"full_name": {
"path_match": "name.*",
"path_unmatch": "*.middle",
"mapping": {
"type": "text",
"copy_to": "full_name"
}
}
}
]
}
}
PUT my-index/_doc/1
{
"name": {
"first": "John",
"middle": "Winston",
"last": "Lennon"
}
}
```
- 先針對字串欄位給予定義，再來針對所有非字串的欄位，關閉 `doc_values`。
```
PUT my-index
{
"mappings": {
"dynamic_templates": [
{
"named_analyzers": {
"match_mapping_type": "string",
"match": "*",
"mapping": {
"type": "text",
"analyzer": "{name}"
}
}
},
{
"no_doc_values": {
"match_mapping_type":"*",
"mapping": {
"type": "{dynamic_type}",
"doc_values": false
}
}
}
]
}
}
PUT my-index/_doc/1
{
"english": "Some English text",
"count": 5
}
```
### Sprint 3: 建立適用的 Dynamic Template
```
# 定義好新的 Dynamic template 設定，同時移除不必要的 mapping 宣告，只剩下 `total_unique_products` 這個例外狀況需要特別宣告。
PUT _index_template/devopsdays-taipei-2023-ec-order
{
"index_patterns": [
"devopsdays-taipei-2023-ec-order*"
],
"template": {
"aliases": {
"devopsdays-taipei-2023-ec-order": {}
},
"mappings": {
"_meta": {
"version": 1,
"version_creation_date": "2023-09-26"
},
"dynamic_templates": [
{
"id": {
"match": "*_id",
"mapping": {
"type": "keyword"
}
}
},
{
"number_double": {
"match": ["*_price", "price"],
"mapping": {
"type": "double"
}
}
},
{
"number_float": {
"match": ["*_amount", "*_percentage", "quantity", "*_quantity"],
"mapping": {
"type": "float"
}
}
},
{
"date": {
"match": ["*_date", "created_on"],
"match_mapping_type": "date",
"mapping": {
"type": "date"
}
}
},
{
"string_as_text": {
"match": ["*_name"],
"match_mapping_type": "string",
"mapping": {
"type": "text",
"fields": {
"keyword": { "type": "keyword", "ignore_above": 256 }
}
}
}
},
{
"string_as_keyword": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
],
"properties": {
"total_unique_products": { "type": "float" }
}
}
},
"priority": 100,
"version": 1,
"_meta": {
"description": "devopsdays workshop",
"version_creation_date": "2023-09-26"
}
}
# 偷偷刪掉資料
DELETE devopsdays-taipei-2023-ec-order_v1
# 重新匯入資料測試一下
PUT devopsdays-taipei-2023-ec-order_v1/_bulk
{ "index" : { "_id" : "584677" } }
{"currency":"EUR","customer_first_name":"Eddie","customer_gender":"MALE","customer_id":38,"customer_last_name":"Underwood","customer_phone":"","order_date":"2023-10-09T09:28:48+00:00","order_id":584677,"products":[{"base_price":11.99,"discount_percentage":0,"quantity":1,"manufacturer":"Elitelligence","tax_amount":0,"product_id":6283,"category":"Men's Clothing","sku":"ZO0549605496","taxless_price":11.99,"unit_discount_amount":0,"min_price":6.35,"_id":"sold_product_584677_6283","discount_amount":0,"created_on":"2016-12-26T09:28:48+00:00","product_name":"Basic T-shirt - dark blue/white","price":12,"taxful_price":12,"base_unit_price":12},{"base_price":24.99,"discount_percentage":0,"quantity":1,"manufacturer":"Oceanavigations","tax_amount":0,"product_id":19400,"category":"Men's Clothing","sku":"ZO0299602996","taxless_price":25,"unit_discount_amount":0,"min_price":11.75,"_id":"sold_product_584677_19400","discount_amount":0,"created_on":"2016-12-26T09:28:48+00:00","product_name":"Sweatshirt - grey multicolor","price":25,"taxful_price":25,"base_unit_price":25}],"taxful_total_price":37,"taxless_total_price":37,"total_quantity":2,"total_unique_products":2,"type":"order","user":"eddie"}
{ "index" : { "_id" : "584021" } }
{"currency":"EUR","customer_first_name":"Mary","customer_gender":"FEMALE","customer_id":20,"customer_last_name":"Bailey","customer_phone":"","order_date":"2023-10-08T21:59:02+00:00","order_id":584021,"products":[{"base_price":24.99,"discount_percentage":0,"quantity":1,"manufacturer":"Champion Arts","tax_amount":0,"product_id":11238,"category":"Women's Clothing","sku":"ZO0489604896","taxless_price":24.99,"unit_discount_amount":0,"min_price":11.75,"_id":"sold_product_584021_11238","discount_amount":0,"created_on":"2016-12-25T21:59:02+00:00","product_name":"Denim dress - black denim","price":24.99,"taxful_price":24.99,"base_unit_price":24.99},{"base_price":28.99,"discount_percentage":0,"quantity":1,"manufacturer":"Pyramidustries","tax_amount":0,"product_id":20149,"category":"Women's Clothing","sku":"ZO0185501855","taxless_price":28.99,"unit_discount_amount":0,"min_price":15.65,"_id":"sold_product_584021_20149","discount_amount":0,"created_on":"2016-12-25T21:59:02+00:00","product_name":"Shorts - black","price":28.99,"taxful_price":28.99,"base_unit_price":28.99}],"taxful_total_price":53.98,"taxless_total_price":53.98,"total_quantity":2,"total_unique_products":2,"type":"order","user":"mary"}
{ "index" : { "_id" : "584058" } }
{"currency":"EUR","customer_first_name":"Gwen","customer_gender":"FEMALE","customer_id":26,"customer_last_name":"Butler","customer_phone":"","order_date":"2023-10-04T22:32:10+00:00","order_id":584058,"products":[{"base_price":99.99,"discount_percentage":0,"quantity":1,"manufacturer":"Low Tide Media","tax_amount":0,"product_id":22794,"category":"Women's Shoes","sku":"ZO0374603746","taxless_price":99.99,"unit_discount_amount":0,"min_price":46.01,"_id":"sold_product_584058_22794","discount_amount":0,"created_on":"2016-12-25T22:32:10+00:00","product_name":"Boots - Midnight Blue","price":99.99,"taxful_price":99.99,"base_unit_price":99.99},{"base_price":99.99,"discount_percentage":0,"quantity":1,"manufacturer":"Oceanavigations","tax_amount":0,"product_id":23386,"category":"Women's Clothing","sku":"ZO0272202722","taxless_price":99.99,"unit_discount_amount":0,"min_price":53.99,"_id":"sold_product_584058_23386","discount_amount":0,"created_on":"2016-12-25T22:32:10+00:00","product_name":"Short coat - white/black","price":99.99,"taxful_price":99.99,"base_unit_price":99.99}],"taxful_total_price":199.98,"taxless_total_price":199.98,"total_quantity":2,"total_unique_products":2,"type":"order","user":"gwen"}
# 檢查 mapping 的型態
GET devopsdays-taipei-2023-ec-order/_mapping
```
# 無法事先定義好 Data Model
## Runtime Field 的使用方式
### 在 Searching 時指定 Runtime Field
- Searching 時指定 runtime field
```
# 定義 `@timestamp` 欄位為 date
PUT rf_test
{
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
}
}
}
}
# Import Test Data
POST rf_test/_bulk?refresh=true
{"index":{}}
{"@timestamp":1516729294000,"model_number":"QVKC92Q","measures":{"voltage":"5.2","start": "300","end":"8675309"}}
{"index":{}}
{"@timestamp":1516642894000,"model_number":"QVKC92Q","measures":{"voltage":"5.8","start": "300","end":"8675309"}}
{"index":{}}
{"@timestamp":1516556494000,"model_number":"QVKC92Q","measures":{"voltage":"5.1","start": "300","end":"8675309"}}
{"index":{}}
{"@timestamp":1516470094000,"model_number":"QVKC92Q","measures":{"voltage":"5.6","start": "300","end":"8675309"}}
{"index":{}}
{"@timestamp":1516383694000,"model_number":"HG537PU","measures":{"voltage":"4.2","start": "400","end":"8625309"}}
{"index":{}}
{"@timestamp":1516297294000,"model_number":"HG537PU","measures":{"voltage":"4.0","start": "400","end":"8625309"}}
GET rf_test
# 定義 `runtime_mapping` 並編寫從 `doc_value` 將資料拿出並進行處理的規則。
GET rf_test/_search
{
"runtime_mappings": {
"day_of_week": {
"type": "keyword",
"script": {
"source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
}
},
"fields": [
"day_of_week"
]
}
# 定義 `runtime_mapping` 並編寫從 `_source` 將資料拿出並進行處理的規則。
GET rf_test/_search
{
"runtime_mappings": {
"mode_number": {
"type": "keyword",
"script": {
"source": "emit(params._source.model_number)"
}
}
},
"fields": [
"model_number"
]
}
```
### 在 Mapping 中定義 Runtime Field
```
# 使用 update mapping API 將 runtime fields 定義進 index mapping 之中
PUT rf_test/_mapping
{
"runtime": {
"day_of_week": {
"type": "keyword",
"script": {
"source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
}
},
"properties": {
"@timestamp": {
"type": "date"
}
}
}
# 使用 _search 將 fields 查出
GET rf_test/_search?docvalue_fields=day_of_week
```
### Sprint 4: 事先所定義的欄位不足
```
# 早期的做法
GET devopsdays-taipei-2023-ec-order/_search
{
"query": {
"script": {
"script": "doc['order_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT) == 'Wednesday'"
}
}
}
GET devopsdays-taipei-2023-ec-order/_search
{
"size": 0,
"aggs": {
"day_of_week": {
"terms": {
"script": {
"source": "doc['order_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)"
}
}
}
}
}
# 現在可以定義成 Runtime Field
GET devopsdays-taipei-2023-ec-order/_search
{
"runtime_mappings": {
"day_of_week": {
"type": "keyword",
"script": "emit(doc['order_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
},
"fields": [
"day_of_week"
]
}
# Runtime Field 可以使用在 query
GET devopsdays-taipei-2023-ec-order/_search
{
"runtime_mappings": {
"day_of_week": {
"type": "keyword",
"script": "emit(doc['order_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
},
"fields": [
"day_of_week"
],
"query": {
"term": {
"day_of_week": "Wednesday"
}
}
}
# Runtime Field 也可以使用在 aggregation
GET devopsdays-taipei-2023-ec-order/_search
{
"runtime_mappings": {
"day_of_week": {
"type": "keyword",
"script": "emit(doc['order_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
},
"fields": [
"day_of_week"
],
"size": 0,
"aggs": {
"day_of_week": {
"terms": {
"field": "day_of_week"
}
}
}
}
# 確認好用法用，我們可以將 Runtime Field 定義在 Mapping 中，就不用每次 Search 時都要宣告
PUT devopsdays-taipei-2023-ec-order/_mapping
{
"runtime": {
"day_of_week": {
"type": "keyword",
"script": {
"source": "emit(doc['order_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
}
}
}
# 記得也要修改 Index Template，並且讓 Version 進版。
PUT _index_template/devopsdays-taipei-2023-ec-order
{
"index_patterns": [
"devopsdays-taipei-2023-ec-order*"
],
"template": {
"aliases": {
"devopsdays-taipei-2023-ec-order": {}
},
"mappings": {
"_meta": {
"version": 2,
"version_creation_date": "2023-09-26"
},
"dynamic_templates": [
{
"id": {
"match": "*_id",
"mapping": {
"type": "keyword"
}
}
},
{
"number_double": {
"match": ["*_price", "price"],
"mapping": {
"type": "double"
}
}
},
{
"number_float": {
"match": ["*_amount", "*_percentage", "quantity", "*_quantity"],
"mapping": {
"type": "float"
}
}
},
{
"date": {
"match": ["*_date", "created_on"],
"match_mapping_type": "date",
"mapping": {
"type": "date"
}
}
},
{
"string_as_text": {
"match": ["*_name"],
"match_mapping_type": "string",
"mapping": {
"type": "text",
"fields": {
"keyword": { "type": "keyword", "ignore_above": 256 }
}
}
}
},
{
"string_as_keyword": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
],
"runtime": {
"day_of_week": {
"type": "keyword",
"script": {
"source": "emit(doc['order_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
}
},
"properties": {
"total_unique_products": { "type": "float" }
}
}
},
"priority": 100,
"version": 2,
"_meta": {
"description": "devopsdays workshop",
"version_creation_date": "2023-09-26"
}
}
```
### Sprint 5: 當我們想將 Runtime Field 正式定義
```
# 在 Index Template 增加 `day_of_week` 的 schema on-write 欄位宣告，並且進版
PUT _index_template/devopsdays-taipei-2023-ec-order
{
"index_patterns": [
"devopsdays-taipei-2023-ec-order*"
],
"template": {
"aliases": {
"devopsdays-taipei-2023-ec-order": {}
},
"mappings": {
"_meta": {
"version": 3,
"version_creation_date": "2023-09-26"
},
"dynamic_templates": [
{
"id": {
"match": "*_id",
"mapping": {
"type": "keyword"
}
}
},
{
"number_double": {
"match": ["*_price", "price"],
"mapping": {
"type": "double"
}
}
},
{
"number_float": {
"match": ["*_amount", "*_percentage", "quantity", "*_quantity"],
"mapping": {
"type": "float"
}
}
},
{
"date": {
"match": ["*_date", "created_on"],
"match_mapping_type": "date",
"mapping": {
"type": "date"
}
}
},
{
"string_as_text": {
"match": ["*_name"],
"match_mapping_type": "string",
"mapping": {
"type": "text",
"fields": {
"keyword": { "type": "keyword", "ignore_above": 256 }
}
}
}
},
{
"string_as_keyword": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
],
"runtime": {
"day_of_week": {
"type": "keyword",
"script": {
"source": "emit(doc['order_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
}
},
"properties": {
"total_unique_products": { "type": "float" },
"order_date": { "type": "date" },
"day_of_week": {
"type": "keyword",
"on_script_error": "fail",
"script": {
"source": "emit(doc['order_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
}
}
}
},
"priority": 100,
"version": 3,
"_meta": {
"description": "devopsdays workshop",
"version_creation_date": "2023-09-26"
}
}
# 將新的資料寫入到新的 index v2 之中
POST devopsdays-taipei-2023-ec-order_v2/_bulk?refresh=true
{ "index" : { "_id" : "584093" } }
{"currency":"EUR","customer_first_name":"Diane","customer_gender":"FEMALE","customer_id":22,"customer_last_name":"Chandler","customer_phone":"","order_date":"2023-10-08T22:58:05+00:00","order_id":584093,"products":[{"base_price":74.99,"discount_percentage":0,"quantity":1,"manufacturer":"Primemaster","tax_amount":0,"product_id":12304,"category":"Women's Shoes","sku":"ZO0360303603","taxless_price":74.99,"unit_discount_amount":0,"min_price":34.5,"_id":"sold_product_584093_12304","discount_amount":0,"created_on":"2016-12-25T22:58:05+00:00","product_name":"High heeled sandals - argento","price":74.99,"taxful_price":74.99,"base_unit_price":74.99},{"base_price":99.99,"discount_percentage":0,"quantity":1,"manufacturer":"Oceanavigations","tax_amount":0,"product_id":19587,"category":"Women's Clothing","sku":"ZO0272002720","taxless_price":99.99,"unit_discount_amount":0,"min_price":47.01,"_id":"sold_product_584093_19587","discount_amount":0,"created_on":"2016-12-25T22:58:05+00:00","product_name":"Classic coat - black","price":99.99,"taxful_price":99.99,"base_unit_price":99.99}],"taxful_total_price":174.98,"taxless_total_price":174.98,"total_quantity":2,"total_unique_products":2,"type":"order","user":"diane"}
{ "index" : { "_id" : "574916" } }
{"currency":"EUR","customer_first_name":"Eddie","customer_gender":"MALE","customer_id":38,"customer_last_name":"Weber","customer_phone":"","order_date":"2023-10-11T03:48:58+00:00","order_id":574916,"products":[{"base_price":59.99,"discount_percentage":0,"quantity":1,"manufacturer":"Elitelligence","tax_amount":0,"product_id":11262,"category":"Men's Clothing","sku":"ZO0542505425","taxless_price":59.99,"unit_discount_amount":0,"min_price":28.2,"_id":"sold_product_574916_11262","discount_amount":0,"created_on":"2016-12-19T03:48:58+00:00","product_name":"Winter jacket - black","price":59.99,"taxful_price":59.99,"base_unit_price":59.99},{"base_price":20.99,"discount_percentage":0,"quantity":1,"manufacturer":"Elitelligence","tax_amount":0,"product_id":15713,"category":"Men's Accessories","sku":"ZO0601306013","taxless_price":20.99,"unit_discount_amount":0,"min_price":10.7,"_id":"sold_product_574916_15713","discount_amount":0,"created_on":"2016-12-19T03:48:58+00:00","product_name":"Watch - green","price":20.99,"taxful_price":20.99,"base_unit_price":20.99}],"taxful_total_price":80.98,"taxless_total_price":80.98,"total_quantity":2,"total_unique_products":2,"type":"order","user":"eddie"}
{ "index" : { "_id" : "574586" } }
{"currency":"EUR","customer_first_name":"Diane","customer_gender":"FEMALE","customer_id":22,"customer_last_name":"Goodwin","customer_phone":"","order_date":"2023-10-04T21:44:38+00:00","order_id":574586,"products":[{"base_price":59.99,"discount_percentage":0,"quantity":1,"manufacturer":"Low Tide Media","tax_amount":0,"product_id":5419,"category":"Women's Shoes","sku":"ZO0376303763","taxless_price":59.99,"unit_discount_amount":0,"min_price":31.79,"_id":"sold_product_574586_5419","discount_amount":0,"created_on":"2016-12-18T21:44:38+00:00","product_name":"Winter boots - brown","price":59.99,"taxful_price":59.99,"base_unit_price":59.99},{"base_price":11.99,"discount_percentage":0,"quantity":1,"manufacturer":"Pyramidustries","tax_amount":0,"product_id":19325,"category":"Women's Clothing","sku":"ZO0212402124","taxless_price":11.99,"unit_discount_amount":0,"min_price":6.47,"_id":"sold_product_574586_19325","discount_amount":0,"created_on":"2016-12-18T21:44:38+00:00","product_name":"Shorts - dark blue/pink/dark green","price":11.99,"taxful_price":11.99,"base_unit_price":11.99}],"taxful_total_price":71.98,"taxless_total_price":71.98,"total_quantity":2,"total_unique_products":2,"type":"order","user":"diane"}
# 同時查詢 v1 與 v2 的資料，使用端沒有任何影響
GET devopsdays-taipei-2023-ec-order/_search
{
"query": {
"term": {
"day_of_week": "Wednesday"
}
}
}
# Aggregation 也一樣，使用端沒有任何影響
GET devopsdays-taipei-2023-ec-order/_search
{
"size": 0,
"aggs": {
"day_of_week": {
"terms": {
"field": "day_of_week"
}
}
}
}
```
## Async Search
```
# 我們嘗試使用 _async_search 但是發現執行太快，不會觸發 async_search
POST devopsdays-taipei-2023-ec-order/_async_search
{
"size": 0,
"aggs": {
"test": {
"date_histogram": {
"field": "order_date",
"fixed_interval": "2d"
}
}
}
}
# 強制執行 `wait_for_completion_timeout=0` 讓查詢立刻先返回，即出現 async_search id
POST devopsdays-taipei-2023-ec-order/_async_search?wait_for_completion_timeout=0
{
"size": 0,
"aggs": {
"test": {
"date_histogram": {
"field": "order_date",
"fixed_interval": "2d"
}
}
}
}
# 使用 id 去查詢執行狀態
GET _async_search/status/<responsed_async_search_id>
# 再使用 id 去取得結果
GET _async_search/<responsed_async_search_id>
# 也可以使用 `keep_on_completion`
POST devopsdays-taipei-2023-ec-order/_async_search?keep_on_completion
{
"size": 0,
"aggs": {
"test": {
"date_histogram": {
"field": "order_date",
"fixed_interval": "2d"
}
}
}
}
# 用完記得要刪除，不然會留 5 天
DELETE _async_search/<responsed_async_search_id>
# 可使用 .async_search 觀察 docs 數量的變化
GET .async-search/_search?size=0
```
# Data Model 的事後修改
## Update by Query
```
# 加上 `joe` 的 tag
POST kibana_sample_data_logs/_update_by_query
{
"script": {
"source": "ctx._source.tags.add(params.tag)",
"lang": "painless",
"params": {
"tag": "joe"
}
},
"query": {
"match_all": {}
}
}
# 移掉 `joe 的 tag，使用 `wait_for_completion=false` 成為 background task
POST kibana_sample_data_logs/_update_by_query?wait_for_completion=false
{
"script": {
"source": "if (ctx._source.tags.contains(params.tag)) { ctx._source.tags.remove(ctx._source.tags.indexOf(params.tag)) }",
"lang": "painless",
"params": {
"tag": "joe"
}
},
"query": {
"match_all": {}
}
}
# 取得 task 結果
GET /_tasks/<task_id>
```
### Sprint 6: 將舊資料也轉成 Schema on Write
```
# 針對 `_v1` 的 index，加上 `day_of_week` 的 explicit mapping 宣告
PUT devopsdays-taipei-2023-ec-order_v1/_mapping
{
"properties": {
"day_of_week": {
"type": "keyword",
"on_script_error": "fail",
"script": {
"source": "emit(doc['order_date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
}
}
}
}
# 接著使用 _update_by_query 將資料重新讀出來並再 indexing 一遍。
POST devopsdays-taipei-2023-ec-order_v1/_update_by_query
{
"query": {
"match_all": {}
}
}
# 刪除先前定義的 runtime field - `day_of_week`
PUT devopsdays-taipei-2023-ec-order_v1/_mapping
{
"runtime": {
"day_of_week": null
}
}
# 我們可以看到最後的 mapping 定義
GET devopsdays-taipei-2023-ec-order_v1/_mapping
# 同時也可以看到查詢出來的結果，是包含 `day_of_week` 欄位的
GET devopsdays-taipei-2023-ec-order_v1/_search?docvalue_fields=day_of_week
```
Sign in with Wallet
Connect another wallet