LinkedIn

Use YAML to define the pipelines

https://ci.concourse-ci.org/

一些使用上的條件限制

• 只支援 Docker container
• 只支援 amd64/x86_64 架構
• pipeline 不支援 if else 分支
• 要熟悉 YAML 及 JSON 語法
• 學習曲線高 (官方自己說的)

簡介請參考 Introducing Concourse CI by $4

pipeline.yml

resource_types:
- name: webdav
  type: registry-image
  source:
    repository: ghcr.io/fourdollars/webdav-resource
  defaults:
    domain: webdav
    username: webdav
    password: webdav
    ssl: false
resources:
- name: demo-storage
  type: webdav
  icon: nas
  source:
    path: demo
- name: ubuntu-focal
  type: registry-image
  icon: ubuntu
  source:
    repository: ubuntu
    tag: focal
jobs:
- name: check-demo-message
  plan:
  - get: ubuntu-focal
  - get: demo-storage
    trigger: true
  - task: check-message
    image: ubuntu-focal
    config:
      platform: linux
      inputs:
      - name: demo-storage
      run:
        path: cat
        args:
        - demo-storage/msg.log

Live Demo

入門請參考 TOSSUG 2021/05/04 Concourse CI 幼幼班

Goal: Print the content of HTTP file when it changed.

$ python3 -m http.server 9527
$ date > msg.log
$ curl http://0.0.0.0:9527/msg.log

resources:
- name: message
  type: http
  icon: nas
  source:
    url: http://0.0.0.0:9527/msg.log

How to generate the digest?

$ curl --head http://0.0.0.0:9527/msg.log
HTTP/1.0 200 OK
Server: SimpleHTTP/0.6 Python/3.8.10
Date: Tue, 13 Jul 2021 14:12:18 GMT
Content-type: application/octet-stream
Content-Length: 53
Last-Modified: Tue, 13 Jul 2021 13:59:01 GMT

$ curl --head http://0.0.0.0:9527/msg.log | sha256sum
ec8af3a926d2fe396d9b8d1d0e7a077bfa064937a9d02158321d1...

This digest is unreliable.

$ curl --head http://0.0.0.0:9527/msg.log | sha256sum
efdaa9fc9f4b5f5d6d0106500621e7c5193efd1288785040809e0...
$ curl --head http://0.0.0.0:9527/msg.log | sha256sum
a326c45b04b5a756906882cdac6477689a72fed551db8afb6d565...
$ curl --head http://0.0.0.0:9527/msg.log | sha256sum
8ab03fab7a6d5a86a290f54aa5b32978e0240517f757ae50a9cd6...

Generate the reliable digest

Find out the immutable parts

$ curl --head http://0.0.0.0:9527/msg.log | \
  grep -e ^Last-Modified -e ^Content-Length
Content-Length: 53
Last-Modified: Tue, 13 Jul 2021 13:59:01 GMT

$ curl --head http://0.0.0.0:9527/msg.log | \
  grep -e ^Last-Modified -e ^Content-Length | sha256sum
c3327a0c5bccfb7524f320bea52861c56542c9fd5e56603e3825c83...

Dockerfile

FROM alpine:latest
RUN apk update
RUN apk upgrade
RUN apk add bash jq curl 
# check step
ADD /check /opt/resource/check
# put step
ADD /out /opt/resource/out
# get step
ADD /in /opt/resource/in
# make sure they are executable
RUN chmod +x /opt/resource/*

• bash: deal with shell script
• jq: deal with JSON
• curl: deal with HTTP

Trinity: check / in / out

$ ln -s in check
$ ln -s in out
$ vim in

Template

#!/bin/bash
 
# Bash strict mode
set -euo pipefail
IFS=$'\n\t'
 
# make stdout available as fd 3 for the result
exec 3>&1
# redirect all output to stderr for logging
exec 1>&2
 
# read the json from /dev/stdin and save to /tmp/input.json
jq -M -S . < /dev/stdin > /tmp/input.json
# '-M' monochrome (don't colorize JSON);
# '-S' sort keys of objects on output;
 
case "$0" in
    ('/opt/resource/check')
        json='[]'
        ;;
    ('/opt/resource/in')
        cd "$1" # /tmp/build/get
        json='{}'
        ;;
    ('/opt/resource/out')
        cd "$1" # /tmp/build/put
        json='{}' # noop
        ;;
esac
 
# output the json to fd 3
jq -n "$json" >&3
# '-n' Don't read from stdin

Read the url from json

# '-S' sort keys of objects on output;

url=$(jq -r .source.url < /tmp/input.json)

case "$0" in

resources:
- name: message
  type: http
  icon: nas
  source:
    url: http://0.0.0.0:9527/msg.log

Generate the reliable digest in check step

case "$0" in
    ('/opt/resource/check')
        sha256=$(curl --head "$url" \
            | grep -i -e ^Last-Modified -e ^Content-Length \
            | sha256sum | awk '{print $1}')
        json=$(cat <<ENDLINE
[
  {
    "digest": "sha256:$sha256"
  }
]
ENDLINE
)
        ;;

Download the file in get step

    ('/opt/resource/in')
        cd "$1" # /tmp/build/get
        curl --dump-header /tmp/header.log "$url" > "$(basename "$url")"
        sha256=$(grep -i -e ^Last-Modified -e ^Content-Length /tmp/header.log \
            | sha256sum | awk '{print $1}')
        length=$(grep -i -e ^Content-Length /tmp/header.log \
            | awk '{print $2}' | tr -d '\r')
        modified=$(grep -i -e ^Last-Modified /tmp/header.log \
            | cut -d ' ' -f 2- | tr -d '\r')
        json=$(cat <<ENDLINE
{
  "version": {
    "digest": "sha256:$sha256"
  },
  "metadata": [
    {
      "name": "url",
      "value": "$url"
    },
    {
      "name": "Last-Modified",
      "value": "$modified"
    },
    {
      "name": "Content-Length",
      "value": "$length"
    }
  ]
}
ENDLINE
)
        ;;

實作請參考 1.9.1 Implementing a Resource Type

pipeline.yml

resources:
  - name: concourse-demo
    type: git
    source:
      uri: https://github.com/fourdollars/concourse-demo
      branch: coscup
  - name: resource-image
    type: docker-image
    source:
      repository: registry:5000/simple-http-resource
      username: registry
      password: registry
      insecure_registries: [ "registry:5000" ]
 
jobs:
  - name: build-simple-http-resource
    plan:
      - get: concourse-demo
        trigger: true
      - put: resource-image
        params:
          build: concourse-demo/simple-http-resource

Live Demo

pipeline.yml

resource_types:
- name: simple-http
  type: docker-image
  source:
    repository: registry:5000/simple-http-resource
    username: registry
    password: registry
    insecure_registries: [ "registry:5000" ]
 
resources:
- name: message
  icon: nas
  type: simple-http
  source:
    url: https://people.debian.org/~fourdollars/coscup-2021.log
- name: ubuntu-focal
  type: registry-image
  icon: ubuntu
  source:
    repository: ubuntu
    tag: focal
 
jobs:
- name: check-message
  plan:
  - get: ubuntu-focal
  - get: message
    trigger: true
  - task: check
    image: ubuntu-focal
    config:
      platform: linux
      inputs:
      - name: message
      run:
        path: cat
        args:
            - message/coscup-2021.log

Live Demo