# Create VM with Terraform in Proxmox Cluster

## 1. 環境準備

### 1.1. 製作 Alpine Cloud-init VM Template

# 1. 下載 alpine cloud init 硬碟檔
$ wget https://dl-cdn.alpinelinux.org/alpine/v3.19/releases/cloud/nocloud_alpine-3.19.1-x86_64-bios-cloudinit-r0.qcow2

# 2. 安裝必要套件
$ apt update; apt install -y libguestfs-tools

# 3. 在 Alpine cloud init 硬碟檔安裝 qemu-guest-agent、bash、sudo 套件
$ virt-customize --install qemu-guest-agent,bash,sudo -a nocloud_alpine-3.19.1-x86_64-bios-cloudinit-r0.qcow2

# 4. 建立 Alpine 虛擬主機
$ vmid="9000"
$ qm create "$vmid" --name alp-cloudinit-v3.19.1 --memory 4096 --sockets 1 --cores 4 --net0 virtio,bridge=vmbr0

# 5. 匯入 alpine cloud init 硬碟檔, 
$ qm importdisk "$vmid" nocloud_alpine-3.19.1-x86_64-bios-cloudinit-r0.qcow2 local-lvm

# 6. 設定使用 SCSI 介面連接 虛擬硬碟檔
$ qm set "$vmid" --scsihw virtio-scsi-pci --scsi0 local-lvm:vm-"$vmid"-disk-0

# 7. 改變硬碟大小
$ qm resize "$vmid" scsi0 50G

# 8. 加入 Cloud Init 功能
$ qm set "$vmid" --ide2 local-lvm:cloudinit

# 9. Set boot device
$ qm set "$vmid" --boot c --bootdisk scsi0

# 10. Enable serial device
$ qm set "$vmid" --serial0 socket --vga serial0

# 11. 撰寫 cloud-init User data 設定檔
$ mkdir -p /var/lib/vz/snippets/

$ nano /var/lib/vz/snippets/user"$vmid".yml
#cloud-config
hostname: alpine
package_update: true
packages:
  - nano
write_files:
  - path: /etc/resolv.conf
    permissions: "0644"
    owner: root
    content: |
      nameserver 8.8.8.8
users:
#  - default
  - name: bigred
#    groups: bigred
    no_user_group: true
    lock_passwd: false
    sudo: ALL=(ALL) NOPASSWD:ALL
    plain_text_passwd: 'bigred'
    homedir: /home/bigred
    shell: /bin/bash
#    ssh_authorized_keys:
#      - 
ssh_pwauth: True
#bootcmd:
#  - echo -e "bigred\nbigred" | adduser -s /bin/bash bigred
#  - echo -e "rancher\nrancher" | adduser -s /bin/bash rancher
#  - echo -e "bigred\nbigred" | passwd root

# 12. 撰寫 cloud-init Network data 設定檔
$ nano /var/lib/vz/snippets/network"$vmid".yml
version: 1
config:
    - type: physical
      name: eth0
      subnets:
      - type: static
        address: '192.168.61.21'
        netmask: '255.255.255.0'
        gateway: '192.168.61.2'

# 12. 設定 VM 套用設定檔
$ qm set "$vmid" --cicustom "user=local:snippets/user"$vmid".yml,network=local:snippets/network"$vmid".yml"

# 13. test
$ qm start "$vmid"
$ ssh bigred@192.168.61.21
Welcome to Alpine!
...

$ exit
$ qm stop "$vmid"

# 14. 將 VM 製作成 Template
$ qm template "$vmid"

### 1.2. 使用 API keys 認證

# 1. 創建一個名為 tfuser 的使用者
$ pveum user add terraform@pve

# 2. 創建一個名為 Terraform 的新角色，並賦予這個角色一系列與虛擬機管理和數據存儲管理相關的權限。這些權限允許用戶使用該角色來創建、配置、克隆、監控和管理虛擬機，以及管理數據存儲空間
$ pveum role add Terraform -privs "Datastore.AllocateSpace Datastore.Audit Pool.Allocate Sys.Audit Sys.Console Sys.Modify VM.Allocate VM.Audit VM.Clone VM.Config.CDROM VM.Config.Cloudinit VM.Config.CPU VM.Config.Disk VM.Config.HWType VM.Config.Memory VM.Config.Network VM.Config.Options VM.Migrate VM.Monitor VM.PowerMgmt SDN.Use"

# 3. 將 TerraformProv 角色分配給 tfuser，讓它擁有相應的權限。
$ pveum aclmod / -user terraform@pve -role Terraform

# 4. 產生 Terraform 要使用的 API Token
$ pveum user token add terraform@pve provider --privsep=0
┌──────────────┬──────────────────────────────────────┐
│ key          │ value                                │
╞══════════════╪══════════════════════════════════════╡
│ full-tokenid │ terraform@pve!provider               │
├──────────────┼──────────────────────────────────────┤
│ info         │ {"privsep":"0"}                      │
├──────────────┼──────────────────────────────────────┤
│ value        │ 1fb0b04e-5673-4d5c-b580-ae602c718d89 │
└──────────────┴──────────────────────────────────────┘

> 要儲存 Token Secrt，因為之後無法取得它。

### 1.3. 安裝 Terraform

$ apt-get update && apt-get install -y gnupg software-properties-common

$ wget -O- https://apt.releases.hashicorp.com/gpg | \
gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg

$ echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \
https://apt.releases.hashicorp.com $(lsb_release -cs) main" | \
tee /etc/apt/sources.list.d/hashicorp.list

$ apt update && apt install -y terraform

## 2. 建立 Terraform 設定檔

# 1. 建立儲存設定檔的工作目錄
$ mkdir terraform && cd terraform

# 2. 產生 backend config
$ cat <<EOF > backend.tf
terraform {
  required_providers {
    proxmox = {
      source  = "telmate/proxmox"
      version = "3.0.1-rc2"
    }
  }
  backend "local" {
  }
}
EOF

# 3. 產生 provider config
$ cat <<EOF > provider.tf
provider "proxmox" {
  pm_api_url = "https://192.168.61.11:8006/api2/json"
  # api token id is in the form of: <username>@pam!<tokenId>
  pm_api_token_id = "terraform@pve!provider"
  # this is the full secret wrapped in quotes.
  pm_api_token_secret = var.PROXMOX_API_SECRET
  pm_tls_insecure     = true

  # debug log
  #  pm_log_enable = true
  #  pm_log_file   = "terraform-plugin-proxmox.log"
  #  pm_debug      = true
  #  pm_log_levels = {
  #    _default    = "debug"
  #    _capturelog = ""
  #  }
}
EOF

# 4. 產生變數設定檔
$ cat <<EOF > vars.tf
variable "PROXMOX_API_SECRET" {
  type = string
}

variable "ssh_key" {
  default = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCa5vPXuVVZOQJw+mvQizMqOY+sJryVg/yPQHHq1NfLVZREbjUHvIMJcTVzEVpy6oO/v6pQ4wYbgUQqPmCP/Np/VvX6YBhdnhc99V8Jn9H99QhmXHVfrG3GgBQ5scupP5VUaPZ372gVsidu63uNyDvnSwZ0cAXUxvB3hANQBH0eJ13/zVKBkWVGQTd/eQJBQx5sxpwXW4sBcPaYdpt+Njtc31GqxwgSzTO9HAHp1aSDq4SA3VEjUNg7/ysW/MwuuslZBsz0wmi55nBa9dkHG1ao72c9viVWc+fz+nLjMRq6QDUxJd2TjQH4k7QexibDXCZydxac2tMRKRo3ukrdSs1TzjcnUdWtTnFM+wDEkWnObagkp9feRg5+7RWh+3qc2zq1/OCEMqGd6Gu17VUxWK4L3QEq+k13vgphuzQpziRiolh8ETvSM0EXajuHj5RoJQKAEhfURvFgH+qa5nUw8DTjtR2gfkevKXOJBTDT9GauTItZJi03fQhVAng1blP1d8OkD+yItHthiylBYW+UdMT/HCgc9JtkDzjtmfRH2HvDZnFJ1O46EX74wycEs7Lgyeu3vHIYC+AP7/XprT9jIT5UHo3MOJ4vZwP0OcFhqg0x+8ptjzWRtn1SubcNRsYlJlc59gqAPSyrsAtYL5FGohgGokxlRX8hDjUhcLmknFQhiw== root@pve"
}

variable "proxmox_host" {
  default = "pve"
}
variable "template_name" {
  default = "alp-cloudinit-v3.19.1"
}
EOF

# 5. 產生 Terraform VM 設定檔 main.tf
$ nano main.tf 
resource "proxmox_vm_qemu" "ckad" {
  count       = 1
  name        = "alp-${count.index + 1}"
  target_node = var.proxmox_host
  vmid        = "101${count.index + 1}"
  clone       = var.template_name
  os_type     = "cloud-init"
  cpu         = "host"
  cores       = 4
  sockets     = 1
  memory      = 4096
  scsihw      = "virtio-scsi-pci"
  bootdisk    = "scsi0"

  disks {
    scsi {
      scsi0 {
        disk {
          size     = "50G"
          storage  = "local-lvm"
        }
      }
    }
    ide {
      ide3 {
        cloudinit {
          storage = "local-lvm"
        }
      }
    }
  }

  network {
    model     = "virtio"
    bridge    = "vmbr0"
    tag       = -1
    firewall  = false
    link_down = false
  }

  ipconfig0 = "ip=192.168.61.2${count.index + 1}/22,gw=192.168.61.2"
  ciuser    = "terraform"
  sshkeys   = <<EOF
  ${var.ssh_key}
  EOF
}

### 2.1. 檢查目錄結構

$ tree ../terraform
../terraform
├── backend.tf
├── main.tf
├── provider.tf
└── vars.tf

1 directory, 4 files

## 3. 建立 VM

$ export PROXMOX_API_SECRET="1fb0b04e-5673-4d5c-b580-ae602c718d89"

# inicialise the terraform modules
$ terraform init --upgrade

# generate a terraform plan
$ terraform plan -out ckad.tfplan

# deploy the VMs
$ terraform apply -parallelism=1 -auto-approve ckad.tfplan

# cleanup
$ terraform plan -destroy -out ckad.tfplan
$ terraform apply "ckad.tfplan"

## 4. 參考文章

- [Cloud Init Guide - Terraform Docs](https://registry.terraform.io/providers/Telmate/proxmox/latest/docs/guides/cloud_init)