HackMDBuild all-in-one Ceph cluster via cephadm
tags: ceph
Deploy all-in-one ceph cluster for testing purpose.
VM spec.
- Hostname: ceph-aio
- OS: Ubuntu 20.04
- IP: 192.168.122.90
- 2 vCPU
- 4G RAM
- 16GB vda (OS disk)
- 10GB vdb
- 10GB vdc
- 10GB vdd
The cluster will be configured with no replica because cluster has single node only.
build steps
1 - download cephadmin
root@ceph-aio:~# curl --silent --remote-name --location https://github.com/ceph/ceph/raw/quincy/src/cephadm/cephadm
root@ceph-aio:~# chmod +x cephadm
2 - install docker
root@ceph-aio:~# sudo apt update
root@ceph-aio:~# sudo apt -y install ca-certificates curl gnupg lsb-release
root@ceph-aio:~# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
root@ceph-aio:~# echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
> $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
root@ceph-aio:~# sudo apt update
root@ceph-aio:~# sudo apt -y install docker-ce docker-ce-cli containerd.io docker-compose-plugin
root@ceph-aio:~# docker --version
Docker version 20.10.14, build a224086
3 - add ceph repo
root@ceph-aio:~# ./cephadm add-repo --release quincy
Installing repo GPG key from https://download.ceph.com/keys/release.gpg...
Installing repo file at /etc/apt/sources.list.d/ceph.list...
Updating package list...
Completed adding repo.
4 - install cephadm and ceph-common
root@ceph-aio:~# ./cephadm install
Installing packages ['cephadm']...
root@ceph-aio:~# which cephadm
/usr/sbin/cephadm
root@ceph-aio:~# ./cephadm install ceph-common
root@ceph-aio:~# ceph -v
ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)
5 - bootstrap ceph
root@ceph-aio:~# sudo mkdir -p /etc/ceph
root@ceph-aio:~# cephadm bootstrap --mon-ip 192.168.122.90
root@ceph-aio:~# ceph orch ps
NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID
alertmanager.ceph-aio ceph-aio *:9093,9094 running (17m) 5m ago 19m 12.1M - ba2b418f427c 6a0419782df9
crash.ceph-aio ceph-aio running (19m) 5m ago 19m 7740k - 17.2.0 c92c93952052 98879b083c8c
grafana.ceph-aio ceph-aio *:3000 running (17m) 5m ago 17m 39.4M - 8.3.5 dad864ee21e9 c19407f8a7e8
mgr.ceph-aio.nmuhhm ceph-aio *:9283 running (20m) 5m ago 20m 426M - 17.2.0 c92c93952052 d02b89542cb3
mon.ceph-aio ceph-aio running (20m) 5m ago 20m 33.4M 2048M 17.2.0 c92c93952052 10e30a018478
node-exporter.ceph-aio ceph-aio *:9100 running (17m) 5m ago 17m 9247k - 1dbe0e931976 7ea15b5b4410
prometheus.ceph-aio ceph-aio *:9095 running (17m) 5m ago 17m 40.0M - 514e6a882f6e 703ac81e0905
root@ceph-aio:~# ceph orch ps --daemon-type mon
NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID
mon.ceph-aio ceph-aio running (20m) 6m ago 20m 33.4M 2048M 17.2.0 c92c93952052 10e30a018478
root@ceph-aio:~# ceph -s
cluster:
id: b25ef344-c62a-11ec-a3e0-4f9f42678996
health: HEALTH_WARN
OSD count 0 < osd_pool_default_size 3
services:
mon: 1 daemons, quorum ceph-aio (age 19m)
mgr: ceph-aio.nmuhhm(active, since 16m)
osd: 0 osds: 0 up, 0 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
6 - add OSDs
root@ceph-aio:~# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
loop0 7:0 0 55.5M 1 loop /snap/core18/2284
loop1 7:1 0 61.9M 1 loop /snap/core20/1405
loop2 7:2 0 67.9M 1 loop /snap/lxd/22526
loop3 7:3 0 67.8M 1 loop /snap/lxd/22753
loop4 7:4 0 61.9M 1 loop /snap/core20/1376
loop5 7:5 0 55.5M 1 loop /snap/core18/2344
loop6 7:6 0 43.6M 1 loop /snap/snapd/15177
loop7 7:7 0 44.7M 1 loop /snap/snapd/15534
vda 252:0 0 16G 0 disk
├─vda1 252:1 0 1M 0 part
└─vda2 252:2 0 16G 0 part /
vdb 252:16 0 10G 0 disk
vdc 252:32 0 10G 0 disk
vdd 252:48 0 10G 0 disk
root@ceph-aio:~# ceph orch device ls
HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS
ceph-aio /dev/vdb hdd 10.7G Yes
ceph-aio /dev/vdc hdd 10.7G Yes
ceph-aio /dev/vdd hdd 10.7G Yes
root@ceph-aio:~# ceph orch apply osd --all-available-devices
Scheduled osd.all-available-devices update...
root@ceph-aio:~# ceph -s
cluster:
id: b25ef344-c62a-11ec-a3e0-4f9f42678996
health: HEALTH_OK
services:
mon: 1 daemons, quorum ceph-aio (age 25m)
mgr: ceph-aio.nmuhhm(active, since 23m)
osd: 3 osds: 0 up, 3 in (since 4s)
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
root@ceph-aio:~# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 0.02939 root default
-3 0.02939 host ceph-aio
0 hdd 0.00980 osd.0 up 1.00000 1.00000
1 hdd 0.00980 osd.1 up 1.00000 1.00000
2 hdd 0.00980 osd.2 up 1.00000 1.00000
root@ceph-aio:~# ceph osd df
ID CLASS WEIGHT REWEIGHT SIZE RAW USE DATA OMAP META AVAIL %USE VAR PGS STATUS
0 hdd 0.00980 1.00000 10 GiB 19 MiB 160 KiB 0 B 19 MiB 10 GiB 0.19 1.00 0 up
1 hdd 0.00980 1.00000 10 GiB 19 MiB 160 KiB 0 B 19 MiB 10 GiB 0.19 1.00 1 up
2 hdd 0.00980 1.00000 10 GiB 19 MiB 160 KiB 0 B 19 MiB 10 GiB 0.19 1.00 0 up
TOTAL 30 GiB 58 MiB 480 KiB 0 B 58 MiB 30 GiB 0.19
MIN/MAX VAR: 1.00/1.00 STDDEV: 0
root@ceph-aio:~# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
811d374e297c quay.io/ceph/ceph "/usr/bin/ceph-osd -…" 5 minutes ago Up 5 minutes ceph-b25ef344-c62a-11ec-a3e0-4f9f42678996-osd-2
7296b32c2ba9 quay.io/ceph/ceph "/usr/bin/ceph-osd -…" 5 minutes ago Up 5 minutes ceph-b25ef344-c62a-11ec-a3e0-4f9f42678996-osd-1
59dbe29f7152 quay.io/ceph/ceph "/usr/bin/ceph-osd -…" 6 minutes ago Up 5 minutes ceph-b25ef344-c62a-11ec-a3e0-4f9f42678996-osd-0
c19407f8a7e8 quay.io/ceph/ceph-grafana:8.3.5 "/bin/sh -c 'grafana…" 28 minutes ago Up 28 minutes ceph-b25ef344-c62a-11ec-a3e0-4f9f42678996-grafana-ceph-aio
6a0419782df9 quay.io/prometheus/alertmanager:v0.23.0 "/bin/alertmanager -…" 28 minutes ago Up 28 minutes ceph-b25ef344-c62a-11ec-a3e0-4f9f42678996-alertmanager-ceph-aio
703ac81e0905 quay.io/prometheus/prometheus:v2.33.4 "/bin/prometheus --c…" 28 minutes ago Up 28 minutes ceph-b25ef344-c62a-11ec-a3e0-4f9f42678996-prometheus-ceph-aio
7ea15b5b4410 quay.io/prometheus/node-exporter:v1.3.1 "/bin/node_exporter …" 29 minutes ago Up 29 minutes ceph-b25ef344-c62a-11ec-a3e0-4f9f42678996-node-exporter-ceph-aio
98879b083c8c quay.io/ceph/ceph "/usr/bin/ceph-crash…" 30 minutes ago Up 30 minutes ceph-b25ef344-c62a-11ec-a3e0-4f9f42678996-crash-ceph-aio
d02b89542cb3 quay.io/ceph/ceph:v17 "/usr/bin/ceph-mgr -…" 31 minutes ago Up 31 minutes ceph-b25ef344-c62a-11ec-a3e0-4f9f42678996-mgr-ceph-aio-nmuhhm
10e30a018478 quay.io/ceph/ceph:v17 "/usr/bin/ceph-mon -…" 31 minutes ago Up 31 minutes ceph-b25ef344-c62a-11ec-a3e0-4f9f42678996-mon-ceph-aio
a few minutes later, the ceph status become HEALTH_WARN
root@ceph-aio:~# ceph -s
cluster:
id: b25ef344-c62a-11ec-a3e0-4f9f42678996
health: HEALTH_WARN
Reduced data availability: 1 pg inactive
Degraded data redundancy: 1 pg undersized
services:
mon: 1 daemons, quorum ceph-aio (age 27m)
mgr: ceph-aio.nmuhhm(active, since 24m)
osd: 3 osds: 3 up (since 91s), 3 in (since 112s)
data:
pools: 1 pools, 1 pgs
objects: 0 objects, 0 B
usage: 58 MiB used, 30 GiB / 30 GiB avail
pgs: 100.000% pgs not active
1 undersized+peered
This is due to a pool has default size 3 and min_size 2. But the all-in-one cluster is only able to store single replica.
root@ceph-aio:~# ceph osd dump | grep 'replicated size'
pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 19 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr
7 - configure cluster to allow pool size one
root@ceph-aio:~# ceph config set global mon_allow_pool_size_one true
set the '.mgr' pool size to 1 (set size 2 first and then 1)
root@ceph-aio:~# ceph osd pool set .mgr size 2 --yes-i-really-mean-it
set pool 1 size to 2
root@ceph-aio:~# ceph osd pool set .mgr size 1 --yes-i-really-mean-it
set pool 1 size to 1
# this step is not necessary here as the size already decreased to 1
root@ceph-aio:~# ceph osd pool set .mgr min_size 1
set pool 1 min_size to 1
8 - create "rbd" pool
root@ceph-aio:~# ceph osd pool create rbd 64 64 replicated
pool 'rbd' created
set size to 1
root@ceph-aio:~# ceph osd pool set rbd min_size 1
set pool 2 min_size to 1
root@ceph-aio:~# ceph osd pool set rbd size 2 --yes-i-really-mean-it
set pool 2 size to 2
root@ceph-aio:~# ceph osd pool set rbd size 1 --yes-i-really-mean-it
set pool 2 size to 1
config pool for RBD use
# initialise pool used for RBD
root@ceph-aio:~# rbd pool init rbd
# associate pool to application
root@ceph-aio:~# ceph osd pool application enable rbd rbd
enabled application 'rbd' on pool 'rbd'
9 - ignore the no replicas warning
root@ceph-aio:~# ceph -s
cluster:
id: b25ef344-c62a-11ec-a3e0-4f9f42678996
health: HEALTH_WARN
2 pool(s) have no replicas configured
services:
mon: 1 daemons, quorum ceph-aio (age 63m)
mgr: ceph-aio.nmuhhm(active, since 61m)
osd: 3 osds: 3 up (since 37m), 3 in (since 38m)
data:
pools: 2 pools, 65 pgs
objects: 2 objects, 449 KiB
usage: 63 MiB used, 30 GiB / 30 GiB avail
pgs: 65 active+clean
root@ceph-aio:~# ceph health mute POOL_NO_REDUNDANCY
root@ceph-aio:~# ceph -s
cluster:
id: b25ef344-c62a-11ec-a3e0-4f9f42678996
health: HEALTH_OK
(muted: POOL_NO_REDUNDANCY)
services:
mon: 1 daemons, quorum ceph-aio (age 65m)
mgr: ceph-aio.nmuhhm(active, since 63m)
osd: 3 osds: 3 up (since 39m), 3 in (since 40m)
data:
pools: 2 pools, 65 pgs
objects: 2 objects, 449 KiB
usage: 63 MiB used, 30 GiB / 30 GiB avail
pgs: 65 active+clean
test
root@ceph-aio:~# rbd create rbd0 --size 1024 --image-feature layering
root@ceph-aio:~# rbd ls -l
NAME SIZE PARENT FMT PROT LOCK
rbd0 1 GiB 2
setup cephfs
to be updated..
setup rados gateway
to be updated..
other configs
to allow pool deletion
$ ceph config set global mon_allow_pool_delete true
reference
https://docs.docker.com/engine/install/ubuntu/
https://balderscape.medium.com/setting-up-a-virtual-single-node-ceph-storage-cluster-d86d6a6c658e
https://docs.ceph.com/en/quincy/start/quick-rbd/
https://docs.ceph.com/en/latest/rados/operations/pools/
