# Overlay FileSystem (overlayFS)

###### tag: `overlayfs`

Test and understand overlay filesystem.

#### create loopback block device
create images and setup loopback

root@NUC10:~# dd if=/dev/zero of=/mnt/lower.img bs=1M count=1024
1024+0 records in
1024+0 records out
1073741824 bytes (1.1 GB, 1.0 GiB) copied, 0.473045 s, 2.3 GB/s
root@NUC10:~# dd if=/dev/zero of=/mnt/overlay.img bs=1M count=1024
1024+0 records in
1024+0 records out
1073741824 bytes (1.1 GB, 1.0 GiB) copied, 0.472078 s, 2.3 GB/s

root@NUC10:~# losetup -fP /mnt/lower.img
root@NUC10:~# losetup -fP /mnt/overlay.img
root@NUC10:~# losetup -a | grep "/mnt/"
/dev/loop26: [66305]:52166664 (/mnt/overlay.img)
/dev/loop20: [66305]:52166663 (/mnt/lower.img)

format image files

root@NUC10:~# mkfs.ext4 /mnt/lower.img
mke2fs 1.46.5 (30-Dec-2021)
Discarding device blocks: done
Creating filesystem with 262144 4k blocks and 65536 inodes
Filesystem UUID: b467fd35-0ac6-4f1a-9c6e-d622fe6b90f1
Superblock backups stored on blocks:
	32768, 98304, 163840, 229376

Allocating group tables: done
Writing inode tables: done
Creating journal (8192 blocks): done
Writing superblocks and filesystem accounting information: done

root@NUC10:~# mkfs.ext4 /mnt/overlay.img
mke2fs 1.46.5 (30-Dec-2021)
Discarding device blocks: done
Creating filesystem with 262144 4k blocks and 65536 inodes
Filesystem UUID: dceb523e-57a3-4051-9a49-d3e38f16dfbc
Superblock backups stored on blocks:
	32768, 98304, 163840, 229376

Allocating group tables: done
Writing inode tables: done
Creating journal (8192 blocks): done
Writing superblocks and filesystem accounting information: done

#### create directories

root@NUC10:/mnt# mkdir test-overlay
root@NUC10:/mnt# cd test-overlay/
root@NUC10:/mnt/test-overlay# mkdir lower overlay merged
root@NUC10:/mnt/test-overlay# ls -l
total 12
drwxr-xr-x 3 root root 4096 Nov 15 00:29 lower
drwxr-xr-x 2 root root 4096 Nov 15 00:48 merged
drwxr-xr-x 5 root root 4096 Nov 15 00:45 overlay

#### mount lower loopback device and create some files inside.

root@NUC10:~# mount -o rw /dev/loop20 /mnt/test-overlay/lower/
root@NUC10:~# echo "lower data1 line" > /mnt/test-overlay/lower/data1
root@NUC10:~# echo "lower data2 line" > /mnt/test-overlay/lower/data2
root@NUC10:~# ls -l /mnt/test-overlay/lower
total 24
-rw-r--r-- 1 root root    17 Nov 15 00:28 data1
-rw-r--r-- 1 root root    17 Nov 15 00:29 data2
drwx------ 2 root root 16384 Nov 15 00:22 lost+found

root@NUC10:~# sync
root@NUC10:~# umount /dev/loop20

#### overlayFS usage
mount lower layer

root@NUC10:~# cd /mnt/test-overlay/
root@NUC10:/mnt/test-overlay# mount -o ro /dev/loop20 ./lower/
root@NUC10:/mnt/test-overlay# ls -l ./lower/
total 24
-rw-r--r-- 1 root root    17 Nov 15 00:28 data1
-rw-r--r-- 1 root root    17 Nov 15 00:29 data2
drwx------ 2 root root 16384 Nov 15 00:22 lost+found

mount overlay layer and create "upper" and "work" directories

root@NUC10:/mnt/test-overlay# mount /dev/loop26 ./overlay/
root@NUC10:/mnt/test-overlay# mkdir ./overlay/{upper,work}
root@NUC10:/mnt/test-overlay# ls -l ./overlay/
total 24
drwx------ 2 root root 16384 Nov 15 00:23 lost+found
drwxr-xr-x 2 root root  4096 Nov 15 00:45 upper
drwxr-xr-x 2 root root  4096 Nov 15 00:45 work

create overlayFS on /mnt/test-overlay/merged

root@NUC10:/mnt/test-overlay# mount overlay -t overlay -o lowerdir=/mnt/test-overlay/lower,upperdir=/mnt/test-overlay/overlay/upper/,workdir=/mnt/test-overlay/overlay/work/ /mnt/test-overlay/merged

> -t specify type of filesystem to mount
> -o set mount options, in this case, "lowerdir", "workdir" and "upperdir"

A overlay filesytem is mounted on "/mnt/test-overlay/merged"

root@NUC10:/mnt/test-overlay# df -h | grep overlay
/dev/loop20     974M   32K  907M   1% /mnt/test-overlay/lower
/dev/loop26     974M   36K  907M   1% /mnt/test-overlay/overlay
overlay         974M   36K  907M   1% /mnt/test-overlay/merged

root@NUC10:/mnt/test-overlay# mount | grep -i overlay
/dev/loop20 on /mnt/test-overlay/lower type ext4 (ro,relatime)
/dev/loop26 on /mnt/test-overlay/overlay type ext4 (rw,relatime)
overlay on /mnt/test-overlay/merged type overlay (rw,relatime,lowerdir=/mnt/test-overlay/lower,upperdir=/mnt/test-overlay/overlay/upper/,workdir=/mnt/test-overlay/overlay/work/)

root@NUC10:/mnt/test-overlay# ls -l ./merged/
total 24
-rw-r--r-- 1 root root    17 Nov 15 00:28 data1
-rw-r--r-- 1 root root    17 Nov 15 00:29 data2
drwx------ 2 root root 16384 Nov 15 00:22 lost+found

Add extra line to data1 file in the merged directory.

root@NUC10:/mnt/test-overlay# echo "overlay data1 line" >> ./merged/data1
root@NUC10:/mnt/test-overlay# cat ./merged/data1
lower data1 line
overlay data1 line

In the lower directory, data1 has no change

root@NUC10:/mnt/test-overlay# cat ./lower/data1
lower data1 line

In the upper directory, a same file is present which is same with one in merged directory.

root@NUC10:/mnt/test-overlay# ls -l ./overlay/upper/
total 4
-rw-r--r-- 1 root root 36 Nov 15 00:53 data1
root@NUC10:/mnt/test-overlay# cat ./overlay/upper/data1
lower data1 line
overlay data1 line

Delete data2 file in merged directory

root@NUC10:/mnt/test-overlay# rm ./merged/data2
root@NUC10:/mnt/test-overlay# ls -l ./merged/
total 20
-rw-r--r-- 1 root root    36 Nov 15 00:53 data1
drwx------ 2 root root 16384 Nov 15 00:22 lost+found

In upper directory, same filename is created with file mode "character" device.
This

root@NUC10:/mnt/test-overlay# ls -l ./overlay/upper/
total 4
-rw-r--r-- 1 root root   36 Nov 15 00:53 data1
c--------- 2 root root 0, 0 Nov 15 00:55 data2

> A whiteout is created as a character device with 0/0 device number. When a whiteout is found in the upper level of a merged directory, any matching name in the lower level is ignored, and the whiteout itself is also hidden.

#### Verify with a docker container overlay filesystem

root@m900:~# docker inspect gitlab | grep overlay
        "Driver": "overlay2",
                "LowerDir": "/var/lib/docker/overlay2/74f18d3b7d672328d102d1e6f1cbfceb4cebbda38732e068c3514b4041a96552-init/diff:/var/lib/docker/overlay2/3c955f9235c05722aa60de2aa7e93817540ead7ed273b5d936f9d9d18a3ae4c1/diff:/var/lib/docker/overlay2/2eb1fa2b58ddf6ea5b7f059c7acef741144504200f9fed1289dc71ad98d26055/diff:/var/lib/docker/overlay2/8266245b479f72ed4596f9f1b9c65cb3e20dab74ebe3c80474447fe3d53c8e83/diff:/var/lib/docker/overlay2/ed7dd462df2cdf4163e202a959b5629e6cdd105ad78018b532ae16bc0f589728/diff:/var/lib/docker/overlay2/336930137b4564c7cdc73f67ad53817481ee3f3bc32aacc2273bd807984df436/diff:/var/lib/docker/overlay2/bf34ce7cadb43f6e2572e6a081f868ae682b8e9b80b05892bf36c6c0d09604dc/diff:/var/lib/docker/overlay2/5e6ff6719b97ef488787171809ec6999c24d47b5022fbffe47a9435f24a39c17/diff:/var/lib/docker/overlay2/c8e4af5dad4021a3d17b38dc0dee4dbff43ab8ece5036fd8e07181b21d4a2217/diff",
                "MergedDir": "/var/lib/docker/overlay2/74f18d3b7d672328d102d1e6f1cbfceb4cebbda38732e068c3514b4041a96552/merged",
                "UpperDir": "/var/lib/docker/overlay2/74f18d3b7d672328d102d1e6f1cbfceb4cebbda38732e068c3514b4041a96552/diff",
                "WorkDir": "/var/lib/docker/overlay2/74f18d3b7d672328d102d1e6f1cbfceb4cebbda38732e068c3514b4041a96552/work"
            "Name": "overlay2"

root@m900:~# df -h | grep /var/lib/docker/overlay2/74f18d3b7d672328d102d1e6f1cbfceb4cebbda38732e068c3514b4041a96552/merged
overlay         916G   35G  835G   5% /var/lib/docker/overlay2/74f18d3b7d672328d102d1e6f1cbfceb4cebbda38732e068c3514b4041a96552/merged

root@m900:~# mount | grep /var/lib/docker/overlay2/74f18d3b7d672328d102d1e6f1cbfceb4cebbda38732e068c3514b4041a96552/merged
overlay on /var/lib/docker/overlay2/74f18d3b7d672328d102d1e6f1cbfceb4cebbda38732e068c3514b4041a96552/merged type overlay (rw,relatime,lowerdir=/var/lib/docker/overlay2/l/KMTT27XT4SJWTURLJ53IAI73QZ:/var/lib/docker/overlay2/l/N25TEKVF3L5MMWTVEVMYIC2HBJ:/var/lib/docker/overlay2/l/2QTL77FSXAKS42FKED4HQAHNWQ:/var/lib/docker/overlay2/l/Q462WZSPHS4ZO6PEO7COEDOSHQ:/var/lib/docker/overlay2/l/RNP4ERXAPLMHLSHM6M6NRGCDSC:/var/lib/docker/overlay2/l/V25GWMW6HVAYD2WRKBGGOAY7VC:/var/lib/docker/overlay2/l/3DGWPPXXQYPYLFNJ2VGOG6EBHE:/var/lib/docker/overlay2/l/EH2CU2CCQFVKLCBLFSYSP2UGXW:/var/lib/docker/overlay2/l/PGDBRHCU2KLMSXOKVSO4FBYQF2,upperdir=/var/lib/docker/overlay2/74f18d3b7d672328d102d1e6f1cbfceb4cebbda38732e068c3514b4041a96552/diff,workdir=/var/lib/docker/overlay2/74f18d3b7d672328d102d1e6f1cbfceb4cebbda38732e068c3514b4041a96552/work)

> Multiple lower layers can now be given using the colon (":") as a separator character between the directory names. For example:
> mount -t overlay overlay -olowerdir=/lower1:/lower2:/lower3 /merged

#### reference
https://docs.kernel.org/filesystems/overlayfs.html
https://linuxconfig.org/introduction-to-the-overlayfs
https://linuxhandbook.com/create-virtual-block-device/
https://stackoverflow.com/questions/31044982/how-to-use-multiple-lower-layers-in-overlayfs