# Linux Project 1
### Team 27 :
- 108602532 王鼎元
- 109502002 黃文翰
- 109502519 張傑誌
> [題目](https://staff.csie.ncu.edu.tw/hsufh/COURSES/FALL2023/linux_project_1.html)
## Kernel & OS version, VM
- Linux Kernel 5.15.137
- [Ubuntu 20.04 LTS](https://releases.ubuntu.com/focal/ubuntu-20.04.6-desktop-amd64.iso)
- [Virtual Box 7.0.12](https://download.virtualbox.org/virtualbox/7.0.12/VirtualBox-7.0.12-159484-Win.exe)
## 新增system call 至kernal 原始碼中
#### 主要流程參考助教公布的教學:
> [Add a system call to kernel](https://hackmd.io/aist49C9R46-vaBIlP3LDA?view)
**桌面 > 右鍵 > Open in terminal**
#### 準備系統
```bash
# 管理員權限
su
# 更新apt
sudo apt update && sudo apt upgrade -y
# 安裝需要的package
sudo apt install build-essential libncurses-dev libssl-dev libelf-dev bison flex -y
# 清除安裝的package
sudo apt clean && sudo apt autoremove -y
# 下載 kernel source code & 解壓縮
wget -P ~/ https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.15.137.tar.xz
tar -xvf linux-5.15.137.tar.xz -C /usr/src
```
#### 新增system call
```bash
# 進入source code資料夾
cd /usr/src/linux-5.15.137
# 新增system call程式資料夾
mkdir my_get_physical_addresses
# 進入資料夾
cd my_get_physical_addresses
# 建立system call 程式碼
nano my_get_physical_addresses.c
```
System call 目的為將傳入的 virtual address 通過一層一層 page table lookup，得到 physical address
此程式碼為 /usr/src/get_virtual_address.c 對應到助教的教學網站為 /usr/src/hello.c 的步驟:
#### System call 程式碼 (Kernel space code):
```c
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/uaccess.h>
#include <linux/init_task.h>
#include <linux/syscalls.h>
SYSCALL_DEFINE2(my_get_physical_addresses, unsigned long *, initial,
unsigned long *, result)
{
// page table pointer
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
unsigned long page_addr = 0;
unsigned long page_offset = 0;
// 動態配置儲存 virtual address & physical address 的變數
unsigned long *virtualAddress = kmalloc(1 * sizeof(unsigned long), GFP_KERNEL); //need to use vmalloc in kernel
unsigned long *physicalAddress = kmalloc(1 * sizeof(unsigned long), GFP_KERNEL);
//copy from user 傳進kernel space
unsigned long check = copy_from_user(virtualAddress, initial, 1 * sizeof(unsigned long));
printk("cfu_fail: %lu", check);
// 一層一層查 page table 進行 address 轉換
pgd = pgd_offset(current->mm, *(virtualAddress));
p4d = p4d_offset(pgd, *(virtualAddress));
pud = pud_offset(p4d, *(virtualAddress));
pmd = pmd_offset(pud, *(virtualAddress));
pte = pte_offset_kernel(pmd, *(virtualAddress));
// 組合
page_addr = pte_val(*pte) & PAGE_MASK;
page_offset = *(virtualAddress) & ~PAGE_MASK;
*(physicalAddress) = page_addr | page_offset;
// copytouser 返回 userspace
copy_to_user(result, physicalAddress, 1 * sizeof(unsigned long));
kfree(virtualAddress);
kfree(physicalAddress);
return 0;
}
```
> [Reference:   pgd pud pmd pte](https://www.796t.com/content/1537503608.html)
#### 修改 kernel 原始碼
```bash
# 建立新的 Makefile
nano Makefile
# 在Makefile中輸入以下內容並儲存
obj-y := my_get_physical_addresses.o
# 修改上個目錄的Makefile
cd ..
nano Makefile
# 將此行最後新增 my_get_physical_addresses/
core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/
變成
core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ my_get_physical_addresses/
(nano 中可以用 Ctrl+Q 進行搜尋)
# 修改 syscall_64.tbl 檔
nano arch/x86/entry/syscalls/syscall_64.tbl
# 增加新的system call，在process_mrelease下一行新增system call，編號應為449
449 common my_get_physical_addresses sys_my_get_physical_addresses
# 修改 syscalls.h 檔
nano include/linux/syscalls.h
# 在最下面一行的 #endif 之前加入下段文字 (Ctrl+Q)
asmlinkage long sys_my_get_physical_addresses(void);
```
## Kernel 編譯
#### 亦參考助教公布的:
> [Add a system call to kernel](https://hackmd.io/aist49C9R46-vaBIlP3LDA?view)
```bash
# 設定組態
make localmodconfig
# 查看邏輯核心數量
nproc
# make 時輸入剛剛看到的核心數量
make -j14
# 安裝kernel
sudo make modules_install -j12
sudo make install -j12
# 更新作業系統的bootloader成新的kernel
sudo update-grub
# 重新啟動
reboot
# 查看有沒有成功更新kernel
uname -r
```
**更新成功的話應該會顯示 5.15.137**
### 我們遇到的問題:
1. [No rule to make target ‘debian/canonical-certs.pem‘, needed by ‘certs/x509_certificate_list](https://blog.csdn.net/m0_47696151/article/details/121574718)
2. 一開始使用 WSL 一直失敗，經過查詢後發現應該是因為 WSL 使用的 kernel 不是 Linux
而是 WSL 自己的 kernel，想要在 WSL 當中新增新的 System call
應該要下載 WSL 的 kernel source code，WSL 也是 open source，因此理論上應該也是做的到
只是因為教學比較少，所以就沒有特別嘗試了，而是改回用 Virtual Box 編譯一般的 Linux
3. [BTF: .tmp_vmlinux.btf: pahole (pahole) is not available](https://stackoverflow.com/questions/61657707/btf-tmp-vmlinux-btf-pahole-pahole-is-not-available)
## Project 1: Multi-thread program showing how the memory areas are shared
#### 測試內容為:
- Code segments
- Data segments
- BSS segments
- Heap segments
- Libraries
- Stack segments
- Thread local storages
#### 測試檔程式碼 (User space code):
```c
#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include <unistd.h>
#include <sys/syscall.h>
#include <string.h>
unsigned long global_data = 4591;
static unsigned long global_bss;
__thread unsigned long thread_local_storage;
// get physical address
unsigned long gpa(unsigned long virtual_address){
unsigned long physical_address;
syscall(449, &virtual_address, &physical_address);
return physical_address;
}
void* threadFunction(void* arg, unsigned long fun){
void* heap_memory = malloc(10);
unsigned long local_data, pid = syscall(SYS_gettid);
thread_local_storage = (unsigned long)arg;
printf("Thread %lu - Code Segment : %lx\n", pid, gpa((unsigned long)fun));
printf("Thread %lu - Data Segment : %lx\n", pid, gpa((unsigned long)&global_data));
printf("Thread %lu - BSS Segment : %lx\n", pid, gpa((unsigned long)&global_bss));
printf("Thread %lu - Heap Segment : %lx\n", pid, gpa((unsigned long)heap_memory));
printf("Thread %lu - Library Address : %lx\n", pid, gpa((unsigned long)&printf));
printf("Thread %lu - Stack Segment : %lx\n", pid, gpa((unsigned long)&local_data));
printf("Thread %lu - Local Storage : %lx\n", pid, gpa((unsigned long)&thread_local_storage));
free(heap_memory);
return NULL;
}
void *func1(void *arg){
threadFunction((void*)1, (unsigned long)&func1);
while(1)sleep(3);
}
void *func2(void *arg){
threadFunction((void*)2, (unsigned long)&func2);
while(1)sleep(2);
}
void *func3(void *arg){
threadFunction((void*)3, (unsigned long)&func3);
while(1)sleep(2);
}
int main(){
pthread_t thread1, thread2, thread3;
threadFunction((void*)0, (unsigned long)&main);
pthread_create(&thread1, NULL, func1, (void*)1);
pthread_create(&thread2, NULL, func2, (void*)2);
pthread_create(&thread3, NULL, func3, (void*)3);
pthread_join(thread1, NULL);
pthread_join(thread2, NULL);
pthread_join(thread3, NULL);
return 0;
}
```
需要注意的是 `unsigned long gpa(unsigned long virtual_address)`
當中的 `syscall(449 , &virtual_address, &physical_address);`
**449** 需替換為修改kernal時 `arch/x86/entry/syscalls/syscall_64.tbl` 所輸入的新 system call 編號
### 測試結果
| Thread | 4125 (Main) | 4126 | 4128 | 4127 |
|--------|-----------------|-----------------|-----------------|-----------------|
| Code Segment | 1434d6499 | 1434d6409 | 1434d6469 | 1434d6439 |
| Data Segment | 8000000162192010| 8000000162192010| 8000000162192010| 8000000162192010|
| BSS Segment | 8000000162192020| 8000000162192020| 8000000162192020| 8000000162192020|
| Heap Segment | 80000001633f82a0| 8000000162fa7b60| 80000001652b8b60| 8000000163a6bb60|
| Library Address | 10bc56c90 | 10bc56c90 | 10bc56c90 | 10bc56c90 |
| Stack Segment | 80000001609b5460| 8000000165267eb0| 80000001633f5eb0| 800000016418aeb0|
| Local Storage | 8000000163037738| 80000001388ab6f8| 8000000162eb86f8| 800000016221a6f8|
### 結論
#### 在不同thread當中:
| Memory Segment | Shared? |
|-------------------------|-----------------|
| Code Segment | Shared |
| Data Segment | Shared |
| BSS Segment | Shared |
| Heap Segment | Not Shared(?) |
| Library Address | Shared |
| Stack Segment | Not Shared |
| Thread Local Storage | Not Shared |
**Heap理論上是要Shared，但是我們測試出來是在不同區段，目前還沒找到原因**
#### 以下為使用gdb得到的memory mapping，用來觀察virtual address
| Start Addr | End Addr | Size | Offset | Perms | objfile |
|------------------|------------------|-----------|---------|-------|------------------------------------------------------|
| 0x555555554000 | 0x555555555000 | 0x1000 | 0x0 | r--p | /home/blade/3th |
| 0x555555555000 | 0x555555556000 | 0x1000 | 0x1000 | r-xp | /home/blade/3th |
| 0x555555556000 | 0x555555557000 | 0x1000 | 0x2000 | r--p | /home/blade/3th |
| 0x555555557000 | 0x555555558000 | 0x1000 | 0x2000 | r--p | /home/blade/3th |
| 0x555555558000 | 0x555555559000 | 0x1000 | 0x3000 | rw-p | /home/blade/3th |
| 0x555555559000 | 0x55555557a000 | 0x21000 | 0x0 | rw-p | [heap] |
| 0x7fffe8000000 | 0x7fffe8021000 | 0x21000 | 0x0 | rw-p | |
| 0x7fffe8021000 | 0x7fffec000000 | 0x3fdf000 | 0x0 | ---p | |
| 0x7fffec000000 | 0x7fffec021000 | 0x21000 | 0x0 | rw-p | |
| 0x7fffec021000 | 0x7ffff0000000 | 0x3fdf000 | 0x0 | ---p | |
| 0x7ffff0000000 | 0x7ffff0021000 | 0x21000 | 0x0 | rw-p | |
| 0x7ffff0021000 | 0x7ffff4000000 | 0x3fdf000 | 0x0 | ---p | |
| 0x7ffff657c000 | 0x7ffff657d000 | 0x1000 | 0x0 | ---p | |
| 0x7ffff657d000 | 0x7ffff6d7d000 | 0x800000 | 0x0 | rw-p | |
| 0x7ffff6d7d000 | 0x7ffff6d7e000 | 0x1000 | 0x0 | ---p | |
| 0x7ffff6d7e000 | 0x7ffff757e000 | 0x800000 | 0x0 | rw-p | |
| 0x7ffff757e000 | 0x7ffff757f000 | 0x1000 | 0x0 | ---p | |
| 0x7ffff757f000 | 0x7ffff7d82000 | 0x803000 | 0x0 | rw-p | |
| 0x7ffff7d82000 | 0x7ffff7daa000 | 0x28000 | 0x0 | r--p | /usr/lib/x86_64-linux-gnu/libc.so.6 |
| 0x7ffff7daa000 | 0x7ffff7f3f000 | 0x195000 | 0x28000 | r-xp | /usr/lib/x86_64-linux-gnu/libc.so.6 |
| 0x7ffff7f3f000 | 0x7ffff7f97000 | 0x58000 | 0x1bd000| r--p | /usr/lib/x86_64-linux-gnu/libc.so.6 |
| 0x7ffff7f97000 | 0x7ffff7f9b000 | 0x4000 | 0x214000| r--p | /usr/lib/x86_64-linux-gnu/libc.so.6 |
| 0x7ffff7f9b000 | 0x7ffff7f9d000 | 0x2000 | 0x218000| rw-p | /usr/lib/x86_64-linux-gnu/libc.so.6 |
| 0x7ffff7f9d000 | 0x7ffff7faa000 | 0xd000 | 0x0 | rw-p | |
| 0x7ffff7fbb000 | 0x7ffff7fbd000 | 0x2000 | 0x0 | rw-p | |
| 0x7ffff7fbd000 | 0x7ffff7fc1000 | 0x4000 | 0x0 | r--p | [vvar] |
| 0x7ffff7fc1000 | 0x7ffff7fc3000 | 0x2000 | 0x0 | r-xp | [vdso] |
| 0x7ffff7fc3000 | 0x7ffff7fc5000 | 0x2000 | 0x0 | r--p | /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 |
| 0x7ffff7fc5000 | 0x7ffff7fef000 | 0x2a000 | 0x2000 | r-xp | /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 |
| 0x7ffff7fef000 | 0x7ffff7ffa000 | 0xb000 | 0x2c000 | r--p | /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 |
| 0x7ffff7ffb000 | 0x7ffff7ffd000 | 0x2000 | 0x37000 | r--p | /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 |
| 0x7ffff7ffd000 | 0x7ffff7fff000 | 0x2000 | 0x39000 | rw-p | /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 |
| 0x7ffffffde000 | 0x7ffffffff000 | 0x21000 | 0x0 | rw-p | [stack] |
| 0xffffffffff600000| 0xffffffffff601000| 0x1000 | 0x0 | --xp | [vsyscall] |
> [Reference: 64-bit layout](https://unix.stackexchange.com/questions/509607/how-a-64-bit-process-virtual-address-space-is-divided-in-linux)
Sign in with Wallet
Connect another wallet