---
# System prepended metadata

title: Linux Project 1

---

# Linux Project 1  
### Team 27 :  
- 108602532 王鼎元 
- 109502002 黃文翰 
- 109502519 張傑誌
> [題目](https://staff.csie.ncu.edu.tw/hsufh/COURSES/FALL2023/linux_project_1.html)
## Kernel & OS version, VM
- Linux Kernel 5.15.137
- [Ubuntu 20.04 LTS](https://releases.ubuntu.com/focal/ubuntu-20.04.6-desktop-amd64.iso)
- [Virtual Box 7.0.12](https://download.virtualbox.org/virtualbox/7.0.12/VirtualBox-7.0.12-159484-Win.exe)

## 新增system call 至kernal 原始碼中
#### 主要流程參考助教公布的教學:
> [Add a system call to kernel](https://hackmd.io/aist49C9R46-vaBIlP3LDA?view)   

**桌面 > 右鍵 > Open in terminal**
#### 準備系統

```bash
# 管理員權限
su

# 更新apt
sudo apt update && sudo apt upgrade -y

# 安裝需要的package
sudo apt install build-essential libncurses-dev libssl-dev libelf-dev bison flex -y

# 清除安裝的package
sudo apt clean && sudo apt autoremove -y

# 下載 kernel source code & 解壓縮
wget -P ~/ https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.15.137.tar.xz
tar -xvf linux-5.15.137.tar.xz -C /usr/src
```

#### 新增system call

```bash
# 進入source code資料夾
cd /usr/src/linux-5.15.137

# 新增system call程式資料夾
mkdir my_get_physical_addresses

# 進入資料夾
cd my_get_physical_addresses

# 建立system call 程式碼
nano my_get_physical_addresses.c
```

System call 目的為將傳入的 virtual address 通過一層一層 page table lookup，得到 physical address  
此程式碼為 /usr/src/get_virtual_address.c 對應到助教的教學網站為 /usr/src/hello.c 的步驟:
#### System call 程式碼 (Kernel space code):
```c
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/uaccess.h>
#include <linux/init_task.h>
#include <linux/syscalls.h>

SYSCALL_DEFINE2(my_get_physical_addresses, unsigned long *, initial,
		unsigned long *, result)
{
    // page table pointer
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	unsigned long page_addr = 0;
	unsigned long page_offset = 0;

    // 動態配置儲存 virtual address & physical address 的變數
	unsigned long *virtualAddress = kmalloc(1 * sizeof(unsigned long), GFP_KERNEL); //need to use vmalloc in kernel
	unsigned long *physicalAddress = kmalloc(1 * sizeof(unsigned long), GFP_KERNEL);

	//copy from user 傳進kernel space
	unsigned long check = copy_from_user(virtualAddress, initial, 1 * sizeof(unsigned long));
    printk("cfu_fail: %lu", check);

    // 一層一層查 page table 進行 address 轉換
	pgd = pgd_offset(current->mm, *(virtualAddress));
	p4d = p4d_offset(pgd, *(virtualAddress));
	pud = pud_offset(p4d, *(virtualAddress));
	pmd = pmd_offset(pud, *(virtualAddress));
	pte = pte_offset_kernel(pmd, *(virtualAddress));

    // 組合
	page_addr = pte_val(*pte) & PAGE_MASK; 
	page_offset = *(virtualAddress) & ~PAGE_MASK;
	*(physicalAddress) = page_addr | page_offset;

	// copytouser 返回 userspace
	copy_to_user(result, physicalAddress, 1 * sizeof(unsigned long));
	kfree(virtualAddress);
	kfree(physicalAddress);

	return 0;
}
```
> [Reference: &ensp; pgd pud pmd pte](https://www.796t.com/content/1537503608.html)

#### 修改 kernel 原始碼

```bash
# 建立新的 Makefile
nano Makefile

# 在Makefile中輸入以下內容並儲存
obj-y := my_get_physical_addresses.o

# 修改上個目錄的Makefile
cd ..
nano Makefile

# 將此行最後新增 my_get_physical_addresses/
core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/
變成
core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ my_get_physical_addresses/
(nano 中可以用 Ctrl+Q 進行搜尋)

# 修改 syscall_64.tbl 檔
nano arch/x86/entry/syscalls/syscall_64.tbl

# 增加新的system call，在process_mrelease下一行新增system call，編號應為449
449 common  my_get_physical_addresses sys_my_get_physical_addresses

# 修改 syscalls.h 檔
nano include/linux/syscalls.h

# 在最下面一行的 #endif 之前加入下段文字 (Ctrl+Q)
asmlinkage long sys_my_get_physical_addresses(void);
```

## Kernel 編譯
#### 亦參考助教公布的: 
> [Add a system call to kernel](https://hackmd.io/aist49C9R46-vaBIlP3LDA?view)

```bash
# 設定組態
make localmodconfig

# 查看邏輯核心數量
nproc

# make 時輸入剛剛看到的核心數量
make -j14

# 安裝kernel
sudo make modules_install -j12
sudo make install -j12

# 更新作業系統的bootloader成新的kernel
sudo update-grub

# 重新啟動
reboot

# 查看有沒有成功更新kernel
uname -r
```

**更新成功的話應該會顯示 5.15.137**


### 我們遇到的問題:  
1. [No rule to make target ‘debian/canonical-certs.pem‘, needed by ‘certs/x509_certificate_list](https://blog.csdn.net/m0_47696151/article/details/121574718)  
2. 一開始使用 WSL 一直失敗，經過查詢後發現應該是因為 WSL 使用的 kernel 不是 Linux  
   而是 WSL 自己的 kernel，想要在 WSL 當中新增新的 System call  
   應該要下載 WSL 的 kernel source code，WSL 也是 open source，因此理論上應該也是做的到  
   只是因為教學比較少，所以就沒有特別嘗試了，而是改回用 Virtual Box 編譯一般的 Linux
3. [BTF: .tmp_vmlinux.btf: pahole (pahole) is not available](https://stackoverflow.com/questions/61657707/btf-tmp-vmlinux-btf-pahole-pahole-is-not-available)
## Project 1: Multi-thread program showing how the memory areas are shared
#### 測試內容為:
- Code segments
- Data segments
- BSS segments
- Heap segments
- Libraries
- Stack segments
- Thread local storages  
#### 測試檔程式碼 (User space code):
```c
#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include <unistd.h>
#include <sys/syscall.h>
#include <string.h>

unsigned long global_data = 4591;
static unsigned long global_bss;
__thread unsigned long thread_local_storage;

// get physical address
unsigned long gpa(unsigned long virtual_address){
    unsigned long physical_address;
    syscall(449, &virtual_address, &physical_address);
    return physical_address;
}

void* threadFunction(void* arg, unsigned long fun){
    void* heap_memory = malloc(10);
    unsigned long local_data, pid = syscall(SYS_gettid);
    thread_local_storage = (unsigned long)arg;
    
    printf("Thread %lu - Code Segment    : %lx\n", pid, gpa((unsigned long)fun));
    printf("Thread %lu - Data Segment    : %lx\n", pid, gpa((unsigned long)&global_data));
    printf("Thread %lu - BSS Segment     : %lx\n", pid, gpa((unsigned long)&global_bss));
    printf("Thread %lu - Heap Segment    : %lx\n", pid, gpa((unsigned long)heap_memory));
    printf("Thread %lu - Library Address : %lx\n", pid, gpa((unsigned long)&printf));    
    printf("Thread %lu - Stack Segment   : %lx\n", pid, gpa((unsigned long)&local_data));
    printf("Thread %lu - Local Storage   : %lx\n", pid, gpa((unsigned long)&thread_local_storage));
    free(heap_memory);
    return NULL;
}

void *func1(void *arg){
	threadFunction((void*)1, (unsigned long)&func1);
	while(1)sleep(3);
}

void *func2(void *arg){
	threadFunction((void*)2, (unsigned long)&func2);
	while(1)sleep(2);
}

void *func3(void *arg){
	threadFunction((void*)3, (unsigned long)&func3);
	while(1)sleep(2);
}

int main(){
    pthread_t thread1, thread2, thread3;
    
    threadFunction((void*)0, (unsigned long)&main);
    pthread_create(&thread1, NULL, func1, (void*)1);
    pthread_create(&thread2, NULL, func2, (void*)2);
    pthread_create(&thread3, NULL, func3, (void*)3);
    pthread_join(thread1, NULL);
    pthread_join(thread2, NULL);
    pthread_join(thread3, NULL);

    return 0;
}

```
需要注意的是 `unsigned long gpa(unsigned long virtual_address)`  
當中的 `syscall(449 , &virtual_address, &physical_address);`  
**449** 需替換為修改kernal時 `arch/x86/entry/syscalls/syscall_64.tbl` 所輸入的新 system call 編號
### 測試結果
| Thread | 4125 (Main)     | 4126            | 4128            | 4127            |
|--------|-----------------|-----------------|-----------------|-----------------|
| Code Segment    | 1434d6499       | 1434d6409       | 1434d6469       | 1434d6439       |
| Data Segment    | 8000000162192010| 8000000162192010| 8000000162192010| 8000000162192010|
| BSS Segment     | 8000000162192020| 8000000162192020| 8000000162192020| 8000000162192020|
| Heap Segment    | 80000001633f82a0| 8000000162fa7b60| 80000001652b8b60| 8000000163a6bb60|
| Library Address | 10bc56c90       | 10bc56c90       | 10bc56c90       | 10bc56c90       |
| Stack Segment   | 80000001609b5460| 8000000165267eb0| 80000001633f5eb0| 800000016418aeb0|
| Local Storage   | 8000000163037738| 80000001388ab6f8| 8000000162eb86f8| 800000016221a6f8|

### 結論
#### 在不同thread當中:
| Memory Segment          | Shared?         |
|-------------------------|-----------------|
| Code Segment            | Shared          |
| Data Segment            | Shared          |
| BSS Segment             | Shared          |
| Heap Segment            | Not Shared(?)   |
| Library Address         | Shared          |
| Stack Segment           | Not Shared      |
| Thread Local Storage    | Not Shared      |


**Heap理論上是要Shared，但是我們測試出來是在不同區段，目前還沒找到原因**

![示意圖](https://hackmd.io/_uploads/HyffILxBa.png)

#### 以下為使用gdb得到的memory mapping，用來觀察virtual address

| Start Addr       | End Addr         | Size      | Offset  | Perms | objfile                                             |
|------------------|------------------|-----------|---------|-------|------------------------------------------------------|
| 0x555555554000   | 0x555555555000   | 0x1000    | 0x0     | r--p  | /home/blade/3th                                    |
| 0x555555555000   | 0x555555556000   | 0x1000    | 0x1000  | r-xp  | /home/blade/3th                                    |
| 0x555555556000   | 0x555555557000   | 0x1000    | 0x2000  | r--p  | /home/blade/3th                                    |
| 0x555555557000   | 0x555555558000   | 0x1000    | 0x2000  | r--p  | /home/blade/3th                                    |
| 0x555555558000   | 0x555555559000   | 0x1000    | 0x3000  | rw-p  | /home/blade/3th                                    |
| 0x555555559000   | 0x55555557a000   | 0x21000   | 0x0     | rw-p  | [heap]                                               |
| 0x7fffe8000000   | 0x7fffe8021000   | 0x21000   | 0x0     | rw-p  |                                                      |
| 0x7fffe8021000   | 0x7fffec000000   | 0x3fdf000 | 0x0     | ---p  |                                                      |
| 0x7fffec000000   | 0x7fffec021000   | 0x21000   | 0x0     | rw-p  |                                                      |
| 0x7fffec021000   | 0x7ffff0000000   | 0x3fdf000 | 0x0     | ---p  |                                                      |
| 0x7ffff0000000   | 0x7ffff0021000   | 0x21000   | 0x0     | rw-p  |                                                      |
| 0x7ffff0021000   | 0x7ffff4000000   | 0x3fdf000 | 0x0     | ---p  |                                                      |
| 0x7ffff657c000   | 0x7ffff657d000   | 0x1000    | 0x0     | ---p  |                                                      |
| 0x7ffff657d000   | 0x7ffff6d7d000   | 0x800000  | 0x0     | rw-p  |                                                      |
| 0x7ffff6d7d000   | 0x7ffff6d7e000   | 0x1000    | 0x0     | ---p  |                                                      |
| 0x7ffff6d7e000   | 0x7ffff757e000   | 0x800000  | 0x0     | rw-p  |                                                      |
| 0x7ffff757e000   | 0x7ffff757f000   | 0x1000    | 0x0     | ---p  |                                                      |
| 0x7ffff757f000   | 0x7ffff7d82000   | 0x803000  | 0x0     | rw-p  |                                                      |
| 0x7ffff7d82000   | 0x7ffff7daa000   | 0x28000   | 0x0     | r--p  | /usr/lib/x86_64-linux-gnu/libc.so.6                 |
| 0x7ffff7daa000   | 0x7ffff7f3f000   | 0x195000  | 0x28000 | r-xp  | /usr/lib/x86_64-linux-gnu/libc.so.6                 |
| 0x7ffff7f3f000   | 0x7ffff7f97000   | 0x58000   | 0x1bd000| r--p  | /usr/lib/x86_64-linux-gnu/libc.so.6                 |
| 0x7ffff7f97000   | 0x7ffff7f9b000   | 0x4000    | 0x214000| r--p  | /usr/lib/x86_64-linux-gnu/libc.so.6                 |
| 0x7ffff7f9b000   | 0x7ffff7f9d000   | 0x2000    | 0x218000| rw-p  | /usr/lib/x86_64-linux-gnu/libc.so.6                 |
| 0x7ffff7f9d000   | 0x7ffff7faa000   | 0xd000    | 0x0     | rw-p  |                                                      |
| 0x7ffff7fbb000   | 0x7ffff7fbd000   | 0x2000    | 0x0     | rw-p  |                                                      |
| 0x7ffff7fbd000   | 0x7ffff7fc1000   | 0x4000    | 0x0     | r--p  | [vvar]                                               |
| 0x7ffff7fc1000   | 0x7ffff7fc3000   | 0x2000    | 0x0     | r-xp  | [vdso]                                               |
| 0x7ffff7fc3000   | 0x7ffff7fc5000   | 0x2000    | 0x0     | r--p  | /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2      |
| 0x7ffff7fc5000   | 0x7ffff7fef000   | 0x2a000   | 0x2000  | r-xp  | /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2      |
| 0x7ffff7fef000   | 0x7ffff7ffa000   | 0xb000    | 0x2c000 | r--p  | /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2      |
| 0x7ffff7ffb000   | 0x7ffff7ffd000   | 0x2000    | 0x37000 | r--p  | /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2      |
| 0x7ffff7ffd000   | 0x7ffff7fff000   | 0x2000    | 0x39000 | rw-p  | /usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2      |
| 0x7ffffffde000   | 0x7ffffffff000   | 0x21000   | 0x0     | rw-p  | [stack]                                              |
| 0xffffffffff600000| 0xffffffffff601000| 0x1000    | 0x0     | --xp  | [vsyscall]                                          |

> [Reference: 64-bit layout](https://unix.stackexchange.com/questions/509607/how-a-64-bit-process-virtual-address-space-is-divided-in-linux)