---
# System prepended metadata

title: Linux Project 1
tags: [Linux]

---

# Linux Project 1 Write up
### 組別資訊 - 第41組
112522116 洪琪懿
112522103 顏嘉鈺
110502565 歐政修
## Kernel 與 OS 版本
* kernel: 5.15.137
* Ubuntu 22.04.3 LTS

## system call
```cpp=
// my_get_physical_addresses.c
#include<linux/kernel.h>
#include<linux/syscalls.h>
#include<linux/sched.h>
#include<asm/pgtable.h>

SYSCALL_DEFINE1 (my_get_physical_address, unsigned long, vaddr){
    pgd_t *pgd;
    p4d_t *p4d;
    pud_t *pud;
    pmd_t *pmd;
    pte_t *pte;


    unsigned long paddr=0;
    unsigned long page_addr=0;     /* from pte*/
    unsigned long page_offset=0;   /* from virtual addr*/

    /* linuxLecture_3_9-4*/
    pgd=pgd_offset(current->mm,vaddr);  /*獲得addr對應的pgd的地址*/
    if(pgd_none(*pgd) || unlikely(pgd_bad(*pgd))){
        return 0;
    }

    p4d=p4d_offset(pgd,vaddr);  /*獲得addr對應的p4d的地址*/
    if(p4d_none(*p4d) || unlikely(p4d_bad(*p4d))){
        return 0;
    }

    pud=pud_offset(p4d,vaddr);
    if(pud_none(*pud) || unlikely(pud_bad(*pud))){
        return 0;
    }

    pmd=pmd_offset(pud,vaddr);
    if(pmd_none(*pmd) || unlikely(pmd_bad(*pmd))){
        return 0;
    }

    pte=pte_offset_kernel(pmd,vaddr);
    if(pte_none(*pte)){
        return 0;
    }
   
    /* PAGE_MASK屏蔽掉offset區域的值*/
    /* PFN: Page Frame Number
     * Extracts the PFN from a (pte|pmd|pud|pgd)val_t of a 4KB page */
    page_addr=pte_val(*pte) & PAGE_MASK & PTE_PFN_MASK;
    page_offset= vaddr & ~PAGE_MASK;
    paddr=page_addr | page_offset;
    printk("page_addr=0x%lx,page_offset=0x%lx\n",page_addr,page_offset);
    printk("vaddr=%lx,paddr=%lx\n",vaddr,paddr);

    return paddr;
}

```

## 編譯kernel過程
### 建立虛擬機
用VMware建立VM時，預設的核心數是2個
建議設大一點後面編譯kernel才不會當掉
![image](https://hackmd.io/_uploads/Hy7F864Vp.png)
能設多大就多大，不要超過host的核心數就好
![image](https://hackmd.io/_uploads/ByfCLTNVT.png)

### 安裝相關套件
下載 Kernel Source
```python
# 進入 root 模式

sudo su

# 用這個指令來下載kernel

wget -P ~/ https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.15.137.tar.xz
```
解壓縮
```python
# 切換到壓縮檔所在目錄

cd /root

# 把檔案解壓縮到 /usr/src 目錄底下

tar -xvf linux-5.15.137.tar.xz -C /usr/src
```
更新apt
```python
sudo apt update && sudo apt upgrade -y
```
先把會用到的套件安裝好
```python
# 這個套件可以幫我們 build 出 kernel-pakage

sudo apt install build-essential libncurses-dev libssl-dev libelf-dev bison flex -y

# 這個套件可以避免後面編譯kernel的bug

sudo apt install dwarves

# 安裝vim

sudo apt install vim -y
```
### 編譯前的準備
```python
# 把目錄轉到剛剛解壓縮完的 kernel 檔案夾
cd /usr/src/linux-5.15.137

# 在裡面創建一個名叫 my_get_physical_addresses 的資料夾

mkdir my_get_physical_addresses

# 把目錄轉到 my_get_physical_addresses 資料夾

cd my_get_physical_addresses

# 建立 my_get_physical_addresses.c

vim my_get_physical_addresses.c

# 填上 my_get_physical_addresses.c 內容

# 於同一個目錄下再建立一個 Makefile

vim Makefile

# Makefile 的內容

obj-y := my_get_physical_addresses.o

# 接著回上個目錄改 Makefile

cd ..

# 打開此目錄下的 Makefile

vim Makefile

# 在後面補上 my_get_physical_addresses 這樣 kernel 在編譯時才會到 my_get_physical_addresses 目錄

core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ my_get_physical_addresses/

# 接下來要去修改 syscall_64.tbl 檔

vim arch/x86/entry/syscalls/syscall_64.tbl

# 在最後一行添加上 system call，然後請把編號記住, 等一下會用

449    common    my_get_physical_addresses    sys_my_get_physical_addresses

# 把目錄轉回去
cd /usr/src/linux-5.15.137

# 編輯 syscalls.h 檔，將 syscall 的原型添加進檔案 (#endif之前)
    
vim include/linux/syscalls.h

# 加入一行(這行直接加在檔案的最下面就好)

asmlinkage long sys_my_get_physical_addresses(unsigned long);
```
### 開始編譯
```python=
#全部按enter跳過詢問
make localmodconfig

# 先設定/usr/src/linux-5.15.137目錄下的.config
vim .config
```
修改`CONFIG_SYSTEM_TRUSTED_KEYS`和`CONFIG_SYSTEM_REVOCATION_KEYS`讓他們的值=""
![image](https://hackmd.io/_uploads/rklJjaV4T.png)
![image](https://hackmd.io/_uploads/ryeloT4VT.png)
```python
# 查看機器幾核心
nproc

# 看自己的核心數，要取1/2 ~ 3/4的核心數
make -j6

sudo make modules_install -j6

sudo make install -j6
```

先看 kernel 的版本
```
uname -r
```
如果版本比5.15.137高，要設定grub的開機引導介面
```
sudo su

cd /root

vim /etc/default/grub
```
把`GRUB_TIMEOUT_STYLE`從`hidden`改成`menu`
再把`GRUB_TIMEOUT`從`0`改成`-1`

最後更新一下再重啟
```python
sudo update-grub

reboot
```

選擇Advanced options for Ubuntu
![image](https://hackmd.io/_uploads/SJ9s2aEVa.png)
選擇我們的版本5.15.137
![image](https://hackmd.io/_uploads/BJOJTpEVT.png)
開機後確認版本，應該會是5.15.137

## user code
```c=
#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include <string.h>
#include <sys/syscall.h>      /* Definition of SYS_* constants */
#include <unistd.h>

#define get_call 449

extern void *func1(void *);
extern void *func2(void *);
extern void *func3(void *);
extern int main();

struct data_{
  int  id ;
  char name[16] ;
};

typedef struct data_ sdata ;
static __thread  sdata  tx ;  //thread local variable

int a=123;  //global variable
int c;

void hello(int pid){
   int b=10;   //local varialbe
   b=b+pid;    //global variable
   char*str=(char*)malloc(60*sizeof(char));
   strcpy(str, "heap_str");
   printf("In thread %d \nthe value of global variable a is %d, the offset of the logical address of  a is %p, ", pid, a, &a);
   printf("the physical address of global variable a is %p\n", (void *)syscall(get_call,&a));
                //local variable
   printf("the value of local varialbe b is %d, the offset of the logical address of b is %p, ", b, &b);
   printf("the physical address of local variable b is %p\n", (void *)syscall(get_call,&b));
                 //thread local variable
   printf("the value of uninitialized data c (bss) is %d, the offset of the logical address of c is %p, ", c, &c);
   printf("the physical address of uninitialized data c (bss) is %p\n", (void *)syscall(get_call,&c));
                 //bss segments
   printf("the offset of the logical address of str (heap) is %p\n",str);
   printf("the physical address of str (heap) is %p\n",(void *)syscall(get_call,str));
                 //heap
   printf("the offset of the logical address of thread local varialbe tx is %p, ", &tx);
   printf("the physical address of thread local variable tx is %p\n", (void *)syscall(get_call,&tx));
                 //function
   printf("the offset of the logical address of function hello is %p, ", hello);
   printf("the physical address of function hello is %p\n", (void *)syscall(get_call,hello));

   printf("the offset of the logical address of function func1 is %p, ", func1);
   printf("the physical address of function func1 is %p\n", (void *)syscall(get_call,func1));

   printf("the offset of the logical address of function func2 is %p, ", func2);
   printf("the physical address of function func2 is %p\n", (void *)syscall(get_call,func2));

   printf("the offset of the logical address of function func3 is %p, ", func3);
   printf("the physical address of function func3 is %p\n", (void *)syscall(get_call,func3));

   printf("the offset of the logical address of function main is %p, ", main);
   printf("the physical address of function main is %p\n", (void *)syscall(get_call,main));
                 //library function
   printf("the offset of the logical address of library function printf is %p, ", printf);
   printf("the physical address of library function printf is %p\n",(void *)syscall(get_call,printf));
   printf("====================================================================================================================\n");
   free(str);
}  

void *func1(void *arg){
   char *p = (char*) arg ;
   int pid  ;
   pid =  syscall( __NR_gettid );
   tx.id = pid ;
   strcpy(tx.name,p) ;
   printf("I am thread with ID %d executing func1().\n",pid);
   hello(pid);

   while(1){
     printf("(%d)(%s)\n",tx.id,tx.name) ;
     sleep(2) ;
   }
}

void *func2(void *arg){
  char *p = (char*) arg ;
  int pid  ;
  pid =  syscall( __NR_gettid );
  tx.id = pid ;
  strcpy(tx.name,p) ;
  printf("I am thread with ID %d executing func2().\n",pid);
  hello(pid);

  while(1){
     printf("(%d)(%s)\n",tx.id,tx.name) ;
     sleep(2) ;
  }
}

void *func3(void *arg){
  char *p = (char*) arg ;
  int pid  ;
  pid =  syscall( __NR_gettid );
  tx.id = pid ;
  strcpy(tx.name,p) ;
  printf("I am thread with ID %d executing func3().\n",pid);
  hello(pid);
  while(1){
     //printf("(%d)(%s)\n",tx.id,tx.name) ;
     sleep(2) ;
  }
}
int main(){ 
  pthread_t id[3];
  char p[3][16] ;
  strcpy(p[0],"Thread1") ;
  pthread_create(&id[0],NULL,func1,(void *)p[0]);
  sleep(1);

  strcpy(p[1],"Thread2") ;
  pthread_create(&id[1],NULL,func2,(void *)p[1] );
  sleep(1);

  strcpy(p[2],"Thread3") ;
  pthread_create(&id[2],NULL,func3,(void *)p[2] );
  sleep(1);

  int pid  ;
  pid =  syscall( __NR_gettid );
  tx.id = pid ;
  strcpy(tx.name,"MAIN") ;
  printf("I am main thread with ID %d.\n", pid);
  hello(pid);
  sleep(1);
  while(1){
    printf("(%d)(%s)\n",tx.id,tx.name) ;
    sleep(50) ;
  }

}
```
## 執行結果&圖示
### 截圖

![messageImage_1701073645445-3](https://hackmd.io/_uploads/HJz1NCZBa.jpg)

### virtual address

Thread 1

![image](https://hackmd.io/_uploads/SyTSGZMr6.png)

Thread2

![image](https://hackmd.io/_uploads/ryS_zbGBT.png)

Thread3

![image](https://hackmd.io/_uploads/S1eKfWzBa.png)

### physical address mapping

local

![image](https://hackmd.io/_uploads/rk3dNmGSa.png)
shared

![image](https://hackmd.io/_uploads/rkDFNXzBa.png)


## 遇到的問題與解法
### 編譯kernel: make -j* 常見報錯
**只要有出現sed: No such file or directory**就代表make失敗
* 確認有無安裝dwarves
* 去看自定義的system call有沒有問題

### kernel code: pud_offset
`pud_offset`問題的提示訊息說輸入的data type應為`p4d_t *` 而非`pgd_t *`
推測此處是64位元的改動，教材上的是32位元
![image](https://hackmd.io/_uploads/S12PN7oEa.png)
`pud_offset`定義在`<asm/pgtable.h>`
![image](https://hackmd.io/_uploads/B1pk77oE6.png)

![image](https://hackmd.io/_uploads/Skleeh4S6.png)

會用到p4d是因為[paging有5層而非4層](https://blog.csdn.net/pwl999/article/details/109453180)
一樣在`<asm/pgtable.h>`可以找到`p4d_offset`的定義
![image](https://hackmd.io/_uploads/rymf8Xs46.png)

將source code補上p4d的部分就好

### kernel code: copy_from_user問題
寫測試的user space code印出的資訊，global variable跟local variable位置一樣，值也很奇怪，很明顯是錯的
![image](https://hackmd.io/_uploads/HyXxkSh4a.png)
用dmesg印不出printk


Call Trace
![image](https://hackmd.io/_uploads/ByFnOFpET.png)
結果好像是因為要allocate一個memory給讀進來的user space data - [CL.](https://stackoverflow.com/questions/24848253/copy-from-user-gives-null-pointer)
[kmalloc用法](https://blog.csdn.net/lu_embedded/article/details/51588902)

```cpp=
// 僅顯示部分程式碼
SYSCALL_DEFINE1(my_get_physical_addresses, void *, laddr){
    // 先配置空間
    unsigned long buf = NULL;
    buf = kmalloc(sizeof(unsigned long), GFP_KERNEL);
    
    long copied = copy_from_user(buf, laddr, sizeof(buf));
```

### kernel code: physical addr output null


![image](https://hackmd.io/_uploads/BJKwG5T4p.png)

![image](https://hackmd.io/_uploads/Sk-ZYy0N6.png)
copy_from_user return 0，代表是成功的
但我的source code判定寫反，重新修正就可以了

### kernel code: global variable's physical addr not consistent


![image](https://hackmd.io/_uploads/rkwlWdZS6.png)

![image](https://hackmd.io/_uploads/HJH7-uWH6.png)



用`copy_from_user`時遇到了很多data type的問題。copy前後的virtual address也不一致。
因此重新審視了[copy_from_user的必要性](https://stackoverflow.com/questions/29397364/copy-to-user-and-copy-from-user-for-basic-data-type)
如果傳入的只是單純的data而不是pointer就可以不用用到`copy_from_user`

將 system call 的 source code 參數改成`unsigned long`
```cpp=
// 僅顯示部分程式碼
SYSCALL_DEFINE1(my_get_physical_addresses, unsigned long, laddr){
    // 不需要copy_from_user了
```

`include/linux/syscalls.h`的最後一行也要把參數類別改成`unsigned long`
```
asmlinkage long sys_my_get_physical_addresses(unsigned long);
```
### kernel code: 新增判斷式後無法編譯
新增的判斷式如下
```
if(pgd_none(*pgd) || unlikely(pgd_bad(*pgd))){
        return 0;
    }
```
* `pxx_none` returns 1 if the corresponding entry does not exist
* [unlikely(參數)](https://hackmd.io/@rickywu0421/unlikely): 一個優化的程式碼，通常認為參數為0，不會進入if的body。增加branch hit機率。
* [pgd_bad](https://hackmd.io/@linD026/Linux-kernel-COW-Copy-on-Write): 
    * making sure the page entry is marked as present and accessed are the two most important checks
![image](https://hackmd.io/_uploads/S1Y2kwBr6.png)


make時的錯誤訊息如下
![image](https://hackmd.io/_uploads/HkVi9PZBp.png)
make時它去找 `__x64_sys_my_get_physical_addresses` 這個system call卻找不到，因為我們是用`sys_my_get_physical_addresses`而非 __x64 開頭。

推測是新增的程式碼有用到x64的功能

如果要正常運行必須把`arch/x86/entry/syscalls/syscall_64.tbl`改成：
```
449    64    my_get_physical_addresses    __x64_sys_my_get_physical_addresses
```

而`include/linux/syscalls.h`估計也要改為
```
asmlinkage long __x64_sys_my_get_physical_addresses(unsigned long);
```

實際上改完這兩個檔案就可以make了