# 奔跑吧 CH 3.1 進程的誕生
## Process and thread
process為OS分配系統資源(memory)的單位，thread為OS分配CPU time的單位
**Process:**
* 一個elf binary檔案在 disk 上叫 user program， load進memory 開始執行後就變成 process
* 有自己的address space =>code section, data section, bss section、stack、heap
* Linux 通常把process當做是**task**，**PCB (processing control block)** 通常也稱為 **struct tast_struct**
**thread:**
* Thread 被稱為輕量級process，它是系統調度最小單位，thread與process差別在於process有獨立的資源空間，而thread則共享process的address
* 共享process的address space => code section, data section
* thread有自己的**program counter, cpu register, stack**
Linux 並沒有特定的data structure來標示thread or process，thread與process都使用process的PCB
<img src="https://i.imgur.com/i3FDa7Y.png" style="height:350px;display: block;margin:auto;">
### Process and task_struct
* Linux用task_struct來描述一個process，透過task_struct->mm->mmap可以找到vm_area_struct，linux用利用VMA(virtual memory area)來描述每個segment特性，大小
<img src="https://i.imgur.com/hWJh8f8.png" style="height:350px;display: block;margin:auto;">
## How kernel execute a binary file
kernel執行第一個user program => init
```clike=
//init/main.c
static int __ref kernel_init(void *unused)
{
....
1148 if (!try_to_run_init_process("/sbin/init") ||
1149 !try_to_run_init_process("/etc/init") ||
1150 !try_to_run_init_process("/bin/init") ||
1151 !try_to_run_init_process("/bin/sh"))
--------------
static int run_init_process(const char *init_filename)
{
argv_init[0] = init_filename;
pr_info("Run %s as init process\n", init_filename);
return do_execve(getname_kernel(init_filename),
(const char __user *const __user *)argv_init,
(const char __user *const __user *)envp_init);
}
```
do_execve -> do_execveat_common -> __do_execve_file
```clike=
static int __do_execve_file(int fd, struct filename *filename,
struct user_arg_ptr argv,
struct user_arg_ptr envp,
int flags, struct file *file)
{
retval = bprm_mm_init(bprm);//alloc mm_struct
if (retval)
....
//search_binary_handler這裡會去呼叫load_elf_binary
//回來之後regs->sepc 再start_thread已經
//設定好user program的entry point
retval = exec_binprm(bprm);
if (retval < 0)
goto out;
```
### mm_struct
```clike=
//fs/exec.c
359 static int bprm_mm_init(struct linux_binprm *bprm)
360 {
......
364 bprm->mm = mm = mm_alloc();
365 err = -ENOMEM;
-------------
//kernel/fork.c
struct mm_struct *mm_alloc(void)
{
struct mm_struct *mm;
mm = allocate_mm();//分配mm_struct
.....
memset(mm, 0, sizeof(*mm));
return mm_init(mm, current, current_user_ns());//current就是task_struct => $tp
}
---------------
// arch/riscv/include/asm/current.h
27 static __always_inline struct task_struct *get_current(void)
28 {
29 register struct task_struct *tp __asm__("tp");
30 return tp;
31 }
32
33 #define current get_current() //task_struct
---------------
static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
struct user_namespace *user_ns)
{
...
mm->mmap = NULL;
mm->mm_rb = RB_ROOT;
mm->vmacache_seqnum = 0;
.....
if (mm_alloc_pgd(mm)) //分配process page directory
---------------
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
......
pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
if (likely(pgd != NULL)) {
memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
/* Copy kernel mappings */ //把kernel page table 給process pgd
memcpy(pgd + USER_PTRS_PER_PGD,
init_mm.pgd + USER_PTRS_PER_PGD,
(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
local_flush_tlb_all();
}
return pgd;
}
-------------
struct mm_struct init_mm = {
.mm_rb = RB_ROOT,
.pgd = swapper_pg_dir, //kernel page table directory
.mm_users = ATOMIC_INIT(2),
.mm_count = ATOMIC_INIT(1),
.mmap_sem = __RWSEM_INITIALIZER(init_mm.mmap_sem),
........
};
```
每個process的kernel page table都會一樣
**task_struct**
* context_switch、schedule ... 都會用到此結構
```clike=
//include/linux/sched.h
struct task_struct {
#ifdef CONFIG_THREAD_INFO_IN_TASK
/*
* For reasons of header soup (see current_thread_info()), this
* must be the first element of task_struct.
*/
struct thread_info thread_info;
#endif
/* -1 unrunnable, 0 runnable, >0 stopped: */
volatile long state;
...
struct mm_struct *mm;
struct mm_struct *active_mm;
...
/* Namespaces: */
struct nsproxy *nsproxy; //container 相關資訊
#ifdef CONFIG_PERF_EVENTS //perf 相關資訊
struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
struct mutex perf_event_mutex;
struct list_head perf_event_list;
#endif
struct sched_entity se; //scheder 相關資訊
#ifdef CONFIG_NUMA_BALANCING //NUMA 相關資訊
int numa_scan_seq;
unsigned int numa_scan_period;
unsigned int numa_scan_period_max;
.....
-----------
(gdb) p/x $tp
$34 = 0xffffffe000171300
(gdb) p/x *((struct task_struct*)0xffffffe000171300)
$37 = {thread_info = {flags = 0x0, preempt_count = 0x0, addr_limit = {se
g = 0x4000000000}, kernel_sp = 0xffffffe03eb91ab0, user_sp = 0xf
fffffe03eb91ab0, cpu = 0x0}, state = 0x0, stack = 0xfff
fffe03eb90000, usage = {refs = {counter = 0x1}}, flags =
0x400000, ptrace = 0x0, wake_entry = {next = 0x0}
----------------
(gdb) p/x *((struct task_struct*)0xffffffe000171300)->mm
$40 = {{mmap = 0xffffffe03afc1790, mm_rb = {rb_node = 0xffffffe03afc1de0
}, vmacache_seqnum = 0x0, get_unmapped
_area = 0xffffffe0014a9270, mmap_base = 0x3ff7bc1000, mm
ap_legacy_base = 0x0, task_size = 0x4000000000, highest_vm_end = 0
x3fffd6a000, pgd = 0xffffffe000010000, membarrier_
state = {counter = 0x0}, mm_users = {counter = 0x1}, mm_c
ount = {counter = 0x1}, pgtables_bytes = {counter = 0x2000}, map
_count = 0x3,
...
```
如果是kernel thread就不會有mm_struct
```clike=
arch_cpu_idle () at arch/riscv/kernel/process.c:31
31 local_irq_enable();
(gdb) p/x $tp
$41 = 0xffffffe0019b1b00
(gdb) p/x *((struct task_struct*)0xffffffe0019b1b00)->mm
$42 = {{mmap = 0x0, mm_rb = {rb_node = 0x0}, vmacache_seqnum = 0x0
(gdb) p/x ((struct task_struct*)0xffffffe0019b1b00)->mm
$43 = 0x0
```
### load_elf_binary
* 以32bit來看，user address sapce有3GB虛擬空間，遠大於實體記憶體空間，Linux kernel 用vma結構來管理process的虛擬空間
```clike=
//fs/binfmt_elf.c
static int load_elf_binary(struct linux_binprm *bprm)
{
struct file *interpreter = NULL; /* to shut gcc up */
....
982 //建立檔案的VMA
983 error = elf_map(bprm->file, load_bias + vaddr, elf_ppnt,
984 elf_prot, elf_flags, total_size);
...
//設定regs->sepc = elf_entry
1154 start_thread(regs, elf_entry, bprm->p);
----------
348 static unsigned long elf_map(struct file *filep, unsigned long addr,
349 const struct elf_phdr *eppnt, int prot, int type,
350 unsigned long total_size)
351 {
352 unsigned long map_addr;
353 unsigned long size = eppnt->p_filesz + ELF_PAGEOFFSET(eppnt->p_vaddr);
354 unsigned long off = eppnt->p_offset - ELF_PAGEOFFSET(eppnt->p_vaddr);
355 addr = ELF_PAGESTART(addr);
356 size = ELF_PAGEALIGN(size);
...
376 } else
377 map_addr = vm_mmap(filep, addr, size, prot, type, off);
378
----------
//elf_map->vm_mmap->vm_mmap_pgoff->do_mmap_pgoff->do_mmap
//建立檔案的 vma (每個process 虛擬區段都需要有對應的vma來管理)
unsigned long do_mmap(struct file *file, unsigned long addr,
unsigned long len, unsigned long prot,
unsigned long flags, vm_flags_t vm_flags,
unsigned long pgoff, unsigned long *populate,
struct list_head *uf)
{
----------
/*在process virtual address 尋找一塊空間 */
addr = get_unmapped_area(file, addr, len, pgoff, flags);
if (offset_in_page(addr))
return addr;
-----------
//alloc 一塊vma給此addr
unsigned long mmap_region(struct file *file, unsigned long addr,
unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
struct list_head *uf)
{
......
vma = vm_area_alloc(mm);
......
//將vma資訊對應到addr
vma->vm_start = addr;
vma->vm_end = addr + len;
vma->vm_flags = vm_flags;
vma->vm_page_prot = vm_get_page_prot(vm_flags);
vma->vm_pgoff = pgoff;
......
```
兩塊LOAD區段分別對應到兩個VMA
**start_thread**
設定spec為elf_entry(程式的進入點)，當返回user space時候就會從此開始執行，如果有interpreter(dynamic loader)，就會從interpreter(dynamic loader)開始執行
```clike=
///arch/riscv/kernel/process.c
void start_thread(struct pt_regs *regs, unsigned long pc,
unsigned long sp)
{
regs->sstatus = SR_SPIE;
.....
regs->sepc = pc;
regs->sp = sp;
set_fs(USER_DS);
}
```
至此只有建立init program的虛擬位址vma與每個section對應關係，尚未建立physical memory，等到cpu開始執行到program的elf_entry時候會產生page fault，這時候才會建立phiscal page的對應
呼叫流程
```clike=
#0 start_thread (regs=0xffffffe03aa47ee0, pc=91625975520, sp=274873105968) at arch/riscv/kernel/process.c:85
#1 0xffffffe000b4d26c in load_elf_binary (bprm=0xffffffe038d59b00) at fs/binfmt_elf.c:1171
#2 0xffffffe000b0dfa0 in search_binary_handler (bprm=0xffffffe03aa47ee0) at fs/exec.c:1653
#3 0xffffffe000b0e7d8 in exec_binprm (bprm=<optimized out>) at fs/exec.c:1695
#4 do_execveat_common (fd=<optimized out>, filename=0x1555556ee0, argv=..., envp=..., flags=<optimized out>) at f
s/exec.c:1817
#5 0xffffffe000b0e94c in do_execve (filename=<optimized out>, __argv=<optimized out>, __envp=<optimized
out>) at fs/exec.c:1862
#6 0xffffffe000a59028 in run_init_process (init_filename=<optimized out>) at init/main.c:1004
#7 0xffffffe000e21248 in kernel_init (unused=<optimized out>) at init/main.c:1074
#8 0xffffffe000a59f22 in handle_exception () at arch/riscv/kernel/entry.S:230
```
```clike=
1706 /*
1707 * sys_execve() executes a new program.
1708 */
1709 static int do_execveat_common(int fd, struct filename *filename,
1710 struct user_arg_ptr argv,
1711 struct user_arg_ptr envp,
1712 int flags)
1713 {
```
## fork
在Linux中，Process或是Thread (kernel_thread也是) 都是通過fork、vfork、clone等系統呼叫來建立的，它們都是透過do_fork來建立
* fork使用了SIGCHLD，在子進程終止後發送SIGCHLD通知父進程
* fork主要為子進程建立一個父進程的副本，為了減少資源採用COW(copy on write)機制，子進程只複製父進程的page table，不複製page內容，當子進程需要寫入時候，才會觸發write copy機制，為子進程建立一個副本
* vfork使用了CLONE_VFORK(父進程會被掛起，直到子進程釋放了虛擬資源)，CLONE_VM(父子進程共享內存空間)
```clike=
//kernel thread
2444 pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
2445 {
2446 struct kernel_clone_args args = {
2447 .flags = ((flags | CLONE_VM | CLONE_UNTRACED) & ~CSIGNAL),
2448 .exit_signal = (flags & CSIGNAL),
2449 .stack = (unsigned long)fn,
2450 .stack_size = (unsigned long)arg,
2451 };
2452
2453 return _do_fork(&args);
-------------
//fork
2457 SYSCALL_DEFINE0(fork)
2458 {
2459 #ifdef CONFIG_MMU
2460 struct kernel_clone_args args = {
2461 .exit_signal = SIGCHLD,
2462 };
2463
2464 return _do_fork(&args);
---------------
//vfork
2473 SYSCALL_DEFINE0(vfork)
2474 {
2475 struct kernel_clone_args args = {
2476 .flags = CLONE_VFORK | CLONE_VM,
2477 .exit_signal = SIGCHLD,
2478 };
2479
2480 return _do_fork(&args);
---------------
//clone
2502 SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
2503 int __user *, parent_tidptr,
2504 int __user *, child_tidptr,
2505 unsigned long, tls)
....
2508 struct kernel_clone_args args = {
2509 .flags = (clone_flags & ~CSIGNAL),
2510 .pidfd = parent_tidptr,
2511 .child_tid = child_tidptr, //child pid
2512 .parent_tid = parent_tidptr,//parent pid
2513 .exit_signal = (clone_flags & CSIGNAL),
2514 .stack = newsp,
2515 .tls = tls,
2516 };
2517
......
2521 return _do_fork(&args);
```
```clojure=
22 /*
23 * This only works because "struct thread_info" is at offset 0 from "struct
24 * task_struct". This constraint seems to be necessary on other architectures
25 * as well, but __switch_to enforces it. We can't check TASK_TI here because
26 * <asm/asm-offsets.h> includes this, and I can't get the definition of "struct
27 * task_struct" here due to some header ordering problems.
28 */
29 static __always_inline struct task_struct *get_current(void)
30 {
31 return riscv_current_is_tp;
32 }
33
34 #define current get_current()
```
```clojure
commit cad6967ac10843a70842cd39c7b53412901dd21f
Author: Christian Brauner <brauner@kernel.org>
Date: Wed Aug 19 12:46:45 2020 +0200
fork: introduce kernel_clone()
The old _do_fork() helper doesn't follow naming conventions of in-kernel
helpers for syscalls. The process creation cleanup in [1] didn't change the
name to something more reasonable mainly because _do_fork() was used in quite a
few places. So sending this as a separate series seemed the better strategy.
This commit does two things:
1. renames _do_fork() to kernel_clone() but keeps _do_fork() as a simple static
inline wrapper around kernel_clone().
2. Changes the return type from long to pid_t. This aligns kernel_thread() and
kernel_clone(). Also, the return value from kernel_clone that is surfaced in
fork(), vfork(), clone(), and clone3() is taken from pid_vrn() which returns
a pid_t too.
```
```clike=
//include/uapi/linux/sched.h
//父子進程共享內存空間
#define CLONE_VM 0x00000100 /* set if VM shared between processes */
//父子進程共享相同的file system
#define CLONE_FS 0x00000200 /* set if fs info shared between processes */
//父子進程共享相同的file descriptor
#define CLONE_FILES 0x00000400 /* set if open files shared between processes */
//父子進程共享相同的signal資訊
#define CLONE_SIGHAND 0x00000800 /* set if signal handlers and blocked signals shared */
////父進程被trace，子進程也被trace
#define CLONE_PTRACE 0x00002000 /* set if we want to let tracing continue on the child too */
//父進程被掛起，直到子進程釋放了虛擬資源
#define CLONE_VFORK 0x00004000 /* set if the parent wants the child to wake it up on mm_release */
```
### do_fork
```clike=
2339 long _do_fork(struct kernel_clone_args *args)
2340 {
2......
1771 struct task_struct *p;
2366 p = copy_process(NULL, trace, NUMA_NO_NODE, args);
2367 add_latent_entropy();
----------
//做memory、file system、signal、I/O等內容的copy
1764 static __latent_entropy struct task_struct *copy_process(
1765 struct pid *pid,
1766 int trace,
1767 int node,
1768 struct kernel_clone_args *args)
1769 {
...
1850 retval = -ENOMEM;
//把原本父進程的task_struct內容複製給子進程的task_strcut p
1851 p = dup_task_struct(current, node);
1852 if (!p)
...
1981 /* Perform scheduler related setup. Assign this task to a CPU. */
1982 retval = sched_fork(clone_flags, p);
1983 if (retval)
...
2012 retval = copy_mm(clone_flags, p);//copy 父進程 mm
2013 if (retval)
2000 retval = copy_files(clone_flags, p);
..
2003 retval = copy_fs(clone_flags, p);
.
2009 retval = copy_signal(clone_flags, p);
.
2015 retval = copy_namespaces(clone_flags, p);
...
2018 retval = copy_io(clone_flags, p);
---------
2839 int sched_fork(unsigned long clone_flags, struct task_struct *p)
2840 {
2843 __sched_fork(clone_flags, p);
2844 /*
2845 * We mark the process as NEW here. This guarantees that
2846 * nobody will actually run it, and a signal or other external
2847 * event cannot wake it up and insert it on the runqueue either.
2848 */
2849 p->state = TASK_NEW;
....
2897 * We're setting the CPU for the first time, we don't migrate,
2898 * so use __set_task_cpu().
2899 */ //把task assign 給一個cpu
_cpu(p, smp_processor_id());
-------------------
//copy_mm->dup_mm()
1346 static struct mm_struct *dup_mm(struct task_struct *tsk,
1347 struct mm_struct *oldmm)
1348 {
.....
1352 mm = allocate_mm();
.....
1358 if (!mm_init(mm, tsk, mm->user_ns))//呼叫mm_alloc_pgd 有kernel page table
1356 memcpy(mm, oldmm, sizeof(*mm));
.....
1361 err = dup_mmap(mm, oldmm);//old_mm為父進程
------------------
// 主要是走訪父進程所有的VMA，然後複製父進程VMA對應的pte entry到子進程MA對應的pte entry
478 static __latent_entropy int dup_mmap(struct mm_struct *mm,
479 struct mm_struct *oldmm)
480 {
.....
492 flush_cache_dup_mm(oldmm);
493 uprobe_dup_mmap(oldmm, mm);
601 retval = copy_page_range(mm, oldmm, mpnt);
602
-----------------
937 int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
938 struct vm_area_struct *vma)
939 {
.....
986 dst_pgd = pgd_offset(dst_mm, addr);
987 src_pgd = pgd_offset(src_mm, addr);
988 do {
989 next = pgd_addr_end(addr, end);
990 if (pgd_none_or_clear_bad(src_pgd))
991 continue;
992 if (unlikely(copy_p4d_range(dst_mm, src_mm, dst_pgd, src_pgd,
993 vma, addr, next))) {
994 ret = -ENOMEM;
995 break;
996 }
997 } while (dst_pgd++, src_pgd++, addr = next, addr != end);
998
999 if (is_cow)
1000 mmu_notifier_invalidate_range_end(&range);
-----------------
copy_p4d_range->copy_pud_range->copy_pmd_range->copy_pte_range->copy_one_pte
671 /*
672 * copy one vm_area from one task to the other. Assumes the page tables
673 * already present in the new task to be cleared in the whole range
674 * covered by this vma.
675 */
676
677 static inline unsigned long
678 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
679 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
680 unsigned long addr, int *rss)
681 {
...
780 set_pte_at(dst_mm, addr, dst_pte, pte);//把pte設給dst_pte
----------------
static inline void set_pte(pte_t *ptep, pte_t pteval)
{
*ptep = pteval;
}
static inline void set_pte_at(struct mm_struct *mm,
unsigned long addr, pte_t *ptep, pte_t pteval)
{
if (pte_present(pteval) && pte_exec(pteval))
flush_icache_pte(pteval);
set_pte(ptep, pteval);
}
```
當copy_process完成後就成功創建一個子進程
### 3.1問題
Kernel 如何獲取當前process的task_struct ?
User space process page table什麼時候被建立?
fork, vfork, clone有什麼差別?
**Ref:**
[linux进程地址空间(3) 内存映射(1)mmap与do_mmap](https://blog.csdn.net/u010246947/article/details/10472587?utm_medium=distribute.pc_relevant.none-task-blog-title-2&spm=1001.2101.3001.4242)
[how-the-kernel-manages-your-memory](https://manybutfinite.com/post/how-the-kernel-manages-your-memory/)
[Copy On Write機制瞭解一下](https://www.itread01.com/content/1544029393.html)
Sign in with Wallet
Connect another wallet