# 2021q3 Homework1 (quiz1)
contributed by < `9m77fans` >

###### tags: `linux2021`

## 執行環境

$ uname -r
5.8.0-63-generic

## 題目

### 1. 解釋上述程式碼運作原理，包含 [ftrace][ftrace.txt] 的使用

- 想要了解這個程式原理必須要先有一些觀察,首先可以用`strace`觀察一下`pidof cron`,會發現一堆輸出,看不出跟我們hook的點有什麼關係,思考很久如果只看strace完全看不出來到底是怎麼走訪process的.

mmap(NULL, 2036952, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc78b77f000
mprotect(0x7fc78b7a4000, 1847296, PROT_NONE) = 0
mmap(0x7fc78b7a4000, 1540096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7fc78b7a4000
mmap(0x7fc78b91c000, 303104, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19d000) = 0x7fc78b91c000
mmap(0x7fc78b967000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7fc78b967000
mmap(0x7fc78b96d000, 13528, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc78b96d000
close(3)                                = 0
arch_prctl(ARCH_SET_FS, 0x7fc78b972580) = 0
mprotect(0x7fc78b967000, 12288, PROT_READ) = 0
mprotect(0x561d598ba000, 4096, PROT_READ) = 0
mprotect(0x7fc78b9b9000, 4096, PROT_READ) = 0
munmap(0x7fc78b973000, 98779)           = 0
chdir("/proc")                          = 0
openat(AT_FDCWD, ".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
fstat(3, {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
brk(NULL)                               = 0x561d5a1e2000
brk(0x561d5a203000)                     = 0x561d5a203000
getdents64(3, /* 310 entries */, 32768) = 8088 
openat(AT_FDCWD, "1/stat", O_RDONLY)    = 4  
fstat(4, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(4, "1 (systemd) S 0 1 1 0 -1 4194560"..., 4096) = 193
read(4, "", 3072)                       = 0
close(4)                                = 0
openat(AT_FDCWD, "1/cmdline", O_RDONLY) = 4

- 仔細觀察會發現`openat`跟`getdents64`與proc filesystem有關,這是一個值得懷疑的點,接著我們可以用trace-cmd搭配kernel code來確認流程,這非常關鍵如果沒有ftrace(trace-cmd)就卡關了

sudo trace-cmd record -p function_graph -l iterate_dir -l proc_root_readdir -l proc_root_readdir -l next_tgid -l find_ge_pid -l __x64_sys_getdents64 -l proc_pid_readdir pidof cronq
sudo trace-cmd report|less

`trace-cmd`的結果我可以觀察到getdents64會去走訪所有的process最終會呼叫到"find_ge_pid"因此我們hook它就可以達到隱藏process的目的啦！

CPU 1 is empty
cpus=2
           pidof-16527 [000]  7084.355464: funcgraph_entry:                   |  __x64_sys_getdents64() {
           pidof-16527 [000]  7084.355465: funcgraph_entry:                   |    iterate_dir() {
           pidof-16527 [000]  7084.355465: funcgraph_entry:                   |      proc_root_readdir() {
           pidof-16527 [000]  7084.355476: funcgraph_entry:                   |        proc_pid_readdir() {
           pidof-16527 [000]  7084.355476: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355477: funcgraph_entry:        0.644 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355478: funcgraph_exit:         1.428 us   |          }
           pidof-16527 [000]  7084.355479: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355479: funcgraph_entry:        0.256 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355480: funcgraph_exit:         0.765 us   |          }
           pidof-16527 [000]  7084.355480: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355480: funcgraph_entry:        0.206 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355481: funcgraph_exit:         0.629 us   |          }
           pidof-16527 [000]  7084.355481: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355482: funcgraph_entry:        0.199 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355482: funcgraph_exit:         0.619 us   |          }
           pidof-16527 [000]  7084.355482: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355483: funcgraph_entry:        0.236 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355483: funcgraph_exit:         0.677 us   |          }
           pidof-16527 [000]  7084.355484: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355484: funcgraph_entry:        0.229 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355484: funcgraph_exit:         0.661 us   |          }
           pidof-16527 [000]  7084.355485: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355485: funcgraph_entry:        0.219 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355485: funcgraph_exit:         0.644 us   |          }
           pidof-16527 [000]  7084.355486: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355486: funcgraph_entry:        0.199 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355487: funcgraph_exit:         0.618 us   |          }
           pidof-16527 [000]  7084.355487: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355487: funcgraph_entry:        0.197 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355488: funcgraph_exit:         0.636 us   |          }
           pidof-16527 [000]  7084.355488: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355488: funcgraph_entry:        0.200 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355489: funcgraph_exit:         0.703 us   |          }
           pidof-16527 [000]  7084.355490: funcgraph_entry:                   |          next_tgid() {
           pidof-16527 [000]  7084.355490: funcgraph_entry:        0.199 us   |            find_ge_pid();
           pidof-16527 [000]  7084.355490: funcgraph_exit:         0.611 us   |          }

:::warning
最後我們肯定要追一下程式碼驗證是不是能找到我們的流程
`__x64_sys_getdents64->iterate_dir->proc_root_readdir->proc_pid_readdir->next_tgid->find_ge_pid`
完美的在frace流程搭配程式碼可以看到proc_pid_readdir會走訪process也就是tgid來取得proc下的資訊

:::

int proc_pid_readdir(struct file *file, struct dir_context *ctx)
{
	struct tgid_iter iter;
	struct proc_fs_info *fs_info = proc_sb_info(file_inode(file)->i_sb);
	struct pid_namespace *ns = proc_pid_ns(file_inode(file)->i_sb);
	loff_t pos = ctx->pos;

	if (pos >= PID_MAX_LIMIT + TGID_OFFSET)
		return 0;

	if (pos == TGID_OFFSET - 2) {
		struct inode *inode = d_inode(fs_info->proc_self);
		if (!dir_emit(ctx, "self", 4, inode->i_ino, DT_LNK))
			return 0;
		ctx->pos = pos = pos + 1;
	}
	if (pos == TGID_OFFSET - 1) {
		struct inode *inode = d_inode(fs_info->proc_thread_self);
		if (!dir_emit(ctx, "thread-self", 11, inode->i_ino, DT_LNK))
			return 0;
		ctx->pos = pos = pos + 1;
	}
	iter.tgid = pos - TGID_OFFSET;
	iter.task = NULL;
	for (iter = next_tgid(ns, iter);
	     iter.task;
	     iter.tgid += 1, iter = next_tgid(ns, iter)) {
		char name[10 + 1];
		unsigned int len;

		cond_resched();
		if (!has_pid_permissions(fs_info, iter.task, HIDEPID_INVISIBLE))
			continue;

		len = snprintf(name, sizeof(name), "%u", iter.tgid);
		ctx->pos = iter.tgid + TGID_OFFSET;
		if (!proc_fill_cache(file, ctx, name, len,
				     proc_pid_instantiate, iter.task, NULL)) {
			put_task_struct(iter.task);
			return 0;
		}
	}
	ctx->pos = PID_MAX_LIMIT + TGID_OFFSET;
	return 0;
}

- 現在linux太複雜一定要會用工具並且搭配程式碼才會事半功倍！！
- 至於要簡化linux來觀察又是另一個故事了...

參考這篇文章
https://ops.tips/blog/how-is-proc-able-to-list-pids/

### 2. 本程式僅在 Linux v5.4 測試，若你用的核心較新，請試著找出替代方案 ###
> 2020 年的變更 [Unexporting kallsyms_lookup_name()][kallsyms_lookup_name]
> [Access to kallsyms on Linux 5.7+][kallsyms-mod]

kallsyms_lookup_name不能用的解決方式我是採用參考文章中的搜尋法

https://github.com/xcellerator/linux_kernel_hacking/issues/3

unsigned long kaddr_lookup_name(const char *fname_raw)
{
    int i;
    unsigned long kaddr;
    char *fname_lookup, *fname;

    fname_lookup = kzalloc(NAME_MAX, GFP_KERNEL);
    if (!fname_lookup)
        return 0;

    fname = kzalloc(strlen(fname_raw)+4, GFP_KERNEL);
    if (!fname)
        return 0;

    /*
     * We have to add "+0x0" to the end of our function name
     * because that's the format that sprint_symbol() returns
     * to us. If we don't do this, then our search can stop
     * prematurely and give us the wrong function address!
     */
    strcpy(fname, fname_raw);
    strcat(fname, "+0x0");

    /*
     * Get the kernel base address:
     * sprint_symbol() is less than 0x100000 from the start of the kernel, so
     * we can just AND-out the last 3 bytes from it's address to the the base
     * address.
     * There might be a better symbol-name to use?
     */
    kaddr = (unsigned long) &sprint_symbol;
    kaddr &= 0xffffffffff000000;

    /*
     * All the syscalls (and all interesting kernel functions I've seen so far)
     * are within the first 0x100000 bytes of the base address. However, the kernel
     * functions are all aligned so that the final nibble is 0x0, so we only
     * have to check every 16th address.
     */
    for ( i = 0x0 ; i < 0x100000 ; i++ )
    {
        /*
         * Lookup the name ascribed to the current kernel address
         */
        sprint_symbol(fname_lookup, kaddr);

        /*
         * Compare the looked-up name to the one we want
         */
        if ( strncmp(fname_lookup, fname, strlen(fname)) == 0 )
        {
            /*
             * Clean up and return the found address
             */
            kfree(fname_lookup);
            return kaddr;
        }
        /*
         * Jump 16 addresses to next possible address
         */
        kaddr += 0x10;
    }
    /*
     * We didn't find the name, so clean up and return 0
     */
    kfree(fname_lookup);
    return 0;
}

### 4. 指出程式碼可改進的地方，並動手實作

- 這個module需要正確釋放資源,剛開始測試的時候rmmod直接當機...
1.釋放之前隱藏的process
2.移除之前的hook function
3.char device 資源釋放並移除

    static void _hideproc_exit(void)
    {
        pid_node_t *proc, *tmp_proc;

        /* free pid_node_t allocated form kmalloc */
        list_for_each_entry_safe(proc, tmp_proc,         &hidden_proc, list_node)
        {   
            list_del(&proc->list_node);
            kfree(proc);
        }   
        hook_remove(&hook);
        dev = cdev.dev;
        device_destroy(hideproc_class, dev);
        cdev_del(&cdev);
        class_destroy(hideproc_class);
        unregister_chrdev_region(dev, MINOR_VERSION);

        printk(KERN_INFO "@ %s\n", __func__);
    }