Linux Kernel Project 1

OS: Ubuntu 22.04
ARCH: X86_64
Source Version: 6.6

Build Linux Kernel

# download the kernel code
wget https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.6.tar.xz
tar -xvf linux-6.6.tar.xz
cd linux-6.6
# Install required dependencies
sudo apt update && sudo apt install make gcc libncurses-dev flex bison
make allnoconfig

# Initialize kernel config
make menuconfig
64-bit kernel -> Enable
Executable file formats -> Enable all
Device Drivers > Character devices > Serial drivers and 8250/16550 and compatible serial support -> Enable
Device Drivers > Character devices > Console on 8250/16550 and compatible serial port -> Enable
General Setup > Initial RAM filesystem and RAM disk (initramfs/initrd) support -> Enable
Process type and features -> Linux guest support -> Support for running PVH guests -> Enable

# Install required dependencies
sudo apt install libssl-dev libelf-dev
make -j <num_cpu>

Build Root FS

wget https://busybox.net/downloads/busybox-1.36.1.tar.bz2
tar -xf busybox-1.36.1.tar.bz2
cd busybox-1.36.1
make menuconfig # Select build static binary
make install
cd _install
mkdir -p lib lib64 proc sys etc etc/init.d
cat > ./etc/init.d/rcS << EOF
#!/bin/sh
 # Mount the /proc and /sys filesystems
mount -t proc none /proc
mount -t sysfs none /sys
 # Populate /dev
/sbin/mdev -s
EOF

chmod +x etc/init.d/rcS
find . | cpio -o --format=newc | gzip > ../../linux-6.6/rootfs.img.gz

Run Kernel

sudo apt install qemu qemu-kvm
qemu-system-x86_64 -kernel vmlinux -nographic -initrd rootfs.img.gz -append "root=/dev/ram rdinit=/sbin/init console=ttyS0"

Add System Call

arch/x86/entry/syscall_64.tbl

Add line after 377 line(453 64 map_shadow_stack sys_map_shadow_stack)

454 common  my_get_physical_addresses   sys_my_get_physical_addresses   

這個檔案會在 compile 階段被讀取後轉為 header file(arch/x86/include/generated/asm/syscalls_64.h)

include/linux/syscalls.h

Add line after 942(asmlinkage long sys_map_shadow_stack(unsigned long addr, unsigned long size, unsigned int flags);)

asmlinkage long sys_my_get_physical_addresses(void *);

Create kernel/project1.c and add it to makefile

#include <linux/syscalls.h>
// #define DEBUG

SYSCALL_DEFINE1(my_get_physical_addresses, void *, addr_p)
{
	unsigned long vaddr = (unsigned long)addr_p;
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
	unsigned long paddr = 0;
	unsigned long page_addr = 0;
	unsigned long page_offset = 0;

	pgd = pgd_offset(current->mm, vaddr);
#ifdef DEBUG
	printk("pgd_val = 0x%lx\n", pgd_val(*pgd));
	printk("pgd_index = %lu\n", pgd_index(vaddr));
#endif
	if (pgd_none(*pgd)) {
		printk("not mapped in pgd\n");
		return 0;
	}

	p4d = p4d_offset(pgd, vaddr);
#ifdef DEBUG
	printk("p4d_val = 0x%lx\n", p4d_val(*p4d));
	printk("p4d_index = %lu\n", p4d_index(vaddr));
#endif
	if (p4d_none(*p4d)) {
		printk("not mapped in p4d\n");
		return 0;
	}

	pud = pud_offset(p4d, vaddr);
#ifdef DEBUG
	printk("pud_val = 0x%lx\n", pud_val(*pud));
	printk("pud_index = %lu\n", pud_index(vaddr));
#endif
	if (pud_none(*pud)) {
		printk("not mapped in pud\n");
		return 0;
	}

	pmd = pmd_offset(pud, vaddr);
#ifdef DEBUG
	printk("pmd_val = 0x%lx\n", pmd_val(*pmd));
	printk("pmd_index = %lu\n", pmd_index(vaddr));
#endif
	if (pmd_none(*pmd)) {
		printk("not mapped in pmd\n");
		return 0;
	}

	pte = pte_offset_kernel(pmd, vaddr);
#ifdef DEBUG
	printk("pte_val = 0x%lx\n", pte_val(*pte));
	printk("pte_index = %lu\n", pte_index(vaddr));
#endif
	if (pte_none(*pte)) {
		printk("not mapped in pte\n");
		return 0;
	}

	/* Page frame physical address mechanism | offset */
	page_addr = pte_val(*pte) & PAGE_MASK;
	page_offset = vaddr & ~PAGE_MASK;
	paddr = page_addr | page_offset;
#ifdef DEBUG
	printk("page_addr = %lx, page_offset = %lx\n", page_addr, page_offset);
	printk("vaddr = %lx, paddr = %lx\n", vaddr, paddr);
#endif

	return paddr;
}

virt_to_phys 只能轉 kernel space 的 virtual address，因此必須從頭用 page table 查找

kernel/Makefile

obj-y     = fork.o exec_domain.o panic.o \
	    cpu.o exit.o softirq.o resource.o \
	    sysctl.o capability.o ptrace.o user.o \
	    signal.o sys.o umh.o workqueue.o pid.o task_work.o \
	    extable.o params.o \
	    kthread.o sys_ni.o nsproxy.o \
	    notifier.o ksysfs.o cred.o reboot.o \
	    async.o range.o smpboot.o ucount.o regset.o ksyms_common.o \
		project1.o

Build Test Binary

Add project1.c to anywhere

#include <stdio.h>
#include <pthread.h>
#include <string.h>
#include <sys/syscall.h> /* Definition of SYS_* constants */
#include <unistd.h>
#include <stdlib.h>

extern void *func1(void *);
extern void *func2(void *);
extern int main();

void *my_get_physical_addresses(void *vaddr)
{
  return syscall(454, vaddr);
}

struct data_
{
  int id;
  char name[16];
};
typedef struct data_ sdata;
static __thread sdata tx; // thread local variable

int a = 123; // global variable
int *c;      // heap variable

void hello(int pid)
{
  int b = 10; // local variable

  b = b + pid;
  // global variable
  printf("In thread %d \nthe value of global variable a is %d, the offset of the logical address of a is %p, ", pid, a, &a);
  printf("the physical address of global variable a is %p\n", my_get_physical_addresses(&a));
  // local variable
  printf("the value of local variable b is %d, the offset of the logical address of b is %p, ", b, &b);
  printf("the physical address of local variable b is %p\n", my_get_physical_addresses(&b));
  // heap variable
  printf("the value of heap variable c is %d, the offset of the logical address of c is %p, ", *c, c);
  printf("the physical address of heap variable c is %p\n", my_get_physical_addresses(c));
  // thread local variable
  printf("the offset of the logical address of thread local variable tx is %p, ", &tx);
  printf("the physical address of thread local variable tx is %p\n", my_get_physical_addresses(&tx));
  // function
  printf("the offset of the logical address of function hello is %p, ", hello);
  printf("the physical address of function hello is %p\n", my_get_physical_addresses(hello));
  printf("the offset of the logical address of function func1 is %p, ", func1);
  printf("the physical address of function func1 is %p\n", my_get_physical_addresses(func1));
  printf("the offset of the logical address of function func2 is %p, ", func2);
  printf("the physical address of function func2 is %p\n", my_get_physical_addresses(func2));
  printf("the offset of the logical address of function main is %p, ", main);
  printf("the physical address of function main is %p\n", my_get_physical_addresses(main));
  // library function
  printf("the offset of the logical address of library function printf is %p, ", printf);
  printf("the physical address of library function printf is %p\n", my_get_physical_addresses(printf));
  printf("====================================================================================================================\n");
}

void *func1(void *arg)
{
  char *p = (char *)arg;
  int pid;
  pid = syscall(__NR_gettid);
  tx.id = pid;
  strcpy(tx.name, p);
  printf("I am thread with ID %d executing func1().\n", pid);
  hello(pid);
  /*
  while (1)
  {
    // printf("(%d)(%s)\n",tx.id,tx.name) ;
    sleep(1);
  }
  */
}

void *func2(void *arg)
{
  char *p = (char *)arg;
  int pid;
  pid = syscall(__NR_gettid);
  tx.id = pid;
  strcpy(tx.name, p);
  printf("I am thread with ID %d executing func2().\n", pid);
  hello(pid);
  /*
  while (1)
  {
    // printf("(%d)(%s)\n",tx.id,tx.name) ;
    sleep(2);
  }
  */
}
int main()
{
  pthread_t id[2];
  char p[2][16];
  c = (int *)malloc(sizeof(int));
  *c = 456;
  strcpy(p[0], "Thread1");
  pthread_create(&id[0], NULL, func1, (void *)p[0]);
  strcpy(p[1], "Thread2");
  pthread_create(&id[1], NULL, func2, (void *)p[1]);

  int pid;
  pid = syscall(__NR_gettid);
  tx.id = pid;
  strcpy(tx.name, "MAIN");
  printf("I am main thread with ID %d.\n", pid);
  hello(pid);
  pthread_join(id[0], NULL);
  pthread_join(id[1], NULL);
  free(c);
  /*
  while (1)
  {
    // printf("(%d)(%s)\n",tx.id,tx.name) ;
    sleep(5);
  }
  */
}

gcc -static project1.c -o project1

這個檔案 build 完要將其加入 rootfs 中才能被 guest 執行

References

Linux kernel on QEMU
Minimal kernel on QEMU
embed linux bootstrap
initrd
Page table
Get physical memory