# Reading [The Linux Kernel Module Programming Guide](https://sysprog21.github.io/lkmpg/) (11)
## Modify QEMU RISCV virt machine to run bh_thread.c

前一篇提到直接改dts重新編譯dtb是不夠的，所以要先去clone QEMU下來並compile(有很多東西要裝，這邊去問chatgpt就知道要怎麼做了)

git clone https://github.com/qemu/qemu.git
cd qemu
mkdir build
cd build
../configure --target-list=riscv64-softmmu
make install
make -j$(nproc)

要編譯蠻久的可能要等一下，之後要重新編譯就在build路徑下

make clean
make -j$(nproc)

首先要改的地方就是`qemu/include/hw/riscv/virt.h`，裡面有關GPIO的部份都是新加的

/*
 * QEMU RISC-V VirtIO machine interface
 *
 * Copyright (c) 2017 SiFive, Inc.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2 or later, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef HW_RISCV_VIRT_H
#define HW_RISCV_VIRT_H

#include "hw/boards.h"
#include "hw/riscv/riscv_hart.h"
#include "hw/sysbus.h"
#include "hw/block/flash.h"
#include "hw/intc/riscv_imsic.h"
#include "hw/gpio/sifive_gpio.h" // for add gpio chip
#include "hw/misc/gpio-timer.h"  // for trigger button interrupt

#define VIRT_CPUS_MAX_BITS 9
#define VIRT_CPUS_MAX (1 << VIRT_CPUS_MAX_BITS)
#define VIRT_SOCKETS_MAX_BITS 2
#define VIRT_SOCKETS_MAX (1 << VIRT_SOCKETS_MAX_BITS)

#define TYPE_RISCV_VIRT_MACHINE MACHINE_TYPE_NAME("virt")
typedef struct RISCVVirtState RISCVVirtState;
DECLARE_INSTANCE_CHECKER(RISCVVirtState, RISCV_VIRT_MACHINE,
                         TYPE_RISCV_VIRT_MACHINE)

typedef enum RISCVVirtAIAType
{
    VIRT_AIA_TYPE_NONE = 0,
    VIRT_AIA_TYPE_APLIC,
    VIRT_AIA_TYPE_APLIC_IMSIC,
} RISCVVirtAIAType;

struct RISCVVirtState
{
    /*< private >*/
    MachineState parent;

    /*< public >*/
    Notifier machine_done;
    DeviceState *platform_bus_dev;
    RISCVHartArrayState soc[VIRT_SOCKETS_MAX];
    DeviceState *irqchip[VIRT_SOCKETS_MAX];
    PFlashCFI01 *flash[2];
    FWCfgState *fw_cfg;

    SIFIVEGPIOState gpio;      // for add gpio chip
    GpioTimerState gpio_timer; // for trigger button interrupt
    int fdt_size;
    bool have_aclint;
    RISCVVirtAIAType aia_type;
    int aia_guests;
    char *oem_id;
    char *oem_table_id;
    OnOffAuto acpi;
    const MemMapEntry *memmap;
    struct GPEXHost *gpex_host;
    OnOffAuto iommu_sys;
    uint16_t pci_iommu_bdf;
};

enum
{
    VIRT_DEBUG,
    VIRT_MROM,
    VIRT_TEST,
    VIRT_RTC,
    VIRT_CLINT,
    VIRT_ACLINT_SSWI,
    VIRT_PLIC,
    VIRT_APLIC_M,
    VIRT_APLIC_S,
    VIRT_UART0,
    VIRT_VIRTIO,
    VIRT_FW_CFG,
    VIRT_IMSIC_M,
    VIRT_IMSIC_S,
    VIRT_FLASH,
    VIRT_DRAM,
    VIRT_PCIE_MMIO,
    VIRT_PCIE_PIO,
    VIRT_PLATFORM_BUS,
    VIRT_PCIE_ECAM,
    VIRT_IOMMU_SYS,
    VIRT_GPIO, // for add gpio chip
};

enum
{
    UART0_IRQ = 10,
    RTC_IRQ = 11,
    VIRTIO_IRQ = 1,
    VIRTIO_COUNT = 8,
    PCIE_IRQ = 0x20,
    IOMMU_SYS_IRQ = 0x24,
    VIRT_PLATFORM_BUS_IRQ = 64,
    VIRT_GPIO_IRQ = 96 // for add gpio chip
};

#define VIRT_PLATFORM_BUS_NUM_IRQS 32

#define VIRT_IRQCHIP_NUM_MSIS 255
#define VIRT_IRQCHIP_NUM_SOURCES 127 // 增加一些，避免interrupt不夠
#define VIRT_IRQCHIP_NUM_PRIO_BITS 3
#define VIRT_IRQCHIP_MAX_GUESTS_BITS 3
#define VIRT_IRQCHIP_MAX_GUESTS ((1U << VIRT_IRQCHIP_MAX_GUESTS_BITS) - 1U)

#define VIRT_PLIC_PRIORITY_BASE 0x00
#define VIRT_PLIC_PENDING_BASE 0x1000
#define VIRT_PLIC_ENABLE_BASE 0x2000
#define VIRT_PLIC_ENABLE_STRIDE 0x80
#define VIRT_PLIC_CONTEXT_BASE 0x200000
#define VIRT_PLIC_CONTEXT_STRIDE 0x1000
#define VIRT_PLIC_SIZE(__num_context) \
    (VIRT_PLIC_CONTEXT_BASE + (__num_context) * VIRT_PLIC_CONTEXT_STRIDE)

#define FDT_PCI_ADDR_CELLS 3
#define FDT_PCI_INT_CELLS 1
#define FDT_PLIC_ADDR_CELLS 0
#define FDT_PLIC_INT_CELLS 1
#define FDT_APLIC_INT_CELLS 2
#define FDT_APLIC_ADDR_CELLS 0
#define FDT_IMSIC_INT_CELLS 0
#define FDT_MAX_INT_CELLS 2
#define FDT_MAX_INT_MAP_WIDTH (FDT_PCI_ADDR_CELLS + FDT_PCI_INT_CELLS + \
                               1 + FDT_MAX_INT_CELLS)
#define FDT_PLIC_INT_MAP_WIDTH (FDT_PCI_ADDR_CELLS + FDT_PCI_INT_CELLS + \
                                1 + FDT_PLIC_INT_CELLS)
#define FDT_APLIC_INT_MAP_WIDTH (FDT_PCI_ADDR_CELLS + FDT_PCI_INT_CELLS + \
                                 1 + FDT_APLIC_INT_CELLS)

bool virt_is_acpi_enabled(RISCVVirtState *s);
bool virt_is_iommu_sys_enabled(RISCVVirtState *s);
void virt_acpi_setup(RISCVVirtState *vms);
uint32_t imsic_num_bits(uint32_t count);

/*
 * The virt machine physical address space used by some of the devices
 * namely ACLINT, PLIC, APLIC, and IMSIC depend on number of Sockets,
 * number of CPUs, and number of IMSIC guest files.
 *
 * Various limits defined by VIRT_SOCKETS_MAX_BITS, VIRT_CPUS_MAX_BITS,
 * and VIRT_IRQCHIP_MAX_GUESTS_BITS are tuned for maximum utilization
 * of virt machine physical address space.
 */

#define VIRT_IMSIC_GROUP_MAX_SIZE (1U << IMSIC_MMIO_GROUP_MIN_SHIFT)
#if VIRT_IMSIC_GROUP_MAX_SIZE < \
    IMSIC_GROUP_SIZE(VIRT_CPUS_MAX_BITS, VIRT_IRQCHIP_MAX_GUESTS_BITS)
#error "Can't accommodate single IMSIC group in address space"
#endif

#define VIRT_IMSIC_MAX_SIZE (VIRT_SOCKETS_MAX * \
                             VIRT_IMSIC_GROUP_MAX_SIZE)
#if 0x4000000 < VIRT_IMSIC_MAX_SIZE
#error "Can't accommodate all IMSIC groups in address space"
#endif

#endif
// 底下是列出GPIO IRQ對應的PLIC line，一共16個
#define GPIO_IRQ0 96
#define GPIO_IRQ1 97
#define GPIO_IRQ2 98
#define GPIO_IRQ3 99
#define GPIO_IRQ4 100
#define GPIO_IRQ5 101
#define GPIO_IRQ6 102
#define GPIO_IRQ7 103
#define GPIO_IRQ8 104
#define GPIO_IRQ9 105
#define GPIO_IRQ10 106
#define GPIO_IRQ11 107
#define GPIO_IRQ12 108
#define GPIO_IRQ13 109
#define GPIO_IRQ14 110
#define GPIO_IRQ15 111

接下來要動的就是qemu/hw/riscv/virt.c
加入create_fdt_gpio和sifive_gpio_create

static void create_fdt_gpio(RISCVVirtState *s,
                            uint32_t *phandle,
                            uint32_t irq_virtio_phandle)
{
    uint32_t gpio_phandle = (*phandle)++;
    char *nodename = g_strdup_printf("/soc/gpio@%lx",
                                     (long)s->memmap[VIRT_GPIO].base);
    void *fdt = MACHINE(s)->fdt;
    qemu_fdt_add_subnode(fdt, nodename);
    qemu_fdt_setprop_cell(fdt, nodename, "phandle", gpio_phandle);
    // qemu_fdt_setprop_cells(fdt, nodename, "clocks",
    //     prci_phandle, PRCI_CLK_TLCLK);
    qemu_fdt_setprop_cell(fdt, nodename, "#interrupt-cells", 2);
    qemu_fdt_setprop(fdt, nodename, "interrupt-controller", NULL, 0);
    qemu_fdt_setprop_cell(fdt, nodename, "#gpio-cells", 2);
    qemu_fdt_setprop(fdt, nodename, "gpio-controller", NULL, 0);
    qemu_fdt_setprop_cells(fdt, nodename, "reg",
                           0x0, s->memmap[VIRT_GPIO].base,
                           0x0, s->memmap[VIRT_GPIO].size);
    qemu_fdt_setprop_cells(fdt, nodename, "interrupts", GPIO_IRQ0,
                           GPIO_IRQ1, GPIO_IRQ2, GPIO_IRQ3,
                           GPIO_IRQ4, GPIO_IRQ5, GPIO_IRQ6,
                           GPIO_IRQ7, GPIO_IRQ8, GPIO_IRQ9,
                           GPIO_IRQ10, GPIO_IRQ11, GPIO_IRQ12,
                           GPIO_IRQ13, GPIO_IRQ14, GPIO_IRQ15);
    qemu_fdt_setprop_cell(fdt, nodename, "interrupt-parent", irq_virtio_phandle);
    qemu_fdt_setprop_string(fdt, nodename, "compatible", "sifive,gpio0");
    g_free(nodename);
}

static void sifive_gpio_create(MachineState *machine, RISCVVirtState *s, DeviceState *virtio_irqchip)
{
    int i;
    object_initialize_child(OBJECT(machine), "gpio", &s->gpio, TYPE_SIFIVE_GPIO);
    qdev_prop_set_uint32(DEVICE(&s->gpio), "ngpio", 16);
    if (!sysbus_realize(SYS_BUS_DEVICE(&s->gpio), &error_fatal))
    {
        return;
    }
    sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpio), 0, s->memmap[VIRT_GPIO].base);

    /* Pass all GPIOs to the SOC layer so they are available to the board */
    // qdev_pass_gpios(DEVICE(&s->gpio), DEVICE(machine), NULL);

    /* Connect GPIO interrupts to the PLIC */
    for (i = 0; i < 16; i++)
    {
        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpio), i,
                           qdev_get_gpio_in(virtio_irqchip,
                                            VIRT_GPIO_IRQ + i));
    }
    object_initialize_child(OBJECT(machine), "gpio-timer", &s->gpio_timer, TYPE_GPIO_TIMER);

    if (!sysbus_realize(SYS_BUS_DEVICE(&s->gpio_timer), &error_fatal))
    {
        return;
    }

    // Connect output to gpio[2] input of sifive-gpio
    qdev_connect_gpio_out(DEVICE(&s->gpio_timer), 0,
                          qdev_get_gpio_in(DEVICE(&s->gpio), 2));
}

`create_fdt_gpio`可以放在`finalize_fdt`的最後，負責生成設備樹中gpio的node

static void finalize_fdt(RISCVVirtState *s)
{
    ...
    create_fdt_rtc(s, irq_mmio_phandle);

    create_fdt_gpio(s, &phandle, irq_virtio_phandle);
}

`sifive_gpio_create`要放在`virt_machine_init`的中間，負責將GPIO在QEMU的實體初始化和PLIC連接起來

static void virt_machine_init(MachineState *machine)
{
    ...
/* VirtIO MMIO devices */
    for (i = 0; i < VIRTIO_COUNT; i++)
    {
        sysbus_create_simple("virtio-mmio",
                             s->memmap[VIRT_VIRTIO].base + i * s->memmap[VIRT_VIRTIO].size,
                             qdev_get_gpio_in(virtio_irqchip, VIRTIO_IRQ + i));
    }
    // create sifive_gpio here
    sifive_gpio_create(machine, s, virtio_irqchip);

    gpex_pcie_init(system_memory, pcie_irqchip, s);
    ...

上面那兩個function是從`qemu/hw/riscv/sifive_u`抄出來的，另外gpio_timer則是我叫chatgpt寫的一個sysbus device，這個device有一個GPIO output port，每10秒會改變一次寫到GPIO值0->1 or 1->0，在sifive_gpio_create的最後一部份初始化並接上gpio的第二個pin。
這樣設定，只要bh_thread.ko這個kernel module裝上之後，每10秒會觸發一次interrupt handler。

接下來就是gpio_timer的部分，要新增的檔案有
* qemu/hw/misc/gpio_timer.c
* qemu/include/hw/misc/gpio-timer.h
要改動的檔案有
* qemu/hw/misc/meson.build

// gpio_timer.h
#ifndef HW_MISC_GPIO_TIMER_H
#define HW_MISC_GPIO_TIMER_H

#include "hw/sysbus.h"

#define TYPE_GPIO_TIMER "gpio-timer"
OBJECT_DECLARE_SIMPLE_TYPE(GpioTimerState, GPIO_TIMER)
struct GpioTimerState
{
    SysBusDevice parent_obj;

    QEMUTimer *timer;
    qemu_irq irq_out;

    uint64_t period_ns; // 10s in ns
    bool level;
};
#endif

// gpio_timer.c
#include "qemu/osdep.h"
#include "hw/sysbus.h"
#include "qapi/error.h"
#include "qemu/timer.h"
#include "hw/qdev-core.h"
#include "hw/irq.h"
#include "hw/misc/gpio-timer.h"

static void gpio_timer_cb(void *opaque)
{
    GpioTimerState *s = opaque;

    s->level = !s->level;
    qemu_set_irq(s->irq_out, s->level);
    timer_mod(s->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + s->period_ns);
}

static void gpio_timer_realize(DeviceState *dev, Error **errp)
{
    GpioTimerState *s = GPIO_TIMER(dev);

    s->period_ns = 10ULL * 1000 * 1000 * 1000; // 10 seconds
    s->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, gpio_timer_cb, s);
    timer_mod(s->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + s->period_ns);
}

static void gpio_timer_init(Object *obj)
{
    GpioTimerState *s = GPIO_TIMER(obj);
    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
    qdev_init_gpio_out(DEVICE(obj), &s->irq_out, 1);
    sysbus_init_irq(sbd, &s->irq_out);
}
static void gpio_timer_class_init(ObjectClass *klass, const void *data)
{
    DeviceClass *dc = DEVICE_CLASS(klass);
    dc->realize = gpio_timer_realize;
    dc->user_creatable = true;
}

static const TypeInfo gpio_timer_info = {
    .name = TYPE_GPIO_TIMER,
    .parent = TYPE_SYS_BUS_DEVICE,
    .instance_size = sizeof(GpioTimerState),
    .instance_init = gpio_timer_init,
    .class_init = gpio_timer_class_init,
};

static void gpio_timer_register_types(void)
{
    type_register_static(&gpio_timer_info);
}

type_init(gpio_timer_register_types)

之後有空再來解釋以上的code在做什麼

最後要改qemu/hw/misc/meson.build才能成功編譯gpio_timer，我自己是在第25左右加上下面這行，告訴compiler要去編譯這個device
system_ss.add(files('gpio_timer.c'))

然後rebuild，QEMU最麻煩的部分就結束了。

接下來還有要調整的東西，bh_thread.c裡面指定gpio number要改，但是要改的數值不是2，還要加上一段offset才行。
具體數值可以在QEMU開機後執行

cat /dev/kmsg | grep gpio

看到
`7,140,394987,-;gpio gpiochip0: registered GPIOs 512 to 527 on 10010000.gpio`
所以在bh_thread.c的gpio leds和buttons會像這樣

/* Define GPIOs for LEDs.
 * FIXME: Change the numbers for the GPIO on your board.
 */
static struct gpio leds[] = {{512, GPIOF_OUT_INIT_LOW, "LED 1"}};

/* Define GPIOs for BUTTONS
 * FIXME: Change the numbers for the GPIO on your board.
 */
static struct gpio buttons[] = {
    {514, GPIOF_IN, "LED 1 ON BUTTON"},
    {516, GPIOF_IN, "LED 1 OFF BUTTON"},
};

其中buttons number 514會被gpio_timer每10秒改變訊號一次，觸發interrupt handler。
### 執行結果

/workspace # insmod bh_thread.ko
[  539.120880] bh_thread: loading out-of-tree module taints kernel.
[  539.132947] bottomhalf_init
[  539.133812] Current button1 value: 1
[  539.134884] Successfully requested BUTTON1 IRQ # 31
[  539.140767] Successfully requested BUTTON2 IRQ # 32
/workspace # [  539.797935] Bottom half task starts
[  540.301821] Bottom half task ends
[  549.797565] Bottom half task starts
[  550.299475] Bottom half task ends
[  559.797880] Bottom half task starts
[  560.299556] Bottom half task ends
[  569.798899] Bottom half task starts
[  570.303493] Bottom half task ends
...

到這邊就差不多了，有空再補充這些更動做了甚麼，為何要這些改動，參考了哪些資料。

## murmur
這整個實驗花了好多時間，即便有chatgpt還是很困難，尤其是一開始甚麼都不知道的情況，常常被chatgpt騙，特別是在提出一個看似可行的目標，但實際上QEMU做不到的事情，chatgpt就會給一堆鬼打牆的方案，繞了一圈了解QEMU在做啥之後才發現chatgpt常常會亂支持我的不正確的提案。
特別是不要再想要做的目標前面加上自己亂七八糟的推理(人類的幻覺，或者說書沒讀透在考卷上胡說八道)，會某種程度上誤導chatgpt，然後chatgpt再誤導你做一些奇奇怪怪的事情。所以整個實驗最大部份的改動基本上是我讀完sifive_u這個machine之後，將裡面的邏輯複製到virt machine上。不過chatgpt的幫助還是很多的，幫我生出gpio_timer，還有compile的過程語法有問題幫我修正，以及一開始甚麼都不了解幫我找到一些可以嘗試的方向。
體感上來說chatgpt對我的幫助大概分兩類
* 知識理解
* 工作效率
前者我認為chatgpt能做到的大概是上一個固定值的buff，如果是從0開始，chatgpt確實能給幾十幾百倍的效益讓我了解一個雛形，但是開始知道得越多，chatgpt能給的幫助效益就快速遞減，甚至會附和我錯誤的理解。
後者則是當我了解問題到一定程度，能給出精確的目的和要求之後，chatgpt就能快速地打好草稿，基於自身的理解可以快速修正chatgpt的錯誤後，工作的效率確實會極大程度的提升。
總結就是:
1. 先透過chatgpt掌握一個輪廓
2. 根據chatgpt提供的資訊廣泛搜索閱讀
3. 深入source code並嘗試編譯執行(這邊chatgpt的輔助效益最大)
4. 掌握一定知識後，將任務細分餵給chatgpt(這邊必須要對問題有一定的理解，否則會被chatgpt搞)
5. 修正chatgpt的結果並完成目標

Reading The Linux Kernel Module Programming Guide主要的部分就到這邊啦。之後看要不要補上debug的過程，和compile kernel時可以打開的一些選項，和qemu的一些理解。
下一個要看的應該會是下一個要看的應該會是`Linux containers in 500 lines of code`