---
tags: linux2023
---

# 2023q1 Homework7 (ktcp)
contributed by < `YSRossi` >

## 開發環境

$ gcc --version
gcc (Ubuntu 11.3.0-1ubuntu1~22.04) 11.3.0

$ lscpu
Architecture:            x86_64
  CPU op-mode(s):        32-bit, 64-bit
  Address sizes:         48 bits physical, 48 bits virtual
  Byte Order:            Little Endian
CPU(s):                  8
  On-line CPU(s) list:   0-7
Vendor ID:               AuthenticAMD
  Model name:            AMD FX-8320E Eight-Core Processor
    CPU family:          21
    Model:               2
    Thread(s) per core:  2
    Core(s) per socket:  4
    Socket(s):           1
    Stepping:            0
    Frequency boost:     enabled
    CPU max MHz:         3200.0000
    CPU min MHz:         1400.0000
    BogoMIPS:            6429.64

## 開發紀錄

### 使用 Ftrace 觀察 kHTTPd
:::spoiler Ftrace 原始 kHTTPd 結果

$ sudo cat /sys/kernel/debug/tracing/trace
# tracer: function_graph
#
# CPU  DURATION                  FUNCTION CALLS
# |     |   |                     |   |   |   |
 5)               |  http_server_worker [khttpd]() {
 5)               |    kernel_sigaction() {
 5)   0.744 us    |      _raw_spin_lock_irq();
 5)   0.560 us    |      _raw_spin_unlock_irq();
 5)   3.674 us    |    }
 5)               |    kernel_sigaction() {
 5)   0.553 us    |      _raw_spin_lock_irq();
 5)   0.544 us    |      _raw_spin_unlock_irq();
 5)   2.878 us    |    }
 5)               |    kmalloc_trace() {
 5)               |      __kmem_cache_alloc_node() {
 5)   0.676 us    |        __cond_resched();
 5)   0.599 us    |        should_failslab();
 5)   4.725 us    |      }
 5)   5.874 us    |    }
 5)   0.567 us    |    http_parser_init [khttpd]();
 5)   0.563 us    |    kthread_should_stop();
 5)               |    http_server_recv.constprop.0 [khttpd]() {
 5)               |      kernel_recvmsg() {
 5)               |        sock_recvmsg() {
 5)   2.242 us    |          security_socket_recvmsg();
 5)   7.902 us    |          inet_recvmsg();
 5) + 12.048 us   |        }
 5) + 13.354 us   |      }
 5) + 14.520 us   |    }
 5)               |    kernel_sock_shutdown() {
 5)               |      inet_shutdown() {
 5)               |        lock_sock_nested() {
 5)   0.540 us    |          __cond_resched();
 5)   0.601 us    |          _raw_spin_lock_bh();
 5)   0.792 us    |          _raw_spin_unlock_bh();
 5)   4.161 us    |        }
 5)               |        tcp_shutdown() {
 5)   1.042 us    |          tcp_set_state();
 5) ! 168.549 us  |          tcp_send_fin();
 5) ! 171.821 us  |        }
 5)               |        sock_def_wakeup() {
 5)   0.562 us    |          __rcu_read_lock();
 5)   0.569 us    |          __rcu_read_unlock();
 5)   2.945 us    |        }
 5)               |        release_sock() {
 5)   0.562 us    |          _raw_spin_lock_bh();
 5) + 25.014 us   |          __release_sock();
 5)   0.601 us    |          tcp_release_cb();
 5)   0.833 us    |          _raw_spin_unlock_bh();
 5) + 29.973 us   |        }
 5) ! 211.963 us  |      }
 5) ! 213.299 us  |    }
 3)               |  http_server_worker [khttpd]() {
 5)               |    sock_release() {
 5)               |      inet_release() {
 3)               |    kernel_sigaction() {
 3)   0.374 us    |      _raw_spin_lock_irq();
 5)   0.639 us    |        ip_mc_drop_socket();
 3)   0.221 us    |      _raw_spin_unlock_irq();
 3)   1.805 us    |    }
 5)               |        tcp_close() {
 3)               |    kernel_sigaction() {
 3)   0.228 us    |      _raw_spin_lock_irq();
 5)   1.423 us    |          lock_sock_nested();
 3)   0.229 us    |      _raw_spin_unlock_irq();
 3)   1.230 us    |    }
 3)               |    kmalloc_trace() {
 3)               |      __kmem_cache_alloc_node() {
 3)   0.353 us    |        __cond_resched();
 5)   9.202 us    |          __tcp_close();
 3)   0.220 us    |        should_failslab();
 3)   1.555 us    |      }
 3)   2.018 us    |    }
 3)   0.235 us    |    http_parser_init [khttpd]();
 3)   0.228 us    |    kthread_should_stop();
 3)               |    http_server_recv.constprop.0 [khttpd]() {
 3)               |      kernel_recvmsg() {
 3)               |        sock_recvmsg() {
 3)   1.047 us    |          security_socket_recvmsg();
 3)   5.441 us    |          inet_recvmsg();
 5)   1.437 us    |          release_sock();
 3)   7.222 us    |        }
 3)   7.689 us    |      }
 3)   8.213 us    |    }
 3)               |    http_parser_execute [khttpd]() {
 3)   0.244 us    |      http_parser_callback_message_begin [khttpd]();
 5)   7.970 us    |          sk_free();
 3)   0.458 us    |      parse_url_char [khttpd]();
 3)   0.310 us    |      http_parser_callback_request_url [khttpd]();
 3)   0.217 us    |      http_parser_callback_header_field [khttpd]();
 3)   0.214 us    |      http_parser_callback_header_value [khttpd]();
 3)   0.222 us    |      http_parser_callback_headers_complete [khttpd]();
 3)   0.229 us    |      http_message_needs_eof [khttpd]();
 3)   0.234 us    |      http_should_keep_alive [khttpd]();
 3)               |      http_parser_callback_message_complete [khttpd]() {
 3)   0.223 us    |        http_should_keep_alive [khttpd]();
 3)               |        _printk() {
 3) + 13.850 us   |          vprintk();
 5) + 22.892 us   |        }
 5) + 25.432 us   |      }
 5)   0.593 us    |      module_put();
 5)               |      iput() {
 5)   0.579 us    |        _raw_spin_lock();
 5)   0.550 us    |        _raw_spin_unlock();
 5)               |        evict() {
 5)   1.672 us    |          inode_wait_for_writeback();
 5)   1.120 us    |          truncate_inode_pages_final();
 5)   1.026 us    |          clear_inode();
 5)   0.567 us    |          _raw_spin_lock();
 3) + 14.461 us   |        }
 5)   0.690 us    |          wake_up_bit();
 3)               |        http_server_send.isra.0 [khttpd]() {
 5)   0.546 us    |          _raw_spin_unlock();
 3) + 52.964 us   |          kernel_sendmsg();
 5)   6.136 us    |          destroy_inode();
 5) + 16.597 us   |        }
 5) + 20.794 us   |      }
 5) + 49.266 us   |    }
 5)               |    kfree() {
 5)   0.635 us    |      __kmem_cache_free();
 5)   2.001 us    |    }
 5) ! 301.312 us  |  }
 3) + 53.702 us   |        }
 3) + 70.234 us   |      }
 3) + 78.295 us   |    }
 3)   0.271 us    |    http_should_keep_alive [khttpd]();
 3)               |    kernel_sock_shutdown() {
 3)               |      inet_shutdown() {
 3)               |        lock_sock_nested() {
 3)   0.231 us    |          __cond_resched();
 3)   0.269 us    |          _raw_spin_lock_bh();
 3)   0.328 us    |          _raw_spin_unlock_bh();
 3)   1.710 us    |        }
 3)               |        tcp_shutdown() {
 3)   0.334 us    |          tcp_set_state();
 3) + 21.255 us   |          tcp_send_fin();
 3) + 22.467 us   |        }
 3)               |        sock_def_wakeup() {
 3)   0.222 us    |          __rcu_read_lock();
 3)   0.236 us    |          __rcu_read_unlock();
 3)   4.224 us    |        }
 3)               |        release_sock() {
 3)   0.233 us    |          _raw_spin_lock_bh();
 3)   0.224 us    |          tcp_release_cb();
 3)   0.339 us    |          _raw_spin_unlock_bh();
 3)   1.669 us    |        }
 3) + 31.272 us   |      }
 3) + 31.804 us   |    }
 3)               |    sock_release() {
 3)               |      inet_release() {
 3)   0.216 us    |        ip_mc_drop_socket();
 3)               |        tcp_close() {
 3)   0.571 us    |          lock_sock_nested();
 3)   1.498 us    |          __tcp_close();
 3)   0.581 us    |          release_sock();
 3)   3.505 us    |        }
 3)   4.421 us    |      }
 3)   0.251 us    |      module_put();
 3)               |      iput() {
 3)   0.338 us    |        _raw_spin_lock();
 3)   0.228 us    |        _raw_spin_unlock();
 3)               |        evict() {
 3)   0.612 us    |          inode_wait_for_writeback();
 3)   0.541 us    |          truncate_inode_pages_final();
 3)   0.519 us    |          clear_inode();
 3)   0.218 us    |          _raw_spin_lock();
 3)   0.295 us    |          wake_up_bit();
 3)   0.217 us    |          _raw_spin_unlock();
 3)   2.220 us    |          destroy_inode();
 3)   7.052 us    |        }
 3)   8.712 us    |      }
 3) + 14.356 us   |    }
 3)               |    kfree() {
 3)   0.254 us    |      __kmem_cache_free();
 3)   0.703 us    |    }
 3) ! 144.009 us  |  }

:::
:::spoiler Ftrace CMWQ + kHTTPd 結果

# tracer: function_graph
#
# CPU  DURATION                  FUNCTION CALLS
# |     |   |                     |   |   |   |
 3)               |  http_server_worker_CMWQ [khttpd]() {
 3)               |    kernel_sigaction() {
 3)   0.944 us    |      _raw_spin_lock_irq();
 3)   0.549 us    |      _raw_spin_unlock_irq();
 3)   3.686 us    |    }
 3)               |    kernel_sigaction() {
 3)   0.505 us    |      _raw_spin_lock_irq();
 3)   0.525 us    |      _raw_spin_unlock_irq();
 3)   2.628 us    |    }
 3)               |    kmalloc_trace() {
 3)               |      __kmem_cache_alloc_node() {
 3)   0.588 us    |        __cond_resched();
 3)   0.589 us    |        should_failslab();
 3)   4.440 us    |      }
 3)   5.607 us    |    }
 3)   0.583 us    |    http_parser_init [khttpd]();
 3)               |    kernel_recvmsg() {
 3)               |      sock_recvmsg() {
 3)               |        security_socket_recvmsg() {
 3)   1.393 us    |          apparmor_socket_recvmsg();
 3)   2.558 us    |        }
 3)               |        inet_recvmsg() {
 3)   6.998 us    |          tcp_recvmsg();
 3)   8.240 us    |        }
 3) + 12.681 us   |      }
 3) + 13.952 us   |    }
 3)               |    kernel_sock_shutdown() {
 3)               |      inet_shutdown() {
 3)               |        lock_sock_nested() {
 3)   0.530 us    |          __cond_resched();
 3)   0.510 us    |          _raw_spin_lock_bh();
 3)   0.844 us    |          _raw_spin_unlock_bh();
 3)   3.947 us    |        }
 3)               |        tcp_shutdown() {
 3)   0.969 us    |          tcp_set_state();
 3) ! 153.584 us  |          tcp_send_fin();
 3) ! 156.750 us  |        }
 3)               |        sock_def_wakeup() {
 3)   0.587 us    |          __rcu_read_lock();
 3)   0.575 us    |          __rcu_read_unlock();
 3)   3.094 us    |        }
 3)               |        release_sock() {
 3)   0.512 us    |          _raw_spin_lock_bh();
 3) + 24.762 us   |          __release_sock();
 3)   0.559 us    |          tcp_release_cb();
 3)   0.832 us    |          _raw_spin_unlock_bh();
 3) + 29.627 us   |        }
 3) ! 196.324 us  |      }
 3) ! 197.551 us  |    }
 3)               |    sock_release() {
 3)               |      inet_release() {
 3)   0.569 us    |        ip_mc_drop_socket();
 3)               |        tcp_close() {
 3)   1.312 us    |          lock_sock_nested();
 3)   9.004 us    |          __tcp_close();
 3)   1.447 us    |          release_sock();
 3)   7.757 us    |          sk_free();
 3) + 22.409 us   |        }
 3) + 24.835 us   |      }
 3)   0.570 us    |      module_put();
 3)               |      iput() {
 3)   0.572 us    |        _raw_spin_lock();
 3)   0.546 us    |        _raw_spin_unlock();
 3)               |        evict() {
 3)   1.555 us    |          inode_wait_for_writeback();
 3)   1.207 us    |          truncate_inode_pages_final();
 3)   1.085 us    |          clear_inode();
 3)   0.564 us    |          _raw_spin_lock();
 3)   0.784 us    |          wake_up_bit();
 3)   0.523 us    |          _raw_spin_unlock();
 3)   6.016 us    |          destroy_inode();
 3) + 16.321 us   |        }
 3) + 20.081 us   |      }
 3) + 48.087 us   |    }
 3)               |    kfree() {
 3)   0.570 us    |      __kmem_cache_free();
 3)   1.830 us    |    }
 3) ! 281.959 us  |  }
 3)               |  http_server_worker_CMWQ [khttpd]() {
 3)               |    kernel_sigaction() {
 3)   0.596 us    |      _raw_spin_lock_irq();
 3)   0.571 us    |      _raw_spin_unlock_irq();
 3)   2.960 us    |    }
 3)               |    kernel_sigaction() {
 3)   0.634 us    |      _raw_spin_lock_irq();
 3)   0.604 us    |      _raw_spin_unlock_irq();
 3)   2.940 us    |    }
 3)               |    kmalloc_trace() {
 3)               |      __kmem_cache_alloc_node() {
 3)   0.515 us    |        __cond_resched();
 3)   0.493 us    |        should_failslab();
 3)   3.327 us    |      }
 3)   4.432 us    |    }
 3)   0.555 us    |    http_parser_init [khttpd]();
 3)               |    kernel_recvmsg() {
 3)               |      sock_recvmsg() {
 3)               |        security_socket_recvmsg() {
 3)   1.079 us    |          apparmor_socket_recvmsg();
 3)   2.153 us    |        }
 3)               |        inet_recvmsg() {
 3) + 63.993 us   |          tcp_recvmsg();
 3) + 65.393 us   |        }
 3) + 69.175 us   |      }
 3) + 70.229 us   |    }
 3)               |    http_parser_execute [khttpd]() {
 3)   0.715 us    |      http_parser_callback_message_begin [khttpd]();
 3)   0.800 us    |      parse_url_char [khttpd]();
 3)   0.635 us    |      http_parser_callback_request_url [khttpd]();
 3)   0.587 us    |      http_parser_callback_header_field [khttpd]();
 3)   0.548 us    |      http_parser_callback_header_value [khttpd]();
 3)   0.550 us    |      http_parser_callback_headers_complete [khttpd]();
 3)   0.506 us    |      http_message_needs_eof [khttpd]();
 3)   0.531 us    |      http_should_keep_alive [khttpd]();
 3)               |      http_parser_callback_message_complete [khttpd]() {
 3)   0.569 us    |        http_should_keep_alive [khttpd]();
 3)               |        http_server_send.isra.0 [khttpd]() {
 3) ! 108.584 us  |          kernel_sendmsg();
 3) ! 110.310 us  |        }
 3) ! 113.065 us  |      }
 3) ! 128.950 us  |    }
 3)   0.579 us    |    http_should_keep_alive [khttpd]();
 3)               |    kernel_sock_shutdown() {
 3)               |      inet_shutdown() {
 3)               |        lock_sock_nested() {
 3)   0.518 us    |          __cond_resched();
 3)   0.555 us    |          _raw_spin_lock_bh();
 3)   0.746 us    |          _raw_spin_unlock_bh();
 3)   3.922 us    |        }
 3)               |        tcp_shutdown() {
 3)   0.836 us    |          tcp_set_state();
 3) + 64.025 us   |          tcp_send_fin();
 3) + 66.696 us   |        }
 3)               |        sock_def_wakeup() {
 3)   0.501 us    |          __rcu_read_lock();
 3)   0.594 us    |          __rcu_read_unlock();
 3)   2.812 us    |        }
 3)               |        release_sock() {
 3)   0.511 us    |          _raw_spin_lock_bh();
 3)   0.493 us    |          tcp_release_cb();
 3)   0.737 us    |          _raw_spin_unlock_bh();
 3)   4.004 us    |        }
 3) + 80.010 us   |      }
 3) + 81.076 us   |    }
 3)               |    sock_release() {
 3)               |      inet_release() {
 3)   0.505 us    |        ip_mc_drop_socket();
 3)               |        tcp_close() {
 3)   1.352 us    |          lock_sock_nested();
 3)   3.370 us    |          __tcp_close();
 3)   1.392 us    |          release_sock();
 3)   8.174 us    |        }
 3) + 10.206 us   |      }
 3)   0.553 us    |      module_put();
 3)               |      iput() {
 3)   0.662 us    |        _raw_spin_lock();
 3)   0.510 us    |        _raw_spin_unlock();
 3)               |        evict() {
 3)   1.404 us    |          inode_wait_for_writeback();
 3)   0.853 us    |          truncate_inode_pages_final();
 3)   0.960 us    |          clear_inode();
 3)   0.525 us    |          _raw_spin_lock();
 3)   0.587 us    |          wake_up_bit();
 3)   1.395 us    |          _raw_spin_unlock();
 3)   4.778 us    |          destroy_inode();
 3) + 14.774 us   |        }
 3) + 18.100 us   |      }
 3) + 30.975 us   |    }
 3)               |    kfree() {
 3)   0.582 us    |      __kmem_cache_free();
 3)   1.689 us    |    }
 3) ! 331.912 us  |  }

:::
### khttpd 執行流程
[Daemon](https://en.wikipedia.org/wiki/Daemon_(computing))
> In multitasking computer operating systems, a daemon is a computer program that runs as a background process, rather than being under the direct control of an interactive user.

khttp_init -> http_server_daemon -> http_server_worker
### khttp_init
建立 socket 等待連線，建立執行緒執行 `http_server_daemon`
### http_server_daemon
等待來自 client 的連線請求 `kernel_accept`，建立執行緒執行 `http_server_worker`
### http_server_worker
解析收到的資料，執行指定任務

| | requests/sec|
|:----------------------------------- | -------------------:|
| 原始 khttpd |17256.942 |
| 原始 khttpd 移除 pr_info |17901.643 |
| 引入 CMWQ 的 khttpd 移除 pr_info |38070.105|

[GCC documentation : unused](https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html)
> This attribute, attached to a variable or structure field, means that the variable or field is meant to be possibly unused. GCC does not produce a warning for this variable or field.

### workqueue_struct

struct workqueue_struct {  
    struct cpu_workqueue_struct *cpu_wq; //該數組每一項對應系統中的一個處理器  
    struct list_head list;  
    const char *name;  
    int singlethread;  
    int freezeable;     /* Freeze threads during suspend */  
    int rt;  
#ifdef CONFIG_LOCKDEP  
    struct lockdep_map lockdep_map;  
#endif  
}

### work_struct

struct work_struct {
    atomic_long_t data;
    struct list_head entry;
    work_func_t func; // 要處理的函數
}

一個 `work_struct` 實例代表一個"工作"，工作包含想要執行的任務。

### 測試 HTTP keep-alive 模式

輸入下指令進行連線

$ telnet localhost 8081

輸入 `GET / HTTP/1.0` （注意：請求訊息輸入完後要多按一次 Enter）

HTTP/1.1 200 OK
Server: khttpd
Content-Type: text/plain
Content-Length: 12
Connection: Close

輸入 `GET / HTTP/1.1`

HTTP/1.1 200 OK
Server: khttpd
Content-Type: text/plain
Content-Length: 12
Connection: Keep-Alive

根據回傳的 Connection 欄位得知，kHTTPd 本身有 keep-alive 的功能。