###### tags: `DPDK Test`
# Mellanox CX6 100G DPDK Setting

1. 查看設備屬於哪個NUMA節點
cat /sys/class/net/enp5s0f0/device/numa_node
2. 查看PCI的樹狀關係
lspci -t -v

cat /proc/cpuinfo | grep MHz|uniq

## RedHat 7.9
Linux localhost.localdomain 3.10.0-1160.el7.x86_64 #1 SMP Tue Aug 18 14:50:17 EDT 2020 x86_64 x86_64 x86_64 GNU/Linux
### install MLNX_OFED
yum install tcl
yum install gcc-gfortran
yum install tk

[root@localhost MLNX_OFED_LINUX-5.4-3.0.3.0-rhel7.9-x86_64]# ./mlnxofedinstall

### build DPDK

apt-get update
apt-get install librte-net-i40e21
apt-get install meson
apt-get install gcc
apt-get install python3-pyelftools

sudo apt-get install make gcc python libpcap-dev libnuma-dev pkg-config -y

sudo apt-get update
### 用編輯方式複製
sudo apt-get install -y python libpcap-dev build-essential linux-headers-`uname -r` libnuma-dev pkg-config gcc libnuma-dev libpcre3 libpcre3-dev libssl-dev

### Build DPDK
https://doc.dpdk.org/guides-20.11/linux_gsg/build_dpdk.html

apt-get install ninja-build meson
rpm -ivh https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/n/ninja-build-1.10.2-3.el7.x86_64.rpm
rpm -ivh https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/m/meson-0.55.1-1.el7.noarch.rpm
cd dpdk-20.11.3
meson -Dexamples=12fwd,13fwd build
cd build
ninja
if error
rm ./build -rf
meson -Dexamples=l2fwd,l3fwd build

cd ./build
ninja

ninja install
ldconfig

## 關閉update
我們開啟「Software Updater」後進行以下的調整，
第一個圈選處改成「Never」，表示我們不再 check update
第二個圈選處改成 「“Display immediately” 取代 “Download and install automatically”」，表示我們顯示重要更新。但不自動下載更新。
## command line 的方法」關閉自動更新
sudo vim /etc/apt/apt.conf.d/20auto-upgrades

我們一共要改兩個值，0 就是取消的意思。
第一個值是不自動 check update
第二個值是自動安裝 update
APT::Periodic::Update-Package-Lists "0";
APT::Periodic::Unattended-Upgrade "0";

CP Rocket
Linux test-RL-20-00 5.8.0-43-generic #49~20.04.1-Ubuntu SMP Fri Feb 5 09:57:56 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux

$lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 20.04.2 LTS
Release: 20.04
Codename: focal

https://www.mellanox.com/products/infiniband-drivers/linux/mlnx_ofed

### MLNX_OFED v5.4-3.0.3.0 Supported Operating Systems
https://docs.mellanox.com/display/MLNXOFEDv543030/General+Support
### MLNX_OFED v5.4-3.0.3.0 Release Notes
https://docs.mellanox.com/display/MLNXOFEDv543030/Release+Notes

cd MLNX_OFED_LINUX-5.4-1.0.3.0-ubuntu20.04-x86_64
./mlnxofedinstall -h
./mlnxofedinstall -h |grep -i add
ls
./mlnxofedinstall
### build_dpdk (dpdk-stable-20.11.3)
http://core.dpdk.org/download/
https://doc.dpdk.org/guides-20.11/linux_gsg/build_dpdk.html
Updated values /l3fwd/l3fwd.h:

cd dpdk-20.11.3
apt install meson
meson -Dexamples=12fwd,13fwd build
cd build
ninja
ninja install
ldconfig

rm ./build -rf
meson -Dexamples=l2fwd,l3fwd build
cd ./build
ninja
ninja install
ldconfig
cd build

mst start
mst status -v
ofed_info
### BOOT Settings
vi /etc/default/grub
sudo update-grub
reboot
cat /proc/cmdline

### L3fwd settings
Updated values /l3fwd/l3fwd.h:
#define RTE_TEST_RX_DESC_DEFAULT 4096
#define RTE_TEST_TX_DESC_DEFAULT 4096
#define MAX_PKT_BURST 64

cd examples/
vi /l3fwd/l3fwd.h
add
#define RTE_TEST_RX_DESC_DEFAULT 4096
#define RTE_TEST_TX_DESC_DEFAULT 4096
#define MAX_PKT_BURST 64

meson -Dexamples=12fwd,13fwd build
cd build
ninja
ninja install
ldconfig

sudo apt-get install libnuma-dev

#### 0p <-> 0p
./build/examples/dpdk-l3fwd -c 0xfff000000000 -n 4 -a 0000:ca:00.0,mprq_en=1,mprq_log_stride_num=8 --socket-mem=0,8192 -- -p 0x1 -P --config='(0,0,47),(0,1,46),(0,2,45),(0,3,44),(0,4,43),(0,5,42),(0,6,41),(0,7,40),(0,8,39),(0,9,38),(0,10,37),(0,11,36)'

-c 0xfff000000000 = -l 36-47
-p 0x1 只有0 port
-p 0x3 0 and 1 port
-a, --allow <[domain:]bus:devid.func>
-l 0-3 先確認numa幾, 設定哪幾個core

https://doc.dpdk.org/guides/sample_app_ug/l3_forward.html
https://doc.dpdk.org/guides/linux_gsg/build_sample_apps.html