setup nvidia dgx

# setup nvidia dgx ssh admin@140.118.122.123 # Find the MAC addresses of NIC/CX7 interfaces ``` $ sudo apt-get install jq -y $ for n in $(ls /sys/class/net); do if ethtool -i "$n" 2>/dev/null | grep -q '^driver: mlx5'; then mac=$(cat /sys/class/net/$n/address) echo "\"MT2910 Family [ConnectX-7], MAC: $mac\"" fi done ``` output MAC:.... use the MAC addresses to fill into the blanks => MAC_address_found1~4MAC_address_found4 # Configure the Interfaces: ``` $ sudo nano /etc/systemd/network/20-aerial00.link [Match] MACAddress=[MAC_address_found1] [Link] Name=aerial00 $ sudo nano /etc/systemd/network/20-aerial01.link [Match] MACAddress=[MAC_address_found2] [Link] Name=aerial01 $ sudo nano /etc/systemd/network/20-aerial02.link [Match] MACAddress=[MAC_address_found3] [Link] Name=aerial02 $ sudo nano /etc/systemd/network/20-aerial03.link [Match] MACAddress=[MAC_address_found4] [Link] Name=aerial03 ``` # Apply the changes ``` $ sudo netplan apply ``` # Disable Auto Upgrade Edit the /etc/apt/apt.conf.d/20auto-upgrades system fileand change the “1” to “0” for both lines. This prevents the installed version of the low latency kernel from being accidentally changed with a subsequent software upgrade. ``` $ sudo nano /etc/apt/apt.conf.d/20auto-upgrades APT::Periodic::Update-Package-Lists "0"; APT::Periodic::Unattended-Upgrade "0"; ``` # Update NVIDIA Optimized Ubuntu Kernel ``` # Install the specific linux kernel $ sudo apt update $ sudo apt install linux-image-6.14.0-1015-nvidia-64k $ sudo sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT="Advanced options for Ubuntu>Ubuntu, with linux-image-6.14.0-1015-nvidia-64k"/' /etc/default/grub ``` # Configure Linux Kernel Command-line iommy.passthrough configuration: ``` # Check if the iommu.passthrough=y is present $ grep iommu.passthrough=y /proc/cmdline # If the parameter is present. find the file containing this parameter and remove it. Otherwise, skip 2 steps below. $ grep -rns iommu.passthrough /etc/default/grub* $ sudo sed -i 's/ iommu.passthrough=y//' /etc/default/<found file> ``` # Update the command-line cat <<"EOF" | sudo tee /etc/default/grub.d/cmdline.cfg GRUB_CMDLINE_LINUX="$GRUB_CMDLINE_LINUX pci=realloc=off pci=pcie_bus_safe default_hugepagesz=512M hugepagesz=512M hugepages=24 tsc=reliable processor.max_cstate=0 audit=0 idle=poll rcu_nocb_poll nosoftlockup irqaffinity=0-3 kthread_cpus=0-3 isolcpus=managed_irq,domain,4-19 nohz_full=4-19 rcu_nocbs=4-19 earlycon module_blacklist=nouveau acpi_power_meter.force_cap_on=y numa_balancing=disable init_on_alloc=0 preempt=none ras=off iommu=off mitgations=off kpti=off skew_tick=1 nowatchdog mce=ignore_ce transparent_hugepage=never" EOF # List all installed kernel image ``` $ dpkg -l | grep linux-image ``` # 把 GRUB 預設 kernel 鎖定成 64k（之後每次 reboot 都自動用 64k，不再跳回普通 kernel）很好，這一步就是把 **GRUB 預設 kernel 鎖定成 64k**（之後每次 reboot 都自動用 64k，不再跳回普通 kernel）。我們一步一步來做 **永久固定**。 --- # ⭐ Step 1 — 找到 64k kernel 的 GRUB 完整名稱先列出 GRUB menu： ```bash awk -F"'" '/menuentry /{print $2}' /boot/grub/grub.cfg | nl -ba ``` 你會看到像之前這樣： ``` 1 DGX OS GNU/Linux 2 DGX OS GNU/Linux, with Linux 6.14.0-1015-nvidia-64k 3 DGX OS GNU/Linux, with Linux 6.14.0-1015-nvidia-64k (recovery mode) 4 DGX OS GNU/Linux, with Linux 6.14.0-1015-nvidia ... ``` 👉 我們要用 **第 2 行那個完整字串** 假設你的目標是： ``` DGX OS GNU/Linux, with Linux 6.14.0-1015-nvidia-64k ``` --- # ⭐ Step 2 — 找到 submenu 名稱（Advanced options） DGX 的 kernel 在 submenu 裡： ```bash grep "submenu '" /boot/grub/grub.cfg ``` 通常會看到： ``` submenu 'Advanced options for DGX OS GNU/Linux' ``` --- # ⭐ Step 3 — 設定 GRUB_DEFAULT（最關鍵🔥）執行這行（整行複製）： ```bash sudo sed -i 's|^GRUB_DEFAULT=.*|GRUB_DEFAULT="Advanced options for DGX OS GNU/Linux>DGX OS GNU/Linux, with Linux 6.14.0-1015-nvidia-64k"|' /etc/default/grub ``` 這行的意思： ``` submenu名稱 > menuentry名稱 ``` --- # ⭐ Step 4 — 產生新的 grub.cfg ```bash sudo update-grub ``` 應該會看到： ``` Found linux image: /boot/vmlinuz-6.14.0-1015-nvidia-64k ``` --- # ⭐ Step 5 — Reboot 測試 ```bash sudo reboot ``` 開機後檢查： ```bash uname -r ``` 應該變成： ``` 6.14.0-1015-nvidia-64k ``` # 啟動nvidia-smi ``` # 先看 Ubuntu 官方建議 driver ubuntu-drivers devices # 尋找哪個版本有 recommended (ex: nvidia-driver-590-open) sudo apt-get update sudo apt-get install -y nvidia-driver-590-open # 安裝完成後重開機 sudo reboot ``` # 設定hugepage = 24 ## ① 編輯 GRUB ```bash sudo nano /etc/default/grub ``` 找到這一行（通常是空的）： ```bash GRUB_CMDLINE_LINUX="" ``` 把它改成： ```bash GRUB_CMDLINE_LINUX="default_hugepagesz=512M hugepagesz=512M hugepages=24" ``` ⚠️ 注意：不要改 DEFAULT 那行了。 --- ## ② 重新產生 grub ```bash sudo update-grub ``` --- ## ③ 重開機 ```bash sudo reboot ``` --- ## ④ 驗證 kernel cmdline ```bash cat /proc/cmdline ``` 現在應該會看到： ``` ... default_hugepagesz=512M hugepagesz=512M hugepages=24 ``` --- ## ⑤ 再驗證 HugePages ```bash grep -i huge /proc/meminfo ``` 應該變成： ``` HugePages_Total: 24 HugePages_Free: 24 Hugetlb: 12582912 kB ``` # Install DOCA-Host deb: Enter the following commands to install DOCA OFED #Install DOCA OFED $ wget https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.1/host/doca- host_3.2.1-044000-25.10-ubuntu2404_arm64.deb $ sudo dpkg -i doca-host_3.2.1-044000-25.10-ubuntu2404_arm64.deb $ sudo apt-get update $ sudo apt-get -y install doca-tools doca-ofed mlnx-fw-updater

Syntax	Example	Reference
# Header	Header	基本排版
- Unordered List	Unordered List
1. Ordered List	Ordered List
- [ ] Todo List	Todo List
> Blockquote	Blockquote
Bold font	Bold font
Italics font	Italics font
~~Strikethrough~~	~~Strikethrough~~
19^th^	19^th
H~2~O	H₂O
++Inserted text++	Inserted text
==Marked text==	Marked text
[link text](https:// "title")	Link
![image alt](https:// "title")	Image
`Code`	`Code`	在筆記中貼入程式碼
```javascript var i = 0; ```	`var i = 0;`
:smile:		Emoji list
{%youtube youtube_id %}	Externals
$L^aT_eX$	L^aT_eX
:::info This is a alert area. :::	This is a alert area.