# Docker

## 1. Install Docker

sudo apt install curl

## Uninstall docker

sudo apt-get remove docker docker-engine docker.io containerd runc

# Add Docker's official GPG key:
sudo apt-get update
sudo apt-get install ca-certificates curl
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc

# Add the repository to Apt sources:
echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update
sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
  && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
    sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list \
  && \
    sudo apt-get update
sudo apt-get install -y nvidia-container-toolkit

sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

sudo nvidia-ctk runtime configure --runtime=containerd
sudo systemctl restart containerd

sudo nvidia-ctk runtime configure --runtime=crio
sudo systemctl restart crio

sudo systemctl enable docker.service
sudo systemctl enable containerd.service

sudo docker run --rm --runtime=nvidia --gpus all nvidia/cuda:11.6.2-base-ubuntu20.04 nvidia-smi

### Add permission user

sudo chmod 666 /var/run/docker.sock
sudo groupadd docker
sudo usermod -aG docker $USER
newgrp docker

### Modify to compile dockers with Nvidia
If you're going to be building containers, you need to set Docker's default-runtime to nvidia, so that the NVCC compiler and GPU are available during docker build operations. Add "default-runtime": "nvidia" to your /etc/docker/daemon.json configuration file before attempting to build the containers:

{
    "runtimes": {
        "nvidia": {
            "path": "nvidia-container-runtime",
            "runtimeArgs": []
        }
    },

    "default-runtime": "nvidia"
}

sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

sudo nvidia-ctk runtime configure --runtime=containerd
sudo systemctl restart containerd

sudo nvidia-ctk runtime configure --runtime=crio
sudo systemctl restart crio

sudo systemctl enable docker.service
sudo systemctl enable containerd.service

## 1.2. Check with nvidia-smi

docker run --rm --runtime=nvidia --gpus all nvidia/cuda:13.2.0-base-ubuntu24.04 nvidia-smi

### Increase swap

sudo systemctl disable nvzramconfig
sudo fallocate -l 300G /mnt/300GB.swap
sudo mkswap /mnt/300GB.swap
sudo swapon /mnt/300GB.swap

### Run vLLM

# Run VLLM Thor & Spark

1. Install uv

curl -LsSf https://astral.sh/uv/install.sh | sh

2. Create environment

sudo apt install python3-dev
uv venv .vllm --python 3.12
source .vllm/bin/activate

3. Install vllm

uv pip install -U vllm --torch-backend=auto --extra-index-url https://wheels.vllm.ai/nightly/cu130

4. Export variables

export TORCH_CUDA_ARCH_LIST=8.7

5. Clean memory

sudo sysctl -w vm.drop_caches=3

6. Run Gemma4

vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 \
  --port 9000 \
  --max-num-seqs 8 \
  --tensor-parallel-size 1 \
  --max-model-len 8000 \
  --trust-remote-code \
  --enable-auto-tool-choice \
  --gpu-memory-utilization 0.75 \
  --tool-call-parser qwen3_coder \
  --reasoning-parser-plugin nano_v3_reasoning_parser.py \
  --reasoning-parser nano_v3 \
  --kv-cache-dtype fp8