配置gpu驱动、docker和vllm过程

ubuntu-drivers devices

sudo ubuntu-drivers autoinstall

sudo apt update
sudo apt install nvidia-driver-590-open

sudo reboot

nvidia-smi

sudo modprobe nvidia
sudo systemctl enable nvidia-persistenced

sudo apt-get remove docker docker-engine docker.io containerd runc

sudo apt update && sudo apt upgrade -y

sudo apt-get install ca-certificates curl gnupg lsb-release

curl -fsSL http://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -

sudo add-apt-repository "deb [arch=amd64] http://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"

sudo apt-get update
sudo apt-get install docker-ce docker-ce-cli containerd.io

sudo usermod -aG docker $USER

sudo reboot

# 启动服务
sudo systemctl start docker

# 设置开机自启
sudo systemctl enable docker

docker run hello-world

docker pull vllm/vllm-openai:latest

python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='Qwen/Qwen3-8B', local_dir='./qwen3-docker/models/Qwen3-8B')"

services:
  vllm:
    container_name: vllm
    restart: no
    image: vllm/vllm-openai:latest
    ipc: host
    volumes:
      - ./models:/models
    command: ["--model", "/models/Qwen3-8B",  "--served-model-name", "qwen3-8b",  "--gpu-memory-utilization", "0.90", "--max-model-len", "8192"]
    ports:
      - 8000:8000
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    environment:
      - LD_LIBRARY_PATH=/lib/x86_64-linux-gnu:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}

docker compose up -d

docker compose logs -f

docker exec -it 【contrainer id】 /bin/bash

docker compose exec dl bash

curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "qwen3-8b",
    "messages": [
      {
        "role": "system",
        "content": "You are a helpful assistant."
      },
      {
        "role": "user",
        "content": "Who won the World Series in 2020?"

      }
    ],
    "stream": true
  }'
 

watch -n 1 "netstat -i | grep ens21f0"

sudo apt install iftop -y 
sudo iftop -i ens21f0