linux系统使用vllm搭建本地模型的详细步骤

# 下载 Miniconda (Python 3.9 版本)
wget https://repo.anaconda.com/miniconda/Miniconda3-py39_25.1.1-2-Linux-x86_64.sh
bash Miniconda3-py39_25.1.1-2-Linux-x86_64.sh -p /path/to/conda_base
# 配置环境变量
echo 'export PATH="/path/to/conda_base/bin:$PATH"' >> ~/.bashrc
source ~/.bashrc
# 创建新的虚拟环境 (Python 3.10 ~ 3.12 均可)
conda create -n vllm_env python=3.10 -y
conda activate vllm_env

# 安装 PyTorch (官方推荐)
pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124
# 直接安装 vLLM (会同步安装依赖)
pip install vllm

# 安装 modelscope
pip install modelscope
# 下载模型示例：Qwen2.5-7B-Instruct 或 DeepSeek-R1
# 模型会默认下载到 ~/.cache/modelscope/hub/models/
modelscope download --model Qwen/Qwen2.5-7B-Instruct
# 或者下载更大参数的模型
modelscope download --model deepseek-ai/DeepSeek-R1-Distill-Qwen-32B

modelscope download --model <模型ID> --local_dir <你想要的路径>

modelscope download --model Qwen/Qwen2.5-7B-Instruct --local_dir /home/user/models/Qwen2.5-7B-Instruct

modelscope download --model Qwen/Qwen2.5-7B-Instruct config.json --local_dir /home/user/models/

export MODELSCOPE_CACHE=/home/user/my_model_cache
modelscope download --model Qwen/Qwen2.5-7B-Instruct

echo "export MODELSCOPE_CACHE=/home/user/my_model_cache" >> ~/.bashrc
source ~/.bashrc
# 然后直接下载
modelscope download --model Qwen/Qwen2.5-7B-Instruct

# 安装 huggingface_hub
pip install huggingface_hub

# 登录 (如果需要 gated 模型)
huggingface-cli login

# 下载模型
huggingface-cli download deepseek-ai/DeepSeek-R1-Distill-Qwen-32B --local-dir ./models/DeepSeek-R1-Distill-Qwen-32B

python -m vllm.entrypoints.openai.api_server \
    --model /root/.cache/modelscope/hub/models/Qwen/Qwen2.5-7B-Instruct \
    --served-model-name Qwen2.5-7B \
    --tensor-parallel-size 1 \
    --gpu-memory-utilization 0.9 \
    --max-model-len 4096 \
    --port 8000 \
    --host 0.0.0.0

# 假设有 4 张 GPU
export CUDA_VISIBLE_DEVICES=0,1,2,3
python -m vllm.entrypoints.openai.api_server --model Qwen/Qwen2.5-72B-Instruct --tensor-parallel-size 4

export VLLM_USE_V1=0

nohup python -m vllm.entrypoints.openai.api_server --model ... > vllm.log 2>&1 &

curl http://localhost:8000/v1/chat/completions \
    -H "Content-Type: application/json" \
    -d '{
        "model": "Qwen2.5-7B",
        "messages": [
            {"role": "user", "content": "你好，请介绍一下你自己"}
        ],
        "temperature": 0.7
    }'

from openai import OpenAI
client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="EMPTY"  # vLLM 默认不需要 API Key
)
response = client.chat.completions.create(
    model="Qwen2.5-7B",
    messages=[{"role": "user", "content": "写一首关于秋天的诗"}]
)
print(response.choices[0].message.content)

# 安装依赖
pip install pandas datasets
# 运行压测 (需要提前下载 benchmark 脚本)
python3 vllm/benchmarks/benchmark_serving.py \
    --backend vllm \
    --model /path/to/model \
    --dataset-name sharegpt \
    --request-rate inf \
    --num-prompts 100