Serve a Fine-Tuned LLM as a Production API with vLLM

python -m venv vllm-env && source vllm-env/bin/activate

# vLLM 0.6.3+ supports native LoRA adapter loading (no merge required)
pip install vllm==0.6.3

# For testing
pip install openai>=1.0 aiohttp

cat my-falcon-lora/adapter_config.json

{
  "base_model_name_or_path": "tiiuae/falcon-7b-instruct",
  "r": 16,
  "lora_alpha": 32,
  "target_modules": ["query_key_value", "dense"],
  "task_type": "CAUSAL_LM"
}

vllm serve tiiuae/falcon-7b-instruct \
    --host 0.0.0.0 \
    --port 8000 \
    --enable-lora \
    --max-loras 2 \
    --max-lora-rank 64 \
    --lora-modules my-falcon-lora=./my-falcon-lora \
    --max-model-len 2048 \
    --gpu-memory-utilization 0.90 \
    --dtype float16 \
    --trust-remote-code \
    --served-model-name falcon-7b-lora

curl http://localhost:8000/v1/models

{
  "object": "list",
  "data": [{"id": "falcon-7b-lora", "object": "model"}]
}

curl -s http://localhost:8000/v1/chat/completions \
    -H "Content-Type: application/json" \
    -d '{
        "model": "falcon-7b-lora",
        "messages": [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "What is the capital of France?"}
        ],
        "temperature": 0.7,
        "max_tokens": 128
    }' | python -m json.tool

from openai import OpenAI
import time

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="not-needed",  # vLLM doesn't require auth by default
)

start = time.time()
response = client.chat.completions.create(
    model="falcon-7b-lora",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Explain the difference between FP16 and BF16 in one sentence."},
    ],
    temperature=0.7,
    max_tokens=128,
)
elapsed = time.time() - start

print(f"Response: {response.choices[0].message.content}")
print(f"Tokens: {response.usage.total_tokens}")
print(f"Latency: {elapsed:.2f}s")

stream = client.chat.completions.create(
    model="falcon-7b-lora",
    messages=[{"role": "user", "content": "Count from 1 to 10."}],
    stream=True,
    max_tokens=64,
)

first_token_time = None
for chunk in stream:
    if chunk.choices[0].delta.content:
        if first_token_time is None:
            first_token_time = time.time()
        print(chunk.choices[0].delta.content, end="", flush=True)

"""
Simple concurrent load test. Measures throughput, latency percentiles, TTFT.
Usage: python load_test.py --concurrency 8 --requests 100
"""
import time, asyncio, statistics, argparse
import aiohttp

async def make_request(session, url, payload, results):
    start = time.time()
    try:
        async with session.post(url, json=payload, timeout=aiohttp.ClientTimeout(total=120)) as resp:
            if resp.status != 200:
                results["errors"] += 1
                return
            data = await resp.json()
            elapsed = time.time() - start
            results["latencies"].append(elapsed)
            results["tokens"] += data.get("usage", {}).get("total_tokens", 0)
    except Exception:
        results["errors"] += 1

async def run(url, concurrency, total_requests, max_tokens=128):
    payload = {
        "model": "falcon-7b-lora",
        "messages": [{"role": "user", "content": "Write a short paragraph about the history of computing."}],
        "max_tokens": max_tokens,
        "temperature": 0.7,
    }
    results = {"latencies": [], "tokens": 0, "errors": 0}
    sem = asyncio.Semaphore(concurrency)

    async def bounded(session):
        async with sem:
            await make_request(session, url, payload, results)

    connector = aiohttp.TCPConnector(limit=concurrency)
    async with aiohttp.ClientSession(connector=connector) as session:
        start = time.time()
        await asyncio.gather(*[bounded(session) for _ in range(total_requests)])
        wall = time.time() - start

    lats = sorted(results["latencies"])
    n = len(lats)
    print(f"Requests: {total_requests} | Success: {n} | Errors: {results['errors']}")
    print(f"Wall time: {wall:.1f}s | RPS: {total_requests/wall:.1f}")
    print(f"Aggregate throughput: {results['tokens']/wall:.0f} tokens/s")
    if n:
        print(f"P50: {lats[n//2]:.2f}s | P95: {lats[int(n*0.95)]:.2f}s | P99: {lats[int(n*0.99)]:.2f}s")

if __name__ == "__main__":
    p = argparse.ArgumentParser()
    p.add_argument("--concurrency", type=int, default=8)
    p.add_argument("--requests", type=int, default=100)
    p.add_argument("--max-tokens", type=int, default=128)
    args = p.parse_args()
    asyncio.run(run("http://localhost:8000/v1/chat/completions",
                    args.concurrency, args.requests, args.max_tokens))

pip install aiohttp
python load_test.py --concurrency 8 --requests 100

FROM vllm/vllm-openai:0.6.3

RUN pip install --no-cache-dir peft==0.11.1 aiohttp

COPY my-falcon-lora/ /opt/model/my-falcon-lora/
COPY entrypoint.py /opt/entrypoint.py

WORKDIR /opt
EXPOSE 8000

HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \
    CMD curl -f http://localhost:8000/health || exit 1

ENTRYPOINT ["python", "/opt/entrypoint.py"]

import subprocess, sys, os, signal

def main():
    model = os.environ.get("VLLM_MODEL", "tiiuae/falcon-7b-instruct")
    lora_path = os.environ.get("VLLM_LORA_PATH", "/opt/model/my-falcon-lora")
    lora_name = os.environ.get("VLLM_LORA_NAME", "my-falcon-lora")

    cmd = [
        "vllm", "serve", model,
        "--host", "0.0.0.0", "--port", "8000",
        "--enable-lora", "--max-loras", "2", "--max-lora-rank", "64",
        "--lora-modules", f"{lora_name}={lora_path}",
        "--max-model-len", os.environ.get("VLLM_MAX_MODEL_LEN", "2048"),
        "--gpu-memory-utilization", os.environ.get("VLLM_GPU_MEM_UTIL", "0.90"),
        "--dtype", "float16",
        "--trust-remote-code",
        "--served-model-name", "falcon-7b-lora",
    ]

    print(f"Starting: {' '.join(cmd)}", flush=True)
    proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)

    def shutdown(sig, frame):
        proc.terminate()
        try: proc.wait(timeout=30)
        except: proc.kill()

    signal.signal(signal.SIGTERM, shutdown)
    signal.signal(signal.SIGINT, shutdown)
    proc.wait()

if __name__ == "__main__":
    main()

docker build -t falcon-7b-lora:v1.0 .

docker run -d \
    --name falcon-lora \
    --gpus all \
    --shm-size 16g \
    -p 8000:8000 \
    --restart unless-stopped \
    falcon-7b-lora:v1.0

# Verify
docker logs -f falcon-lora
curl http://localhost:8000/health

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(
    "tiiuae/falcon-7b-instruct",
    torch_dtype="auto",
    device_map="auto",
)
model = PeftModel.from_pretrained(model, "./my-falcon-lora")
model = model.merge_and_unload()
model.save_pretrained("./falcon-7b-merged")

tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b-instruct")
tokenizer.save_pretrained("./falcon-7b-merged")

groups:
  - name: vllm
    rules:
      - alert: VLLMHighTTFT
        expr: histogram_quantile(0.95, rate(vllm:time_to_first_token_seconds_bucket[5m])) > 2.0
        for: 2m
        labels: {severity: warning}

      - alert: VLLMQueueBacklog
        expr: vllm:num_requests_waiting > 10
        for: 1m
        labels: {severity: warning}

      - alert: VLLMCachePressure
        expr: vllm:gpu_cache_usage > 0.95
        for: 5m
        labels: {severity: critical}