Building a vLLM Model Serving Management API

Developing a vLLM Model Serving Management API

Deploying a trained model to production requires model serving infrastructure. XGen uses vLLM to serve models and exposes a management API on top of it.

Why vLLM

• PagedAttention: High memory efficiency enables handling more concurrent requests
• OpenAI-compatible API: Drop-in replacement with no changes to existing code
• Continuous Batching: Dynamic batching maximizes throughput
• Broad model support (Llama, Mistral, Qwen, etc.)

Serving Instance Model

class ServingInstance(Base):
    __tablename__ = "serving_instances"

    id = Column(String, primary_key=True)
    model_name = Column(String, nullable=False)
    model_path = Column(String, nullable=False)
    gpu_instance_id = Column(Integer, nullable=True)
    port = Column(Integer, default=8000)
    status = Column(String(20), default="stopped")
    max_model_len = Column(Integer, default=4096)
    tensor_parallel_size = Column(Integer, default=1)
    gpu_memory_utilization = Column(Float, default=0.9)
    created_at = Column(DateTime, default=datetime.utcnow)

vLLM Serving Manager

class VLLMManager:
    async def start_serving(self, config: dict) -> dict:
        instance = await self.gpu_service.get_instance(config["gpu_instance_id"])
        ssh = await self.get_ssh_connection(instance)

        cmd = (
            f"python -m vllm.entrypoints.openai.api_server "
            f"--model {config['model_path']} "
            f"--host 0.0.0.0 "
            f"--port {config.get('port', 8000)} "
            f"--max-model-len {config.get('max_model_len', 4096)} "
            f"--tensor-parallel-size {config.get('tensor_parallel_size', 1)} "
            f"--gpu-memory-utilization {config.get('gpu_memory_utilization', 0.9)} "
            f"--dtype auto"
        )

        # Run the vLLM server in the background
        await ssh.execute(f"nohup {cmd} > /var/log/vllm.log 2>&1 &")

        # Wait for the server to become ready
        await self._wait_for_ready(instance["ip"], config.get("port", 8000))

        return {"status": "running", "endpoint": f"http://{instance['ip']}:{config['port']}"}

    async def _wait_for_ready(self, host: str, port: int, timeout: int = 300):
        import httpx
        start = time.time()
        while time.time() - start < timeout:
            try:
                async with httpx.AsyncClient() as client:
                    resp = await client.get(f"http://{host}:{port}/health")
                    if resp.status_code == 200:
                        return
            except Exception:
                pass
            await asyncio.sleep(5)
        raise TimeoutError("vLLM server startup timed out")

    async def stop_serving(self, instance_id: str):
        instance = await self.get_serving_instance(instance_id)
        ssh = await self.get_ssh_connection(instance)
        await ssh.execute("pkill -f vllm")
        instance.status = "stopped"

Model Serving API

router = APIRouter(prefix="/api/serving", tags=["serving"])

@router.post("/start")
async def start_model_serving(
    model_name: str,
    gpu_instance_id: int,
    max_model_len: int = 4096,
):
    result = await vllm_manager.start_serving({
        "model_path": f"/models/{model_name}",
        "gpu_instance_id": gpu_instance_id,
        "max_model_len": max_model_len,
    })
    return result

@router.post("/{instance_id}/stop")
async def stop_model_serving(instance_id: str):
    await vllm_manager.stop_serving(instance_id)
    return {"status": "stopped"}

@router.get("/{instance_id}/metrics")
async def get_serving_metrics(instance_id: str):
    metrics = await vllm_manager.get_metrics(instance_id)
    return metrics

Proxying Inference Requests

XGen proxies requests to vLLM serving instances, letting users call custom models through the same OpenAI-compatible interface.

@router.post("/api/inference/{instance_id}/chat/completions")
async def proxy_inference(instance_id: str, request: Request):
    instance = await vllm_manager.get_serving_instance(instance_id)
    endpoint = f"http://{instance.ip}:{instance.port}/v1/chat/completions"

    async with httpx.AsyncClient() as client:
        body = await request.json()
        resp = await client.post(endpoint, json=body, timeout=120)
        return resp.json()

Integrating vLLM model serving into the platform closes the loop on XGen's MLOps pipeline, covering everything from training to serving in one place.