Function chat

examples/streaming_chat/main.py:28–61

Stream a chat response using Server-Sent Events. The response is streamed token-by-token, simulating LLM inference. Each token is sent as an SSE event with JSON data. Args: request: Chat request with prompt and optional thinking mode Returns: StreamingResponse with

(request: ChatRequest)

@app.post("/chat")
async def chat(request: ChatRequest):
    """Stream a chat response using Server-Sent Events.

    The response is streamed token-by-token, simulating LLM inference.
    Each token is sent as an SSE event with JSON data.

    Args:
        request: Chat request with prompt and optional thinking mode

    Returns:
        StreamingResponse with text/event-stream content type
    """
    client = await get_dirty_client_async()
    action = "generate_with_thinking" if request.thinking else "generate"

    async def stream():
        async for token in client.stream_async(
            "streaming_chat.chat_app:ChatApp",
            action,
            request.prompt
        ):
            data = json.dumps({"token": token})
            yield f"data: {data}\n\n"
        yield "data: [DONE]\n\n"

    return StreamingResponse(
        stream(),
        media_type="text/event-stream",
        headers={
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "X-Accel-Buffering": "no",  # Disable nginx buffering
        }
    )


@app.post("/chat/sync", response_model=ChatResponse)