Create a completion for the provided prompt and parameters.
(request: CompletionRequest, req: Request)
| 575 | @app.post("/v1/completions") |
| 576 | @with_cancellation |
| 577 | async def create_completion(request: CompletionRequest, req: Request): |
| 578 | """ |
| 579 | Create a completion for the provided prompt and parameters. |
| 580 | """ |
| 581 | api_server_logger.info(f"Completion Received request: {request.model_dump_json()}") |
| 582 | if envs.TRACES_ENABLE: |
| 583 | if req.headers: |
| 584 | headers = dict(req.headers) |
| 585 | trace_context = extract(headers) |
| 586 | request.trace_context = trace_context |
| 587 | if app.state.dynamic_load_weight: |
| 588 | status, msg = app.state.engine_client.is_workers_alive() |
| 589 | if not status: |
| 590 | return JSONResponse(content={"error": "Worker Service Not Healthy"}, status_code=304) |
| 591 | try: |
| 592 | async with connection_manager(): |
| 593 | tracing.label_span(request) |
| 594 | generator = await app.state.completion_handler.create_completion(request) |
| 595 | if isinstance(generator, ErrorResponse): |
| 596 | connection_semaphore.release() |
| 597 | return JSONResponse(content=generator.model_dump(), status_code=500) |
| 598 | elif isinstance(generator, CompletionResponse): |
| 599 | connection_semaphore.release() |
| 600 | return JSONResponse(content=generator.model_dump()) |
| 601 | else: |
| 602 | wrapped_generator = wrap_streaming_generator(generator) |
| 603 | return StreamingResponse(content=wrapped_generator(), media_type="text/event-stream") |
| 604 | except HTTPException as e: |
| 605 | return JSONResponse(status_code=e.status_code, content={"detail": e.detail}) |
| 606 | |
| 607 | |
| 608 | @app.get("/v1/models") |
nothing calls this directly
no test coverage detected