(self, request: GenerationRequest)
| 80 | atexit.register(self.shutdown) |
| 81 | |
| 82 | def submit(self, request: GenerationRequest) -> GenerationResult: |
| 83 | request.set_id(self._get_next_client_id()) |
| 84 | logprob_params = self._get_logprob_params(request) |
| 85 | |
| 86 | # submit is a fire-and-forget operation, don't need to wait for response |
| 87 | with nvtx_range_debug("RPCExecutor.submit", color="green", category="Proxy"): |
| 88 | self.rpc_client.submit(request).remote(need_response=False) |
| 89 | |
| 90 | result = GenerationResult( |
| 91 | request, |
| 92 | background_error_handler=self._handle_background_error, |
| 93 | executor=self, |
| 94 | disaggregated_params=request.disaggregated_params, |
| 95 | logprob_params=logprob_params, |
| 96 | ) |
| 97 | self._results[request.id] = result |
| 98 | |
| 99 | return result |
| 100 | |
| 101 | def handle_responses(self, responses: list[GenerationResult]) -> bool: |
| 102 | async_queues = [] |
nothing calls this directly
no test coverage detected