Method submit

tensorrt_llm/executor/ray_executor.py:231–250

Low-level API to the executor. Return a "future" GenerationResult which can be waited. Forwards the request to the workers through RPC.

(self, request: "GenerationRequest")

        return await asyncio.gather(*refs)

    def submit(self, request: "GenerationRequest") -> "GenerationResult":
        """
        Low-level API to the executor. Return a "future" GenerationResult
        which can be waited. Forwards the request to the workers through RPC.
        """
        request.set_id(self._get_next_client_id())
        logprob_params = self._get_logprob_params(request)

        with nvtx_range_debug("rpc_submit"):
            self.rpc_client.submit(request).remote(need_response=False)

        result = GenerationResult(
            request,
            background_error_handler=self._handle_background_error,
            executor=self,
            disaggregated_params=request.disaggregated_params,
            logprob_params=logprob_params)
        self._results[request.id] = result

        return result

    def start(self):
        pass