Generate output for the given prompt token ids in the asynchronous mode. Asynchronous generation accepts single prompt only.
(
self,
prompt_token_ids: List[int],
sampling_params: SamplingParams,
query_token_ids: Optional[Union[torch.Tensor, np.ndarray, list]] = None,
lora_request: Optional[LoRARequest] = None,
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
streaming: bool = False,
kv_cache_retention_config: Optional[KvCacheRetentionConfig] = None,
disaggregated_params: Optional[DisaggregatedParams] = None,
trace_headers: Optional[Mapping[str, str]] = None,
postproc_params: Optional[PostprocParams] = None,
multimodal_params: Optional[MultimodalParams] = None,
scheduling_params: Optional[SchedulingParams] = None,
cache_salt_id: Optional[int] = None,
arrival_time: Optional[float] = None,
)
| 112 | pass |
| 113 | |
| 114 | def generate_async( |
| 115 | self, |
| 116 | prompt_token_ids: List[int], |
| 117 | sampling_params: SamplingParams, |
| 118 | query_token_ids: Optional[Union[torch.Tensor, np.ndarray, list]] = None, |
| 119 | lora_request: Optional[LoRARequest] = None, |
| 120 | prompt_adapter_request: Optional[PromptAdapterRequest] = None, |
| 121 | streaming: bool = False, |
| 122 | kv_cache_retention_config: Optional[KvCacheRetentionConfig] = None, |
| 123 | disaggregated_params: Optional[DisaggregatedParams] = None, |
| 124 | trace_headers: Optional[Mapping[str, str]] = None, |
| 125 | postproc_params: Optional[PostprocParams] = None, |
| 126 | multimodal_params: Optional[MultimodalParams] = None, |
| 127 | scheduling_params: Optional[SchedulingParams] = None, |
| 128 | cache_salt_id: Optional[int] = None, |
| 129 | arrival_time: Optional[float] = None, |
| 130 | ) -> GenerationResult: |
| 131 | """Generate output for the given prompt token ids in the asynchronous mode. |
| 132 | Asynchronous generation accepts single prompt only. |
| 133 | """ |
| 134 | assert isinstance(prompt_token_ids[0], int) |
| 135 | assert isinstance(sampling_params, SamplingParams) |
| 136 | |
| 137 | self._maybe_initialize_iteration_results() |
| 138 | |
| 139 | if postproc_params: |
| 140 | postproc_params.postproc_args.num_prompt_tokens = len( |
| 141 | prompt_token_ids) |
| 142 | request = GenerationRequest( |
| 143 | prompt_token_ids, |
| 144 | sampling_params=sampling_params, |
| 145 | postproc_params=postproc_params, |
| 146 | query_token_ids=query_token_ids, |
| 147 | lora_request=lora_request, |
| 148 | prompt_adapter_request=prompt_adapter_request, |
| 149 | streaming=streaming, |
| 150 | kv_cache_retention_config=kv_cache_retention_config, |
| 151 | disaggregated_params=disaggregated_params, |
| 152 | trace_headers=trace_headers, |
| 153 | multimodal_params=multimodal_params, |
| 154 | scheduling_params=scheduling_params, |
| 155 | cache_salt_id=cache_salt_id, |
| 156 | arrival_time=arrival_time) |
| 157 | result = self.submit(request) |
| 158 | # release memory in time |
| 159 | if hasattr(request, "multimodal_params"): |
| 160 | del request.multimodal_params |
| 161 | return result |
| 162 | |
| 163 | def generate( |
| 164 | self, |
no test coverage detected