(self,
num_postprocess_workers: int = 0,
postprocess_tokenizer_dir: Optional[str] = None,
is_llm_executor: Optional[bool] = None)
| 78 | class GenerationExecutor(ABC): |
| 79 | |
| 80 | def __init__(self, |
| 81 | num_postprocess_workers: int = 0, |
| 82 | postprocess_tokenizer_dir: Optional[str] = None, |
| 83 | is_llm_executor: Optional[bool] = None): |
| 84 | self.postproc_config = PostprocWorkerConfig( |
| 85 | num_postprocess_workers=num_postprocess_workers, |
| 86 | postprocess_tokenizer_dir=postprocess_tokenizer_dir) |
| 87 | |
| 88 | self.kv_events_queues = IterationResultQueue() |
| 89 | self.stats_queues = IterationResultQueue() |
| 90 | |
| 91 | atexit.register(self.shutdown) |
| 92 | |
| 93 | # This is used to capture the exceptions from the threads. |
| 94 | self._error_queue = Queue() |
| 95 | |
| 96 | # A flag to avoid calling shutdown() recursively. This happens when the background threads raise errors. |
| 97 | self.doing_shutdown = False |
| 98 | |
| 99 | self._last_client_id: int = 1 |
| 100 | |
| 101 | # whether it's the executor instance of LLM API |
| 102 | self._is_llm_executor = is_llm_executor |
| 103 | self._iter_kv_events_result: IterationResult | None = None |
| 104 | self._iter_stats_result: IterationResult | None = None |
| 105 | |
| 106 | @abstractmethod |
| 107 | def submit(self, request: GenerationRequest) -> GenerationResult: |
nothing calls this directly
no test coverage detected