(self)
| 290 | self.request_queue.put_noblock(None, retry=4) |
| 291 | |
| 292 | def shutdown(self): |
| 293 | if not self.workers_started: |
| 294 | return |
| 295 | |
| 296 | if not self.doing_shutdown: |
| 297 | self.pre_shutdown() |
| 298 | |
| 299 | logger_debug('Proxy.shutdown...\n', "yellow") |
| 300 | |
| 301 | for f in self.mpi_futures: |
| 302 | try: |
| 303 | f.result() |
| 304 | except: |
| 305 | # The errors are already captured in mpi_done_callback, ignored |
| 306 | # here |
| 307 | pass |
| 308 | |
| 309 | # step2: notify the background threads to quit |
| 310 | if self.dispatch_result_thread is not None and self.dispatch_result_thread.is_alive( |
| 311 | ): |
| 312 | self.dispatch_result_thread.stop() |
| 313 | self.dispatch_result_thread.join() |
| 314 | |
| 315 | # step3: finish all remaining work |
| 316 | |
| 317 | # close the RPC client |
| 318 | if self.rpc_client is not None: |
| 319 | self.rpc_client.close() |
| 320 | self.rpc_client = None |
| 321 | |
| 322 | # close all the sockets |
| 323 | self.request_queue.close() |
| 324 | self.worker_init_status_queue.close() |
| 325 | self.result_queue.close() |
| 326 | |
| 327 | self.workers_started = False |
| 328 | self.mpi_session.shutdown() |
| 329 | |
| 330 | # Process the errors in-case error during shutting down the threads |
| 331 | self._handle_background_error() |
| 332 | |
| 333 | if enable_llm_debug(): |
| 334 | print_alive_threads() |
| 335 | |
| 336 | def submit(self, request: GenerationRequest) -> GenerationResult: |
| 337 | """ |
no test coverage detected