start gpu worker service
(self)
| 503 | return command_prefix |
| 504 | |
| 505 | def _start_worker_service(self): |
| 506 | """ |
| 507 | start gpu worker service |
| 508 | |
| 509 | """ |
| 510 | console_logger.debug("Start worker process...") |
| 511 | log_dir = os.getenv("FD_LOG_DIR", default="log") |
| 512 | command_prefix = self._setting_environ_variables() |
| 513 | current_file_path = os.path.abspath(__file__) |
| 514 | current_dir_path = os.path.split(current_file_path)[0] |
| 515 | # TODO |
| 516 | uncache_worker_stdout = "" if os.getenv("UNCACHE_WORKER_STDOUT", "0") == 1 else "-u" |
| 517 | pd_cmd = f"{command_prefix} {sys.executable} {uncache_worker_stdout} -m paddle.distributed.launch" |
| 518 | pd_cmd = pd_cmd + f" --log_dir {log_dir}" |
| 519 | |
| 520 | worker_path = "../worker/worker_process.py" |
| 521 | py_script = os.path.join(current_dir_path, worker_path) |
| 522 | |
| 523 | ori_vocab_size = ( |
| 524 | len(self.engine.data_processor.tokenizer.sp_model) |
| 525 | if hasattr(self.engine.data_processor.tokenizer, "sp_model") |
| 526 | else len(self.engine.data_processor.tokenizer.vocab) |
| 527 | ) |
| 528 | |
| 529 | think_end_id = self.data_processor.tokenizer.get_vocab().get("</think>", -1) |
| 530 | if think_end_id > 0: |
| 531 | llm_logger.info(f"Get think_end_id {think_end_id} from vocab.") |
| 532 | else: |
| 533 | llm_logger.info("No </think> token found in vocabulary, the model can not do reasoning.") |
| 534 | image_patch_id = self.data_processor.tokenizer.get_vocab().get("<|IMAGE_PLACEHOLDER|>", -1) |
| 535 | line_break_id = self.data_processor.tokenizer.get_vocab().get("\n", -1) |
| 536 | |
| 537 | try: |
| 538 | think_truncate_prompt_ids = self.data_processor.tokenizer.convert_tokens_to_ids( |
| 539 | self.data_processor.tokenizer.tokenize(self.data_processor.tokenizer.think_truncate_prompt) |
| 540 | ) |
| 541 | except Exception: |
| 542 | think_truncate_prompt_ids = self.data_processor.tokenizer.convert_tokens_to_ids( |
| 543 | self.data_processor.tokenizer.tokenize(envs.FD_LIMIT_THINKING_CONTENT_TRUNCATE_STR) |
| 544 | ) |
| 545 | llm_logger.info(f"Get think_truncate_prompt_ids {think_truncate_prompt_ids} from tokenizer.") |
| 546 | |
| 547 | ports = ",".join(map(str, self.cfg.parallel_config.engine_worker_queue_port)) |
| 548 | ips = None |
| 549 | if self.cfg.ips is not None: |
| 550 | ips = ",".join(self.cfg.ips) |
| 551 | arguments = ( |
| 552 | f" --devices {self.cfg.parallel_config.device_ids} {py_script}" |
| 553 | f" --max_num_seqs {self.cfg.scheduler_config.max_num_seqs} --max_model_len {self.cfg.model_config.max_model_len}" |
| 554 | f" --gpu_memory_utilization {self.cfg.cache_config.gpu_memory_utilization}" |
| 555 | f" --model {self.cfg.model_config.model!s}" |
| 556 | f" --device_ids {self.cfg.parallel_config.device_ids}" |
| 557 | f" --tensor_parallel_size {self.cfg.parallel_config.tensor_parallel_size}" |
| 558 | f" --engine_worker_queue_port {ports}" |
| 559 | f" --pod_ip {self.cfg.master_ip}" |
| 560 | f" --block_size {self.cfg.cache_config.block_size}" |
| 561 | f" --enc_dec_block_num {self.cfg.cache_config.enc_dec_block_num}" |
| 562 | f" --eos_tokens_lens {self.engine.data_processor.eos_token_id_len}" |
no test coverage detected