| 159 | |
| 160 | |
| 161 | class DataProcessor(BaseDataProcessor): |
| 162 | def __init__(self, model_name_or_path, reasoning_parser_obj=None, tool_parser_obj=None): |
| 163 | """ |
| 164 | Initializes the DecodeStatus object. |
| 165 | |
| 166 | Args: |
| 167 | model_name_or_path (str): The name or path of the pre-trained model to be loaded. |
| 168 | Can also be a path to a directory containing the pre-trained model file. |
| 169 | |
| 170 | Returns: |
| 171 | None. |
| 172 | |
| 173 | Raises: |
| 174 | None. |
| 175 | """ |
| 176 | |
| 177 | self.model_name_or_path = model_name_or_path |
| 178 | |
| 179 | # Generation config |
| 180 | try: |
| 181 | self.generation_config = GenerationConfig.from_pretrained(self.model_name_or_path) |
| 182 | except Exception as e: |
| 183 | data_processor_logger.warning( |
| 184 | f"Can't find generation config: {e}, so it will not use generation_config field in the model config" |
| 185 | ) |
| 186 | self.generation_config = None |
| 187 | |
| 188 | self.decode_status = dict() |
| 189 | self.model_status_dict = dict() |
| 190 | self.tool_parser_dict = dict() |
| 191 | self.tokenizer = self._load_tokenizer() |
| 192 | data_processor_logger.info( |
| 193 | f"tokenizer information: bos_token is {self.tokenizer.bos_token}, {self.tokenizer.bos_token_id}, \ |
| 194 | eos_token is {self.tokenizer.eos_token}, {self.tokenizer.eos_token_id} " |
| 195 | ) |
| 196 | |
| 197 | try: |
| 198 | from paddleformers.trl.llm_utils import get_eos_token_id |
| 199 | except Exception: |
| 200 | from paddleformers.cli.utils.llm_utils import get_eos_token_id |
| 201 | |
| 202 | self.eos_token_ids = get_eos_token_id(self.tokenizer, self.generation_config) |
| 203 | data_processor_logger.info( |
| 204 | f"The eos_token_ids obtained by merging tokenizer and generation_config is {self.eos_token_ids}" |
| 205 | ) |
| 206 | self.eos_token_id_len = len(self.eos_token_ids) |
| 207 | self.pad_token_id = self.get_pad_id() |
| 208 | self.reasoning_parser = None |
| 209 | self.tool_parser_obj = tool_parser_obj |
| 210 | if reasoning_parser_obj: |
| 211 | self.reasoning_parser = reasoning_parser_obj(self.tokenizer) |
| 212 | self.tokenizer.pad_token_id = self.pad_token_id |
| 213 | |
| 214 | def process_request(self, request, max_model_len=None, **kwargs): |
| 215 | """ |
| 216 | Preprocess the request |
| 217 | |
| 218 | Args: |
no outgoing calls