Determine if the model should be saved based on the evaluation metrics. Returns: bool: True if a new best metric was found, else False
(self, metrics: dict[str, float], trial: "optuna.Trial | dict[str, Any] | None")
| 3127 | ) |
| 3128 | |
| 3129 | def _determine_best_metric(self, metrics: dict[str, float], trial: "optuna.Trial | dict[str, Any] | None") -> bool: |
| 3130 | """ |
| 3131 | Determine if the model should be saved based on the evaluation metrics. |
| 3132 | |
| 3133 | Returns: |
| 3134 | bool: True if a new best metric was found, else False |
| 3135 | """ |
| 3136 | is_new_best_metric = False |
| 3137 | |
| 3138 | if self.args.metric_for_best_model is not None: |
| 3139 | metric_to_check = self.args.metric_for_best_model |
| 3140 | |
| 3141 | if not metric_to_check.startswith("eval_"): |
| 3142 | metric_to_check = f"eval_{metric_to_check}" |
| 3143 | |
| 3144 | try: |
| 3145 | metric_value = metrics[metric_to_check] |
| 3146 | except KeyError as exc: |
| 3147 | raise KeyError( |
| 3148 | f"The `metric_for_best_model` training argument is set to '{metric_to_check}', which is not found in the evaluation metrics. " |
| 3149 | f"The available evaluation metrics are: {list(metrics.keys())}. Consider changing the `metric_for_best_model` via the TrainingArguments." |
| 3150 | ) from exc |
| 3151 | |
| 3152 | operator = np.greater if self.args.greater_is_better else np.less |
| 3153 | |
| 3154 | if self.state.best_metric is None: |
| 3155 | self.state.best_metric = float("-inf") if self.args.greater_is_better else float("inf") |
| 3156 | |
| 3157 | if operator(metric_value, self.state.best_metric): |
| 3158 | self.state.best_metric = metric_value |
| 3159 | |
| 3160 | if self.args.save_strategy in [SaveStrategy.STEPS, SaveStrategy.EPOCH, SaveStrategy.BEST]: |
| 3161 | self.state.best_global_step = self.state.global_step |
| 3162 | |
| 3163 | is_new_best_metric = True |
| 3164 | |
| 3165 | return is_new_best_metric |
| 3166 | |
| 3167 | def _save_rng_state(self, output_dir: str) -> None: |
| 3168 | """Save random number generator states for reproducible resumption.""" |
no test coverage detected