Veco evaluates the datasets one by one.
(self, checkpoint_path=None)
| 147 | class VecoTrainer(NlpEpochBasedTrainer): |
| 148 | |
| 149 | def evaluate(self, checkpoint_path=None): |
| 150 | """Veco evaluates the datasets one by one. |
| 151 | |
| 152 | """ |
| 153 | from modelscope.msdatasets.dataset_cls.custom_datasets import VecoDataset |
| 154 | if checkpoint_path is not None: |
| 155 | from modelscope.trainers.hooks import LoadCheckpointHook |
| 156 | LoadCheckpointHook.load_checkpoint(checkpoint_path, self) |
| 157 | self.model.eval() |
| 158 | self._mode = ModeKeys.EVAL |
| 159 | metric_values = {} |
| 160 | |
| 161 | if self.eval_dataset is None: |
| 162 | self.eval_dataset = self.build_dataset_from_cfg( |
| 163 | model_cfg=self.cfg, |
| 164 | mode=self._mode, |
| 165 | preprocessor=self.eval_preprocessor) |
| 166 | |
| 167 | idx = 0 |
| 168 | dataset_cnt = 1 |
| 169 | if isinstance(self.eval_dataset, VecoDataset): |
| 170 | self.eval_dataset.switch_dataset(idx) |
| 171 | dataset_cnt = len(self.eval_dataset.datasets) |
| 172 | |
| 173 | while True: |
| 174 | self.eval_dataloader = self._build_dataloader_with_dataset( |
| 175 | self.eval_dataset, **self.cfg.evaluation.get('dataloader', {})) |
| 176 | self.data_loader = self.eval_dataloader |
| 177 | |
| 178 | metric_classes = [build_metric(metric) for metric in self.metrics] |
| 179 | for m in metric_classes: |
| 180 | m.trainer = self |
| 181 | self.evaluation_loop(self.eval_dataloader, metric_classes) |
| 182 | |
| 183 | for m_idx, metric_cls in enumerate(metric_classes): |
| 184 | if f'eval_dataset[{idx}]' not in metric_values: |
| 185 | metric_values[f'eval_dataset[{idx}]'] = {} |
| 186 | metric_values[f'eval_dataset[{idx}]'][ |
| 187 | self.metrics[m_idx]] = metric_cls.evaluate() |
| 188 | |
| 189 | idx += 1 |
| 190 | if idx < dataset_cnt: |
| 191 | self.eval_dataset.switch_dataset(idx) |
| 192 | else: |
| 193 | break |
| 194 | |
| 195 | for metric_name in self.metrics: |
| 196 | all_metrics = [m[metric_name] for m in metric_values.values()] |
| 197 | for key in all_metrics[0].keys(): |
| 198 | metric_values[key] = np.average( |
| 199 | [metric[key] for metric in all_metrics]) |
| 200 | |
| 201 | return metric_values |
nothing calls this directly
no test coverage detected