(
args, model_name, ref_model, eval_model, example_inputs
)
| 239 | |
| 240 | |
| 241 | def _evaluate( |
| 242 | args, model_name, ref_model, eval_model, example_inputs |
| 243 | ) -> dict[str, Any]: |
| 244 | evaluators: list[Evaluator] = [] |
| 245 | |
| 246 | # Add evaluator for compression ratio of TOSA file |
| 247 | intermediates_path = Path(args.intermediates) |
| 248 | tosa_paths = list(intermediates_path.glob("*.tosa")) |
| 249 | if tosa_paths: |
| 250 | evaluators.append(FileCompressionEvaluator(model_name, str(tosa_paths[0]))) |
| 251 | else: |
| 252 | logging.warning( |
| 253 | f"No TOSA file found in {args.intermediates} for compression evaluation" |
| 254 | ) |
| 255 | |
| 256 | # Add user-specified evaluators |
| 257 | for evaluator_name in args.evaluators: |
| 258 | evaluator: Evaluator |
| 259 | match evaluator_name: |
| 260 | case "numerical": |
| 261 | evaluator = NumericalModelEvaluator( |
| 262 | model_name, |
| 263 | ref_model, |
| 264 | eval_model, |
| 265 | example_inputs, |
| 266 | eval_dtype=_DTYPE_MAP.get(args.dtype, None), |
| 267 | ) |
| 268 | case "imagenet": |
| 269 | evaluator = ImageNetEvaluator( |
| 270 | model_name, |
| 271 | eval_model, |
| 272 | batch_size=args.batch_size, |
| 273 | validation_dataset_path=args.evaluation_dataset, |
| 274 | eval_dtype=_DTYPE_MAP.get(args.dtype, None), |
| 275 | ) |
| 276 | case _: |
| 277 | raise AssertionError(f"Unknown evaluator {evaluator_name}") |
| 278 | evaluators.append(evaluator) |
| 279 | |
| 280 | # Run evaluators |
| 281 | metrics: dict[str, Any] = {} |
| 282 | for evaluator in evaluators: |
| 283 | result = evaluator.evaluate() |
| 284 | metrics |= result |
| 285 | |
| 286 | return metrics |
| 287 | |
| 288 | |
| 289 | def main() -> None: |
no test coverage detected