MCPcopy Create free account
hub / github.com/pytorch/executorch / _evaluate

Function _evaluate

backends/arm/scripts/evaluate_model.py:241–286  ·  view source on GitHub ↗
(
    args, model_name, ref_model, eval_model, example_inputs
)

Source from the content-addressed store, hash-verified

239
240
241def _evaluate(
242 args, model_name, ref_model, eval_model, example_inputs
243) -> dict[str, Any]:
244 evaluators: list[Evaluator] = []
245
246 # Add evaluator for compression ratio of TOSA file
247 intermediates_path = Path(args.intermediates)
248 tosa_paths = list(intermediates_path.glob("*.tosa"))
249 if tosa_paths:
250 evaluators.append(FileCompressionEvaluator(model_name, str(tosa_paths[0])))
251 else:
252 logging.warning(
253 f"No TOSA file found in {args.intermediates} for compression evaluation"
254 )
255
256 # Add user-specified evaluators
257 for evaluator_name in args.evaluators:
258 evaluator: Evaluator
259 match evaluator_name:
260 case "numerical":
261 evaluator = NumericalModelEvaluator(
262 model_name,
263 ref_model,
264 eval_model,
265 example_inputs,
266 eval_dtype=_DTYPE_MAP.get(args.dtype, None),
267 )
268 case "imagenet":
269 evaluator = ImageNetEvaluator(
270 model_name,
271 eval_model,
272 batch_size=args.batch_size,
273 validation_dataset_path=args.evaluation_dataset,
274 eval_dtype=_DTYPE_MAP.get(args.dtype, None),
275 )
276 case _:
277 raise AssertionError(f"Unknown evaluator {evaluator_name}")
278 evaluators.append(evaluator)
279
280 # Run evaluators
281 metrics: dict[str, Any] = {}
282 for evaluator in evaluators:
283 result = evaluator.evaluate()
284 metrics |= result
285
286 return metrics
287
288
289def main() -> None:

Callers 1

mainFunction · 0.85

Calls 6

evaluateMethod · 0.95
ImageNetEvaluatorClass · 0.90
appendMethod · 0.45
getMethod · 0.45

Tested by

no test coverage detected