This is the official recommended flow for quantization in pytorch 2.0 export.
(
model: GraphModule,
model_name: str,
compile_specs: ArmCompileSpec,
example_inputs: Tuple[torch.Tensor],
quant_mode: QuantMode = QuantMode.INT8,
calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]] = None,
)
| 335 | |
| 336 | |
| 337 | def quantize( |
| 338 | model: GraphModule, |
| 339 | model_name: str, |
| 340 | compile_specs: ArmCompileSpec, |
| 341 | example_inputs: Tuple[torch.Tensor], |
| 342 | quant_mode: QuantMode = QuantMode.INT8, |
| 343 | calibration_samples: Optional[List[Tuple[torch.Tensor, ...]]] = None, |
| 344 | ) -> GraphModule: |
| 345 | """This is the official recommended flow for quantization in pytorch 2.0 |
| 346 | export. |
| 347 | """ |
| 348 | logging.info("Quantizing Model...") |
| 349 | logging.debug(f"Original model: {model}") |
| 350 | |
| 351 | quantizer = create_quantizer(compile_specs) |
| 352 | |
| 353 | match quant_mode: |
| 354 | case QuantMode.INT8: |
| 355 | operator_config = get_symmetric_quantization_config(is_per_channel=True) |
| 356 | case QuantMode.A16W8: |
| 357 | if compile_specs.tosa_spec.support_extension("int16"): |
| 358 | operator_config = get_symmetric_a16w8_quantization_config( |
| 359 | is_per_channel=True |
| 360 | ) |
| 361 | else: |
| 362 | raise ValueError( |
| 363 | f"Context TOSA spec {compile_specs.tosa_spec} doesn't support int16" |
| 364 | ) |
| 365 | case _: |
| 366 | raise ValueError(f"Unsupported quantization mode: {quant_mode}") |
| 367 | |
| 368 | quantizer.set_global(operator_config) |
| 369 | m = prepare_pt2e(model, quantizer) |
| 370 | |
| 371 | if calibration_samples is None: |
| 372 | calibration_samples = [example_inputs] |
| 373 | |
| 374 | for sample in calibration_samples: |
| 375 | _validate_calibration_sample(sample, example_inputs) |
| 376 | m(*sample) |
| 377 | |
| 378 | m = convert_pt2e(m) |
| 379 | logging.debug(f"Quantized model: {m}") |
| 380 | return m |
| 381 | |
| 382 | |
| 383 | class QuantAddTest(torch.nn.Module): |
no test coverage detected