Test model in EpochBasedTrainer with multiple gpus. This method tests model with multiple gpus and collects the results under two different modes: gpu and cpu modes. By setting ``gpu_collect=True``, it encodes results to gpu tensors and use gpu communication for results collection.
(trainer,
data_loader,
device,
metric_classes=None,
vis_closure=None,
tmpdir=None,
gpu_collect=False,
data_loader_iters_per_gpu=None)
| 78 | |
| 79 | |
| 80 | def multi_gpu_test(trainer, |
| 81 | data_loader, |
| 82 | device, |
| 83 | metric_classes=None, |
| 84 | vis_closure=None, |
| 85 | tmpdir=None, |
| 86 | gpu_collect=False, |
| 87 | data_loader_iters_per_gpu=None): |
| 88 | """Test model in EpochBasedTrainer with multiple gpus. |
| 89 | |
| 90 | This method tests model with multiple gpus and collects the results |
| 91 | under two different modes: gpu and cpu modes. By setting |
| 92 | ``gpu_collect=True``, it encodes results to gpu tensors and use gpu |
| 93 | communication for results collection. On cpu mode it saves the results on |
| 94 | different gpus to ``tmpdir`` and collects them by the rank 0 worker. |
| 95 | |
| 96 | Args: |
| 97 | trainer (modelscope.trainers.EpochBasedTrainer): Trainer to be tested. |
| 98 | data_loader (nn.Dataloader): Pytorch data loader. |
| 99 | device: (str | torch.device): The target device for the data. |
| 100 | tmpdir (str): Path of directory to save the temporary results from |
| 101 | different gpus under cpu mode. |
| 102 | gpu_collect (bool): Option to use either gpu or cpu to collect results. |
| 103 | data_loader_iters_per_gpu (int): Used when dataset has no attribute __len__ or only load part of dataset. |
| 104 | Returns: |
| 105 | list: The prediction results. |
| 106 | """ |
| 107 | dataset = data_loader.dataset |
| 108 | rank, world_size = get_dist_info(trainer.dp_group) |
| 109 | |
| 110 | progress_with_iters = False |
| 111 | if data_loader_iters_per_gpu is None: |
| 112 | try: |
| 113 | data_len = len(dataset) |
| 114 | total_samples = data_len |
| 115 | except Exception as e: |
| 116 | logging.error(e) |
| 117 | raise ValueError( |
| 118 | 'Please implement ``__len__`` method for your dataset, or provide ``data_loader_iters_per_gpu``' |
| 119 | ) |
| 120 | desc = 'Total test samples with multi gpus' |
| 121 | else: |
| 122 | total_samples = 0 |
| 123 | progress_with_iters = True |
| 124 | data_len = data_loader_iters_per_gpu * world_size |
| 125 | desc = 'Total test iterations with multi gpus' |
| 126 | |
| 127 | count = 0 |
| 128 | with tqdm(total=data_len, desc=desc) as pbar: |
| 129 | for i, data in enumerate(data_loader): |
| 130 | data = to_device(data, device) |
| 131 | |
| 132 | evaluate_batch(trainer, data, metric_classes, vis_closure) |
| 133 | |
| 134 | if isinstance(data, Mapping): |
| 135 | if 'nsentences' in data: |
| 136 | batch_size = data['nsentences'] |
| 137 | else: |
no test coverage detected
searching dependent graphs…