| 85 | ] = float(row["result"]) |
| 86 | |
| 87 | def plot(self): |
| 88 | fig, ax = plt.subplots() |
| 89 | title_str = "Time usage" if self.args.is_time else "Memory usage" |
| 90 | title_str = title_str + " for training" if self.args.is_train else title_str + " for inference" |
| 91 | |
| 92 | if not self.args.no_log_scale: |
| 93 | # set logarithm scales |
| 94 | ax.set_xscale("log") |
| 95 | ax.set_yscale("log") |
| 96 | |
| 97 | for axis in [ax.xaxis, ax.yaxis]: |
| 98 | axis.set_major_formatter(ScalarFormatter()) |
| 99 | |
| 100 | for model_name_idx, model_name in enumerate(self.result_dict.keys()): |
| 101 | batch_sizes = sorted(list(set(self.result_dict[model_name]["bsz"]))) |
| 102 | sequence_lengths = sorted(list(set(self.result_dict[model_name]["seq_len"]))) |
| 103 | results = self.result_dict[model_name]["result"] |
| 104 | |
| 105 | (x_axis_array, inner_loop_array) = ( |
| 106 | (batch_sizes, sequence_lengths) if self.args.plot_along_batch else (sequence_lengths, batch_sizes) |
| 107 | ) |
| 108 | |
| 109 | label_model_name = ( |
| 110 | model_name if self.args.short_model_names is None else self.args.short_model_names[model_name_idx] |
| 111 | ) |
| 112 | |
| 113 | for inner_loop_value in inner_loop_array: |
| 114 | if self.args.plot_along_batch: |
| 115 | y_axis_array = np.asarray( |
| 116 | [results[(x, inner_loop_value)] for x in x_axis_array if (x, inner_loop_value) in results], |
| 117 | dtype=np.int, |
| 118 | ) |
| 119 | else: |
| 120 | y_axis_array = np.asarray( |
| 121 | [results[(inner_loop_value, x)] for x in x_axis_array if (inner_loop_value, x) in results], |
| 122 | dtype=np.float32, |
| 123 | ) |
| 124 | |
| 125 | (x_axis_label, inner_loop_label) = ( |
| 126 | ("batch_size", "len") if self.args.plot_along_batch else ("in #tokens", "bsz") |
| 127 | ) |
| 128 | |
| 129 | x_axis_array = np.asarray(x_axis_array, np.int)[: len(y_axis_array)] |
| 130 | plt.scatter( |
| 131 | x_axis_array, y_axis_array, label=f"{label_model_name} - {inner_loop_label}: {inner_loop_value}" |
| 132 | ) |
| 133 | plt.plot(x_axis_array, y_axis_array, "--") |
| 134 | |
| 135 | title_str += f" {label_model_name} vs." |
| 136 | |
| 137 | title_str = title_str[:-4] |
| 138 | y_axis_label = "Time in s" if self.args.is_time else "Memory in MB" |
| 139 | |
| 140 | # plot |
| 141 | plt.title(title_str) |
| 142 | plt.xlabel(x_axis_label) |
| 143 | plt.ylabel(y_axis_label) |
| 144 | plt.legend() |