(self)
| 187 | return prompt |
| 188 | |
| 189 | def generate_samples(self) -> Iterable[tuple]: |
| 190 | for subject in self.SUBJECT_TO_SUBCATEGORIES.keys(): |
| 191 | dev_df = pd.read_csv(f"{self.dataset_path}/dev/{subject}_dev.csv", |
| 192 | header=None) |
| 193 | train_prompt = self.gen_prompt(dev_df, subject, self.num_fewshot) |
| 194 | |
| 195 | test_df = pd.read_csv( |
| 196 | f"{self.dataset_path}/test/{subject}_test.csv", header=None) |
| 197 | if self.num_samples_per_subject is not None and self.num_samples_per_subject < test_df.shape[ |
| 198 | 0]: |
| 199 | test_df = test_df.sample(self.num_samples_per_subject) |
| 200 | |
| 201 | for i in range(test_df.shape[0]): |
| 202 | prompt_end = self.format_example(test_df, |
| 203 | i, |
| 204 | include_answer=False) |
| 205 | prompt = train_prompt + prompt_end |
| 206 | label = test_df.iloc[i, test_df.shape[1] - 1] |
| 207 | yield prompt, None, label, subject |
| 208 | |
| 209 | def compute_score(self, outputs: List[RequestOutput], references: List[str], |
| 210 | subjects: List[str]) -> float: |
nothing calls this directly
no test coverage detected