(sample, problems, example_test=False, is_mbpp=False, language="python")
| 108 | return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]) |
| 109 | |
| 110 | def process_humaneval_test(sample, problems, example_test=False, is_mbpp=False, language="python"): |
| 111 | task_id = sample["task_id"] |
| 112 | |
| 113 | if is_mbpp: |
| 114 | return sample["generation"] + "\n" + "\n".join(problems[task_id]["test"]) |
| 115 | #language = task_id.split("/")[0].lower() |
| 116 | |
| 117 | prompt = sample.get("prompt", "") |
| 118 | if example_test and "example_test" in problems[task_id] and problems[task_id]["example_test"] != "": |
| 119 | test = problems[task_id]["example_test"] |
| 120 | else: |
| 121 | test = problems[task_id]["test"] |
| 122 | code = sample["generation"] |
| 123 | |
| 124 | # Pre-process for different languages |
| 125 | if language == "python": |
| 126 | '''code_ = [] |
| 127 | for line in code.split("\n"): |
| 128 | if (len(line.strip()) > 0 and line[0] != ' ' and line[0] != '\t'): |
| 129 | break |
| 130 | code_.append(line) |
| 131 | code = "\n".join(code_)''' |
| 132 | test_setup = "\n".join(IMPORT_HELPER["python"]) + "\n" |
| 133 | test_string = test_setup + code + "\n" + test + "\n" |
| 134 | elif language == "cpp": |
| 135 | test_set_up = "" |
| 136 | for s in IMPORT_HELPER["cpp"]: |
| 137 | if s not in prompt: |
| 138 | test_set_up += s + "\n" |
| 139 | test_string = test_set_up + "\n" + code + "\n" + test |
| 140 | elif language == "java": |
| 141 | test_string = code + "\n" + test |
| 142 | elif language in ["js", "javascript", "ts", "cs", "sh"]: |
| 143 | test_string = code + "\n" + test |
| 144 | elif language == "go": |
| 145 | import_string = problems[task_id]["import"] |
| 146 | prompt = prompt.replace(import_string, "") |
| 147 | if example_test and "example_test" in problems[task_id]: |
| 148 | test = problems[task_id]["example_test"] |
| 149 | else: |
| 150 | test = problems[task_id]["test"] |
| 151 | test_setup = problems[task_id]["test_setup"] |
| 152 | other_pkgs = [] |
| 153 | for pkg in IMPORT_HELPER["go"]: |
| 154 | if pkg not in test_setup: |
| 155 | p = pkg.split("/")[-1] |
| 156 | if p + "." in code: |
| 157 | other_pkgs.append(f"\"{pkg}\"") |
| 158 | if other_pkgs: |
| 159 | import_other_pkgs = "import (\n" + " ".join([p + "\n" for p in other_pkgs]) + ")" |
| 160 | test_string = test_setup + "\n" + import_other_pkgs + "\n" + prompt + code + "\n" + test |
| 161 | else: |
| 162 | test_string = test_setup + "\n" + prompt + code + "\n" + test |
| 163 | elif language == "rust": |
| 164 | main = "\nfn main(){ \n } \n" |
| 165 | declaration = problems[task_id]["declaration"] |
| 166 | test_string = main + declaration + prompt + code + test |
| 167 | elif language == "php": |
no test coverage detected