MCPcopy Create free account
hub / github.com/PRIME-RL/PRIME / process_humaneval_test

Function process_humaneval_test

eval/utils/evaluation_leetcode.py:110–169  ·  view source on GitHub ↗
(sample, problems, example_test=False, is_mbpp=False, language="python")

Source from the content-addressed store, hash-verified

108 return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
109
110def process_humaneval_test(sample, problems, example_test=False, is_mbpp=False, language="python"):
111 task_id = sample["task_id"]
112
113 if is_mbpp:
114 return sample["generation"] + "\n" + "\n".join(problems[task_id]["test"])
115 #language = task_id.split("/")[0].lower()
116
117 prompt = sample.get("prompt", "")
118 if example_test and "example_test" in problems[task_id] and problems[task_id]["example_test"] != "":
119 test = problems[task_id]["example_test"]
120 else:
121 test = problems[task_id]["test"]
122 code = sample["generation"]
123
124 # Pre-process for different languages
125 if language == "python":
126 '''code_ = []
127 for line in code.split("\n"):
128 if (len(line.strip()) > 0 and line[0] != ' ' and line[0] != '\t'):
129 break
130 code_.append(line)
131 code = "\n".join(code_)'''
132 test_setup = "\n".join(IMPORT_HELPER["python"]) + "\n"
133 test_string = test_setup + code + "\n" + test + "\n"
134 elif language == "cpp":
135 test_set_up = ""
136 for s in IMPORT_HELPER["cpp"]:
137 if s not in prompt:
138 test_set_up += s + "\n"
139 test_string = test_set_up + "\n" + code + "\n" + test
140 elif language == "java":
141 test_string = code + "\n" + test
142 elif language in ["js", "javascript", "ts", "cs", "sh"]:
143 test_string = code + "\n" + test
144 elif language == "go":
145 import_string = problems[task_id]["import"]
146 prompt = prompt.replace(import_string, "")
147 if example_test and "example_test" in problems[task_id]:
148 test = problems[task_id]["example_test"]
149 else:
150 test = problems[task_id]["test"]
151 test_setup = problems[task_id]["test_setup"]
152 other_pkgs = []
153 for pkg in IMPORT_HELPER["go"]:
154 if pkg not in test_setup:
155 p = pkg.split("/")[-1]
156 if p + "." in code:
157 other_pkgs.append(f"\"{pkg}\"")
158 if other_pkgs:
159 import_other_pkgs = "import (\n" + " ".join([p + "\n" for p in other_pkgs]) + ")"
160 test_string = test_setup + "\n" + import_other_pkgs + "\n" + prompt + code + "\n" + test
161 else:
162 test_string = test_setup + "\n" + prompt + code + "\n" + test
163 elif language == "rust":
164 main = "\nfn main(){ \n } \n"
165 declaration = problems[task_id]["declaration"]
166 test_string = main + declaration + prompt + code + test
167 elif language == "php":

Callers 1

Calls 1

getMethod · 0.45

Tested by

no test coverage detected