MCPcopy Create free account
hub / github.com/PRIME-RL/PRIME / check_correctness

Function check_correctness

eval/utils/execution_leetcode.py:23–568  ·  view source on GitHub ↗

Evaluates the functional correctness of a completion by running the test suite provided in the problem.

(
        task_id: str,
        sample: dict,
        language_type: str,
        timeout: float = 3.0,
        tmp_dir: str = None,
        completion_id: Optional[int] = None,
)

Source from the content-addressed store, hash-verified

21cs_exec = ""
22
23def check_correctness(
24 task_id: str,
25 sample: dict,
26 language_type: str,
27 timeout: float = 3.0,
28 tmp_dir: str = None,
29 completion_id: Optional[int] = None,
30) -> Dict:
31 """
32 Evaluates the functional correctness of a completion by running the test
33 suite provided in the problem.
34 """
35
36 def unsafe_execute(tmp_dir):
37 random_id = random.randint(1, 100000)
38 if "python" in language_type.lower():
39 with create_tempdir():
40
41 # These system calls are needed when cleaning up tempdir.
42 import os
43 import shutil
44 rmtree = shutil.rmtree
45 rmdir = os.rmdir
46 chdir = os.chdir
47
48 # Disable functionalities that can make destructive changes to the test.
49 reliability_guard()
50
51 try:
52 exec_globals = {}
53 with swallow_io():
54 with time_limit(timeout):
55 # WARNING
56 # This program exists to execute untrusted model-generated code. Although
57 # it is highly unlikely that model-generated code will do something overtly
58 # malicious in response to this test suite, model-generated code may act
59 # destructively due to a lack of model capability or alignment.
60 # Users are strongly encouraged to sandbox this evaluation suite so that it
61 # does not perform destructive actions on their host or network.
62 # Once you have read this disclaimer and taken appropriate precautions,
63 # uncomment the following line and proceed at your own risk:
64 exec(sample["test_code"], exec_globals)
65 result.append("passed")
66 except TimeoutException:
67 result.append("timed out")
68 except AssertionError as e:
69 result.append(f"failed: AssertionError")
70 except BaseException as e:
71 result.append(f"failed: {e}")
72 #print(sample["test_code"])
73 #print(result)
74 # Needed for cleaning up.
75 shutil.rmtree = rmtree
76 os.rmdir = rmdir
77 os.chdir = chdir
78
79 elif "go" in language_type.lower():
80 assert tmp_dir is not None, "Go should be evaluated in a dir where necessary module files installed."

Callers

nothing calls this directly

Calls

no outgoing calls

Tested by

no test coverage detected