()
| 21 | """ |
| 22 | |
| 23 | def unsafe_execute(): |
| 24 | |
| 25 | with create_tempdir(): |
| 26 | |
| 27 | # These system calls are needed when cleaning up tempdir. |
| 28 | import os |
| 29 | import shutil |
| 30 | rmtree = shutil.rmtree |
| 31 | rmdir = os.rmdir |
| 32 | chdir = os.chdir |
| 33 | |
| 34 | # Disable functionalities that can make destructive changes to the test. |
| 35 | reliability_guard() |
| 36 | |
| 37 | # Construct the check program and run it. |
| 38 | check_program = ( |
| 39 | problem["prompt"] + completion + "\n" + |
| 40 | problem["test"] + "\n" + |
| 41 | f"check({problem['entry_point']})" |
| 42 | ) |
| 43 | |
| 44 | try: |
| 45 | exec_globals = {} |
| 46 | with swallow_io(): |
| 47 | with time_limit(timeout): |
| 48 | # WARNING |
| 49 | # This program exists to execute untrusted model-generated code. Although |
| 50 | # it is highly unlikely that model-generated code will do something overtly |
| 51 | # malicious in response to this test suite, model-generated code may act |
| 52 | # destructively due to a lack of model capability or alignment. |
| 53 | # Users are strongly encouraged to sandbox this evaluation suite so that it |
| 54 | # does not perform destructive actions on their host or network. For more |
| 55 | # information on how OpenAI sandboxes its code, see the accompanying paper. |
| 56 | # Once you have read this disclaimer and taken appropriate precautions, |
| 57 | # uncomment the following line and proceed at your own risk: |
| 58 | exec(check_program, exec_globals) |
| 59 | result.append("passed") |
| 60 | except TimeoutException: |
| 61 | result.append("timed out") |
| 62 | except BaseException as e: |
| 63 | result.append(f"failed: {e}") |
| 64 | |
| 65 | # Needed for cleaning up. |
| 66 | shutil.rmtree = rmtree |
| 67 | os.rmdir = rmdir |
| 68 | os.chdir = chdir |
| 69 | |
| 70 | manager = multiprocessing.Manager() |
| 71 | result = manager.list() |
nothing calls this directly
no test coverage detected