wehub-resource-sync
267 lines
6.6 KiB
Python
267 lines
6.6 KiB
Python
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# This code is adapted from OpenAI's release
|
|
# https://github.com/openai/human-eval/blob/master/human_eval/execution.py
|
|
|
|
import contextlib
|
|
import faulthandler
|
|
import io
|
|
import multiprocessing
|
|
import os
|
|
import platform
|
|
import signal
|
|
import tempfile
|
|
|
|
|
|
def check_correctness(check_program, timeout, task_id, completion_id):
|
|
"""
|
|
Evaluates the functional correctness of a completion by running the test
|
|
suite provided in the problem.
|
|
|
|
:param completion_id: an optional completion ID so we can match
|
|
the results later even if execution finishes asynchronously.
|
|
"""
|
|
manager = multiprocessing.Manager()
|
|
result = manager.list()
|
|
|
|
p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout))
|
|
p.start()
|
|
p.join(timeout=timeout + 1)
|
|
if p.is_alive():
|
|
p.kill()
|
|
|
|
if not result:
|
|
result.append("timed out")
|
|
|
|
return dict(
|
|
task_id=task_id,
|
|
passed=result[0] == "passed",
|
|
result=result[0],
|
|
completion_id=completion_id,
|
|
)
|
|
|
|
|
|
import io
|
|
import sys
|
|
|
|
|
|
# Function to capture print output
|
|
def capture_print_output(code):
|
|
# Create a StringIO stream to capture output
|
|
output = io.StringIO()
|
|
|
|
# Save the current stdout (standard output)
|
|
current_stdout = sys.stdout
|
|
|
|
# Set the stdout to the StringIO stream
|
|
sys.stdout = output
|
|
|
|
try:
|
|
# Run the function
|
|
exec(code)
|
|
finally:
|
|
# Restore stdout to its original setting
|
|
sys.stdout = current_stdout
|
|
|
|
# Get the content of the StringIO stream
|
|
return output.getvalue()
|
|
|
|
|
|
# Capture the output
|
|
# output = capture_print_output(my_function)
|
|
# print("Captured output:", output)
|
|
|
|
|
|
def unsafe_execute(check_program, result, timeout):
|
|
|
|
with create_tempdir():
|
|
|
|
# These system calls are needed when cleaning up tempdir.
|
|
import os
|
|
import shutil
|
|
|
|
rmtree = shutil.rmtree
|
|
rmdir = os.rmdir
|
|
chdir = os.chdir
|
|
|
|
# Disable functionalities that can make destructive changes to the test.
|
|
reliability_guard()
|
|
|
|
# Run program.
|
|
try:
|
|
exec_globals = {}
|
|
with swallow_io():
|
|
with time_limit(timeout):
|
|
exec(check_program, exec_globals)
|
|
result.append("passed")
|
|
except TimeoutException:
|
|
result.append("timed out")
|
|
except BaseException as e:
|
|
result.append(f"failed: {e}")
|
|
|
|
# Needed for cleaning up.
|
|
shutil.rmtree = rmtree
|
|
os.rmdir = rmdir
|
|
os.chdir = chdir
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def time_limit(seconds):
|
|
def signal_handler(signum, frame):
|
|
raise TimeoutException("Timed out!")
|
|
|
|
signal.setitimer(signal.ITIMER_REAL, seconds)
|
|
signal.signal(signal.SIGALRM, signal_handler)
|
|
try:
|
|
yield
|
|
finally:
|
|
signal.setitimer(signal.ITIMER_REAL, 0)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def swallow_io():
|
|
stream = WriteOnlyStringIO()
|
|
with contextlib.redirect_stdout(stream):
|
|
with contextlib.redirect_stderr(stream):
|
|
with redirect_stdin(stream):
|
|
yield
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def create_tempdir():
|
|
with tempfile.TemporaryDirectory() as dirname:
|
|
with chdir(dirname):
|
|
yield dirname
|
|
|
|
|
|
class TimeoutException(Exception):
|
|
pass
|
|
|
|
|
|
class WriteOnlyStringIO(io.StringIO):
|
|
"""StringIO that throws an exception when it's read from"""
|
|
|
|
def read(self, *args, **kwargs):
|
|
raise OSError
|
|
|
|
def readline(self, *args, **kwargs):
|
|
raise OSError
|
|
|
|
def readlines(self, *args, **kwargs):
|
|
raise OSError
|
|
|
|
def readable(self, *args, **kwargs):
|
|
"""Returns True if the IO object can be read."""
|
|
return False
|
|
|
|
|
|
class redirect_stdin(contextlib._RedirectStream): # type: ignore
|
|
_stream = "stdin"
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def chdir(root):
|
|
if root == ".":
|
|
yield
|
|
return
|
|
cwd = os.getcwd()
|
|
os.chdir(root)
|
|
try:
|
|
yield
|
|
except BaseException as exc:
|
|
raise exc
|
|
finally:
|
|
os.chdir(cwd)
|
|
|
|
|
|
def reliability_guard(maximum_memory_bytes=None):
|
|
"""
|
|
This disables various destructive functions and prevents the generated code
|
|
from interfering with the test (e.g. fork bomb, killing other processes,
|
|
removing filesystem files, etc.)
|
|
|
|
WARNING
|
|
This function is NOT a security sandbox. Untrusted code, including, model-
|
|
generated code, should not be blindly executed outside of one. See the
|
|
Codex paper for more information about OpenAI's code sandbox, and proceed
|
|
with caution.
|
|
"""
|
|
|
|
if maximum_memory_bytes is not None:
|
|
import resource
|
|
|
|
resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
|
|
resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
|
|
if not platform.uname().system == "Darwin":
|
|
resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
|
|
|
|
faulthandler.disable()
|
|
|
|
import builtins
|
|
|
|
builtins.exit = None
|
|
builtins.quit = None
|
|
|
|
import os
|
|
|
|
os.environ["OMP_NUM_THREADS"] = "1"
|
|
|
|
os.kill = None
|
|
os.system = None
|
|
os.putenv = None
|
|
os.remove = None
|
|
os.removedirs = None
|
|
os.rmdir = None
|
|
os.fchdir = None
|
|
os.setuid = None
|
|
os.fork = None
|
|
os.forkpty = None
|
|
os.killpg = None
|
|
os.rename = None
|
|
os.renames = None
|
|
os.truncate = None
|
|
os.replace = None
|
|
os.unlink = None
|
|
os.fchmod = None
|
|
os.fchown = None
|
|
os.chmod = None
|
|
os.chown = None
|
|
os.chroot = None
|
|
os.fchdir = None
|
|
os.lchflags = None
|
|
os.lchmod = None
|
|
os.lchown = None
|
|
os.getcwd = None
|
|
os.chdir = None
|
|
|
|
import shutil
|
|
|
|
shutil.rmtree = None
|
|
shutil.move = None
|
|
shutil.chown = None
|
|
|
|
import subprocess
|
|
|
|
subprocess.Popen = None # type: ignore
|
|
|
|
__builtins__["help"] = None
|
|
|
|
import sys
|
|
|
|
sys.modules["ipdb"] = None
|
|
sys.modules["joblib"] = None
|
|
sys.modules["resource"] = None
|
|
sys.modules["psutil"] = None
|
|
sys.modules["tkinter"] = None |