added deepseekv2

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_scala.py

+import tempfile
+from pathlib import Path
+
+from .safe_subprocess import run
+
+LANG_NAME = "Scala"
+LANG_EXT = ".scala"
+
+
+def eval_script(path: Path):
+    with tempfile.TemporaryDirectory() as outdir:
+        # Each Scala file contains the class with same name `JAVA_CLASS_NAME`
+        # Hence, scalac will same JAVA_CLASS_NAME.class file for each problem
+        # Write class for each problem to a different temp dir
+        build = run(["scalac", "-d", outdir, path], timeout_seconds=45)
+        if build.exit_code != 0:
+            # Well, it's a compile error. May be a type error or
+            # something. But, why break the set convention
+            return {
+                "status": "SyntaxError",
+                "exit_code": build.exit_code,
+                "stdout": build.stdout,
+                "stderr": build.stderr,
+            }
+        # "Problem" is the name of the class we emit.
+        r = run(["scala", "-cp", f"{outdir}", "Problem"])
+        if r.timeout:
+            status = "Timeout"
+        elif r.exit_code == 0 and r.stderr == "":
+            status = "OK"
+        else:
+            # Well, it's a panic
+            status = "Exception"
+    return {
+        "status": status,
+        "exit_code": r.exit_code,
+        "stdout": r.stdout,
+        "stderr": r.stderr,
+    }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_sh.py

+from pathlib import Path
+
+from .safe_subprocess import run
+
+LANG_NAME = "bash"
+LANG_EXT = ".sh"
+
+
+def eval_script(path: Path):
+    # Capture output - will be generated regardless of success, fail, or syntax error
+    p = run(["bash", path])
+    if p.timeout:
+        status = "Timeout"
+    elif p.exit_code == 0:
+        status = "OK"
+    elif "syntax error" in p.stderr:
+        status = "SyntaxError"
+    else:
+        status = "Exception"
+
+    return {
+        "status": status,
+        "exit_code": p.exit_code,
+        "stdout": p.stdout,
+        "stderr": p.stderr,
+    }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_swift.py

+import os
+import subprocess
+from pathlib import Path
+
+from .safe_subprocess import run
+
+
+def eval_script(path: Path):
+    basename = ".".join(str(path).split(".")[:-1])
+    r = run(["swiftc", path, "-o", basename], timeout_seconds=45)
+    if r.timeout:
+        status = "Timeout"
+    elif r.exit_code != 0:
+        # Well, it's a compile error. May be a type error or
+        # something. But, why break the set convention
+        status = "SyntaxError"
+    else:
+        r = run([basename], timeout_seconds=5)
+        if r.timeout:
+            status = "Timeout"
+        elif r.exit_code != 0:
+            # Well, it's a panic
+            status = "Exception"
+        else:
+            status = "OK"
+        os.remove(basename)
+    return {
+        "status": status,
+        "exit_code": r.exit_code,
+        "stdout": r.stdout,
+        "stderr": r.stderr,
+    }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_ts.py

+from pathlib import Path
+
+from .safe_subprocess import run
+
+
+def eval_script(path: Path):
+    r = run(["tsc", "--target", "esnext", str(path)], timeout_seconds=15)
+    if r.exit_code != 0:
+        return {
+            "status": "SyntaxError",
+            "exit_code": r.exit_code,
+            "stdout": r.stdout,
+            "stderr": r.stderr,
+        }
+
+    r = run(["node", str(path).replace(".ts", ".js")], timeout_seconds=15)
+    if r.timeout:
+        status = "Timeout"
+    elif r.exit_code == 0:
+        status = "OK"
+    elif "ERR_ASSERTION" in r.stderr:
+        status = "AssertionError"
+    elif "SyntaxError" in r.stderr:
+        status = "SyntaxError"
+    elif "ReferenceError" in r.stderr:
+        status = "ReferenceError"
+    else:
+        status = "Exception"
+    return {
+        "status": status,
+        "exit_code": r.exit_code,
+        "stdout": r.stdout,
+        "stderr": r.stderr,
+    }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/evaluation.py

+import json
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from threading import Lock
+from typing import Optional
+
+from .containerized_eval import eval_string_script
+
+# Get working directory
+WORKING_DIR = Path(__file__).parent.parent
+
+# program: str => Result
+CACHE = dict()
+CACHE_LOCK = Lock()
+
+
+def cache_get(program: str) -> Optional[dict]:
+    if program in CACHE:
+        result = CACHE[program]
+        return result
+    else:
+        return None
+
+
+def cache_set(program: str, result: dict):
+    if program in CACHE:
+        print("Setting already-existing cache")
+    CACHE[program] = result
+
+
+def cached_eval_script(problem, index) -> dict:
+    # here prompt is already included in completions
+    program = problem["completions"][index] + "\n" + problem["tests"]
+    CACHE_LOCK.acquire(True)
+    cached = cache_get(program)
+    if cached is not None:
+        CACHE_LOCK.release()
+        return cached
+    else:
+        result_yaml = dict()
+        cache_set(program, result_yaml)
+        CACHE_LOCK.release()
+        result_dict = eval_string_script(problem["language"], program)
+        for k in result_dict.keys():
+            result_yaml[k] = result_dict[k]
+            result_yaml["timestamp"] = int(time.time())
+        return result_yaml
+
+
+def get_test_results_json_path(
+    output_dir: str, problem_json_path: str, input_dir: Path
+) -> Path:
+    suffixes = ".results.json"
+    problem_name = problem_json_path[: -len(".json")]
+    if input_dir:
+        raise ValueError("input dir given")
+        return Path(output_dir) / (
+            problem_json_path.relative_to(input_dir).parent / (problem_name + suffixes)
+        )
+    return Path(output_dir) / (problem_name + suffixes)
+
+
+def evaluate_problem(
+    output_dir: str, problem_json_path: str, max_workers: int, input_dir: Path = None
+):
+    with open(problem_json_path, "r") as f:
+        problem = json.load(f)
+    test_results_path = get_test_results_json_path(
+        output_dir, problem_json_path, input_dir
+    )
+    test_results_path.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
+
+    test_results = problem.copy()
+    del test_results["completions"]
+    test_results["results"] = []
+
+    num_problems = len(problem["completions"])
+    min_problem = len(test_results["results"])
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        for j in executor.map(
+            lambda index: cached_eval_script(problem, index),
+            range(min_problem, num_problems),
+        ):
+            test_results["results"].append(j)
+            with open(test_results_path, "w") as f:
+                f.write(json.dumps(test_results, indent=2))

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/generic_eval.py

+# This is a helper script for evaluating benchmarks that have been translated to
+# different languages.
+#
+# To use this script, call eval_lang.py.
+# The --directory argument is required, and tells the script where the benchmarks are located.
+# The --files argument is optional, and takes a list of numbers corresponding to the files to be evaluated.
+#
+# The script will print the results on each benchmark, and also write to results/lang.csv.
+# When the script completes, it will print a summary.
+#
+# Examples
+#
+# To run the entire benchmark suite:
+#   python3 src/eval_php.py --directory datasets/php-keep-code_davinci_001_temp_0.2-0/
+#
+# To run benchmarks 1, 2, and 3:
+#   python3 src/eval_php.py --directory datasets/php-keep-code_davinci_001_temp_0.2-0/ --files 1 2 3
+
+import argparse
+import sys
+from pathlib import Path
+from sys import exit as sysexit
+
+
+def list_files(directory, ext):
+    files_unsorted = directory.glob(f"HumanEval_*{ext}")
+    # assumption: base filenames are in the format of HumanEval_X_*
+    # Where X is a valid number
+    def key(s):
+        return int(str(s.name).split("_")[1])
+
+    files_sorted = sorted(files_unsorted, key=(lambda s: key(s)))
+
+    # assumption: there may be missing files, but no extra files
+    # so we build files_array where the index corresponds to the file's number,
+    # and a missing file is represented by None
+    size = key(files_sorted[-1]) + 1
+    files_array = [None] * size
+    for f in files_sorted:
+        k = key(f)
+        files_array[k] = f
+
+    return files_array
+
+
+def main(eval_script, language, extension):
+    args = argparse.ArgumentParser()
+
+    args.add_argument(
+        "--directory", type=str, required=True, help="Directory to read benchmarks from"
+    )
+    args.add_argument(
+        "--files",
+        type=int,
+        nargs="*",
+        default=[],
+        help="Specify the benchmarks to evaluate by their number, e.g. --files 0 1 2",
+    )
+    args = args.parse_args()
+
+    directory = Path(args.directory).resolve()
+
+    files_sorted = list_files(directory, extension)
+
+    # the directory you specified does not contain the right language
+    if len(files_sorted) == 0:
+        print(f"The specified directory does not contain files of type {extension}")
+        sysexit(1)
+
+    files_index = []
+    if len(args.files) > 0:
+        files_index = args.files
+    else:
+        files_index = range(len(files_sorted))
+
+    total = 0
+    passed = 0
+    syntax_error = 0
+
+    results_file = Path(
+        Path(__file__).parent, "..", "results", language.lower() + ".csv"
+    ).resolve()
+
+    with open(results_file, "w") as f:
+        for i in files_index:
+            filepath = files_sorted[i]
+            if filepath is None:
+                print("File {} does not exist!".format(i))
+                continue
+            res = eval_script(filepath)
+            output = f"{language},{filepath.stem},{res['status']}\n"
+            f.write(output)
+            print(output, end="")
+            total += 1
+            if res["status"] == "OK":
+                passed += 1
+            elif res["status"] == "SyntaxError":
+                syntax_error += 1
+    print(f"Total {total}, Syntax Error {syntax_error}, Passed {passed}")
+
+
+def main_check_stubs(check_script, language, extension):
+    args = argparse.ArgumentParser()
+
+    args.add_argument(
+        "--directory", type=str, required=True, help="Directory to read benchmarks from"
+    )
+    args.add_argument(
+        "--files",
+        type=int,
+        nargs="*",
+        default=[],
+        help="Specify the benchmarks to evaluate by their number, e.g. --files 0 1 2",
+    )
+    args = args.parse_args()
+
+    directory = Path(args.directory).resolve()
+
+    files_sorted = list_files(directory, extension)
+
+    # the directory you specified does not contain the right language
+    if len(files_sorted) == 0:
+        print(f"The specified directory does not contain files of type {extension}")
+        sysexit(1)
+
+    files_index = []
+    if len(args.files) > 0:
+        files_index = args.files
+    else:
+        files_index = range(len(files_sorted))
+
+    total = 0
+    passed = 0
+
+    results_file = Path(
+        Path(__file__).parent, "..", "check_results", language.lower() + ".csv"
+    ).resolve()
+
+    with open(results_file, "w") as f:
+        for i in files_index:
+            filepath = files_sorted[i]
+            if filepath is None:
+                print("File {} does not exist!".format(i))
+                continue
+            res = check_script(filepath)
+            output = f"{language},{filepath.stem},{res['status']}\n"
+            f.write(output)
+            print(output, end="")
+            total += 1
+            if res["status"] == "OK":
+                passed += 1
+    print(f"Total {total}, Passed {passed}")
+
+    if total != passed:
+        sys.exit(1)

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/libeval.py

+import os
+import signal
+import subprocess
+from typing import List
+
+from . import generic_eval
+
+
+def testing_mail(x, y, z):
+    generic_eval.gmain(x, y, z)
+
+
+def run_without_exn(args: List[str]):
+    """
+    Runs the given program with a five second timeout. Does not throw an exception
+    no matter what happens. The output is a dictionary of the format that we expect
+    for our evaluation scripts. The "status" field is "OK" when the exit code is
+    zero. If that isn't enough, you may want to tweak the status based on the
+    captured stderr and stdout.
+    """
+    p = subprocess.Popen(
+        args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True
+    )
+    try:
+        stdout, stderr = p.communicate(timeout=5)
+        exit_code = p.returncode
+        status = "OK" if exit_code == 0 else "Exception"
+    except subprocess.TimeoutExpired as exc:
+        stdout, stderr = p.stdout.read(), p.stderr.read()
+        os.killpg(os.getpgid(p.pid), signal.SIGTERM)
+        exit_code = -1
+        status = "Timeout"
+
+    if stdout is None:
+        stdout = b""
+    if stderr is None:
+        stderr = b""
+    return {
+        "status": status,
+        "exit_code": exit_code,
+        "stdout": stdout.decode("utf-8", errors="ignore"),
+        "stderr": stderr.decode("utf-8", errors="ignore"),
+    }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/.gitignore

+/__pycache__
+/.pytest_cache
\ No newline at end of file

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/__init__.py

+import fcntl
+import os
+import signal
+import subprocess
+import time
+from typing import List
+
+MAX_BYTES_PER_READ = 1024
+SLEEP_BETWEEN_READS = 0.1
+
+
+class Result:
+    timeout: int
+    exit_code: int
+    stdout: str
+    stderr: str
+
+    def __init__(self, timeout, exit_code, stdout, stderr):
+        self.timeout = timeout
+        self.exit_code = exit_code
+        self.stdout = stdout
+        self.stderr = stderr
+
+
+def set_nonblocking(reader):
+    fd = reader.fileno()
+    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
+    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+
+
+def run(
+    args: List[str],
+    timeout_seconds: int = 15,
+    max_output_size: int = 2048,
+    env=None,
+) -> Result:
+    """
+    Runs the given program with arguments. After the timeout elapses, kills the process
+    and all other processes in the process group. Captures at most max_output_size bytes
+    of stdout and stderr each, and discards any output beyond that.
+    """
+    p = subprocess.Popen(
+        args,
+        env=env,
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        start_new_session=True,
+        bufsize=MAX_BYTES_PER_READ,
+    )
+    set_nonblocking(p.stdout)
+    set_nonblocking(p.stderr)
+
+    process_group_id = os.getpgid(p.pid)
+
+    # We sleep for 0.1 seconds in each iteration.
+    max_iterations = timeout_seconds * 10
+    stdout_saved_bytes = []
+    stderr_saved_bytes = []
+    stdout_bytes_read = 0
+    stderr_bytes_read = 0
+
+    for _ in range(max_iterations):
+        this_stdout_read = p.stdout.read(MAX_BYTES_PER_READ)
+        this_stderr_read = p.stderr.read(MAX_BYTES_PER_READ)
+        # this_stdout_read and this_stderr_read may be None if stdout or stderr
+        # are closed. Without these checks, test_close_output fails.
+        if this_stdout_read is not None and stdout_bytes_read < max_output_size:
+            stdout_saved_bytes.append(this_stdout_read)
+            stdout_bytes_read += len(this_stdout_read)
+        if this_stderr_read is not None and stderr_bytes_read < max_output_size:
+            stderr_saved_bytes.append(this_stderr_read)
+            stderr_bytes_read += len(this_stderr_read)
+        exit_code = p.poll()
+        if exit_code is not None:
+            break
+        time.sleep(SLEEP_BETWEEN_READS)
+
+    try:
+        # Kills the process group. Without this line, test_fork_once fails.
+        os.killpg(process_group_id, signal.SIGKILL)
+    except ProcessLookupError:
+        pass
+
+    timeout = exit_code is None
+    exit_code = exit_code if exit_code is not None else -1
+    stdout = b"".join(stdout_saved_bytes).decode("utf-8", errors="ignore")
+    stderr = b"".join(stderr_saved_bytes).decode("utf-8", errors="ignore")
+    return Result(timeout=timeout, exit_code=exit_code, stdout=stdout, stderr=stderr)

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/block_on_inputs.py

+while True:
+    input()

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/close_outputs.py

+import sys
+
+print("This is the end")
+sys.stdout.close()
+sys.stderr.close()
+while True:
+    pass

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/fork_bomb.py

+import os
+
+while True:
+    os.fork()

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/fork_once.py

+import os
+import time
+
+if os.fork() == 0:
+    while True:
+        time.sleep(60)

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/sleep_forever.py

+import time
+
+while True:
+    time.sleep(60)

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/unbounded_output.py

+b = True
+while True:
+    print(b)
+    b = not b

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/module_test.py

+import time
+from pathlib import Path
+
+from safe_subprocess import run
+
+ROOT = Path(__file__).resolve().parent / "evil_programs"
+
+
+def assert_no_running_evil():
+    result = run(["pgrep", "-f", ROOT], timeout_seconds=1, max_output_size=1024)
+    assert (
+        result.exit_code == 1
+    ), f"There are still evil processes running: {result.stdout}"
+    assert len(result.stderr) == 0
+    assert len(result.stdout) == 0
+
+
+def test_fork_once():
+    # The program exits cleanly and immediately. But, it forks a child that runs
+    # forever.
+    result = run(
+        ["python3", ROOT / "fork_once.py"],
+        timeout_seconds=2,
+        max_output_size=1024,
+    )
+    assert result.exit_code == 0
+    assert result.timeout == False
+    assert len(result.stderr) == 0
+    assert len(result.stdout) == 0
+    assert_no_running_evil()
+
+
+def test_close_outputs():
+    # The program prints to stdout, closes its output, and then runs forever.
+    result = run(
+        ["python3", ROOT / "close_outputs.py"],
+        timeout_seconds=2,
+        max_output_size=1024,
+    )
+    assert result.exit_code == -1
+    assert result.timeout == True
+    assert len(result.stderr) == 0
+    assert result.stdout == "This is the end\n"
+    assert_no_running_evil()
+
+
+def test_unbounded_output():
+    result = run(
+        ["python3", ROOT / "unbounded_output.py"],
+        timeout_seconds=3,
+        max_output_size=1024,
+    )
+    assert result.exit_code == -1
+    assert result.timeout == True
+    assert len(result.stderr) == 0
+    assert len(result.stdout) == 1024
+    assert_no_running_evil()
+
+
+def test_sleep_forever():
+    result = run(
+        ["python3", ROOT / "sleep_forever.py"],
+        timeout_seconds=2,
+        max_output_size=1024,
+    )
+    assert result.exit_code == -1
+    assert result.timeout == True
+    assert len(result.stderr) == 0
+    assert len(result.stdout) == 0
+    assert_no_running_evil()
+
+
+def test_fork_bomb():
+    result = run(
+        ["python3", ROOT / "fork_bomb.py"],
+        timeout_seconds=2,
+        max_output_size=1024,
+    )
+    assert result.exit_code == -1
+    assert result.timeout == True
+    assert len(result.stderr) == 0
+    assert len(result.stdout) == 0
+    # Unfortunately, this sleep seems to be necessary. My theories:
+    # 1. os.killpg doesn't block until the whole process group is dead.
+    # 2. pgrep can produce stale output
+    time.sleep(2)
+    assert_no_running_evil()
+
+
+def test_block_on_inputs():
+    # We run the subprocess with /dev/null as input. So, any program that tries
+    # to read input will error.
+    result = run(
+        ["python3", ROOT / "block_on_inputs.py"],
+        timeout_seconds=2,
+        max_output_size=1024,
+    )
+    assert result.exit_code == 1
+    assert result.timeout == False
+    assert len(result.stdout) == 0
+    assert "EOF when reading a line" in result.stderr
+    assert_no_running_evil()

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/single_experiment_pass_k.py

+import json
+
+import numpy as np
+
+
+def estimator(n: int, c: int, k: int) -> float:
+    """
+    Calculates 1 - comb(n - c, k) / comb(n, k).
+    """
+    if n - c < k:
+        return 1.0
+    return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+
+def for_file(path):
+    with open(path, "r") as f:
+        data = json.load(f)
+    n = len(data["results"])
+    c = len(
+        [True for r in data["results"] if r["status"] == "OK" and r["exit_code"] == 0]
+    )
+    return np.array([estimator(n, c, 1), estimator(n, c, 10), estimator(n, c, 100)])

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/pal_metric/pal_code_exec.py

+import os
+import warnings
+from collections import Counter, defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from bigcode_eval.tasks.custom_metrics.pal_metric.python_executor import run_program
+
+# adapted from https://github.com/huggingface/evaluate/blob/main/metrics/code_eval/code_eval.py
+
+_WARNING = """
+################################################################################
+                                  !!!WARNING!!!
+################################################################################
+The "code_eval" metric executes untrusted model-generated code in Python.
+Although it is highly unlikely that model-generated code will do something
+overtly malicious in response to this test suite, model-generated code may act
+destructively due to a lack of model capability or alignment.
+Users are strongly encouraged to sandbox this evaluation suite so that it
+does not perform destructive actions on their host or network. For more
+information on how OpenAI sandboxes its code, see the paper "Evaluating Large
+Language Models Trained on Code" (https://arxiv.org/abs/2107.03374).
+Once you have read this disclaimer and taken appropriate precautions,
+set the environment variable HF_ALLOW_CODE_EVAL="1". Within Python you can to this
+with:
+>>> import os
+>>> os.environ["HF_ALLOW_CODE_EVAL"] = "1"
+################################################################################\
+"""
+
+
+def compute(
+    predictions,
+    references,
+    num_workers=4,
+    timeout=3.0,
+    majority_voting=False,
+    answer_symbol=None,
+):
+    """
+    Returns the scores
+
+    :param majority_voting: bool
+        Takes majority voted answer to evaluate against the reference , defaults to False
+
+    :param answer_symbol: str
+        If speficifed the result of execution is fetched from the program's global context,
+        the program is expected to have the variable name mentioned in `answer_symbol` that is available in globals.
+        if not specified, the result are fetched from the stdout of the execution
+        defaults to None.
+
+    """
+
+    if os.getenv("HF_ALLOW_CODE_EVAL", 0) != "1":
+        raise ValueError(_WARNING)
+
+    if os.name == "nt":
+        raise NotImplementedError("This metric is currently not supported on Windows.")
+
+    with ThreadPoolExecutor(max_workers=num_workers) as executor:
+        futures = []
+        completion_id = Counter()
+        n_samples = 0
+        results = defaultdict(list)
+
+        for task_id, candidates in enumerate(predictions):
+            for candidate in candidates:
+                args = (candidate, timeout, task_id, completion_id[task_id])
+                if answer_symbol:
+                    args += (answer_symbol,)
+                future = executor.submit(run_program, *args)
+                futures.append(future)
+                completion_id[task_id] += 1
+                n_samples += 1
+
+        for future in as_completed(futures):
+            result = future.result()
+            results[result["task_id"]].append((result["completion_id"], result))
+
+    answers = [None] * len(results)
+    for result in results.values():
+        result.sort()
+        task_id = result[0][1]["task_id"]
+        # filtering the failed generations to avoid influencing majority voting
+        eval_answers = [
+            r[1]["result"]
+            for r in result
+            if isinstance(r[1]["result"], str)
+            and not r[1]["result"].startswith("failed:")
+        ]
+        # if all generations are failed - default to empty str for soring
+        eval_answers = [""] if len(eval_answers) == 0 else eval_answers
+        if majority_voting:
+            counter = Counter(eval_answers)
+            eval_answers = [counter.most_common()[0][0]]
+
+        if not majority_voting and len(eval_answers) > 1:
+            warnings.warn(
+                f"Multiple generations found for a task without setting `majority_voting` to True, defaulting answers from first generation"
+            )
+        answers[task_id] = eval_answers[0]
+
+    scores = []
+    # Number of code generated that failed execution.
+    errored = 0
+    for task_id, (ans, ref) in enumerate(zip(answers, references)):
+        try:
+            score = 1 if abs(float(ans) - float(ref)) < 1e-3 else 0
+        except ValueError as e:
+            errored += 1
+            score = 0
+
+        scores.append(score)
+
+    return {"accuracy": sum(scores) / len(scores), "num_failed_execution": errored}

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/pal_metric/python_executor.py

+# This code is adapted from HF evaluate repo
+# https://github.com/huggingface/evaluate/blob/main/metrics/code_eval/execute.py  and from PAL repo
+# https://github.com/reasoning-machines/pal/blob/main/pal/core/runtime.py
+
+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import tempfile
+
+
+def run_program(program, timeout, task_id, completion_id, answer_symbol=None):
+    """
+    Executes the given program and returns the results of the execution fetched from
+    stdout or from a variable in the program context
+
+    :param completion_id: an optional completion ID so we can match
+        the results later even if execution finishes asynchronously.
+    :param answer_symbol: If speficifed the result of execution is fetched from the program global context,
+        the program is expected to have the variable name mentioned in `answer_symbol` that is available in globals.
+        if not specified, the result are fetched from the stdout of the execution
+
+    """
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(
+        target=unsafe_execute, args=(program, result, timeout, answer_symbol)
+    )
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append("failed: timed out")
+
+    return dict(
+        task_id=task_id,
+        result=result[0],
+        completion_id=completion_id,
+    )
+
+
+def unsafe_execute(program, result, timeout, answer_symbol=None):
+
+    with create_tempdir():
+
+        # These system calls are needed when cleaning up tempdir.
+        import os
+        import shutil
+
+        rmtree = shutil.rmtree
+        rmdir = os.rmdir
+        chdir = os.chdir
+
+        # Disable functionalities that can make destructive changes to the test.
+        reliability_guard()
+
+        # Run program.
+        try:
+            exec_globals = {}
+            program_io = io.StringIO()
+            with swallow_io(program_io):
+                with time_limit(timeout):
+                    exec(program, exec_globals)
+            if answer_symbol:
+                result.append(exec_globals[answer_symbol])
+            else:
+                program_io.seek(0)
+                result.append(program_io.readlines()[-1].strip())
+        except TimeoutException:
+            result.append("failed: timed out")
+        except BaseException as e:
+            result.append(f"failed: {e}")
+
+        # Needed for cleaning up.
+        shutil.rmtree = rmtree
+        os.rmdir = rmdir
+        os.chdir = chdir
+
+
+@contextlib.contextmanager
+def time_limit(seconds):
+    def signal_handler(signum, frame):
+        raise TimeoutException("Timed out!")
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io(stream):
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield stream
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """StringIO that throws an exception when it's read from"""
+
+    def read(self, *args, **kwargs):
+        raise OSError
+
+    def readline(self, *args, **kwargs):
+        raise OSError
+
+    def readlines(self, *args, **kwargs):
+        raise OSError
+
+    def readable(self, *args, **kwargs):
+        """Returns True if the IO object can be read."""
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = "stdin"
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == ".":
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes=None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(
+            resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)
+        )
+        resource.setrlimit(
+            resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)
+        )
+        if not platform.uname().system == "Darwin":
+            resource.setrlimit(
+                resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)
+            )
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ["OMP_NUM_THREADS"] = "1"
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__["help"] = None
+
+    import sys
+
+    sys.modules["ipdb"] = None
+    sys.modules["joblib"] = None
+    sys.modules["resource"] = None
+    sys.modules["psutil"] = None
+    sys.modules["tkinter"] = None