v1.0

scripts/lcb_runner/evaluation/compute_code_execution_metrics.py

+import numpy as np
+from concurrent.futures import ProcessPoolExecutor
+import tqdm
+
+from lcb_runner.evaluation.utils_execute import BASE_IMPORTS, check_correctness
+
+def evaluate_score(args) -> list[bool]:
+    gs, (c, i, o) = args
+
+    execution_results = []
+    for g in gs:
+        if i in g:
+            pass
+        else:
+            code_to_execute = f"{BASE_IMPORTS}\n{c}\nassert {o} == {g}"
+            execution_results.append(check_correctness(code_to_execute, 3))
+    if len(execution_results) == 0:
+        execution_results = [False] * len(gs)
+    return execution_results
+
+def pass_at_k(n, c, k):
+    if n - c < k: return 1.0
+    return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+def code_execution_metrics(
+    samples,
+    generations,
+):
+    # execute the code
+    references = [(doc["code"], doc["input"], doc["output"]) for doc in samples]
+    with ProcessPoolExecutor() as executor:
+        args_list = zip(generations, references)
+        results = executor.map(evaluate_score, args_list)
+    all_results = list(results)
+
+    # serial version
+    # all_results = []
+    # for i in range(len(generations)):
+    #     generation = generations[i]
+    #     result = evaluate_score([generation, references[i]])
+    #     all_results.append(result)
+
+    # compute pass@1
+    pass_at_1s = []
+    for execution_result in all_results:
+        c, n = execution_result.count(True), len(execution_result)
+        pass_at_1s.append(pass_at_k(n, c, 1))
+    metrics = {"pass@1": sum(pass_at_1s) / len(pass_at_1s) * 100}
+
+    results = {}
+    for i, r in enumerate(all_results):
+        r_new = []
+        for _r in r:
+            r_new.append([_r])
+        results[i] = r_new
+    return [metrics, results]

scripts/lcb_runner/evaluation/compute_code_generation_metrics.py

+# borrowed and extended from
+# https://github.com/Naman-ntc/codescratch/blob/main/evaluation/bigcode-evaluation-harness/lm_eval/tasks/custom_metrics/apps_custom_metrics/utils.py
+
+import os
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+import json
+import multiprocessing
+from collections import defaultdict
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+
+import numpy as np
+from tqdm import tqdm
+
+from lcb_runner.evaluation.testing_util import run_test
+from lcb_runner.evaluation.pass_k_utils import compute_metrics_from_results
+
+
+def _temp_run(sample, generation, debug, result, metadata_list, timeout):
+    res, metadata = run_test(sample, test=generation, debug=debug, timeout=timeout)
+    result.append(res)
+    metadata_list.append(metadata)
+
+
+def check_correctness(sample, generation, timeout, debug=True):
+    """Check correctness of code generation with a global timeout.
+    The global timeout is to catch some extreme/rare cases not handled by the timeouts
+    inside `run_test`"""
+
+    manager = multiprocessing.Manager()
+    result = manager.list()
+    metadata_list = manager.list()
+    p = multiprocessing.Process(
+        target=_temp_run,
+        args=(sample, generation, debug, result, metadata_list, timeout),
+    )
+    p.start()
+    p.join(
+        timeout=(timeout + 1) * len(json.loads(sample["input_output"])["inputs"]) + 5
+    )
+    if p.is_alive():
+        p.kill()
+    if not result:
+        in_outs = json.loads(sample["input_output"])
+        # consider that all tests failed
+        result = [[-1 for i in range(len(in_outs["inputs"]))]]
+        if debug:
+            print(f"global timeout")
+    return result[0], metadata_list[0]
+
+
+def evaluate_generations_by_problem(args):
+    problem_generations: list[str] = args[0]
+    sample = args[1]
+    debug: bool = args[2]
+    timeout: int = args[3]
+
+    res = []
+    metadata = []
+    for o_idx, o in enumerate(problem_generations):
+        curr_res = [-2]
+        try:
+            curr_res, curr_metadata = check_correctness(
+                sample, o, timeout=timeout, debug=debug
+            )
+            if debug:
+                print(f"\nSuccessful compilation of task {o_idx}!")
+            fixed = []
+            for e in curr_res:
+                if isinstance(e, np.ndarray):
+                    e = e.item(0)
+                if isinstance(e, np.bool_):
+                    e = bool(e)
+                fixed.append(e)
+            curr_res = fixed
+            if not np.all(curr_res):
+                if debug:
+                    print(f"Results were not True for all test cases {curr_res=}\n")
+        except Exception as e:
+            if debug:
+                print(f"Compilation failed, test framework exception = {repr(e)}{e}\n")
+            # break
+            curr_metadata = {
+                "error": repr(e),
+                "error_code": -5,
+                "error_message": "TestRunnerError",
+            }
+        finally:
+            assert isinstance(curr_res, list)
+            assert isinstance(curr_metadata, dict)
+            res.append(curr_res)
+            metadata.append(curr_metadata)
+    if debug:
+        for i, r in enumerate(problem_generations):
+            print("Sample\n")
+            print(r)
+            print("\n")
+            print("Result\n")
+            print(res[i])
+            print("*" * 30 + "\n\n")
+    return res, metadata
+
+
+def evaluate_generations(
+    samples_list: list,
+    generations_list: list[list[str]],
+    debug: bool = False,
+    num_process_evaluate: int = 16,
+    timeout=6,
+):
+    """We take the list of code generations and try to compile them
+     and the run their corresponding unit tests which are retrieved from the APPS dataset.
+
+    Args:
+        generations: list of code generations (same order as samples in APPS dataset)
+        level: difficulty level used in the generation, can be "all", "introductory", "interview" or "competition"
+
+    Returns:
+        results: dictionary of results, key is the problem index, value is a list of results for each generation
+        [-2] = compile error, [-1] = runtime error [False] = failed test case [True] = passed test case
+    """
+
+    # generations are code generations in the same order of the dataset
+
+    inputs = [
+        [(generations_list[index], samples_list[index], debug, timeout), index]
+        for index in range(len(generations_list))
+    ]
+
+    with tqdm(total=len(inputs)) as pbar:
+        with ProcessPoolExecutor(
+            max_workers=1 if debug else num_process_evaluate
+        ) as executor:
+            futures = {
+                executor.submit(evaluate_generations_by_problem, arg): index
+                for arg, index in inputs
+            }
+
+            results = {}
+            metadata = {}
+            for future in as_completed(futures):
+                index = futures[future]
+                results[index], metadata[index] = future.result()
+                pbar.update(1)
+
+    assert len(results) == len(
+        inputs
+    ), f"results = {len(results)} inputs = {len(inputs)} {results=}"
+    # results = {i: r for r, (_, i) in zip(results, inputs)}
+
+    return results, metadata
+
+
+def codegen_metrics(
+    samples_list,
+    generations_list,
+    k_list=[1, 5, 10, 20, 40, 50, 75, 100, 125, 150, 200, 500, 1000],
+    num_process_evaluate=16,
+    timeout=6,
+    debug=False,
+):
+
+    samples_linear = []
+    generations_linear = []
+    remap_index = []
+    results = defaultdict(list)
+    metadatas = defaultdict(list)
+    for idx, (sample, generation_list) in enumerate(
+        zip(samples_list, generations_list)
+    ):
+        assert isinstance(generation_list, list), generations_list[0]
+        for generation in generation_list:
+            assert isinstance(generation, str), generations_list[0]
+            samples_linear.append(sample)
+            generations_linear.append([generation])
+            remap_index.append(idx)
+
+    print(f"Evaluating {len(samples_linear)}...")
+
+    results_linear, metadatas_linear = evaluate_generations(
+        samples_linear,
+        generations_linear,
+        debug=debug,
+        num_process_evaluate=num_process_evaluate,
+        timeout=timeout,
+    )
+
+    for idx, sub_results in sorted(results_linear.items(), key=lambda x: x[0]):
+        results[remap_index[idx]].append(sub_results[0])
+
+    for idx, sub_metadatas in sorted(metadatas_linear.items(), key=lambda x: x[0]):
+        metadatas[remap_index[idx]].append(sub_metadatas[0])
+
+    metrics = compute_metrics_from_results(results, k_list=k_list)
+
+    final_metadata = []
+    for key in sorted(list(metadatas.keys())):
+        final_metadata.append(metadatas[key])
+    for i in range(len(final_metadata)):
+        if type(final_metadata[i]) is not list:
+            final_metadata[i] = [json.dumps(final_metadata[i])]
+        else:
+            final_metadata[i] = [json.dumps(x) for x in final_metadata[i]]
+
+        assert len(final_metadata[i]) == len(
+            generations_list[0]
+        ), f"{len(final_metadata[i])=}"
+
+    return [metrics, results, final_metadata]

scripts/lcb_runner/evaluation/compute_scores.py

+import json
+import argparse
+import numpy as np
+from datetime import datetime
+
+from lcb_runner.lm_styles import LanguageModelStore
+from lcb_runner.evaluation.pass_k_utils import (
+    estimate_pass_at_k,
+    compute_metrics_from_results,
+)
+from lcb_runner.utils.scenarios import Scenario
+from lcb_runner.utils.path_utils import get_eval_all_output_path
+
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model",
+        type=str,
+        default="gpt-3.5-turbo-0301",
+        help="Name of the model to use matching `lm_styles.py`",
+    )
+    parser.add_argument(
+        "--scenario",
+        type=Scenario,
+        default=Scenario.codegeneration,
+        help="Type of scenario to run",
+    )
+    parser.add_argument(
+        "--n", type=int, default=10, help="Number of samples to generate"
+    )
+    parser.add_argument(
+        "--temperature", type=float, default=0.2, help="Temperature for sampling"
+    )
+
+    parser.add_argument(
+        "--eval_all_file",
+        type=str,
+        default=None,
+        help="Alternative way to provide the evaluation file",
+    )
+
+    parser.add_argument(
+        "--start_date",
+        type=str,
+        default=None,
+        help="Start date for the contest to filter the evaluation file (format - YYYY-MM-DD)",
+    )
+    parser.add_argument(
+        "--end_date",
+        type=str,
+        default=None,
+        help="End date for the contest to filter the evaluation file (format - YYYY-MM-DD)",
+    )
+
+    parser.add_argument(
+        "--platform",
+        type=str,
+        default=None,
+        help="Platform to filter the evaluation file",
+    )
+
+    args = parser.parse_args()
+
+    if args.eval_all_file is None:
+        model = LanguageModelStore[args.model]
+        args.eval_all_file = get_eval_all_output_path(model, args)
+
+    return args
+
+
+def compute_scores(args):
+    with open(args.eval_all_file, "r") as f:
+        results = json.load(f)
+
+    for res in results:
+        res["contest_date"] = datetime.fromisoformat(res["contest_date"])
+
+    if args.start_date is not None:
+        args.start_date = datetime.strptime(args.start_date, "%Y-%m-%d")
+        results = [
+            result for result in results if args.start_date <= result["contest_date"]
+        ]
+
+    if args.end_date is not None:
+        args.end_date = datetime.strptime(args.end_date, "%Y-%m-%d")
+        results = [
+            result for result in results if result["contest_date"] <= args.end_date
+        ]
+
+    if args.platform is not None:
+        results = [result for result in results if result["platform"] == args.platform]
+
+    print(len(results))
+    totals = [len(x["graded_list"]) for x in results]
+    corrects = [sum(x["graded_list"]) for x in results]
+
+    easy_totals = [len(x["graded_list"]) for x in results if x["difficulty"] == "easy"]
+    med_totals = [len(x["graded_list"]) for x in results if x["difficulty"] == "medium"]
+    hard_totals = [len(x["graded_list"]) for x in results if x["difficulty"] == "hard"]
+    easy_corrects = [sum(x["graded_list"]) for x in results if x["difficulty"] == "easy"]
+    med_corrects = [sum(x["graded_list"]) for x in results if x["difficulty"] == "medium"]
+    hard_corrects = [sum(x["graded_list"]) for x in results if x["difficulty"] == "hard"]
+    for k in [1, 5, 10, 25, 50, 100, 150, 200]:
+        print(
+            f"Pass@{k} = ",
+            estimate_pass_at_k(totals, corrects, k).mean(),
+            # np.array(
+            #     [estimate_pass_at_k(t, c, k) for t, c in zip(totals, corrects)]
+            # ).mean(),
+        )
+        print(
+            f"Easy Pass@{k} = ",
+            estimate_pass_at_k(easy_totals, easy_corrects, k).mean(),
+        )
+        print(
+            f"Medium Pass@{k} = ",
+            estimate_pass_at_k(med_totals, med_corrects, k).mean(),
+        )
+        print(
+            f"Hard Pass@{k} = ",
+            estimate_pass_at_k(hard_totals, hard_corrects, k).mean(),
+        )
+
+
+    pass_1_list = [result["pass@1"] for result in results]
+    print(f"Pass@1: {sum(pass_1_list) / len(pass_1_list)}")
+
+    easy_pass_1_list = [
+        result["pass@1"]
+        for result in results
+        if "difficulty" in result and result["difficulty"] == "easy"
+    ]
+    if len(easy_pass_1_list) > 0:
+        print(f"Easy Pass@1: {sum(easy_pass_1_list) / len(easy_pass_1_list)}")
+
+    medium_pass_1_list = [
+        result["pass@1"]
+        for result in results
+        if "difficulty" in result and result["difficulty"] == "medium"
+    ]
+    if len(medium_pass_1_list) > 0:
+        print(f"Medium Pass@1: {sum(medium_pass_1_list) / len(medium_pass_1_list)}")
+
+    hard_pass_1_list = [
+        result["pass@1"]
+        for result in results
+        if "difficulty" in result and result["difficulty"] == "hard"
+    ]
+    if len(hard_pass_1_list) > 0:
+        print(f"Hard Pass@1: {sum(hard_pass_1_list) / len(hard_pass_1_list)}")
+
+
+if __name__ == "__main__":
+    compute_scores(get_parser())

scripts/lcb_runner/evaluation/compute_test_output_prediction_metrics.py

+import ast
+import json
+
+import tqdm
+
+from lcb_runner.evaluation.pass_k_utils import compute_metrics_from_results
+
+
+def parse_assert_statement(statement):
+    """
+    Parse a Python assert statement and extract the expected output
+    from the right side of the '==' operator as a string.
+
+    :param statement: A string containing the assert statement.
+    :return: The expected output from the assert statement as a string.
+    """
+    try:
+        parsed = ast.parse(statement, mode="exec")
+    except SyntaxError:
+        return "Invalid syntax"
+
+    if len(parsed.body) == 0:
+        return "Empty statement"
+
+    if not isinstance(parsed.body[0], ast.Assert):
+        return "Not an assert statement"
+
+    comparison = parsed.body[0].test
+
+    if not isinstance(comparison, ast.Compare) or not isinstance(
+        comparison.ops[0], ast.Eq
+    ):
+        return "Not an equality assertion"
+
+    # Extract and return the right side of the '==' operator as a string
+    return ast.get_source_segment(statement, comparison.comparators[0])
+
+
+def check_testcase_output(testcase_str, expected_output):
+
+    if len(testcase_str.splitlines()) > 1:
+        for line in testcase_str.splitlines():
+            if line.startswith("#"):
+                continue
+            if "assert" in line:
+                testcase_str = line
+                break
+
+    testcase_str = testcase_str.strip()
+
+    if "assert" in testcase_str:
+        testcase_output_str = str(parse_assert_statement(testcase_str))
+
+    else:
+        testcase_output_str = testcase_str
+
+    global_result = None
+
+    try:
+        testcase_output_eval = eval(testcase_output_str)
+    except:
+        global_result = False
+        # print("Failed to eval testcase output", testcase_output_str)
+        # breakpoint()
+
+    try:
+        expected_output_eval = json.loads(expected_output)
+    except:
+        global_result = False
+        print("Failed to eval expected testcase output", expected_output)
+
+    if global_result is None:
+        global_result = testcase_output_eval == expected_output_eval
+
+    return global_result
+
+
+def test_output_metrics(
+    samples,
+    generations,
+    k_list=[1, 5],
+):
+    num_samples = len(samples)
+    results = []
+    for idx in tqdm.tqdm(list(range(num_samples))):
+        idx_results = []
+        sample = samples[idx]
+        extracted_generation_list = generations[idx]
+        for extracted_generation in extracted_generation_list:
+            global_result = check_testcase_output(
+                extracted_generation, sample["output"]
+            )
+            idx_results.append([global_result])
+        results.append(idx_results)
+
+    results = {result_idx: results[result_idx] for result_idx in range(len(results))}
+
+    metrics = compute_metrics_from_results(results, k_list=k_list)
+
+    return [metrics, results]

scripts/lcb_runner/evaluation/old_results_check.py

+import numpy as np
+import json
+from lcb_runner.benchmarks import load_generation_dataset, CodeGenerationProblem
+from lcb_runner.evaluation import codegen_metrics
+
+
+dataset = load_generation_dataset()
+
+dataset = sorted(dataset, key=lambda x: x.question_id)
+
+
+def check_model(model_key):
+    path = f"/home/naman/Repos/LiveCodeBench/run_models_outputs/{model_key}/chat_0.2_checked.json"
+    with open(path) as f:
+        old_results = json.load(f)
+    old_results = sorted(old_results, key=lambda x: x["question_id"])
+    assert old_results[0]["question_id"] == dataset[0].question_id
+
+    def debug(idx):
+        codegen_metrics(
+            [dataset[idx].get_evaluation_sample()],
+            [old_results[idx]["code_list"][:1]],
+            debug=True,
+        )
+
+    def run(idx):
+        return codegen_metrics(
+            [dataset[idx].get_evaluation_sample()],
+            [old_results[idx]["code_list"]],
+        )
+
+    debug(380)
+    exit()
+    # debug(196)
+    # debug(352)
+
+    metrics = codegen_metrics(
+        [d.get_evaluation_sample() for d in dataset],
+        [r["code_list"] for r in old_results],
+        num_process_evaluate=12,
+    )
+    old_pass1 = np.mean([np.mean(r["pass1_list"]) for r in old_results])
+
+    print(old_pass1)
+    print(metrics[0]["pass@1"])
+
+    for idx in range(400):
+        old_pass1 = np.mean(old_results[idx]["pass1_list"])
+        new_pass1 = metrics[0]["detail"]["pass@1"][idx]
+        if not abs(old_pass1 - new_pass1) < 1e-4:
+            print(idx, old_pass1, new_pass1)
+
+
+# model_key = "GPT-4-Turbo-1106"
+# check_model(model_key)
+
+model_key = "Claude-3-Opus"
+check_model(model_key)
+
+model_key = "GPT-4-0613"
+check_model(model_key)
+
+model_key = "Mistral-Large"
+check_model(model_key)
+
+model_key = "Claude-3-Sonnet"
+check_model(model_key)
+
+model_key = "GPT-3.5-Turbo-0301"
+check_model(model_key)
+
+model_key = "Gemini-Pro"
+check_model(model_key)

scripts/lcb_runner/evaluation/pass_k_utils.py

+import numpy as np
+
+
+def estimate_pass_at_k(num_samples, num_correct, k):
+    """Estimates pass@k of each problem and returns them in an array."""
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """Calculates 1 - comb(n - c, k) / comb(n, k)."""
+        if n - c < k:
+            return 1.0
+        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+    import itertools
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array(
+        [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]
+    )
+
+
+def compute_metrics_from_results(results, k_list=[1, 5]):
+    total = []
+    correct = []
+    task_ids = []
+    for task_id, res in results.items():
+        all_correct = []
+        for generation in res:
+            gen = np.array(generation)
+            all_correct.append(np.all(gen > 0))
+        task_ids.append(task_id)
+        total.append(len(all_correct))
+        correct.append(sum(all_correct))
+    total = np.array(total)
+    correct = np.array(correct)
+    ks = k_list
+    detail_pass_at_k = {
+        f"pass@{k}": estimate_pass_at_k(total, correct, k).tolist()
+        for k in ks
+        if (total >= k).all()
+    }
+    pass_at_k = {
+        f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
+        for k in ks
+        if (total >= k).all()
+    }
+    detail_metrics = {k: dict(zip(task_ids, v)) for k, v in detail_pass_at_k.items()}
+    pass_at_k["detail"] = detail_metrics
+    return pass_at_k
+
+
+def extract_instance_results(results):
+    instance_wise_grades = {}
+    for task_id, res in results.items():
+        instance_wise_grades[task_id] = []
+        for generation in res:
+            instance_wise_grades[task_id].append(all([g > 0 for g in generation]))
+
+    instance_wise_grades = [
+        v for _, v in sorted(instance_wise_grades.items(), key=lambda item: item[0])
+    ]
+    return instance_wise_grades

scripts/lcb_runner/evaluation/utils_execute.py

+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This code is adapted from OpenAI's release
+# https://github.com/openai/human-eval/blob/master/human_eval/execution.py
+
+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import tempfile
+
+
+BASE_IMPORTS = """from itertools import accumulate, chain, combinations, count, permutations, product, groupby, islice, repeat
+from copy import deepcopy
+from string import ascii_lowercase
+from math import floor, log2, log10, sqrt, comb, gcd, ceil, inf, isqrt
+from collections import defaultdict, deque, Counter
+from bisect import bisect, bisect_left, bisect_right, insort
+from heapq import heappush, heappop, heapify, merge
+from functools import reduce, cache, lru_cache
+from random import randrange, shuffle
+from operator import itemgetter, sub
+from re import search as re_search  # Assuming 're' refers to a regex search
+from os.path import commonprefix
+from typing import List, Tuple, Dict, Set, Optional, Union, Any, Callable, Iterable, Iterator, Generator
+import copy
+import string
+import math
+import collections
+import bisect
+import heapq
+import functools
+import random
+import itertools
+import operator
+import re
+import numpy as np
+import pandas as pd
+from math import log, prod  # 'log' and 'prod' are functions in the math module
+from collections import deque, defaultdict, Counter, OrderedDict
+from itertools import accumulate, permutations, combinations, product, groupby, islice, chain, repeat, zip_longest, cycle
+from functools import lru_cache, reduce, partial
+# from sortedcontainers import SortedList, SortedDict, SortedSet
+# import sortedcontainers
+from operator import iand
+import sys
+"""
+
+def check_correctness(check_program, timeout=3):
+    """
+    Evaluates the functional correctness of a completion by running the test
+    suite provided in the problem.
+
+    :param completion_id: an optional completion ID so we can match
+        the results later even if execution finishes asynchronously.
+    """
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append("timed out")
+
+    return result[0] == "passed"
+
+
+def unsafe_execute(check_program, result, timeout):
+
+    with create_tempdir():
+
+        # These system calls are needed when cleaning up tempdir.
+        import os
+        import shutil
+
+        rmtree = shutil.rmtree
+        rmdir = os.rmdir
+        chdir = os.chdir
+
+        # Disable functionalities that can make destructive changes to the test.
+        reliability_guard()
+
+        # Run program.
+        try:
+            exec_globals = {}
+            with swallow_io():
+                with time_limit(timeout):
+                    exec(check_program, exec_globals)
+            result.append("passed")
+        except TimeoutException:
+            result.append("timed out")
+        except BaseException as e:
+            result.append(f"failed: {e}")
+
+        # Needed for cleaning up.
+        shutil.rmtree = rmtree
+        os.rmdir = rmdir
+        os.chdir = chdir
+
+
+@contextlib.contextmanager
+def time_limit(seconds):
+    def signal_handler(signum, frame):
+        raise TimeoutException("Timed out!")
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """StringIO that throws an exception when it's read from"""
+
+    def read(self, *args, **kwargs):
+        raise OSError
+
+    def readline(self, *args, **kwargs):
+        raise OSError
+
+    def readlines(self, *args, **kwargs):
+        raise OSError
+
+    def readable(self, *args, **kwargs):
+        """Returns True if the IO object can be read."""
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = "stdin"
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == ".":
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes=None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == "Darwin":
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ["OMP_NUM_THREADS"] = "1"
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__["help"] = None
+
+    import sys
+
+    sys.modules["ipdb"] = None
+    sys.modules["joblib"] = None
+    sys.modules["resource"] = None
+    sys.modules["psutil"] = None
+    sys.modules["tkinter"] = None
\ No newline at end of file