v1.0

scripts/lcb_runner/evaluation/compute_code_execution_metrics.py

+import numpy as np
+from concurrent.futures import ProcessPoolExecutor
+import tqdm
+
+from lcb_runner.evaluation.utils_execute import BASE_IMPORTS, check_correctness
+
+def evaluate_score(args) -> list[bool]:
+    gs, (c, i, o) = args
+
+    execution_results = []
+    for g in gs:
+        if i in g:
+            pass
+        else:
+            code_to_execute = f"{BASE_IMPORTS}\n{c}\nassert {o} == {g}"
+            execution_results.append(check_correctness(code_to_execute, 3))
+    if len(execution_results) == 0:
+        execution_results = [False] * len(gs)
+    return execution_results
+
+def pass_at_k(n, c, k):
+    if n - c < k: return 1.0
+    return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+def code_execution_metrics(
+    samples,
+    generations,
+):
+    # execute the code
+    references = [(doc["code"], doc["input"], doc["output"]) for doc in samples]
+    with ProcessPoolExecutor() as executor:
+        args_list = zip(generations, references)
+        results = executor.map(evaluate_score, args_list)
+    all_results = list(results)
+
+    # serial version
+    # all_results = []
+    # for i in range(len(generations)):
+    #     generation = generations[i]
+    #     result = evaluate_score([generation, references[i]])
+    #     all_results.append(result)
+
+    # compute pass@1
+    pass_at_1s = []
+    for execution_result in all_results:
+        c, n = execution_result.count(True), len(execution_result)
+        pass_at_1s.append(pass_at_k(n, c, 1))
+    metrics = {"pass@1": sum(pass_at_1s) / len(pass_at_1s) * 100}
+
+    results = {}
+    for i, r in enumerate(all_results):
+        r_new = []
+        for _r in r:
+            r_new.append([_r])
+        results[i] = r_new
+    return [metrics, results]

scripts/lcb_runner/evaluation/compute_code_generation_metrics.py

+# borrowed and extended from
+# https://github.com/Naman-ntc/codescratch/blob/main/evaluation/bigcode-evaluation-harness/lm_eval/tasks/custom_metrics/apps_custom_metrics/utils.py
+
+import os
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+import json
+import multiprocessing
+from collections import defaultdict
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+
+import numpy as np
+from tqdm import tqdm
+
+from lcb_runner.evaluation.testing_util import run_test
+from lcb_runner.evaluation.pass_k_utils import compute_metrics_from_results
+
+
+def _temp_run(sample, generation, debug, result, metadata_list, timeout):
+    res, metadata = run_test(sample, test=generation, debug=debug, timeout=timeout)
+    result.append(res)
+    metadata_list.append(metadata)
+
+
+def check_correctness(sample, generation, timeout, debug=True):
+    """Check correctness of code generation with a global timeout.
+    The global timeout is to catch some extreme/rare cases not handled by the timeouts
+    inside `run_test`"""
+
+    manager = multiprocessing.Manager()
+    result = manager.list()
+    metadata_list = manager.list()
+    p = multiprocessing.Process(
+        target=_temp_run,
+        args=(sample, generation, debug, result, metadata_list, timeout),
+    )
+    p.start()
+    p.join(
+        timeout=(timeout + 1) * len(json.loads(sample["input_output"])["inputs"]) + 5
+    )
+    if p.is_alive():
+        p.kill()
+    if not result:
+        in_outs = json.loads(sample["input_output"])
+        # consider that all tests failed
+        result = [[-1 for i in range(len(in_outs["inputs"]))]]
+        if debug:
+            print(f"global timeout")
+    return result[0], metadata_list[0]
+
+
+def evaluate_generations_by_problem(args):
+    problem_generations: list[str] = args[0]
+    sample = args[1]
+    debug: bool = args[2]
+    timeout: int = args[3]
+
+    res = []
+    metadata = []
+    for o_idx, o in enumerate(problem_generations):
+        curr_res = [-2]
+        try:
+            curr_res, curr_metadata = check_correctness(
+                sample, o, timeout=timeout, debug=debug
+            )
+            if debug:
+                print(f"\nSuccessful compilation of task {o_idx}!")
+            fixed = []
+            for e in curr_res:
+                if isinstance(e, np.ndarray):
+                    e = e.item(0)
+                if isinstance(e, np.bool_):
+                    e = bool(e)
+                fixed.append(e)
+            curr_res = fixed
+            if not np.all(curr_res):
+                if debug:
+                    print(f"Results were not True for all test cases {curr_res=}\n")
+        except Exception as e:
+            if debug:
+                print(f"Compilation failed, test framework exception = {repr(e)}{e}\n")
+            # break
+            curr_metadata = {
+                "error": repr(e),
+                "error_code": -5,
+                "error_message": "TestRunnerError",
+            }
+        finally:
+            assert isinstance(curr_res, list)
+            assert isinstance(curr_metadata, dict)
+            res.append(curr_res)
+            metadata.append(curr_metadata)
+    if debug:
+        for i, r in enumerate(problem_generations):
+            print("Sample\n")
+            print(r)
+            print("\n")
+            print("Result\n")
+            print(res[i])
+            print("*" * 30 + "\n\n")
+    return res, metadata
+
+
+def evaluate_generations(
+    samples_list: list,
+    generations_list: list[list[str]],
+    debug: bool = False,
+    num_process_evaluate: int = 16,
+    timeout=6,
+):
+    """We take the list of code generations and try to compile them
+     and the run their corresponding unit tests which are retrieved from the APPS dataset.
+
+    Args:
+        generations: list of code generations (same order as samples in APPS dataset)
+        level: difficulty level used in the generation, can be "all", "introductory", "interview" or "competition"
+
+    Returns:
+        results: dictionary of results, key is the problem index, value is a list of results for each generation
+        [-2] = compile error, [-1] = runtime error [False] = failed test case [True] = passed test case
+    """
+
+    # generations are code generations in the same order of the dataset
+
+    inputs = [
+        [(generations_list[index], samples_list[index], debug, timeout), index]
+        for index in range(len(generations_list))
+    ]
+
+    with tqdm(total=len(inputs)) as pbar:
+        with ProcessPoolExecutor(
+            max_workers=1 if debug else num_process_evaluate
+        ) as executor:
+            futures = {
+                executor.submit(evaluate_generations_by_problem, arg): index
+                for arg, index in inputs
+            }
+
+            results = {}
+            metadata = {}
+            for future in as_completed(futures):
+                index = futures[future]
+                results[index], metadata[index] = future.result()
+                pbar.update(1)
+
+    assert len(results) == len(
+        inputs
+    ), f"results = {len(results)} inputs = {len(inputs)} {results=}"
+    # results = {i: r for r, (_, i) in zip(results, inputs)}
+
+    return results, metadata
+
+
+def codegen_metrics(
+    samples_list,
+    generations_list,
+    k_list=[1, 5, 10, 20, 40, 50, 75, 100, 125, 150, 200, 500, 1000],
+    num_process_evaluate=16,
+    timeout=6,
+    debug=False,
+):
+
+    samples_linear = []
+    generations_linear = []
+    remap_index = []
+    results = defaultdict(list)
+    metadatas = defaultdict(list)
+    for idx, (sample, generation_list) in enumerate(
+        zip(samples_list, generations_list)
+    ):
+        assert isinstance(generation_list, list), generations_list[0]
+        for generation in generation_list:
+            assert isinstance(generation, str), generations_list[0]
+            samples_linear.append(sample)
+            generations_linear.append([generation])
+            remap_index.append(idx)
+
+    print(f"Evaluating {len(samples_linear)}...")
+
+    results_linear, metadatas_linear = evaluate_generations(
+        samples_linear,
+        generations_linear,
+        debug=debug,
+        num_process_evaluate=num_process_evaluate,
+        timeout=timeout,
+    )
+
+    for idx, sub_results in sorted(results_linear.items(), key=lambda x: x[0]):
+        results[remap_index[idx]].append(sub_results[0])
+
+    for idx, sub_metadatas in sorted(metadatas_linear.items(), key=lambda x: x[0]):
+        metadatas[remap_index[idx]].append(sub_metadatas[0])
+
+    metrics = compute_metrics_from_results(results, k_list=k_list)
+
+    final_metadata = []
+    for key in sorted(list(metadatas.keys())):
+        final_metadata.append(metadatas[key])
+    for i in range(len(final_metadata)):
+        if type(final_metadata[i]) is not list:
+            final_metadata[i] = [json.dumps(final_metadata[i])]
+        else:
+            final_metadata[i] = [json.dumps(x) for x in final_metadata[i]]
+
+        assert len(final_metadata[i]) == len(
+            generations_list[0]
+        ), f"{len(final_metadata[i])=}"
+
+    return [metrics, results, final_metadata]

scripts/lcb_runner/evaluation/compute_scores.py

+import json
+import argparse
+import numpy as np
+from datetime import datetime
+
+from lcb_runner.lm_styles import LanguageModelStore
+from lcb_runner.evaluation.pass_k_utils import (
+    estimate_pass_at_k,
+    compute_metrics_from_results,
+)
+from lcb_runner.utils.scenarios import Scenario
+from lcb_runner.utils.path_utils import get_eval_all_output_path
+
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model",
+        type=str,
+        default="gpt-3.5-turbo-0301",
+        help="Name of the model to use matching `lm_styles.py`",
+    )
+    parser.add_argument(
+        "--scenario",
+        type=Scenario,
+        default=Scenario.codegeneration,
+        help="Type of scenario to run",
+    )
+    parser.add_argument(
+        "--n", type=int, default=10, help="Number of samples to generate"
+    )
+    parser.add_argument(
+        "--temperature", type=float, default=0.2, help="Temperature for sampling"
+    )
+
+    parser.add_argument(
+        "--eval_all_file",
+        type=str,
+        default=None,
+        help="Alternative way to provide the evaluation file",
+    )
+
+    parser.add_argument(
+        "--start_date",
+        type=str,
+        default=None,
+        help="Start date for the contest to filter the evaluation file (format - YYYY-MM-DD)",
+    )
+    parser.add_argument(
+        "--end_date",
+        type=str,
+        default=None,
+        help="End date for the contest to filter the evaluation file (format - YYYY-MM-DD)",
+    )
+
+    parser.add_argument(
+        "--platform",
+        type=str,
+        default=None,
+        help="Platform to filter the evaluation file",
+    )
+
+    args = parser.parse_args()
+
+    if args.eval_all_file is None:
+        model = LanguageModelStore[args.model]
+        args.eval_all_file = get_eval_all_output_path(model, args)
+
+    return args
+
+
+def compute_scores(args):
+    with open(args.eval_all_file, "r") as f:
+        results = json.load(f)
+
+    for res in results:
+        res["contest_date"] = datetime.fromisoformat(res["contest_date"])
+
+    if args.start_date is not None:
+        args.start_date = datetime.strptime(args.start_date, "%Y-%m-%d")
+        results = [
+            result for result in results if args.start_date <= result["contest_date"]
+        ]
+
+    if args.end_date is not None:
+        args.end_date = datetime.strptime(args.end_date, "%Y-%m-%d")
+        results = [
+            result for result in results if result["contest_date"] <= args.end_date
+        ]
+
+    if args.platform is not None:
+        results = [result for result in results if result["platform"] == args.platform]
+
+    print(len(results))
+    totals = [len(x["graded_list"]) for x in results]
+    corrects = [sum(x["graded_list"]) for x in results]
+
+    easy_totals = [len(x["graded_list"]) for x in results if x["difficulty"] == "easy"]
+    med_totals = [len(x["graded_list"]) for x in results if x["difficulty"] == "medium"]
+    hard_totals = [len(x["graded_list"]) for x in results if x["difficulty"] == "hard"]
+    easy_corrects = [sum(x["graded_list"]) for x in results if x["difficulty"] == "easy"]
+    med_corrects = [sum(x["graded_list"]) for x in results if x["difficulty"] == "medium"]
+    hard_corrects = [sum(x["graded_list"]) for x in results if x["difficulty"] == "hard"]
+    for k in [1, 5, 10, 25, 50, 100, 150, 200]:
+        print(
+            f"Pass@{k} = ",
+            estimate_pass_at_k(totals, corrects, k).mean(),
+            # np.array(
+            #     [estimate_pass_at_k(t, c, k) for t, c in zip(totals, corrects)]
+            # ).mean(),
+        )
+        print(
+            f"Easy Pass@{k} = ",
+            estimate_pass_at_k(easy_totals, easy_corrects, k).mean(),
+        )
+        print(
+            f"Medium Pass@{k} = ",
+            estimate_pass_at_k(med_totals, med_corrects, k).mean(),
+        )
+        print(
+            f"Hard Pass@{k} = ",
+            estimate_pass_at_k(hard_totals, hard_corrects, k).mean(),
+        )
+
+
+    pass_1_list = [result["pass@1"] for result in results]
+    print(f"Pass@1: {sum(pass_1_list) / len(pass_1_list)}")
+
+    easy_pass_1_list = [
+        result["pass@1"]
+        for result in results
+        if "difficulty" in result and result["difficulty"] == "easy"
+    ]
+    if len(easy_pass_1_list) > 0:
+        print(f"Easy Pass@1: {sum(easy_pass_1_list) / len(easy_pass_1_list)}")
+
+    medium_pass_1_list = [
+        result["pass@1"]
+        for result in results
+        if "difficulty" in result and result["difficulty"] == "medium"
+    ]
+    if len(medium_pass_1_list) > 0:
+        print(f"Medium Pass@1: {sum(medium_pass_1_list) / len(medium_pass_1_list)}")
+
+    hard_pass_1_list = [
+        result["pass@1"]
+        for result in results
+        if "difficulty" in result and result["difficulty"] == "hard"
+    ]
+    if len(hard_pass_1_list) > 0:
+        print(f"Hard Pass@1: {sum(hard_pass_1_list) / len(hard_pass_1_list)}")
+
+
+if __name__ == "__main__":
+    compute_scores(get_parser())

scripts/lcb_runner/evaluation/compute_test_output_prediction_metrics.py

+import ast
+import json
+
+import tqdm
+
+from lcb_runner.evaluation.pass_k_utils import compute_metrics_from_results
+
+
+def parse_assert_statement(statement):
+    """
+    Parse a Python assert statement and extract the expected output
+    from the right side of the '==' operator as a string.
+
+    :param statement: A string containing the assert statement.
+    :return: The expected output from the assert statement as a string.
+    """
+    try:
+        parsed = ast.parse(statement, mode="exec")
+    except SyntaxError:
+        return "Invalid syntax"
+
+    if len(parsed.body) == 0:
+        return "Empty statement"
+
+    if not isinstance(parsed.body[0], ast.Assert):
+        return "Not an assert statement"
+
+    comparison = parsed.body[0].test
+
+    if not isinstance(comparison, ast.Compare) or not isinstance(
+        comparison.ops[0], ast.Eq
+    ):
+        return "Not an equality assertion"
+
+    # Extract and return the right side of the '==' operator as a string
+    return ast.get_source_segment(statement, comparison.comparators[0])
+
+
+def check_testcase_output(testcase_str, expected_output):
+
+    if len(testcase_str.splitlines()) > 1:
+        for line in testcase_str.splitlines():
+            if line.startswith("#"):
+                continue
+            if "assert" in line:
+                testcase_str = line
+                break
+
+    testcase_str = testcase_str.strip()
+
+    if "assert" in testcase_str:
+        testcase_output_str = str(parse_assert_statement(testcase_str))
+
+    else:
+        testcase_output_str = testcase_str
+
+    global_result = None
+
+    try:
+        testcase_output_eval = eval(testcase_output_str)
+    except:
+        global_result = False
+        # print("Failed to eval testcase output", testcase_output_str)
+        # breakpoint()
+
+    try:
+        expected_output_eval = json.loads(expected_output)
+    except:
+        global_result = False
+        print("Failed to eval expected testcase output", expected_output)
+
+    if global_result is None:
+        global_result = testcase_output_eval == expected_output_eval
+
+    return global_result
+
+
+def test_output_metrics(
+    samples,
+    generations,
+    k_list=[1, 5],
+):
+    num_samples = len(samples)
+    results = []
+    for idx in tqdm.tqdm(list(range(num_samples))):
+        idx_results = []
+        sample = samples[idx]
+        extracted_generation_list = generations[idx]
+        for extracted_generation in extracted_generation_list:
+            global_result = check_testcase_output(
+                extracted_generation, sample["output"]
+            )
+            idx_results.append([global_result])
+        results.append(idx_results)
+
+    results = {result_idx: results[result_idx] for result_idx in range(len(results))}
+
+    metrics = compute_metrics_from_results(results, k_list=k_list)
+
+    return [metrics, results]

scripts/lcb_runner/evaluation/old_results_check.py

+import numpy as np
+import json
+from lcb_runner.benchmarks import load_generation_dataset, CodeGenerationProblem
+from lcb_runner.evaluation import codegen_metrics
+
+
+dataset = load_generation_dataset()
+
+dataset = sorted(dataset, key=lambda x: x.question_id)
+
+
+def check_model(model_key):
+    path = f"/home/naman/Repos/LiveCodeBench/run_models_outputs/{model_key}/chat_0.2_checked.json"
+    with open(path) as f:
+        old_results = json.load(f)
+    old_results = sorted(old_results, key=lambda x: x["question_id"])
+    assert old_results[0]["question_id"] == dataset[0].question_id
+
+    def debug(idx):
+        codegen_metrics(
+            [dataset[idx].get_evaluation_sample()],
+            [old_results[idx]["code_list"][:1]],
+            debug=True,
+        )
+
+    def run(idx):
+        return codegen_metrics(
+            [dataset[idx].get_evaluation_sample()],
+            [old_results[idx]["code_list"]],
+        )
+
+    debug(380)
+    exit()
+    # debug(196)
+    # debug(352)
+
+    metrics = codegen_metrics(
+        [d.get_evaluation_sample() for d in dataset],
+        [r["code_list"] for r in old_results],
+        num_process_evaluate=12,
+    )
+    old_pass1 = np.mean([np.mean(r["pass1_list"]) for r in old_results])
+
+    print(old_pass1)
+    print(metrics[0]["pass@1"])
+
+    for idx in range(400):
+        old_pass1 = np.mean(old_results[idx]["pass1_list"])
+        new_pass1 = metrics[0]["detail"]["pass@1"][idx]
+        if not abs(old_pass1 - new_pass1) < 1e-4:
+            print(idx, old_pass1, new_pass1)
+
+
+# model_key = "GPT-4-Turbo-1106"
+# check_model(model_key)
+
+model_key = "Claude-3-Opus"
+check_model(model_key)
+
+model_key = "GPT-4-0613"
+check_model(model_key)
+
+model_key = "Mistral-Large"
+check_model(model_key)
+
+model_key = "Claude-3-Sonnet"
+check_model(model_key)
+
+model_key = "GPT-3.5-Turbo-0301"
+check_model(model_key)
+
+model_key = "Gemini-Pro"
+check_model(model_key)

scripts/lcb_runner/evaluation/pass_k_utils.py

+import numpy as np
+
+
+def estimate_pass_at_k(num_samples, num_correct, k):
+    """Estimates pass@k of each problem and returns them in an array."""
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """Calculates 1 - comb(n - c, k) / comb(n, k)."""
+        if n - c < k:
+            return 1.0
+        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+    import itertools
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array(
+        [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]
+    )
+
+
+def compute_metrics_from_results(results, k_list=[1, 5]):
+    total = []
+    correct = []
+    task_ids = []
+    for task_id, res in results.items():
+        all_correct = []
+        for generation in res:
+            gen = np.array(generation)
+            all_correct.append(np.all(gen > 0))
+        task_ids.append(task_id)
+        total.append(len(all_correct))
+        correct.append(sum(all_correct))
+    total = np.array(total)
+    correct = np.array(correct)
+    ks = k_list
+    detail_pass_at_k = {
+        f"pass@{k}": estimate_pass_at_k(total, correct, k).tolist()
+        for k in ks
+        if (total >= k).all()
+    }
+    pass_at_k = {
+        f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
+        for k in ks
+        if (total >= k).all()
+    }
+    detail_metrics = {k: dict(zip(task_ids, v)) for k, v in detail_pass_at_k.items()}
+    pass_at_k["detail"] = detail_metrics
+    return pass_at_k
+
+
+def extract_instance_results(results):
+    instance_wise_grades = {}
+    for task_id, res in results.items():
+        instance_wise_grades[task_id] = []
+        for generation in res:
+            instance_wise_grades[task_id].append(all([g > 0 for g in generation]))
+
+    instance_wise_grades = [
+        v for _, v in sorted(instance_wise_grades.items(), key=lambda item: item[0])
+    ]
+    return instance_wise_grades

scripts/lcb_runner/evaluation/testing_util.py

+import ast
+import json
+import sys
+import faulthandler
+import platform
+
+# used for debugging to time steps
+from datetime import datetime
+
+# to run the solution files we're using a timing based approach
+import signal
+
+import numpy as np
+
+# for capturing the stdout
+from io import StringIO
+
+# used for testing the code that reads from input
+from unittest.mock import patch, mock_open
+
+from pyext import RuntimeModule
+
+from enum import Enum
+
+
+def truncatefn(s, length=300):
+    assert isinstance(s, str)
+    if len(s) <= length:
+        return s
+
+    return s[: length // 2] + "...(truncated) ..." + s[-length // 2 :]
+
+
+class CODE_TYPE(Enum):
+    call_based = 0
+    standard_input = 1
+
+
+# stuff for setting up signal timer
+class TimeoutException(Exception):
+    pass
+
+
+def timeout_handler(signum, frame):
+    print("alarm went off")
+    # return
+    raise TimeoutException
+
+
+signal.signal(signal.SIGALRM, timeout_handler)
+# timeout = 6  # seconds
+
+
+# used to capture stdout as a list
+# from https://stackoverflow.com/a/16571630/6416660
+# alternative use redirect_stdout() from contextlib
+class Capturing(list):
+    def __enter__(self):
+        self._stdout = sys.stdout
+        sys.stdout = self._stringio = StringIO()
+        # Make closing the StringIO a no-op
+        self._stringio.close = lambda x: 1
+        return self
+
+    def __exit__(self, *args):
+        self.append(self._stringio.getvalue())
+        del self._stringio  # free up some memory
+        sys.stdout = self._stdout
+
+
+def only_int_check(val):
+    return isinstance(val, int)
+
+
+def string_int_check(val):
+    return isinstance(val, str) and val.isdigit()
+
+
+def combined_int_check(val):
+    return only_int_check(val) or string_int_check(val)
+
+
+def run_test(sample, test=None, debug=False, timeout=6):
+    """
+    if test(generated_code) is not None it'll try to run the code.
+    otherwise it'll just return an input and output pair.
+    """
+    # Disable functionalities that can make destructive changes to the test.
+    reliability_guard()
+
+    if debug:
+        print(f"start = {datetime.now().time()}")
+
+    try:
+        in_outs = json.loads(sample["input_output"])
+    except ValueError:
+        in_outs = None
+    if in_outs:
+        if in_outs.get("fn_name") is None:
+            which_type = CODE_TYPE.standard_input  # Standard input
+            method_name = None
+        else:
+            which_type = CODE_TYPE.call_based  # Call-based
+            method_name = in_outs["fn_name"]
+
+    if debug:
+        print(f"loaded input_output = {datetime.now().time()}")
+
+    if test is None:
+        assert False, "should not happen: test code is none"
+        return in_outs, {"error": "no test code provided"}
+    elif test is not None:
+        results = []
+        sol = "from string import *\nfrom re import *\nfrom datetime import *\nfrom collections import *\nfrom heapq import *\nfrom bisect import *\nfrom copy import *\nfrom math import *\nfrom random import *\nfrom statistics import *\nfrom itertools import *\nfrom functools import *\nfrom operator import *\nfrom io import *\nfrom sys import *\nfrom json import *\nfrom builtins import *\nfrom typing import *\nimport string\nimport re\nimport datetime\nimport collections\nimport heapq\nimport bisect\nimport copy\nimport math\nimport random\nimport statistics\nimport itertools\nimport functools\nimport operator\nimport io\nimport sys\nimport json\nsys.setrecursionlimit(6*10**5)\n"
+        if debug:
+            print(f"loading test code = {datetime.now().time()}")
+
+        if which_type == CODE_TYPE.call_based:
+
+            sol += test
+            if debug:
+                print(f"sol = {sol}")
+            signal.alarm(timeout)
+            try:
+                tmp_sol = RuntimeModule.from_string("tmp_sol", "", sol)
+                if "class Solution" not in test:
+                    tmp = tmp_sol
+                else:
+                    tmp = tmp_sol.Solution()
+                signal.alarm(0)
+            except Exception as e:
+                signal.alarm(0)
+                if debug:
+                    print(f"type 0 compilation error = {e}")
+                results.append(-2)
+                return results, {
+                    "error": repr(e),
+                    "error_code": -1,
+                    "error_message": "Compilation Error",
+                }
+            signal.alarm(0)
+
+        elif which_type == CODE_TYPE.standard_input:
+            # sol
+            # if code has if __name__ == "__main__": then remove it
+            try:
+                astree = ast.parse(test)
+                last_block = astree.body[-1]
+                if isinstance(last_block, ast.If):
+                    condition = last_block.test
+                    if ast.unparse(condition).strip() == "__name__ == '__main__'":
+                        test = (
+                            ast.unparse(astree.body[:-1])
+                            + "\n"
+                            + ast.unparse(last_block.body)
+                        )
+            except:
+                pass
+
+            tmp_test = test.split("\n")
+
+            new_test = []
+            for x in tmp_test:
+                if (not x.startswith("from ")) and (not x.startswith("import ")):
+                    new_test.append("\t" + x + "\n")
+                else:
+                    new_test.append(x + "\n")
+            tmp_test = new_test
+
+            new_test = ""
+            started = False
+            for i in tmp_test:
+                if i.startswith("\t") and not started:
+                    new_test += "stdin = sys.stdin\nstdout = sys.stdout\n"
+                    new_test += "def code():\n"
+                    new_test += i
+                    started = True
+                elif started and ((i.startswith("from ")) or (i.startswith("import "))):
+                    new_test += "\t" + i
+                else:
+                    new_test += i
+            tmp_test = new_test
+
+            sol += tmp_test
+            if debug:
+                print(f"sol = {sol}")
+            method_name = "code"
+            signal.alarm(timeout)
+            try:
+                tmp_sol = RuntimeModule.from_string("tmp_sol", "", sol)
+                tmp = tmp_sol
+                signal.alarm(0)
+            except Exception as e:
+                signal.alarm(0)
+                if debug:
+                    print(f"type 1 compilation error = {e}")
+                results.append(-2)
+                return results, {
+                    "error": repr(e),
+                    "error_code": -1,
+                    "error_message": "Compilation Error",
+                }
+            signal.alarm(0)
+        if debug:
+            print(f"get method = {datetime.now().time()}")
+
+        try:
+            method = getattr(tmp, method_name)  # get_attr second arg must be str
+        except:
+            signal.alarm(0)
+            e = sys.exc_info()
+            print(f"unable to get function error = {e}")
+            results.append(-2)
+            return results, {
+                "error": repr(e),
+                "error_code": -1,
+                "error_message": "Unable to extract code",
+            }
+
+        for index, inputs in enumerate(in_outs["inputs"]):
+            raw_inputs = inputs
+            raw_outputs = in_outs["outputs"][index]
+            if which_type == CODE_TYPE.call_based:
+                inputs = [json.loads(line) for line in inputs.split("\n")]
+                in_outs["outputs"][index] = json.loads(in_outs["outputs"][index])
+
+                truncate_line_size = 300 // (raw_inputs.count("\n") + 1)
+                raw_inputs = "\n".join(
+                    [
+                        truncatefn(line, truncate_line_size)
+                        for line in raw_inputs.strip().split("\n")
+                    ]
+                )
+                raw_outputs = truncatefn(raw_outputs, 200)
+            else:
+                raw_inputs = truncatefn(raw_inputs)
+                raw_outputs = truncatefn(raw_outputs, 200)
+            # JSON forces dictionaries to have string keys; this undoes this (assuming a singleton list)
+            try:
+                if isinstance(inputs[0], dict):
+                    inputs = [{int(k): v for k, v in inputs[0].items()}]
+            except:
+                True
+            try:
+                if isinstance(in_outs["outputs"][index], dict):
+                    in_outs["outputs"][index] = [
+                        {int(k): v for k, v in in_outs["outputs"][index].items()}
+                    ]
+            except:
+                True
+            try:
+                if isinstance(in_outs["outputs"][index][0], dict):
+                    in_outs["outputs"][index] = [
+                        {int(k): v for k, v in in_outs["outputs"][index][0].items()}
+                    ]
+            except:
+                True
+
+            if debug:
+                print(
+                    f"time: {datetime.now().time()} testing index = {index}  inputs = {inputs}, {type(inputs)}. type = {which_type}"
+                )
+            if which_type == CODE_TYPE.call_based:  # Call-based
+                signal.alarm(timeout)
+                faulthandler.enable()
+                try:
+                    output = method(*inputs)
+                    raw_true_output = output
+
+                    raw_true_output_copy = json.dumps(output)
+                    raw_true_output_copy = truncatefn(raw_true_output_copy, 200)
+
+                    # ground truth sequences are not tuples
+                    if isinstance(output, tuple):
+                        output = list(output)
+
+                    tmp_result = output == in_outs["outputs"][index]
+                    if (
+                        isinstance(in_outs["outputs"][index], list)
+                        and in_outs["outputs"][index]
+                    ):
+                        tmp_result = tmp_result or (
+                            output == in_outs["outputs"][index][0]
+                        )
+
+                    # ground truth sequences are not tuples
+                    try:
+                        if isinstance(output[0], tuple):
+                            tmp_result = tmp_result or (
+                                [list(x) for x in output]
+                                == in_outs["outputs"][index][0]
+                            )
+                    except:
+                        True
+                    results.append(tmp_result)
+                    if tmp_result != True:
+                        return results, {
+                            "output": raw_true_output_copy,
+                            "expected": raw_outputs,
+                            "inputs": raw_inputs,
+                            "error_code": -2,
+                            "error_message": "Wrong Answer",
+                        }
+                    # reset the alarm
+                    signal.alarm(0)
+                except Exception as e:
+                    signal.alarm(0)
+                    faulthandler.disable()
+                    if debug:
+                        print(
+                            f"Standard input runtime error or time limit exceeded error = {e}"
+                        )
+                    results.append(-1)
+                    if "timeoutexception" in repr(e).lower():
+                        return results, {
+                            "error": repr(e),
+                            "error_code": -3,
+                            "error_message": "Time Limit Exceeded",
+                            "inputs": raw_inputs,
+                            "expected": raw_outputs,
+                        }
+                    else:
+                        return results, {
+                            "error": repr(e),
+                            "error_code": -4,
+                            "error_message": "Runtime Error",
+                            "inputs": raw_inputs,
+                            "expected": raw_outputs,
+                        }
+                faulthandler.disable()
+                signal.alarm(0)
+                if debug:
+                    print(
+                        f"outputs = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"
+                    )
+            elif which_type == CODE_TYPE.standard_input:  # Standard input
+                faulthandler.enable()
+                passed = False
+
+                if isinstance(inputs, list):
+                    inputs = "\n".join(inputs)
+                if isinstance(in_outs["outputs"][index], list):
+                    in_outs["outputs"][index] = "\n".join(in_outs["outputs"][index])
+
+                signal.alarm(timeout)
+                with Capturing() as output:
+                    try:
+                        call_method(method, inputs)
+                        # reset the alarm
+                        signal.alarm(0)
+                        passed = True
+                    except Exception as e:
+                        # runtime error or took too long
+                        signal.alarm(0)
+                        print(
+                            f"Call-based runtime error or time limit exceeded error = {repr(e)}{e}"
+                        )
+                        results.append(-1)
+                        if "timeoutexception" in repr(e).lower():
+                            return results, {
+                                "error": repr(e),
+                                "error_code": -3,
+                                "error_message": "Time Limit Exceeded",
+                                "inputs": raw_inputs,
+                                "expected": raw_outputs,
+                            }
+                        else:
+                            return results, {
+                                "error": repr(e),
+                                "error_code": -4,
+                                "error_message": "Runtime Error",
+                                "inputs": raw_inputs,
+                                "expected": raw_outputs,
+                            }
+                    signal.alarm(0)
+                raw_true_output = output[0]
+                raw_true_output_copy = truncatefn(raw_true_output, 200)
+                output = raw_true_output.splitlines()
+                if not passed:
+                    if debug:
+                        nl = "\n"
+                        if not isinstance(inputs, list):
+                            print(
+                                f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"
+                            )
+                        else:
+                            print(
+                                f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"
+                            )
+                    continue
+
+                if passed and debug:
+                    print(
+                        f"==> output = {output}, test outputs = {in_outs['outputs'][index]}"
+                    )
+
+                if custom_compare_(output, in_outs["outputs"][index]):
+                    tmp_result = True
+                    results.append(tmp_result)
+                    continue
+
+                # ground truth sequences are expressed as lists not tuples
+                if isinstance(output, tuple):
+                    output = list(output)
+
+                tmp_result = False
+                try:
+                    tmp_result = output == [in_outs["outputs"][index]]
+                    if isinstance(in_outs["outputs"][index], list):
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index])
+                        if isinstance(output[0], str):
+                            tmp_result = tmp_result or (
+                                [e.strip() for e in output] == in_outs["outputs"][index]
+                            )
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check1 exception = {e}")
+                    pass
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                # try one more time without \n
+                if isinstance(in_outs["outputs"][index], list):
+                    for tmp_index, i in enumerate(in_outs["outputs"][index]):
+                        in_outs["outputs"][index][tmp_index] = i.split("\n")
+                        in_outs["outputs"][index][tmp_index] = [
+                            x.strip() for x in in_outs["outputs"][index][tmp_index] if x
+                        ]
+                else:
+                    in_outs["outputs"][index] = in_outs["outputs"][index].split("\n")
+                    in_outs["outputs"][index] = list(
+                        filter(len, in_outs["outputs"][index])
+                    )
+                    in_outs["outputs"][index] = list(
+                        map(lambda x: x.strip(), in_outs["outputs"][index])
+                    )
+
+                try:
+                    tmp_result = output == [in_outs["outputs"][index]]
+                    if isinstance(in_outs["outputs"][index], list):
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index])
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check2 exception = {e}")
+                    pass
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                # try by converting the output into a split up list too
+                if isinstance(output, list):
+                    output = list(filter(len, output))
+
+                if debug:
+                    nl = "\n"
+                    if not isinstance(inputs, list):
+                        print(
+                            f"@1 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]} {tmp_result=}"
+                        )
+                    else:
+                        print(
+                            f"@1 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]} {tmp_result=}"
+                        )
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                if debug:
+                    print(f"{tmp_result=} @a")
+
+                try:
+                    tmp_result = output == [in_outs["outputs"][index]]
+                    if isinstance(in_outs["outputs"][index], list):
+                        tmp_result = tmp_result or (output == in_outs["outputs"][index])
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check3 exception = {e}")
+                    pass
+
+                if debug:
+                    print(f"{tmp_result=} @b")
+
+                try:
+                    all_ints = all(
+                        combined_int_check(e1) and combined_int_check(e2)
+                        for e1, e2 in zip(output, in_outs["outputs"][index])
+                    )
+                    if not all_ints:
+                        if debug:
+                            print(
+                                [
+                                    combined_int_check(e1) and combined_int_check(e2)
+                                    for e1, e2 in zip(output, in_outs["outputs"][index])
+                                ]
+                            )
+                        output_float = [float(e) for e in output]
+                        gt_float = [float(e) for e in in_outs["outputs"][index]]
+                        tmp_result = tmp_result or (
+                            (len(output_float) == len(gt_float))
+                            and np.allclose(output_float, gt_float)
+                        )
+                except Exception as e:
+                    pass
+
+                if debug:
+                    print(f"{tmp_result=} @c")
+
+                try:
+                    if isinstance(output[0], list):
+                        all_ints = all(
+                            combined_int_check(e1) and combined_int_check(e2)
+                            for e1, e2 in zip(output[0], in_outs["outputs"][index])
+                        )
+                        if not all_ints:
+                            output_float = [float(e) for e in output[0]]
+                            gt_float = [float(e) for e in in_outs["outputs"][index][0]]
+                            tmp_result = tmp_result or (
+                                (len(output_float) == len(gt_float))
+                                and np.allclose(output_float, gt_float)
+                            )
+                except Exception as e:
+                    pass
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                if debug:
+                    print(f"{tmp_result=} @d")
+                # try by converting the stuff into split up list
+                if isinstance(in_outs["outputs"][index], list):
+                    for tmp_index, i in enumerate(in_outs["outputs"][index]):
+                        in_outs["outputs"][index][tmp_index] = set(i.split())
+                else:
+                    in_outs["outputs"][index] = set(in_outs["outputs"][index].split())
+
+                if debug:
+                    print(f"{tmp_result=} @e")
+
+                try:
+                    tmp_result = output == in_outs["outputs"][index]
+                except Exception as e:
+                    if debug:
+                        print(f"Failed check4 exception = {e}")
+                    continue
+
+                if tmp_result == True:
+                    results.append(tmp_result)
+                    continue
+
+                if debug:
+                    print(f"{tmp_result=} @f")
+
+                # try by converting the output into a split up list too
+                if isinstance(output, list):
+                    for tmp_index, i in enumerate(output):
+                        output[tmp_index] = i.split()
+                    output = list(filter(len, output))
+                    for tmp_index, i in enumerate(output):
+                        output[tmp_index] = set(i)
+                else:
+                    output = output.split()
+                    output = list(filter(len, output))
+                    output = set(output)
+
+                if debug:
+                    print(f"{tmp_result=} @g")
+                # try:
+                #     tmp_result = set(frozenset(s) for s in output) == set(
+                #         frozenset(s) for s in in_outs["outputs"][index]
+                #     )
+                # except Exception as e:
+                #     if debug:
+                #         print(f"Failed check5 exception = {e}")
+
+                # if they are all numbers, round so that similar numbers are treated as identical
+                # try:
+                #     all_ints = all(
+                #         combined_int_check(e1) and combined_int_check(e2)
+                #         for e1, e2 in zip(output, in_outs["outputs"][index])
+                #     )
+                #     tmp_result = tmp_result or (
+                #         set(frozenset(round(float(t), 3) for t in s) for s in output)
+                #         == set(
+                #             frozenset(round(float(t), 3) for t in s)
+                #             for s in in_outs["outputs"][index]
+                #         )
+                #     )
+                # except Exception as e:
+                #     if debug:
+                #         print(f"Failed check6 exception = {e}")
+
+                if debug:
+                    print(f"{tmp_result=} @h")
+
+                if tmp_result == True and debug:
+                    print("PASSED")
+
+                results.append(tmp_result)
+                if tmp_result != True:
+                    return results, {
+                        "output": raw_true_output_copy,
+                        "expected": raw_outputs,
+                        "inputs": raw_inputs,
+                        "error_code": -2,
+                        "error_message": "Wrong Answer",
+                    }
+
+                if debug:
+                    nl = "\n"
+                    if not isinstance(inputs, list):
+                        print(
+                            f"@2 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"
+                        )
+                    else:
+                        print(
+                            f"@2 output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}"
+                        )
+
+                    print(f"results = {results}")
+
+    return results, {}
+
+
+def custom_compare_(output, ground_truth):
+
+    if isinstance(output, list):
+        output_1 = "\n".join(output)
+        if stripped_string_compare(output_1, ground_truth):
+            return True
+
+    if isinstance(output, list):
+        output_2 = [o.lstrip().rstrip() for o in output]
+        output_2 = "\n".join(output_2)
+        if stripped_string_compare(output_2, ground_truth):
+            return True
+
+    return False
+
+
+def stripped_string_compare(s1, s2):
+    s1 = s1.lstrip().rstrip()
+    s2 = s2.lstrip().rstrip()
+    return s1 == s2
+
+
+def call_method(method, inputs):
+
+    if isinstance(inputs, list):
+        inputs = "\n".join(inputs)
+
+    inputs_line_iterator = iter(inputs.split("\n"))
+
+    # sys.setrecursionlimit(10000)
+
+    # @patch('builtins.input', side_effect=inputs.split("\n"))
+    @patch("builtins.open", mock_open(read_data=inputs))
+    @patch("sys.stdin", StringIO(inputs))
+    @patch("sys.stdin.readline", lambda *args: next(inputs_line_iterator))
+    @patch("sys.stdin.readlines", lambda *args: inputs.split("\n"))
+    @patch("sys.stdin.read", lambda *args: inputs)
+    # @patch('sys.stdout.write', print)
+    def _inner_call_method(_method):
+        try:
+            return _method()
+        except SystemExit as e:
+            pass
+        finally:
+            pass
+
+    return _inner_call_method(method)
+
+
+def reliability_guard(maximum_memory_bytes=None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(
+            resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)
+        )
+        resource.setrlimit(
+            resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)
+        )
+        if not platform.uname().system == "Darwin":
+            resource.setrlimit(
+                resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)
+            )
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ["OMP_NUM_THREADS"] = "1"
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__["help"] = None
+
+    import sys
+
+    sys.modules["ipdb"] = None
+    sys.modules["joblib"] = None
+    sys.modules["resource"] = None
+    sys.modules["psutil"] = None
+    sys.modules["tkinter"] = None

scripts/lcb_runner/evaluation/utils_execute.py

+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This code is adapted from OpenAI's release
+# https://github.com/openai/human-eval/blob/master/human_eval/execution.py
+
+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import tempfile
+
+
+BASE_IMPORTS = """from itertools import accumulate, chain, combinations, count, permutations, product, groupby, islice, repeat
+from copy import deepcopy
+from string import ascii_lowercase
+from math import floor, log2, log10, sqrt, comb, gcd, ceil, inf, isqrt
+from collections import defaultdict, deque, Counter
+from bisect import bisect, bisect_left, bisect_right, insort
+from heapq import heappush, heappop, heapify, merge
+from functools import reduce, cache, lru_cache
+from random import randrange, shuffle
+from operator import itemgetter, sub
+from re import search as re_search  # Assuming 're' refers to a regex search
+from os.path import commonprefix
+from typing import List, Tuple, Dict, Set, Optional, Union, Any, Callable, Iterable, Iterator, Generator
+import copy
+import string
+import math
+import collections
+import bisect
+import heapq
+import functools
+import random
+import itertools
+import operator
+import re
+import numpy as np
+import pandas as pd
+from math import log, prod  # 'log' and 'prod' are functions in the math module
+from collections import deque, defaultdict, Counter, OrderedDict
+from itertools import accumulate, permutations, combinations, product, groupby, islice, chain, repeat, zip_longest, cycle
+from functools import lru_cache, reduce, partial
+# from sortedcontainers import SortedList, SortedDict, SortedSet
+# import sortedcontainers
+from operator import iand
+import sys
+"""
+
+def check_correctness(check_program, timeout=3):
+    """
+    Evaluates the functional correctness of a completion by running the test
+    suite provided in the problem.
+
+    :param completion_id: an optional completion ID so we can match
+        the results later even if execution finishes asynchronously.
+    """
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append("timed out")
+
+    return result[0] == "passed"
+
+
+def unsafe_execute(check_program, result, timeout):
+
+    with create_tempdir():
+
+        # These system calls are needed when cleaning up tempdir.
+        import os
+        import shutil
+
+        rmtree = shutil.rmtree
+        rmdir = os.rmdir
+        chdir = os.chdir
+
+        # Disable functionalities that can make destructive changes to the test.
+        reliability_guard()
+
+        # Run program.
+        try:
+            exec_globals = {}
+            with swallow_io():
+                with time_limit(timeout):
+                    exec(check_program, exec_globals)
+            result.append("passed")
+        except TimeoutException:
+            result.append("timed out")
+        except BaseException as e:
+            result.append(f"failed: {e}")
+
+        # Needed for cleaning up.
+        shutil.rmtree = rmtree
+        os.rmdir = rmdir
+        os.chdir = chdir
+
+
+@contextlib.contextmanager
+def time_limit(seconds):
+    def signal_handler(signum, frame):
+        raise TimeoutException("Timed out!")
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """StringIO that throws an exception when it's read from"""
+
+    def read(self, *args, **kwargs):
+        raise OSError
+
+    def readline(self, *args, **kwargs):
+        raise OSError
+
+    def readlines(self, *args, **kwargs):
+        raise OSError
+
+    def readable(self, *args, **kwargs):
+        """Returns True if the IO object can be read."""
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = "stdin"
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == ".":
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes=None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == "Darwin":
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ["OMP_NUM_THREADS"] = "1"
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__["help"] = None
+
+    import sys
+
+    sys.modules["ipdb"] = None
+    sys.modules["joblib"] = None
+    sys.modules["resource"] = None
+    sys.modules["psutil"] = None
+    sys.modules["tkinter"] = None
\ No newline at end of file

scripts/lcb_runner/lm_styles.py

+from dataclasses import dataclass
+from datetime import datetime
+from enum import Enum
+
+
+class LMStyle(Enum):
+    OpenAIChat = "OpenAIChat"
+    OpenAIReason = "OpenAIReason"
+    Claude = "Claude"  # Claude 1 and Claude 2
+    Claude3 = "Claude3"
+    Gemini = "Gemini"
+    MistralWeb = "MistralWeb"
+    CohereCommand = "CohereCommand"
+    DataBricks = "DataBricks"
+    DeepSeekAPI = "DeepSeekAPI"
+
+    GenericBase = "GenericBase"
+
+    DeepSeekCodeInstruct = "DeepSeekCodeInstruct"
+    CodeLLaMaInstruct = "CodeLLaMaInstruct"
+    StarCoderInstruct = "StarCoderInstruct"
+    CodeQwenInstruct = "CodeQwenInstruct"
+
+    Phind = "Phind"
+    WizardCoder = "WizardCoder"
+    MagiCoder = "MagiCoder"
+    OC = "OC"
+    Eurusx = "Eurusx"
+
+    Qwen1point5 = "Qwen1point5"
+    Smaug2 = "Smaug2"
+
+    LLaMa3 = "LLaMa3"
+    DracarysLlama = "DracarysLlama"
+    DracarysQwen = "DracarysQwen"
+
+
+@dataclass
+class LanguageModel:
+    model_name: str
+    model_repr: str
+    model_style: LMStyle
+    release_date: datetime | None  # XXX Should we use timezone.utc?
+    link: str | None = None
+
+    def __hash__(self) -> int:
+        return hash(self.model_name)
+
+
+LanguageModelList: list[LanguageModel] = [
+    LanguageModel(
+        "meta-llama/Meta-Llama-3-70B",
+        "LLama3-70b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/meta-llama/Meta-Llama-3-70B",
+    ),
+    LanguageModel(
+        "meta-llama/Meta-Llama-3-8B",
+        "LLama3-8b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/meta-llama/Meta-Llama-3-8B",
+    ),
+    LanguageModel(
+        "meta-llama/Meta-Llama-3-8B-Instruct",
+        "LLama3-8b-Ins",
+        LMStyle.LLaMa3,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct",
+    ),
+    LanguageModel(
+        "meta-llama/Meta-Llama-3-70B-Instruct",
+        "LLama3-70b-Ins",
+        LMStyle.LLaMa3,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct",
+    ),
+    LanguageModel(
+        "meta-llama/Meta-Llama-3.1-8B",
+        "LLama3.1-8b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/meta-llama/Meta-Llama-3.1-8B",
+    ),
+    LanguageModel(
+        "meta-llama/Meta-Llama-3.1-70B",
+        "LLama3.1-70b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/meta-llama/Meta-Llama-3.1-70B",
+    ),
+    LanguageModel(
+        "meta-llama/Meta-Llama-3.1-405B-FP8",
+        "LLama3.1-405b-Base-FP8",
+        LMStyle.LLaMa3,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/meta-llama/Meta-Llama-3.1-405B-Instruct-FP8",
+    ),
+    LanguageModel(
+        "meta-llama/Meta-Llama-3.1-8B-Instruct",
+        "LLama3.1-8b-Ins",
+        LMStyle.LLaMa3,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct",
+    ),
+    LanguageModel(
+        "meta-llama/Meta-Llama-3.1-70B-Instruct",
+        "LLama3.1-70b-Ins",
+        LMStyle.LLaMa3,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct",
+    ),
+    LanguageModel(
+        "meta-llama/Meta-Llama-3.1-405B-Instruct-FP8",
+        "LLama3.1-405b-Ins-FP8",
+        LMStyle.LLaMa3,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/meta-llama/Meta-Llama-3.1-405B-Instruct-FP8",
+    ),
+    LanguageModel(
+        "deepseek-ai/deepseek-coder-33b-base",
+        "DSCoder-33b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/deepseek-ai/deepseek-coder-33b-base",
+    ),
+    LanguageModel(
+        "deepseek-ai/deepseek-coder-6.7b-base",
+        "DSCoder-6.7b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base",
+    ),
+    LanguageModel(
+        "deepseek-ai/deepseek-coder-1.3b-base",
+        "DSCoder-1.3b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base",
+    ),
+    LanguageModel(
+        "deepseek-ai/deepseek-coder-33b-instruct",
+        "DSCoder-33b-Ins",
+        LMStyle.DeepSeekCodeInstruct,
+        datetime(2023, 9, 1),
+        link="https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct",
+    ),
+    LanguageModel(
+        "deepseek-ai/deepseek-coder-6.7b-instruct",
+        "DSCoder-6.7b-Ins",
+        LMStyle.DeepSeekCodeInstruct,
+        datetime(2023, 9, 1),
+        link="https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
+    ),
+    LanguageModel(
+        "deepseek-ai/deepseek-coder-1.3b-instruct",
+        "DSCoder-1.3b-Ins",
+        LMStyle.DeepSeekCodeInstruct,
+        datetime(2023, 8, 1),
+        link="https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct",
+    ),
+    LanguageModel(
+        "deepseek-chat",
+        "DeepSeek-V2",
+        LMStyle.DeepSeekAPI,
+        datetime(2023, 8, 1),
+        link="https://huggingface.co/deepseek-ai/DeepSeek-V2",
+    ),
+    LanguageModel(
+        "deepseek-coder",
+        "DeepSeekCoder-V2.5",
+        LMStyle.DeepSeekAPI,
+        datetime(2023, 8, 1),
+        link="https://huggingface.co/deepseek-ai/DeepSeek-V2",
+    ),
+    LanguageModel(
+        "deepseek-ai/DeepSeek-V2-Chat",
+        "DeepSeek-V2-Chat",
+        LMStyle.DeepSeekCodeInstruct,
+        datetime(2023, 12, 30),
+        link="https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat",
+    ),
+    LanguageModel(
+        "deepseek-ai/DeepSeek-Coder-V2-Instruct",
+        "DeepSeek-Coder-V2-Instruct",
+        LMStyle.DeepSeekCodeInstruct,
+        datetime(2023, 12, 30),
+        link="https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct",
+    ),
+    LanguageModel(
+        "deepseek-ai/DeepSeek-V2-Chat-0628",
+        "DeepSeek-V2-Chat-0628",
+        LMStyle.DeepSeekCodeInstruct,
+        datetime(2023, 12, 30),
+        link="https://huggingface.co/deepseek-ai/DeepSeek-V2-Chat-0628",
+    ),
+    LanguageModel(
+        "deepseek-ai/DeepSeek-Coder-V2-Instruct-0724",
+        "DeepSeek-Coder-V2-Instruct-0724",
+        LMStyle.DeepSeekCodeInstruct,
+        datetime(2023, 12, 30),
+        link="https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct-0724",
+    ),
+    LanguageModel(
+        "deepseek-ai/DeepSeek-V2-Lite-Chat",
+        "DeepSeek-V2-Lite-Chat",
+        LMStyle.DeepSeekCodeInstruct,
+        datetime(2023, 12, 30),
+        link="https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat",
+    ),
+    LanguageModel(
+        "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",
+        "DeepSeek-Coder-V2-Lite-Instruct",
+        LMStyle.DeepSeekCodeInstruct,
+        datetime(2023, 12, 30),
+        link="https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",
+    ),
+    LanguageModel(
+        "codellama/CodeLlama-70b-hf",
+        "CodeLlama-70b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/codellama/CodeLlama-70b-hf",
+    ),
+    LanguageModel(
+        "codellama/CodeLlama-34b-hf",
+        "CodeLlama-34b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/codellama/CodeLlama-34b-hf",
+    ),
+    LanguageModel(
+        "codellama/CodeLlama-13b-hf",
+        "CodeLlama-13b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/codellama/CodeLlama-13b-hf",
+    ),
+    LanguageModel(
+        "codellama/CodeLlama-7b-hf",
+        "CodeLlama-7b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/codellama/CodeLlama-7b-hf",
+    ),
+    LanguageModel(
+        "codellama/CodeLlama-70b-Instruct-hf",
+        "CodeLlama-70b-Ins",
+        LMStyle.CodeLLaMaInstruct,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/codellama/CodeLlama-70b-hf",
+    ),
+    LanguageModel(
+        "codellama/CodeLlama-34b-Instruct-hf",
+        "CodeLlama-34b-Ins",
+        LMStyle.CodeLLaMaInstruct,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf",
+    ),
+    LanguageModel(
+        "codellama/CodeLlama-13b-Instruct-hf",
+        "CodeLlama-13b-Ins",
+        LMStyle.CodeLLaMaInstruct,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf",
+    ),
+    LanguageModel(
+        "codellama/CodeLlama-7b-Instruct-hf",
+        "CodeLlama-7b-Ins",
+        LMStyle.CodeLLaMaInstruct,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf",
+    ),
+    LanguageModel(
+        "gpt-3.5-turbo-0301",
+        "GPT-3.5-Turbo-0301",
+        LMStyle.OpenAIChat,
+        datetime(2021, 10, 1),
+        link="https://openai.com/blog/new-models-and-developer-products-announced-at-devday",
+    ),
+    LanguageModel(
+        "gpt-3.5-turbo-0125",
+        "GPT-3.5-Turbo-0125",
+        LMStyle.OpenAIChat,
+        datetime(2021, 10, 1),
+        link="https://openai.com/blog/new-embedding-models-and-api-updates#:~:text=Other%20new%20models%20and%20lower%20pricing",
+    ),
+    LanguageModel(
+        "gpt-4-0613",
+        "GPT-4-0613",
+        LMStyle.OpenAIChat,
+        datetime(2021, 10, 1),
+        link="https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4",
+    ),
+    LanguageModel(
+        "gpt-4-1106-preview",
+        "GPT-4-Turbo-1106",
+        LMStyle.OpenAIChat,
+        datetime(2023, 4, 30),
+        link="https://openai.com/blog/new-models-and-developer-products-announced-at-devday",
+    ),
+    LanguageModel(
+        "gpt-4-turbo-2024-04-09",
+        "GPT-4-Turbo-2024-04-09",
+        LMStyle.OpenAIChat,
+        datetime(2023, 4, 30),
+        link="https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4",
+    ),
+    LanguageModel(
+        "gpt-4o-2024-05-13",
+        "GPT-4O-2024-05-13",
+        LMStyle.OpenAIChat,
+        datetime(2023, 4, 30),
+        link="https://openai.com/index/spring-update",
+    ),
+    LanguageModel(
+        "gpt-4o-2024-08-06",
+        "GPT-4O-2024-08-06",
+        LMStyle.OpenAIChat,
+        datetime(2023, 4, 30),
+        link="https://openai.com/index/spring-update",
+    ),
+    LanguageModel(
+        "gpt-4o-mini-2024-07-18",
+        "GPT-4O-mini-2024-07-18",
+        LMStyle.OpenAIChat,
+        datetime(2023, 4, 30),
+        link="https://openai.com/index/spring-update",
+    ),
+    LanguageModel(
+        "chatgpt-4o-latest",
+        "ChatGPT-4O-Latest-08-24",
+        LMStyle.OpenAIChat,
+        datetime(2023, 4, 30),
+        link="https://openai.com/index/spring-update",
+    ),
+    LanguageModel(
+        "o1-preview-2024-09-12",
+        "O1-Preview-2024-09-12 (N=1)",
+        LMStyle.OpenAIReason,
+        datetime(2023, 4, 30),
+        link="https://openai.com/index/spring-update",
+    ),
+    LanguageModel(
+        "o1-mini-2024-09-12",
+        "O1-Mini-2024-09-12 (N=1)",
+        LMStyle.OpenAIReason,
+        datetime(2023, 4, 30),
+        link="https://openai.com/index/spring-update",
+    ),
+    LanguageModel(
+        "claude-instant-1",
+        "Claude-Instant-1",
+        LMStyle.Claude,
+        datetime(2022, 12, 31),
+        link="https://www.anthropic.com/index/introducing-claude",
+    ),
+    LanguageModel(
+        "claude-2",
+        "Claude-2",
+        LMStyle.Claude,
+        datetime(2022, 12, 31),
+        link="https://www.anthropic.com/index/claude-2",
+    ),
+    LanguageModel(
+        "claude-3-opus-20240229",
+        "Claude-3-Opus",
+        LMStyle.Claude3,
+        datetime(2023, 9, 1),
+        link="https://www.anthropic.com/index/claude-3",
+    ),
+    LanguageModel(
+        "claude-3-sonnet-20240229",
+        "Claude-3-Sonnet",
+        LMStyle.Claude3,
+        datetime(2023, 9, 1),
+        link="https://www.anthropic.com/index/claude-3",
+    ),
+    LanguageModel(
+        "claude-3-5-sonnet-20240620",
+        "Claude-3.5-Sonnet",
+        LMStyle.Claude3,
+        datetime(2024, 3, 31),
+        link="https://www.anthropic.com/news/claude-3-5-sonnet",
+    ),
+    LanguageModel(
+        "claude-3-haiku-20240307",
+        "Claude-3-Haiku",
+        LMStyle.Claude3,
+        datetime(2023, 4, 30),
+        link="https://www.anthropic.com/index/claude-3",
+    ),
+    LanguageModel(
+        "gemini-1.5-pro-002",
+        "Gemini-Pro-1.5-002",
+        LMStyle.Gemini,
+        datetime(2023, 4, 30),
+        link="https://blog.google/technology/ai/gemini-api-developers-cloud",
+    ),
+    LanguageModel(
+        "gemini-1.5-flash-002",
+        "Gemini-Flash-1.5-002",
+        LMStyle.Gemini,
+        datetime(2023, 4, 30),
+        link="https://blog.google/technology/ai/gemini-api-developers-cloud",
+    ),
+    LanguageModel(
+        "databricks-dbrx-instruct",
+        "DBRX-Ins",
+        LMStyle.DataBricks,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/databricks/dbrx-instruct",
+    ),
+    LanguageModel(
+        "bigcode/starcoder2-3b",
+        "StarCoder2-3b",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/bigcode/starcoder2-7b-magicoder-instruct/tree/main",
+    ),
+    LanguageModel(
+        "bigcode/starcoder2-7b",
+        "StarCoder2-7b",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/bigcode/starcoder2-7b-magicoder-instruct/tree/main",
+    ),
+    LanguageModel(
+        "bigcode/starcoder2-15b",
+        "StarCoder2-15b",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/bigcode/starcoder2-7b-magicoder-instruct/tree/main",
+    ),
+    LanguageModel(
+        "google/codegemma-7b",
+        "CodeGemma-7b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/google/codegemma-7b",
+    ),
+    LanguageModel(
+        "google/codegemma-2b",
+        "CodeGemma-2b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/google/codegemma-2b",
+    ),
+    LanguageModel(
+        "google/gemma-7b",
+        "Gemma-7b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/google/gemma-7b",
+    ),
+    LanguageModel(
+        "google/gemma-2b",
+        "Gemma-2b-Base",
+        LMStyle.GenericBase,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/google/gemma-2b",
+    ),
+    LanguageModel(
+        "mistral-large-latest",
+        "Mistral-Large",
+        LMStyle.MistralWeb,
+        datetime(2023, 1, 1),
+        link="https://mistral.ai/news/mistral-large/",
+    ),
+    LanguageModel(
+        "open-mixtral-8x22b",
+        "Mixtral-8x22B-Ins",
+        LMStyle.MistralWeb,
+        datetime(2023, 1, 1),
+        link="https://mistral.ai/news/mixtral-8x22b/",
+    ),
+    LanguageModel(
+        "open-mixtral-8x7b",
+        "Mixtral-8x7B-Ins",
+        LMStyle.MistralWeb,
+        datetime(2023, 1, 1),
+        link="https://mistral.ai/news/mixtral-8x7b/",
+    ),
+    LanguageModel(
+        "open-mixtral-8x7b",
+        "Mixtral-8x7B-Ins",
+        LMStyle.MistralWeb,
+        datetime(2023, 1, 1),
+        link="https://mistral.ai/news/mixtral-8x7b/",
+    ),
+    LanguageModel(
+        "codestral-latest",
+        "Codestral-Latest",
+        LMStyle.MistralWeb,
+        datetime(2023, 1, 1),
+        link="https://mistral.ai/news/codestral/",
+    ),
+    LanguageModel(
+        "command-r",
+        "Command-R",
+        LMStyle.CohereCommand,
+        datetime(2023, 1, 1),
+        link="https://docs.cohere.com/docs/models",
+    ),
+    LanguageModel(
+        "command-r-plus",
+        "Command-R+",
+        LMStyle.CohereCommand,
+        datetime(2023, 1, 1),
+        link="https://docs.cohere.com/docs/models",
+    ),
+    LanguageModel(
+        "Qwen/CodeQwen1.5-7B",
+        "CodeQwen15-7B",
+        LMStyle.GenericBase,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/CodeQwen1.5-7B",
+    ),
+    LanguageModel(
+        "Qwen/CodeQwen1.5-7B-Chat",
+        "CodeQwen15-7B-Chat",
+        LMStyle.CodeQwenInstruct,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat",
+    ),
+    LanguageModel(
+        "Qwen/Qwen2-72B",
+        "Qwen2-Base-72B",
+        LMStyle.GenericBase,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/Qwen2-72B",
+    ),
+    LanguageModel(
+        "Qwen/Qwen2-72B-Instruct",
+        "Qwen2-Ins-72B",
+        LMStyle.CodeQwenInstruct,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/Qwen2-72B-Instruct",
+    ),
+    LanguageModel(
+        "Qwen/Qwen2.5-7B",
+        "Qwen2.5-Base-7B",
+        LMStyle.GenericBase,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/Qwen2.5-7B",
+    ),
+    LanguageModel(
+        "Qwen/Qwen2.5-7B-Instruct",
+        "Qwen2.5-Ins-7B",
+        LMStyle.CodeQwenInstruct,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/Qwen2.5-7B-Instruct",
+    ),
+    LanguageModel(
+        "Qwen/Qwen2.5-32B",
+        "Qwen2.5-Base-32B",
+        LMStyle.GenericBase,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/Qwen2.5-32B",
+    ),
+    LanguageModel(
+        "Qwen/Qwen2.5-32B-Instruct",
+        "Qwen2.5-Ins-32B",
+        LMStyle.CodeQwenInstruct,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/Qwen2.5-32B-Instruct",
+    ),
+    LanguageModel(
+        "Qwen/Qwen2.5-72B",
+        "Qwen2.5-Base-72B",
+        LMStyle.GenericBase,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/Qwen2.5-72B",
+    ),
+    LanguageModel(
+        "Qwen/Qwen2.5-72B-Instruct",
+        "Qwen2.5-Ins-72B",
+        LMStyle.CodeQwenInstruct,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/Qwen2.5-72B-Instruct",
+    ),
+    LanguageModel(
+        "Qwen/Qwen2.5-Coder-7B",
+        "Qwen2.5-Coder-Base-7B",
+        LMStyle.GenericBase,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/Qwen2.5-Coder-7B",
+    ),
+    LanguageModel(
+        "Qwen/Qwen2.5-Coder-7B-Instruct",
+        "Qwen2.5-Coder-Ins-7B",
+        LMStyle.CodeQwenInstruct,
+        datetime(2023, 8, 30),
+        link="https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct",
+    ),
+    LanguageModel(
+        "m-a-p/OpenCodeInterpreter-DS-33B",
+        "OC-DS-33B",
+        LMStyle.OC,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B/",
+    ),
+    LanguageModel(
+        "m-a-p/OpenCodeInterpreter-DS-6.7B",
+        "OC-DS-6.7B",
+        LMStyle.OC,
+        datetime(2023, 9, 1),
+        link="https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B/",
+    ),
+    LanguageModel(
+        "m-a-p/OpenCodeInterpreter-DS-1.3B",
+        "OC-DS-1.3B",
+        LMStyle.OC,
+        datetime(2023, 9, 1),
+        link="https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-1.3B/",
+    ),
+    LanguageModel(
+        "stabilityai/stable-code-3b",
+        "StableCode-3B",
+        LMStyle.GenericBase,
+        datetime(2023, 9, 1),
+        link="https://huggingface.co/stabilityai/stable-code-3b/",
+    ),
+    LanguageModel(
+        "bigcode/starcoder2-instruct-15b-v0.1",
+        "StarCoder2-Ins-v0.1",
+        LMStyle.LLaMa3,
+        datetime(2023, 4, 30),
+        link="https://huggingface.co/bigcode/starcoder2-instruct-15b-v0.1",
+    ),
+    LanguageModel(
+        "qwen/Qwen1.5-72B-Chat",
+        "Qwen-1.5-72B-Chat ",
+        LMStyle.Qwen1point5,
+        datetime(2024, 3, 31),
+        link="https://huggingface.co/qwen/Qwen1.5-72B-Chat/",
+    ),
+    LanguageModel(
+        "abacusai/Smaug-2-72B",
+        "Smaug-2-72B ",
+        LMStyle.Smaug2,
+        datetime(2024, 3, 31),
+        link="https://huggingface.co/abacusai/Smaug-2-72B/",
+    ),
+    LanguageModel(
+        "WizardCoderLM/WizardCoderCoder-Python-34B-V1.0",
+        "WCoder-34B-V1",
+        LMStyle.WizardCoder,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/WizardCoderLM/WizardCoderCoder-Python-34B-V1.0",
+    ),
+    LanguageModel(
+        "WizardCoderLM/WizardCoderCoder-33B-V1.1",
+        "WCoder-33B-V1.1",
+        LMStyle.WizardCoder,
+        datetime(2023, 9, 1),
+        link="https://huggingface.co/WizardCoderLM/WizardCoderCoder-33B-V1.1",
+    ),
+    LanguageModel(
+        "Phind/Phind-CodeLlama-34B-v2",
+        "Phind-34B-V2",
+        LMStyle.Phind,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/Phind/Phind-CodeLlama-34B-v2",
+    ),
+    LanguageModel(
+        "ise-uiuc/Magicoder-S-DS-6.7B",
+        "MagiCoderS-DS-6.7B",
+        LMStyle.MagiCoder,
+        datetime(2023, 7, 30),
+        link="https://huggingface.co/ise-uiuc/Magicoder-S-DS-6.7B",
+    ),
+    LanguageModel(
+        "ise-uiuc/Magicoder-S-CL-7B",
+        "MagiCoderS-CL-7B",
+        LMStyle.MagiCoder,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/ise-uiuc/Magicoder-S-CL-7B",
+    ),
+    LanguageModel(
+        "openbmb/Eurus-70b-sft",
+        "Eurus-70B-SFT (n=1)",
+        LMStyle.Eurusx,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/openbmb/Eurus-70b-sft",
+    ),
+    LanguageModel(
+        "openbmb/Eurux-8x22b-nca",
+        "Eurux-8x22b-NCA (n=1)",
+        LMStyle.Eurusx,
+        datetime(2023, 4, 30),
+        link="https://huggingface.co/openbmb/Eurux-8x22b-nca",
+    ),
+    LanguageModel(
+        "abacusai/Dracarys-Llama-3.1-70B-Instruct",
+        "LLama3.1-70b-Ins",
+        LMStyle.DracarysLlama,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/abacusai/Dracarys-Llama-3.1-70B-Instruct",
+    ),
+    LanguageModel(
+        "abacusai/Dracarys-72B-Instruct",
+        "Qwen2-Ins-72B",
+        LMStyle.DracarysQwen,
+        datetime(2023, 1, 1),
+        link="https://huggingface.co/abacusai/Dracarys-72B-Instruct",
+    ),
+]
+
+LanguageModelStore: dict[str, LanguageModel] = {
+    lm.model_name: lm for lm in LanguageModelList
+}
+
+if __name__ == "__main__":
+    print(list(LanguageModelStore.keys()))