first commit

Evaluation/HumanEval/utils/dataset.py

+import os
+import numpy as np
+import json
+
+class HumanEvalDataset:
+
+    def __init__(self, root, sample_num=1, language="python", issft=False):
+        """
+        root: the path to the HumanEval dataset
+        sample_num: the number of samples for each prompt
+        language: the language of the HumanEval dataset
+        issft: whether to use the SFT setting
+        """
+        self.root = root
+        self.data = open(os.path.join(self.root, f"humaneval-{language}.jsonl")).readlines()
+
+        tmp = self.get_qa_only_data(self.data, issft)
+        self.clean_data = []
+        for i in range(len(tmp)):
+            for j in range(sample_num):
+                self.clean_data.append(tmp[i])
+        self.stopwords = self.clean_data[0]["stopwords"]
+        np.random.seed(1234)
+        print(f"Read HumanEval from {root}, number of samples {len(self.clean_data)}")
+
+    def get_qa_only_data(self, data_json, sft=False):
+        """
+        data_json: the jsonl file of HumanEval
+        sft: whether to use the SFT setting
+        return: a list of dict, each dict contains the prompt, task_id and stopwords
+        """
+        ans = []
+        for line in data_json:
+            line = json.loads(line)
+            prompt = line["prompt"].strip()
+            if "prefix" in line:
+                origin_prompt = line["prefix"]
+            else:
+                origin_prompt = line["prompt"]
+
+            if sft:
+                prompt = f"""Below is an instruction that describes a task, paired with an input that provides further context.\nWrite a response that appropriately completes the request.\n\n### Instruction:\nWrite a program to perform the given task.\n\nInput:\n{prompt}\n\n### Response:\n"""
+            if "stop_tokens" in line:
+                s = line["stop_tokens"]
+            else:
+                s = []
+            ans.append({"prompt":prompt, "task_id":line["task_id"], "original_prompt": origin_prompt, "stopwords":s})
+        return ans
+
+    def __len__(self):
+        """
+        return the number of samples in the dataset
+        """
+        return len(self.clean_data)
+
+    def __getitem__(self, index):
+        """
+        return the sample at index
+        """
+        sample = self.clean_data[index]
+        return sample

Evaluation/HumanEval/utils/utils.py

+import re
+
+languge_settings = {
+    'python': {
+        'full_name': 'Python',
+        'indent': 4,
+    },
+    'cpp': {
+        'full_name': 'cpp',
+        'indent': 0,
+        'main': "int main()",
+    },
+    'java': {
+        'full_name': 'Java',
+        'indent': 4,
+        'main': "public static void main",
+    },
+    'cs': {
+        'full_name': "csharp",
+        'indent': 0,
+        'main': "public static void Main",
+    },
+    'php': {
+        'full_name': "PHP",
+        'indent': 0,
+    },
+    'ts': {
+        'full_name': "TypeScript",
+        'indent': 0,
+    },
+    'js': {
+        'full_name': "JavaScript",
+        'indent': 0
+    },
+    'sh': {
+        'full_name': "Bash",
+        'indent': 0
+    }
+}
+
+def get_function_name(question: str, lang: str):
+    func_lines = [x for x in question.strip().split('\n') if x.strip()]
+
+    if lang.lower() == 'python':
+        func_idx = [i for i in range(len(func_lines)) if func_lines[i].startswith("def ")][-1]
+        func_name = func_lines[func_idx].split('(')[0].strip()
+        func_prefix = "\n".join(func_lines[:func_idx])
+        return func_name, func_prefix
+    
+    func_name = func_lines[-1].split('{')[0].strip()
+    func_prefix = "\n".join(func_lines[:-1])
+    return func_name, func_prefix
+
+def extract_generation_code(example: str, lang_code: str, verbose: bool=False):
+    task_id = example['task_id']
+    output = example.get('output', example.get("gpt_completion"))
+    question = example["prompt"].strip()
+    setting = languge_settings[lang_code]
+    lang = setting['full_name']
+    indent = setting['indent']
+
+    try:
+        code_block: str = re.findall(f'```{lang.lower()}\n(.*?)```', output, re.DOTALL | re.IGNORECASE)[0]
+        if verbose:
+            print(">>> Task: {}\n{}".format(task_id, code_block))
+        
+        # Remove main
+        if setting.get('main', None) and setting['main'] in code_block:
+            main_start = code_block.index(setting['main'])
+            code_block = code_block[:main_start]
+        
+        func_name, func_prefix = get_function_name(question, lang)
+
+        try:
+            start = code_block.lower().index(func_name.lower())
+            indent = 0
+            while start - indent >= 0 and code_block[start - indent-1] == ' ':
+                indent += 1
+            
+            try:
+                end = code_block.rindex('\n' + ' '*indent + '}')
+            except:
+                end = len(code_block)
+        except:
+            start = 0
+            try:
+                end = code_block.rindex('\n' + ' '*indent + '}')
+            except:
+                end = len(code_block)
+
+        body = code_block[start:end]
+
+        if lang_code.lower() in ['php', 'ts', 'js']:
+            body += '\n' + ' '*indent + '}'
+    
+        generation = func_prefix + '\n' + body + '\n'
+        example['generation'] = generation
+
+    except Exception as ex:
+        print("Failed to extract code block with error `{}`:\n>>> Task: {}\n>>> Output:\n{}".format(
+            ex, task_id, output
+        ))
+        example['generation'] = example['prompt'] + '\n' + output
+    
+    return example
+
+def cleanup_code(
+    code: str,
+    language_type: str = None,
+    dataset: str = None,
+    issft: bool = False,
+    stop_words = []
+):
+    """
+    Cleans up the generated code.
+    """
+
+    if language_type.lower() == "python":
+        if issft:
+            code = _clean_python_code_for_sft(code)
+        stop_words = ["\ndef", "\nclass", "\nif", "\n#", "\nprint"]
+        code = _truncate_code_at_stopwords(code, stop_words)
+    elif language_type.lower() == "ts":
+        code = _truncate_code_at_stopwords(code, stop_words + ["\nexport", "\nimport", "\nexport default", "\nimport default", "\nconsole.log"])
+    else:
+        code = _truncate_code_at_stopwords(code, stop_words)
+
+    return code
+
+def _clean_python_code_for_sft(code):
+    code = code.replace("\r", "")
+    if "```python" in code:
+        code_start_idx = code.index("```python")
+        code = code[code_start_idx:].replace("```python", "").strip()
+        end_idx = code.find("```") if "```" in code else len(code)
+        code = code[:end_idx].strip()
+
+    return code
+
+def _truncate_code_at_stopwords(code, stop_words):
+    min_stop_idx = len(code)
+    for stop_word in stop_words:
+        stop_index = code.find(stop_word)
+        if 0 <= stop_index < min_stop_idx:
+            min_stop_idx = stop_index
+    return code[:min_stop_idx]
\ No newline at end of file

Evaluation/LeetCode/evaluate_leetcode.py

+import re
+import json
+from pathlib import Path
+from collections import defaultdict
+from human_eval.evaluation import evaluate_functional_correctness
+
+version = "20240121-Jul"
+
+DATA_DIR = Path(__file__).parent / "data"
+
+def extract_python_code(generation: str):
+    generation = generation.replace("[PYTHON]", '```python').replace("[/PYTHON]", '```')
+    if '```python' in generation:
+        p_code = re.compile(r'```python\n(.*?)\n```', flags=re.DOTALL)
+        code_block = p_code.findall(generation)[0]
+        return code_block
+    else:
+        codelist = re.split("\ndef|\nclass|\nif|\n#|\nprint", generation)
+        return codelist[0]
+    
+def evaluate_main(generation_path: str, result_path: str, temp_dir: str):
+    problem_path = (DATA_DIR / f"{version}.jsonl").as_posix()
+
+    print(problem_path)
+    problems = [json.loads(line) for line in open(problem_path, 'r')]
+
+    id2problems = { x['task_id']: x for x in problems }
+
+    results = [json.loads(line) for line in open(generation_path, 'r')]
+    for result in results:
+        if 'task_id' not in result:
+            result['task_id'] = problems[result['index']]['task_id']
+
+        if 'generation' not in result:
+            try:
+                if 'output' not in result:
+                    result['output'] = result['response']
+                if result['output'].startswith("\n        "):
+                    func_code = extract_python_code(result['prompt_sft']).strip()
+                    result['generation'] = func_code + '\n' + result['output']
+                else:
+                    result['generation'] = extract_python_code(result['output'])
+            except:
+                result['generation'] = result['output']
+    
+    with open(result_path, 'w') as fr:
+        for result in results:
+            fr.write(json.dumps(result) + "\n")
+
+    score = evaluate_functional_correctness(
+        input_file=result_path,
+        tmp_dir=temp_dir,
+        problem_file=problem_path,
+        result_path=result_path
+    )
+
+    hardness_results = defaultdict(int)
+    for result in [json.loads(line) for line in open(result_path, 'r')]:
+        problem = id2problems[result['task_id']]
+
+        hardness = problem['meta']['difficulty']
+        hardness_results[hardness] += 1
+        hardness_results[hardness + "_correct"] += result['passed']
+
+    print("="*100)
+    print("Evaluate {} over.".format(generation_path))
+    print("Pass@1: {:.3f}".format(score["pass@1"]))
+    for key in ["Easy", "Medium", "Hard"]:
+        if key.endswith("_correct"):
+            continue
+        acc = hardness_results[key+"_correct"] / hardness_results[key]
+        print("{}: {:.3f}({}/{})".format(key, acc, hardness_results[key+"_correct"],  hardness_results[key]))
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--generation_path", type=str, required=True)
+    parser.add_argument("--result_path", type=str)
+    parser.add_argument("--temp_dir", type=str, default="output/temp")
+    args = parser.parse_args()
+
+    if args.result_path is None:
+        args.result_path = args.generation_path.replace(".jsonl", "_result.jsonl")
+    
+    evaluate_main(args.generation_path, args.result_path, temp_dir=args.temp_dir)
+    pass

Evaluation/LeetCode/human_eval/data.py

+from typing import Iterable, Dict
+import gzip
+import json
+import os
+
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+HUMAN_EVAL = os.path.join(ROOT, "..", "data", "HumanEval.jsonl.gz")
+
+
+def read_problems(evalset_file: str = HUMAN_EVAL) -> Dict[str, Dict]:
+    return {task["task_id"]: task for task in stream_jsonl(evalset_file)}
+
+
+def stream_jsonl(filename: str) -> Iterable[Dict]:
+    """
+    Parses each jsonl line and yields it as a dictionary
+    """
+    if filename.endswith(".gz"):
+        with open(filename, "rb") as gzfp:
+            with gzip.open(gzfp, 'rt') as fp:
+                for line in fp:
+                    if any(not x.isspace() for x in line):
+                        yield json.loads(line)
+    else:
+        with open(filename, "r") as fp:
+            for line in fp:
+                if any(not x.isspace() for x in line):
+                    yield json.loads(line)
+
+
+def write_jsonl(filename: str, data: Iterable[Dict], append: bool = False):
+    """
+    Writes an iterable of dictionaries to jsonl
+    """
+    if append:
+        mode = 'ab'
+    else:
+        mode = 'wb'
+    filename = os.path.expanduser(filename)
+    if filename.endswith(".gz"):
+        with open(filename, mode) as fp:
+            with gzip.GzipFile(fileobj=fp, mode='wb') as gzfp:
+                for x in data:
+                    gzfp.write((json.dumps(x) + "\n").encode('utf-8'))
+    else:
+        with open(filename, mode) as fp:
+            for x in data:
+                fp.write((json.dumps(x) + "\n").encode('utf-8'))

Evaluation/LeetCode/human_eval/evaluation.py

+import os
+import json
+import gzip
+import numpy as np
+import itertools
+
+from typing import *
+from tqdm.auto import tqdm
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from human_eval.data import stream_jsonl
+from human_eval.execution import check_correctness
+
+IMPORT_HELPER = {
+    "python": [
+        "import math",
+        "import re",
+        "import sys",
+        "import copy",
+        "import datetime",
+        "import itertools",
+        "import collections",
+        "import heapq",
+        "import functools",
+        "import hashlib",
+        "import numpy",
+        "import numpy as np",
+        "import string",
+        "from typing import *",
+        "from collections import *",
+        "from functools import *"
+    ],
+    "go"   : [
+        "math",
+        "strings",
+        "fmt",
+        "strconv",
+        "time",
+        "bytes",
+        "regexp",
+        "sort",
+        "math/rand",
+        "crypto/md5",
+    ],
+    "cpp"   : [
+        "#include<stdlib.h>",
+        "#include<algorithm>",
+        "#include<math.h>",
+        "#include<stdio.h>",
+        "#include<vector>",
+        "#include<string>",
+        "#include<climits>",
+        "#include<cstring>",
+        "#include<iostream>",
+    ],
+}
+
+
+LANGUAGE_NAME = {
+    "cpp"   : "CPP",
+    "go"    : "Go",
+    "java"  : "Java",
+    "js"    : "JavaScript",
+    "python": "Python",
+}
+
+
+def read_dataset(
+    data_file: str = None,
+    dataset_type: str = "humaneval",
+    num_shot=None,
+) -> Dict:
+    if num_shot is not None:
+        print(f"{num_shot}-shot setting...")
+    if "humaneval" in dataset_type.lower():
+        if data_file is None:
+            current_path = os.path.dirname(os.path.abspath(__file__))
+            data_file = os.path.join(current_path, "..", "humaneval-x", "python", "data", "humaneval_python.jsonl.gz")
+        dataset = {task["task_id"]: task for task in stream_jsonl(data_file)}
+    else:
+        raise f"Dataset: {dataset_type} not supported."
+
+    return dataset
+
+def estimate_pass_at_k(
+        num_samples: Union[int, List[int], np.ndarray],
+        num_correct: Union[List[int], np.ndarray],
+        k: int
+) -> np.ndarray:
+    """
+    Estimates pass@k of each problem and returns them in an array.
+    """
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """
+        Calculates 1 - comb(n - c, k) / comb(n, k).
+        """
+        if n - c < k:
+            return 1.0
+        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
+
+def process_humaneval_test(sample, problems, example_test=False, is_mbpp=False, language="python"):
+    task_id = sample["task_id"]
+    
+    if is_mbpp:
+        return sample["generation"] + "\n" + "\n".join(problems[task_id]["test"])
+    #language = task_id.split("/")[0].lower()
+
+    prompt = sample.get("prompt", "")
+    if example_test and "example_test" in problems[task_id] and problems[task_id]["example_test"] != "":
+        test = problems[task_id]["example_test"]
+    else:
+        test = problems[task_id]["test"]
+    code = sample["generation"]
+
+    # Pre-process for different languages
+    if language == "python":
+        '''code_ = []
+        for line in code.split("\n"):
+            if (len(line.strip()) > 0 and line[0] != ' ' and line[0] != '\t'):
+                break
+            code_.append(line)
+        code = "\n".join(code_)'''
+        test_setup = "\n".join(IMPORT_HELPER["python"]) + "\n"
+        test_string = test_setup + code + "\n" + test + "\n"
+    elif language == "cpp":
+        test_set_up = ""
+        for s in IMPORT_HELPER["cpp"]:
+            if s not in prompt:
+                test_set_up += s + "\n"
+        test_string = test_set_up + "\n" + code + "\n" + test
+    elif language == "java":
+        test_string = code + "\n" + test
+    elif language in ["js", "javascript", "ts", "cs", "sh"]:
+        test_string = code + "\n" + test
+    elif language == "go":
+        import_string = problems[task_id]["import"]
+        prompt = prompt.replace(import_string, "")
+        if example_test and "example_test" in problems[task_id]:
+            test = problems[task_id]["example_test"]
+        else:
+            test = problems[task_id]["test"]
+        test_setup = problems[task_id]["test_setup"]
+        other_pkgs = []
+        for pkg in IMPORT_HELPER["go"]:
+            if pkg not in test_setup:
+                p = pkg.split("/")[-1]
+                if p + "." in code:
+                    other_pkgs.append(f"\"{pkg}\"")
+        if other_pkgs:
+            import_other_pkgs = "import (\n" + "    ".join([p + "\n" for p in other_pkgs]) + ")"
+            test_string = test_setup + "\n" + import_other_pkgs + "\n" + prompt + code + "\n" + test
+        else:
+            test_string = test_setup + "\n" + prompt + code + "\n" + test
+    elif language == "rust":
+        main = "\nfn main(){ \n } \n"
+        declaration = problems[task_id]["declaration"]
+        test_string = main + declaration + prompt + code + test
+    elif language == "php":
+        test_string = code + "\n" + test + "?>"
+    return test_string
+
+
+def stream_jsonl_all(filename: str) -> Iterable[Dict]:
+    results = []
+    if filename.endswith(".gz"):
+        fp = gzip.open(open(filename, "rb"), "rt")
+    else:
+        fp = open(filename, "r")
+    for line in fp:
+        if any(not x.isspace() for x in line):
+            results.append(json.loads(line))
+    fp.close()
+
+    return results
+
+
+def evaluate_functional_correctness(
+    input_file: str = None,
+    tmp_dir: str = "./",
+    n_workers: int = 32,
+    timeout: float = 10.0,
+    problem_file: str = "../data/humaneval_python.jsonl.gz",
+    result_path: str = None,
+    k: List[int] = [1, 10, 100],
+    test_groundtruth: bool = False,
+    example_test: bool = False,
+    is_mbpp: bool = False,
+    language: str = "python",
+):
+    if example_test:
+        print("Example test...")
+
+    problems = read_dataset(problem_file, dataset_type="humaneval")
+    sample_jsonl = stream_jsonl_all(input_file)
+    with ThreadPoolExecutor(max_workers=n_workers) as executor:
+        futures = []
+        completion_id = Counter()
+        n_samples = 0
+        results = defaultdict(list)
+
+        if test_groundtruth:
+            print("Testing ground truth...")
+            for sample in tqdm(problems.values()):
+                task_id = sample["task_id"]
+                lang = task_id.split("/")[0].lower()
+                if lang == "javascript":
+                    lang = "js"
+                tmp_dir_ = os.path.join(tmp_dir, lang, "evaluation")
+                sample["generation"] = sample["canonical_solution"]
+                sample["test_code"] = process_humaneval_test(sample, problems, example_test, language)
+                if sample["test_code"] is None:
+                    continue
+                args = (task_id, sample, lang, timeout, tmp_dir_, completion_id[task_id])
+                future = executor.submit(check_correctness, *args)
+                futures.append(future)
+                completion_id[task_id] += 1
+                n_samples += 1
+        else:
+            print("Reading Samples...")
+            id2samples = {}
+            for sample in tqdm(sample_jsonl):
+                task_id = sample["task_id"]
+
+                if not is_mbpp:
+                    lang = language
+                if not is_mbpp and lang == "javascript":
+                    lang = "js"
+                if is_mbpp:
+                    lang = "python"
+                tmp_dir_ = os.path.join(tmp_dir, lang, "evaluation")
+                sample["task_id"] = task_id
+                sample["test_code"] = process_humaneval_test(sample, problems, example_test, is_mbpp, language)
+                if sample["test_code"] is None:
+                    continue
+                if "completion_id" in sample:
+                    completion_id_ = sample["completion_id"]
+                else:
+                    completion_id_ = completion_id[task_id]
+                args = (task_id, sample, lang, timeout, tmp_dir_, completion_id_)
+                id2samples[(task_id, completion_id_)] = sample
+                future = executor.submit(check_correctness, *args)
+                futures.append(future)
+                completion_id[task_id] += 1
+                n_samples += 1
+
+        if len(completion_id) == len(problems):
+            evaluate_pass_at_k = True
+        else:
+            evaluate_pass_at_k = False
+
+        print("Running test suites...")
+        sample_with_results = []
+        for future in tqdm(as_completed(futures), total=len(futures)):
+            result = future.result()
+            results[result["task_id"]].append((result["completion_id"], result))
+
+            sample = id2samples[(result["task_id"], result["completion_id"])]
+            sample_with_results.append({
+                'task_id': result['task_id'],
+                'completion_id': result["completion_id"],
+                'passed': result['passed'],
+                'generation': sample['generation']
+            })
+
+            for key in sample:
+                if key not in sample_with_results[-1]:
+                    sample_with_results[-1][key] = sample[key]
+
+    # Calculate pass@k.
+    total, correct = [], []
+    for result in results.values():
+        passed = [r[1]["passed"] for r in result]
+        total.append(len(passed))
+        correct.append(sum(passed))
+
+    total = np.array(total)
+    correct = np.array(correct)
+    if evaluate_pass_at_k:
+        ks = k
+        pass_at_k = {
+            f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
+            for k in ks if (total >= k).all()
+        }
+        print(pass_at_k)
+    else:
+        print("Total:", np.sum(total))
+        print("Correct:", np.sum(correct))
+    
+    if result_path is not None:
+        with open(result_path, 'w', encoding='utf-8') as fw:
+            for sample_with_result in sample_with_results:
+                fw.write(json.dumps(sample_with_result) + '\n')
+            print("Save evaluation results to\n{}".format(result_path))
+
+    return pass_at_k

Evaluation/LeetCode/human_eval/execution.py

+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import random
+import subprocess
+import tempfile
+import gzip
+import json
+from typing import *
+import traceback
+
+java_exec = ""
+node_exec = ""
+tsc_exec = ""
+go_exec = ""
+php_exec = ""
+cs_exec = ""
+
+def check_correctness(
+        task_id: str,
+        sample: dict,
+        language_type: str,
+        timeout: float = 3.0,
+        tmp_dir: str = None,
+        completion_id: Optional[int] = None,
+) -> Dict:
+    """
+    Evaluates the functional correctness of a completion by running the test
+    suite provided in the problem.
+    """
+
+    def unsafe_execute(tmp_dir):
+        random_id = random.randint(1, 100000)
+        if "python" in language_type.lower():
+            with create_tempdir():
+
+                # These system calls are needed when cleaning up tempdir.
+                import os
+                import shutil
+                rmtree = shutil.rmtree
+                rmdir = os.rmdir
+                chdir = os.chdir
+
+                # Disable functionalities that can make destructive changes to the test.
+                reliability_guard()
+
+                try:
+                    exec_globals = {}
+                    with swallow_io():
+                        with time_limit(timeout):
+                            # WARNING
+                            # This program exists to execute untrusted model-generated code. Although
+                            # it is highly unlikely that model-generated code will do something overtly
+                            # malicious in response to this test suite, model-generated code may act
+                            # destructively due to a lack of model capability or alignment.
+                            # Users are strongly encouraged to sandbox this evaluation suite so that it
+                            # does not perform destructive actions on their host or network.
+                            # Once you have read this disclaimer and taken appropriate precautions,
+                            # uncomment the following line and proceed at your own risk:
+                            exec(sample["test_code"], exec_globals)
+                        result.append("passed")
+                except TimeoutException:
+                    result.append("timed out")
+                except AssertionError as e:
+                    result.append(f"failed: AssertionError")
+                except BaseException as e:
+                    result.append(f"failed: {e}")
+                #print(sample["test_code"])
+                #print(result)
+                # Needed for cleaning up.
+                shutil.rmtree = rmtree
+                os.rmdir = rmdir
+                os.chdir = chdir
+
+        elif "go" in language_type.lower():
+            assert tmp_dir is not None, "Go should be evaluated in a dir where necessary module files installed."
+
+            import os
+            import shutil
+
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            origin_path = os.getcwd()
+            os.chdir(tmp_dir)
+            open(f"main_test.go", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                     exec_result = subprocess.run([f"{go_exec}go", "test", f"-timeout={timeout}s", "main_test.go"], timeout=timeout, capture_output=True)
+
+                if exec_result.returncode == 0:
+                    result.append("passed")
+                else:
+                    if exec_result.stderr:
+                        try:
+                            err = exec_result.stderr.decode()
+                        except:
+                            err = exec_result.stderr
+                    else:
+                        try:
+                            err = exec_result.stdout.decode()
+                        except:
+                            err = exec_result.stdout
+                    result.append(f"failed: {err}")
+
+            except TimeoutException:
+                result.append("timed out")
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "js" in language_type.lower():
+            import os
+            import shutil
+
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            origin_path = os.getcwd()
+            os.chdir(tmp_dir)
+            open(f"test.js", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                     exec_result = subprocess.run([f"{node_exec}node", "test.js"], timeout=timeout, capture_output=True)
+
+                if exec_result.stderr.decode():
+                    err = exec_result.stderr.decode()
+                    result.append(f"failed: {err}")
+                elif exec_result.stdout.decode():
+                    err = exec_result.stdout.decode()
+                    result.append(f"failed: {err}")
+                else:
+                    result.append("passed")
+
+            except TimeoutException:
+                result.append("timed out")
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "cpp" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            open(f"test.cpp", 'w').write(sample["test_code"])
+            if "162" in task_id:
+                compilation_result = subprocess.run(["/usr/bin/g++", "-std=c++17", "test.cpp", "-lcrypto", "-lssl"],
+                                                    timeout=timeout,
+                                                    capture_output=True)
+            else:
+                compilation_result = subprocess.run(["/usr/bin/g++", "-std=c++17", "test.cpp"], timeout=timeout,
+                                                    capture_output=True)
+            if compilation_result.returncode != 0:
+                if compilation_result.stderr:
+                    err = compilation_result.stderr.decode()
+                else:
+                    err = compilation_result.stdout.decode()
+                result.append(f"failed: compilation error: {err}")
+            else:
+                try:
+                    exec_result = None
+                    with time_limit(timeout):
+                        # WARNING
+                        # This program exists to execute untrusted model-generated code. Although
+                        # it is highly unlikely that model-generated code will do something overtly
+                        # malicious in response to this test suite, model-generated code may act
+                        # destructively due to a lack of model capability or alignment.
+                        # Users are strongly encouraged to sandbox this evaluation suite so that it
+                        # does not perform destructive actions on their host or network.
+                        # Once you have read this disclaimer and taken appropriate precautions,
+                        # uncomment the following line and proceed at your own risk:
+                         exec_result = subprocess.run(["./a.out"], timeout=timeout, capture_output=True)
+
+                    if exec_result.returncode == 0:
+                        result.append("passed")
+                    else:
+                        if exec_result.stderr:
+                            try:
+                                err = exec_result.stderr.decode()
+                            except:
+                                err = exec_result.stderr
+                        else:
+                            try:
+                                err = exec_result.stdout.decode()
+                            except:
+                                err = exec_result.stdout
+                        result.append(f"failed: {err}")
+                except TimeoutException:
+                    result.append("timed out")
+            #print(result[-1])
+            #print(sample["test_code"])
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "php" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            open(f"test.php", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    cmd = f"{php_exec}php -f test.php"
+                    exec_result = subprocess.run(cmd, timeout=timeout, capture_output=True, shell=True)
+
+                if exec_result.returncode == 0:
+                    result.append("passed")
+                else:
+                    if exec_result.stderr:
+                        try:
+                            err = exec_result.stderr.decode()
+                        except:
+                            err = exec_result.stderr
+                    else:
+                        try:
+                            err = exec_result.stdout.decode()
+                        except:
+                            err = exec_result.stdout
+                    result.append(f"failed: {err}")
+            except TimeoutException:
+                result.append("timed out")
+            print(result[-1])
+            print(sample["test_code"])
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "sh" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            open(f"test.sh", 'w').write(sample["test_code"])
+            try:
+                exec_result = None
+                with time_limit(timeout):
+                    cmd = "/bin/bash test.sh"
+                    exec_result = subprocess.run(cmd, timeout=10, capture_output=True, shell=True)
+
+                if exec_result.returncode == 0:
+                    result.append("passed")
+                else:
+                    if exec_result.stderr:
+                        try:
+                            err = exec_result.stderr.decode()
+                        except:
+                            err = exec_result.stderr
+                    else:
+                        try:
+                            err = exec_result.stdout.decode()
+                        except:
+                            err = exec_result.stdout
+                    result.append(f"failed: {err}")
+            except TimeoutException:
+                result.append("timed out")
+            #print(result[-1])
+            #print(sample["test_code"])
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "ts" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+
+            os.chdir(tmp_dir)
+            env = {"PATH": f"{node_exec}:" + subprocess.os.environ["PATH"]}
+            open(f"test.ts", 'w').write(sample["test_code"])
+            cmd = f"{tsc_exec}tsc test.ts --target ES2015 --lib ES2015,DOM"
+            compilation_result = subprocess.run(cmd, timeout=timeout, capture_output=True, env=env, shell=True)
+            if compilation_result.returncode != 0:
+                if compilation_result.stderr:
+                    err = compilation_result.stderr.decode()
+                else:
+                    err = compilation_result.stdout.decode()
+                result.append(f"failed: compilation error: {err}")
+            else:
+                try:
+                    exec_result = None
+                    with time_limit(timeout):
+                         exec_result = subprocess.run([f"{node_exec}node", "test.js"], timeout=timeout, capture_output=True)
+
+                    if exec_result.returncode == 0:
+                        result.append("passed")
+                    else:
+                        if exec_result.stderr:
+                            try:
+                                err = exec_result.stderr.decode()
+                            except:
+                                err = exec_result.stderr
+                        else:
+                            try:
+                                err = exec_result.stdout.decode()
+                            except:
+                                err = exec_result.stdout
+                        result.append(f"failed: {err}")
+                except TimeoutException:
+                    result.append("timed out")
+            if result[-1] != "passed":
+                env = {"PATH": f"{node_exec}:" + subprocess.os.environ["PATH"]}
+                cmd = f"{tsc_exec}tsc test.ts"
+                compilation_result = subprocess.run(cmd, timeout=timeout, capture_output=True, env=env, shell=True)
+                if compilation_result.returncode != 0:
+                    if compilation_result.stderr:
+                        err = compilation_result.stderr.decode()
+                    else:
+                        err = compilation_result.stdout.decode()
+                    result[-1] = f"failed: compilation error: {err}"
+                else:
+                    try:
+                        exec_result = None
+                        with time_limit(timeout):
+                            exec_result = subprocess.run([f"{node_exec}node", "test.js"], timeout=timeout, capture_output=True)
+
+                        if exec_result.returncode == 0:
+                            result[-1] = "passed"
+                        else:
+                            if exec_result.stderr:
+                                try:
+                                    err = exec_result.stderr.decode()
+                                except:
+                                    err = exec_result.stderr
+                            else:
+                                try:
+                                    err = exec_result.stdout.decode()
+                                except:
+                                    err = exec_result.stdout
+                            result[-1] = f"failed: {err}"
+                    except TimeoutException:
+                        result[-1] = "timed out"
+ 
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "cs" in language_type.lower():
+            import os
+            import shutil
+            origin_path = os.getcwd()
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            os.chdir(tmp_dir)
+            open(f"Program.cs", 'w').write(sample["test_code"])
+            cmd = f"{cs_exec}mcs -d:DEBUG Program.cs"
+            compilation_result = subprocess.run(cmd, shell=True, capture_output=True)
+            if compilation_result.returncode != 0:
+                if compilation_result.stderr:
+                    err = compilation_result.stderr.decode()
+                else:
+                    err = compilation_result.stdout.decode()
+                result.append(f"failed: compilation error: {err}")
+            else:
+                try:
+                    exec_result = None
+                    cmd = f"{cs_exec}mono Program.exe"
+                    env = dict(MONO_TRACE_LISTENER="Console.Error")
+                    with time_limit(timeout):
+                         exec_result = subprocess.run(cmd, timeout=timeout, shell=True, capture_output=True, env=env)
+
+                    if "Fail" not in exec_result.stderr.decode():
+                        result.append("passed")
+                    else:
+                        if exec_result.stderr:
+                            try:
+                                err = exec_result.stderr.decode()
+                            except:
+                                err = exec_result.stderr
+                        else:
+                            try:
+                                err = exec_result.stdout.decode()
+                            except:
+                                err = exec_result.stdout
+                        result.append(f"failed: {err}")
+                except TimeoutException:
+                    result.append("timed out")
+                except Exception as e:
+                    result.append(f"failed: {e}")
+            os.chdir(origin_path)
+            shutil.rmtree(tmp_dir)
+        elif "rust" in language_type.lower():  
+            import os         
+            
+            WD: str = os.path.dirname(os.path.abspath(__file__))
+            RUST_DIR: str = os.path.join(WD, "rust")
+            RUST_SRC: str = os.path.join(RUST_DIR, "src")
+            RUST_BIN: str = os.path.join(RUST_SRC, "bin")
+            RUST_TMP_DIR: str = os.path.join(RUST_DIR, "tmp")
+            RUST_LOGS: str = os.path.join(RUST_TMP_DIR, "logs")
+            RUST_EXT: str = ".rs" 
+
+            # Create mandatory tmp directories
+            os.makedirs(RUST_TMP_DIR, exist_ok=True)
+            os.makedirs(RUST_LOGS, exist_ok=True)
+            os.makedirs(RUST_SRC, exist_ok=True)
+            os.makedirs(RUST_BIN, exist_ok=True)
+
+            with tempfile.NamedTemporaryFile(dir = RUST_BIN, delete=False) as f:
+                #temporal file name
+                file_prefix = sample["task_id"].lower().replace("/", "_")
+                file_name:str =  file_prefix +RUST_EXT
+                
+                os.rename(f.name, os.path.join(RUST_BIN, file_name))
+                
+                # Sample to pure Rust function
+                rust_code: str = sample["test_code"]
+
+                # dump the rust source code in the target temporal file
+                f.write(rust_code.encode('utf-8'))
+
+            # Proceed towards Rust binaries compilation. Therefore move to Rust module root dir.
+            os.chdir(RUST_DIR)
+
+            # Two possible outcomes
+            # Pass OR Fail compilation
+            log_filename: str = file_prefix + ".jsonl"
+            log_path: str = os.path.join(RUST_LOGS, log_filename)
+            cargo_check: str = "cargo check --bin " + file_prefix + " --message-format json >> " + log_path
+            # Compilation build status
+            returned_val_compilation: int
+            
+            # Overwrite file content
+            if os.path.exists(log_path):
+                if(file_size := os.path.getsize(log_path)) >= 0: 
+                    os.remove(log_path)
+                    returned_val_compilation = os.system(cargo_check)
+
+            else: 
+                returned_val_compilation = os.system(cargo_check)
+
+            # 0 means success   
+            if returned_val_compilation == 0:
+
+                #Execution pipeline
+                cargo_test: str = "cargo test --bin " +file_prefix+ " --message-format json >> " + log_path
+                returned_val_execution = os.system(cargo_test)
+                
+                if returned_val_execution == 0:
+                    result.append("passed")
+                else:
+                   result.append(f"failed: execution error") 
+
+            else:
+                result.append(f"failed: compilation error")
+
+
+        elif "java" in language_type.lower():
+            assert tmp_dir is not None, "Java should be evaluated in a temporary dir."
+
+            import os
+            import shutil
+
+            if "tmp" not in tmp_dir:
+                tmp_dir = os.path.join(tmp_dir, "tmp")
+            tmp_dir = os.path.join(tmp_dir, f"{task_id.replace('/', '-')}-{random_id}")
+            if not os.path.exists(tmp_dir):
+                os.makedirs(tmp_dir)
+            open(os.path.join(tmp_dir, "Problem.java"), 'w').write(sample["test_code"])
+            origin_path = os.getcwd()
+            os.system(f"cp ./javatuples-1.2.jar {tmp_dir}/")
+            os.chdir(tmp_dir)
+            res = "failed: unknown error"
+            compile_returncode = -1
+            for _ in range(5):
+                try:
+                    cmd = f"{java_exec}javac -cp javatuples-1.2.jar Problem.java"
+                    compilation_result = subprocess.run(cmd, timeout=60, capture_output=True, shell=True)  
+                    compile_returncode = compilation_result.returncode
+                    break
+                except subprocess.TimeoutExpired as e:
+                    continue
+            if compile_returncode != 0:
+                res = "failed: compilation error"
+            else:
+                exec_result = None
+                try:
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                    cmd = f"{java_exec}java -ea -cp .:javatuples-1.2.jar Problem"
+                    exec_result = subprocess.run(cmd, timeout=timeout, capture_output=True, shell=True)  
+                    if exec_result.returncode == 0:
+                        res = "passed"
+                    elif exec_result.returncode == 1:
+                        if "AssertionError" in exec_result.stderr.decode('unicode-escape'):
+                            res = "failed: wrong answer"
+                        else:
+                            res = f"failed: {exec_result.stderr.decode()}"
+                except subprocess.TimeoutExpired as e:
+                    res = "time out"
+                except BaseException as e:
+                    res = f"failed: {e}"
+
+            result.append(res)  
+            os.chdir(origin_path)          
+            shutil.rmtree(tmp_dir)
+        
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(target=unsafe_execute, args=(tmp_dir,))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append("timed out")
+
+    return {
+        "task_id"      : task_id,
+        "completion_id": completion_id,
+        "result"       : result[0],
+        "passed"       : result[0] == "passed",
+        "finish"       : -1 if "finish" not in sample else sample["finish"],
+        "code"         : sample["test_code"]
+    }
+
+# Copyright (c) OpenAI (https://openai.com)
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# ============================================================================
+@contextlib.contextmanager
+def time_limit(seconds: float):
+    def signal_handler(signum, frame):
+        raise TimeoutException("Timed out!")
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """ StringIO that throws an exception when it's read from """
+
+    def read(self, *args, **kwargs):
+        raise IOError
+
+    def readline(self, *args, **kwargs):
+        raise IOError
+
+    def readlines(self, *args, **kwargs):
+        raise IOError
+
+    def readable(self, *args, **kwargs):
+        """ Returns True if the IO object can be read. """
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = 'stdin'
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == ".":
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes: Optional[int] = None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the 
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == 'Darwin':
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+    os.environ['OMP_NUM_THREADS'] = '1'
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__['help'] = None
+
+    import sys
+    sys.modules['ipdb'] = None
+    sys.modules['joblib'] = None
+    sys.modules['resource'] = None
+    sys.modules['psutil'] = None
+    sys.modules['tkinter'] = None
\ No newline at end of file

Evaluation/LeetCode/readme.md

+## 1. Introduction
+We construct the LeetCode Contest benchmark to to further validate the model's capability in real-world programming problems.
+[LeetCode](https://leetcode.com/) presents competition-level problems, offering significant challenges that test the model's problem understanding and code generation skills. We collected the latest problems from LeetCode Contests to prevent the appearance of both the problems or their solutions in our pre-training data. A total of `180` problems were collected from July 2023 to January 2024. For each problem, we collected `100` test cases. The data format is the same as human-eval. For more details, please refer to [leetcode_contest_data](./data/20240121-Jul.jsonl).
+
+## 2. Evaluation
+Please follow the following two steps to evaluate the model's performance on our LeetCode Contest benchmark:
+
+1. Run `vllm_inference.py` to get generation results.
+```bash
+cd Evaluation/LeetCode
+
+# Set the model or path here
+MODEL="deepseek-ai/deepseek-coder-7b-instruct"
+
+python vllm_inference.py --model_name_or_path $MODEL --saved_path output/20240121-Jul.deepseek-coder-7b-instruct.jsonl
+```
+
+If you want to evaluate the model with COT, please add `--cot` to the command:
+```bash
+python vllm_inference.py --model_name_or_path $MODEL --saved_path output/20240121-Jul.deepseek-coder-7b-instruct.jsonl --cot
+```
+
+2. Run `evaluate_leetcode.py` to get evaluation results.
+```bash
+python evaluate_leetcode.py --generation_path output/20240121-Jul.deepseek-coder-7b-instruct.jsonl --result_path output/20240121-Jul.deepseek-coder-7b-instruct.result.jsonl
+```
+
+## 3. Experimental Results
+We report experimental results here:
+
+| Model                       | Size | Easy (45) | Medium (91) | Hard (44) | Overall(180) |
+|-----------------------------|------|-----------|-------------|-----------|--------------|
+| WizardCoder-V1.0            | 15B  | 17.8%     | 1.1%        | 0.0%      | 5.0%         |
+| CodeLlama-Instruct          | 34B  | 24.4%     | 4.4%        | 4.5%      | 9.4%         |
+| Phind-CodeLlama-V2          | 34B  | 26.7%     | 8.8%        | 9.1%      | 13.3%        |
+| | | | |
+| GPT-3.5-Turbo               | -    | 46.7%     | 15.4 %      | 15.9%     | 23.3%        |
+| GPT-3.5-Turbo + CoT         | -    | 42.2%     | 15.4%       | 20.5%     | 23.3%        |
+| GPT-4-Turbo                 | -    | 73.3%     | 31.9%       | 25.0%     | 40.6%        |
+| GPT-4-Turbo + CoT           | -    | 71.1%     | 35.2%       | 25.0%     | 41.8%        |
+| | | | |
+| DeepSeek-Coder-Instruct     | 1.3B | 22.2%     | 1.1%        | 4.5%      | 7.2%         |
+| DeepSeek-Coder-Instruct + CoT | 1.3B | 22.2%   | 2.2%        | 2.3%      | 7.2%         |
+| DeepSeek-Coder-Instruct     | 6.7B | 44.4%     | 12.1%       | 9.1%      | 19.4%        |
+| DeepSeek-Coder-Instruct + CoT | 6.7B | 44.4%   | 17.6%       | 4.5%      | 21.1%        |
+| DeepSeek-Coder-Instruct     | 33B  | 57.8%     | 22.0%       | 9.1%      | 27.8%        |
+| DeepSeek-Coder-Instruct + CoT | 33B | 53.3%    | 25.3%       | 11.4%     | 28.9%        |
+

Evaluation/LeetCode/vllm_inference.py

+from vllm import LLM, SamplingParams
+import json
+from transformers import AutoTokenizer
+from pathlib import Path
+
+version = "20240121-Jul"
+
+def generate_batch(examples, tokenizer, llm, model: str):
+    stop = None
+    if model == 'deepseekcoder-instruct':
+        prompts = [
+            tokenizer.apply_chat_template([{'role': 'user', 'content': ex['prompt_sft'] }], tokenize=False, add_generation_prompt=True)
+            for ex in examples
+        ]
+    else:
+        raise NotImplementedError()
+
+    # Create a sampling params object.
+    sampling_params = SamplingParams(
+        temperature=0.0,
+        # top_p=0.95,
+        max_tokens=1024,
+        stop=stop
+    )
+
+    print("Sample prompt: {}".format(prompts[0]))
+    outputs = llm.generate(prompts, sampling_params)
+    for i in range(len(examples)):
+        examples[i]['output'] = outputs[i].outputs[0].text
+
+    return examples
+
+def generate_main(data_path: str, model_name_or_path: str, saved_path: str, model_type: str='deepseekcoder-instruct', cot: bool=False):
+    examples = [json.loads(x) for x in open(data_path).readlines()]
+    def _convert_for_sft(ex):
+        ex['prompt_sft'] = ex["prompt_sft"] + "\nYou need first write a step-by-step outline and then write the code."
+        return ex
+    
+    if cot:
+        examples = [_convert_for_sft(x) for x in examples]
+        saved_path = saved_path.replace(".jsonl", ".cot.jsonl")
+
+    print(model_type)
+    print("Model `{}`, COT = {}:{}".format(model_type, cot, model_name_or_path))
+    print("Saved path: {}".format(saved_path))
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
+    print("load tokenizer {} from {} over.".format(tokenizer.__class__, model_name_or_path))
+
+    # Create an LLM.
+    llm = LLM(
+        model=model_name_or_path,
+        pipeline_parallel_size=1,
+        tensor_parallel_size=8,
+        max_num_seqs=512,
+        max_num_batched_tokens=8192,
+        max_model_len=4096,
+        gpu_memory_utilization=0.85,
+        trust_remote_code=True
+    )
+    
+    generated_examples = generate_batch(examples, tokenizer, llm, model_type)    
+    print("Generate all over!!!")
+    with open(saved_path, 'w', encoding='utf-8') as fw:
+        for ex in generated_examples:
+            fw.write(json.dumps(ex) + '\n')
+        print("Save {} processed examples into {} over!".format(len(generated_examples), saved_path))
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--data_path', type=str, default=Path(__file__).parent.joinpath(f"data/{version}.jsonl").as_posix())
+    parser.add_argument('--model_name_or_path', type=str, default='deepseek-ai/deepseek-coder-7b-instruct')
+    parser.add_argument('--saved_path', type=str, default=f'output/{version}.deepseek-coder-7b-instruct.jsonl')
+    parser.add_argument('--cot', action='store_true', default=False)
+    args = parser.parse_args()
+
+    generate_main(
+        data_path=args.data_path,
+        model_name_or_path=args.model_name_or_path,
+        saved_path=args.saved_path,
+        cot=args.cot,
+    )

Evaluation/MBPP/README.md

+## 1. Introduction
+
+We provide a test script to evaluate the performance of the **deepseek-coder** model on code generation benchmarks, [**MBPP**](https://huggingface.co/datasets/mbpp), with 3-shot setting.
+
+
+
+## 2. Setup
+
+```
+pip install accelerate
+pip install attrdict
+pip install transformers
+pip install pytorch
+```
+
+
+
+## 3. Evaluation
+
+We've created a sample script, **eval.sh**, that demonstrates how to test the **deepseek-coder-1.3b-base** model on the MBPP dataset leveraging **8** GPUs.
+
+```bash
+MODEL_NAME_OR_PATH="deepseek-ai/deepseek-coder-1.3b-base"
+DATASET_ROOT="data/"
+LANGUAGE="python"
+python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --dataroot ${DATASET_ROOT} 
+```
+
+## 4. Experimental Results
+
+We report experimental results here for several models. We set the maximum input length to **4096** and the maximum output length to **500**, and employ the **greedy search strategy**.
+
+
+
+#### (1) Multilingual Base Models
+
+| Model             | Size | Pass@1 | 
+|-------------------|------|--------|
+| CodeShell         | 7B   | 38.6%  | 
+| CodeGeeX2         | 6B   | 36.2%  |
+| StarCoder     | 16B  | 42.8%  | 
+| CodeLLama-Base   | 7B   | 38.6%  | 
+| CodeLLama-Base    | 13B  | 47.0%  | 
+| CodeLLama-Base    | 34B  | 55.0%  | 
+| | | | |  |  |  |  |  |  | |
+| DeepSeek-Coder-Base| 1.3B   | 46.8%  |
+| DeepSeek-Coder-Base| 5.7B   | 57.2%  | 
+| DeepSeek-Coder-Base| 6.7B   | 60.6%  | 
+| DeepSeek-Coder-Base|33B  | **66.0%**  |
+
+#### (2) Instruction-Tuned Models
+| Model               | Size | Pass@1  |
+|---------------------|------|--------|
+| GPT-3.5-Turbo            | -    | 70.8%  | 
+| GPT-4               | -    | **80.0%**  |
+| | | | |  |  |  |  |  |  | |
+| DeepSeek-Coder-Instruct | 1.3B  | 49.4%      |
+| DeepSeek-Coder-Instruct  | 6.7B  | 65.4%     |
+| DeepSeek-Coder-Instruct  | 33B | **70.0%**     | 
+
+

Evaluation/MBPP/eval.sh

+MODEL_NAME_OR_PATH="deepseek-ai/deepseek-coder-1b-base/"
+DATASET_ROOT="data/"
+LANGUAGE="python"
+CUDA_VISIBLE_DEVICES=1,2,3 python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT}
\ No newline at end of file

Evaluation/MBPP/eval_instruct.py

+import argparse
+import json
+import os
+import torch
+import re
+from pathlib import Path
+from tqdm import tqdm
+
+data_abs_dir = Path(__file__).parent / "data"
+
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from human_eval.evaluation import evaluate_functional_correctness
+
+def read_test_examples(data_path: str):
+    def format_test_example(q, tests, code: str=None):
+        prompt = ">>> Problem:\n{}\n>>> Test Cases:\n{}\n".format(q.strip(), "\n".join(tests))
+        if code:
+            code = code.replace("\r", "").replace("\t", "    ")
+            prompt += "\n>>> Code:\n```python\n{}\n```".format(code)
+        return prompt
+
+    examples = [json.loads(x) for x in open(data_path)]
+    print("Read all {} examples from {} over!".format(len(examples), data_path))
+
+    # test_cases
+    examples_str = []
+    for i in range(1, 4):
+        ex = examples[i]
+        q, test, code = ex['text'], ex['test_list'], ex['code']
+        ex_prompt = format_test_example(q, test, code)
+        example_prompt = '- Example {}:\n{}'.format(i, ex_prompt)
+        examples_str += [example_prompt]
+
+    for i in range(10, 510):
+        ex = examples[i]
+        q, test, code = ex['text'], ex['test_list'], ex['code']
+        
+        prompt = format_test_example(q, test, code=None)
+
+        prompt_with_shots = '''
+Please refer the given examples and generate a python function for my problem.
+Examples are listed as follows:
+{}
+
+Here is my problem:
+{}
+'''.strip().format('\n\n'.join(examples_str), prompt)
+        yield {
+            'task_id': ex['task_id'],
+            'prompt': prompt_with_shots
+        }
+
+def convert_for_evaluation(example):
+    gpt_completion = example['gpt_completion']
+    generation = gpt_completion
+    try:
+        code_block: str = re.findall(f'```python\n(.*?)```', gpt_completion, re.DOTALL | re.IGNORECASE)[0]
+        generation = code_block
+    except Exception as ex:
+        print("Failed to extract codeblock:\n{}".format(gpt_completion))
+
+    example['generation'] = generation
+    return example
+
+def generate_one(example, tokenizer, model):
+    prompt = example['prompt']
+    inputs = tokenizer.apply_chat_template(
+        [{'role': 'user', 'content': prompt }],
+        return_tensors="pt", add_generation_prompt=True
+    ).to(model.device)
+
+    stop_id = tokenizer.convert_tokens_to_ids("<|EOT|>")
+    assert isinstance(stop_id, int), "Invalid tokenizer, EOT id not found"
+    outputs = model.generate(
+        inputs, 
+        max_new_tokens=512,
+        do_sample=False,
+        # top_p=0.95,
+        # temperature=temperature,
+        pad_token_id=stop_id,
+        eos_token_id=stop_id
+    )
+
+    output = tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True)
+    # print(output)
+    example['gpt_completion'] = output
+    return convert_for_evaluation(example)
+
+def generate_main(args):
+    model_name_or_path = args.model
+    saved_path = args.output_path
+    temp_dir = args.temp_dir
+    os.makedirs(temp_dir, exist_ok=True)
+    problem_file = os.path.join(data_abs_dir, f"mbpp.jsonl")
+
+    print("model", model_name_or_path)
+    tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
+    print("load tokenizer {} from {} over.".format(tokenizer.__class__, model_name_or_path))
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name_or_path,
+        torch_dtype=torch.bfloat16,
+        device_map="auto",
+    )
+    model.eval()
+
+    examples = list(read_test_examples(problem_file))
+    print("Read {} examples for evaluation over.".format(len(examples)))
+
+    generated_examples = []
+    for ex in tqdm(examples, desc='Generating'):
+        gen_example = generate_one(ex, tokenizer, model)
+        generated_examples.append(gen_example)
+        print("Generate {}/{} over...".format(len(generated_examples), len(examples)))
+
+    print("Generate all over!!!")
+    with open(saved_path, 'w', encoding='utf-8') as fw:
+        for ex in generated_examples:
+            fw.write(json.dumps(ex) + '\n')
+        print("Save {} processed examples into {} over!".format(len(generated_examples), saved_path))
+    
+    result = evaluate_functional_correctness(
+        input_file=saved_path,
+        tmp_dir=temp_dir,
+        problem_file=os.path.join(data_abs_dir, f"mbpp_test.jsonl"),
+        language='python',
+        is_mbpp=True
+    )
+    print(result, model_name_or_path)
+    pass
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--model', type=str, help="model name or path")
+    parser.add_argument('--output_path', type=str, help="output path of your generation")
+    parser.add_argument('--temp_dir', type=str, help="temp dir for evaluation", default="tmp")
+    args = parser.parse_args()
+
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+    generate_main(args)
+    pass
\ No newline at end of file

Evaluation/MBPP/eval_pal.py

+import os
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn.functional as F
+import json
+import torch.distributed as dist
+import subprocess
+import sys
+from accelerate import Accelerator
+from accelerate import DistributedDataParallelKwargs
+from pathlib import Path
+from argparse import ArgumentParser
+from mbpp import MBPP as evaltor
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+if __name__ == '__main__':
+    kwargs_handlers = [DistributedDataParallelKwargs(find_unused_parameters=True)]
+    accelerator = Accelerator(mixed_precision="bf16", kwargs_handlers=kwargs_handlers)   
+
+
+    parser = ArgumentParser()
+    parser.add_argument("--logdir", type=str, default="")
+    parser.add_argument("--dataroot", type=str, default="")
+    args = parser.parse_args()
+
+    logdir = args.logdir
+
+    if logdir == "":
+        logdir = "tmp/"
+    tokenizer = dict(
+        cls=AutoTokenizer,
+        model_path=logdir,)
+
+    dataroot = args.dataroot
+
+    evaluator = evaltor(data_root=dataroot, max_seq_len=4096, tokenizer_cfg=tokenizer, log_dir=logdir, n_sample=1, batch_size=1, max_gen_len=500)
+    model = AutoModelForCausalLM.from_pretrained(logdir, device_map=accelerator.device, trust_remote_code=True, torch_dtype=torch.bfloat16)
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+    evaluator.eval_model(model, accelerator)

Evaluation/MBPP/human_eval/data.py

+from typing import Iterable, Dict
+import gzip
+import json
+import os
+
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+HUMAN_EVAL = os.path.join(ROOT, "..", "data", "HumanEval.jsonl.gz")
+
+
+def read_problems(evalset_file: str = HUMAN_EVAL) -> Dict[str, Dict]:
+    return {task["task_id"]: task for task in stream_jsonl(evalset_file)}
+
+
+def stream_jsonl(filename: str) -> Iterable[Dict]:
+    """
+    Parses each jsonl line and yields it as a dictionary
+    """
+    if filename.endswith(".gz"):
+        with open(filename, "rb") as gzfp:
+            with gzip.open(gzfp, 'rt') as fp:
+                for line in fp:
+                    if any(not x.isspace() for x in line):
+                        yield json.loads(line)
+    else:
+        with open(filename, "r", encoding="utf-8") as fp:
+            for line in fp:
+                if any(not x.isspace() for x in line):
+                    yield json.loads(line)
+
+
+def write_jsonl(filename: str, data: Iterable[Dict], append: bool = False):
+    """
+    Writes an iterable of dictionaries to jsonl
+    """
+    if append:
+        mode = 'ab'
+    else:
+        mode = 'wb'
+    filename = os.path.expanduser(filename)
+    if filename.endswith(".gz"):
+        with open(filename, mode) as fp:
+            with gzip.GzipFile(fileobj=fp, mode='wb') as gzfp:
+                for x in data:
+                    gzfp.write((json.dumps(x) + "\n").encode('utf-8'))
+    else:
+        with open(filename, mode) as fp:
+            for x in data:
+                fp.write((json.dumps(x) + "\n").encode('utf-8'))

Evaluation/MBPP/human_eval/evaluate_functional_correctness.py

+import fire
+import sys
+
+from .data import HUMAN_EVAL
+from .evaluation import evaluate_functional_correctness
+
+
+def entry_point(
+    sample_file: str,
+    k: str = "1,10,100",
+    n_workers: int = 4,
+    timeout: float = 3.0,
+    problem_file: str = "",
+    is_mbpp: bool = False,
+):
+    """
+    Evaluates the functional correctness of generated samples, and writes
+    results to f"{sample_file}_results.jsonl.gz"
+    """
+    k = list(map(int, k.split(",")))
+    results = evaluate_functional_correctness(sample_file, k, n_workers, timeout, problem_file, is_mbpp)
+    print(results)
+
+
+def main():
+    fire.Fire(entry_point)
+
+
+sys.exit(main())