first commit

Evaluation/HumanEval/utils/dataset.py

+import os
+import numpy as np
+import json
+
+class HumanEvalDataset:
+
+    def __init__(self, root, sample_num=1, language="python", issft=False):
+        """
+        root: the path to the HumanEval dataset
+        sample_num: the number of samples for each prompt
+        language: the language of the HumanEval dataset
+        issft: whether to use the SFT setting
+        """
+        self.root = root
+        self.data = open(os.path.join(self.root, f"humaneval-{language}.jsonl")).readlines()
+
+        tmp = self.get_qa_only_data(self.data, issft)
+        self.clean_data = []
+        for i in range(len(tmp)):
+            for j in range(sample_num):
+                self.clean_data.append(tmp[i])
+        self.stopwords = self.clean_data[0]["stopwords"]
+        np.random.seed(1234)
+        print(f"Read HumanEval from {root}, number of samples {len(self.clean_data)}")
+
+    def get_qa_only_data(self, data_json, sft=False):
+        """
+        data_json: the jsonl file of HumanEval
+        sft: whether to use the SFT setting
+        return: a list of dict, each dict contains the prompt, task_id and stopwords
+        """
+        ans = []
+        for line in data_json:
+            line = json.loads(line)
+            prompt = line["prompt"].strip()
+            if "prefix" in line:
+                origin_prompt = line["prefix"]
+            else:
+                origin_prompt = line["prompt"]
+
+            if sft:
+                prompt = f"""Below is an instruction that describes a task, paired with an input that provides further context.\nWrite a response that appropriately completes the request.\n\n### Instruction:\nWrite a program to perform the given task.\n\nInput:\n{prompt}\n\n### Response:\n"""
+            if "stop_tokens" in line:
+                s = line["stop_tokens"]
+            else:
+                s = []
+            ans.append({"prompt":prompt, "task_id":line["task_id"], "original_prompt": origin_prompt, "stopwords":s})
+        return ans
+
+    def __len__(self):
+        """
+        return the number of samples in the dataset
+        """
+        return len(self.clean_data)
+
+    def __getitem__(self, index):
+        """
+        return the sample at index
+        """
+        sample = self.clean_data[index]
+        return sample

Evaluation/HumanEval/utils/utils.py

+import re
+
+languge_settings = {
+    'python': {
+        'full_name': 'Python',
+        'indent': 4,
+    },
+    'cpp': {
+        'full_name': 'cpp',
+        'indent': 0,
+        'main': "int main()",
+    },
+    'java': {
+        'full_name': 'Java',
+        'indent': 4,
+        'main': "public static void main",
+    },
+    'cs': {
+        'full_name': "csharp",
+        'indent': 0,
+        'main': "public static void Main",
+    },
+    'php': {
+        'full_name': "PHP",
+        'indent': 0,
+    },
+    'ts': {
+        'full_name': "TypeScript",
+        'indent': 0,
+    },
+    'js': {
+        'full_name': "JavaScript",
+        'indent': 0
+    },
+    'sh': {
+        'full_name': "Bash",
+        'indent': 0
+    }
+}
+
+def get_function_name(question: str, lang: str):
+    func_lines = [x for x in question.strip().split('\n') if x.strip()]
+
+    if lang.lower() == 'python':
+        func_idx = [i for i in range(len(func_lines)) if func_lines[i].startswith("def ")][-1]
+        func_name = func_lines[func_idx].split('(')[0].strip()
+        func_prefix = "\n".join(func_lines[:func_idx])
+        return func_name, func_prefix
+    
+    func_name = func_lines[-1].split('{')[0].strip()
+    func_prefix = "\n".join(func_lines[:-1])
+    return func_name, func_prefix
+
+def extract_generation_code(example: str, lang_code: str, verbose: bool=False):
+    task_id = example['task_id']
+    output = example.get('output', example.get("gpt_completion"))
+    question = example["prompt"].strip()
+    setting = languge_settings[lang_code]
+    lang = setting['full_name']
+    indent = setting['indent']
+
+    try:
+        code_block: str = re.findall(f'```{lang.lower()}\n(.*?)```', output, re.DOTALL | re.IGNORECASE)[0]
+        if verbose:
+            print(">>> Task: {}\n{}".format(task_id, code_block))
+        
+        # Remove main
+        if setting.get('main', None) and setting['main'] in code_block:
+            main_start = code_block.index(setting['main'])
+            code_block = code_block[:main_start]
+        
+        func_name, func_prefix = get_function_name(question, lang)
+
+        try:
+            start = code_block.lower().index(func_name.lower())
+            indent = 0
+            while start - indent >= 0 and code_block[start - indent-1] == ' ':
+                indent += 1
+            
+            try:
+                end = code_block.rindex('\n' + ' '*indent + '}')
+            except:
+                end = len(code_block)
+        except:
+            start = 0
+            try:
+                end = code_block.rindex('\n' + ' '*indent + '}')
+            except:
+                end = len(code_block)
+
+        body = code_block[start:end]
+
+        if lang_code.lower() in ['php', 'ts', 'js']:
+            body += '\n' + ' '*indent + '}'
+    
+        generation = func_prefix + '\n' + body + '\n'
+        example['generation'] = generation
+
+    except Exception as ex:
+        print("Failed to extract code block with error `{}`:\n>>> Task: {}\n>>> Output:\n{}".format(
+            ex, task_id, output
+        ))
+        example['generation'] = example['prompt'] + '\n' + output
+    
+    return example
+
+def cleanup_code(
+    code: str,
+    language_type: str = None,
+    dataset: str = None,
+    issft: bool = False,
+    stop_words = []
+):
+    """
+    Cleans up the generated code.
+    """
+
+    if language_type.lower() == "python":
+        if issft:
+            code = _clean_python_code_for_sft(code)
+        stop_words = ["\ndef", "\nclass", "\nif", "\n#", "\nprint"]
+        code = _truncate_code_at_stopwords(code, stop_words)
+    elif language_type.lower() == "ts":
+        code = _truncate_code_at_stopwords(code, stop_words + ["\nexport", "\nimport", "\nexport default", "\nimport default", "\nconsole.log"])
+    else:
+        code = _truncate_code_at_stopwords(code, stop_words)
+
+    return code
+
+def _clean_python_code_for_sft(code):
+    code = code.replace("\r", "")
+    if "```python" in code:
+        code_start_idx = code.index("```python")
+        code = code[code_start_idx:].replace("```python", "").strip()
+        end_idx = code.find("```") if "```" in code else len(code)
+        code = code[:end_idx].strip()
+
+    return code
+
+def _truncate_code_at_stopwords(code, stop_words):
+    min_stop_idx = len(code)
+    for stop_word in stop_words:
+        stop_index = code.find(stop_word)
+        if 0 <= stop_index < min_stop_idx:
+            min_stop_idx = stop_index
+    return code[:min_stop_idx]
\ No newline at end of file

Evaluation/LeetCode/evaluate_leetcode.py

+import re
+import json
+from pathlib import Path
+from collections import defaultdict
+from human_eval.evaluation import evaluate_functional_correctness
+
+version = "20240121-Jul"
+
+DATA_DIR = Path(__file__).parent / "data"
+
+def extract_python_code(generation: str):
+    generation = generation.replace("[PYTHON]", '```python').replace("[/PYTHON]", '```')
+    if '```python' in generation:
+        p_code = re.compile(r'```python\n(.*?)\n```', flags=re.DOTALL)
+        code_block = p_code.findall(generation)[0]
+        return code_block
+    else:
+        codelist = re.split("\ndef|\nclass|\nif|\n#|\nprint", generation)
+        return codelist[0]
+    
+def evaluate_main(generation_path: str, result_path: str, temp_dir: str):
+    problem_path = (DATA_DIR / f"{version}.jsonl").as_posix()
+
+    print(problem_path)
+    problems = [json.loads(line) for line in open(problem_path, 'r')]
+
+    id2problems = { x['task_id']: x for x in problems }
+
+    results = [json.loads(line) for line in open(generation_path, 'r')]
+    for result in results:
+        if 'task_id' not in result:
+            result['task_id'] = problems[result['index']]['task_id']
+
+        if 'generation' not in result:
+            try:
+                if 'output' not in result:
+                    result['output'] = result['response']
+                if result['output'].startswith("\n        "):
+                    func_code = extract_python_code(result['prompt_sft']).strip()
+                    result['generation'] = func_code + '\n' + result['output']
+                else:
+                    result['generation'] = extract_python_code(result['output'])
+            except:
+                result['generation'] = result['output']
+    
+    with open(result_path, 'w') as fr:
+        for result in results:
+            fr.write(json.dumps(result) + "\n")
+
+    score = evaluate_functional_correctness(
+        input_file=result_path,
+        tmp_dir=temp_dir,
+        problem_file=problem_path,
+        result_path=result_path
+    )
+
+    hardness_results = defaultdict(int)
+    for result in [json.loads(line) for line in open(result_path, 'r')]:
+        problem = id2problems[result['task_id']]
+
+        hardness = problem['meta']['difficulty']
+        hardness_results[hardness] += 1
+        hardness_results[hardness + "_correct"] += result['passed']
+
+    print("="*100)
+    print("Evaluate {} over.".format(generation_path))
+    print("Pass@1: {:.3f}".format(score["pass@1"]))
+    for key in ["Easy", "Medium", "Hard"]:
+        if key.endswith("_correct"):
+            continue
+        acc = hardness_results[key+"_correct"] / hardness_results[key]
+        print("{}: {:.3f}({}/{})".format(key, acc, hardness_results[key+"_correct"],  hardness_results[key]))
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--generation_path", type=str, required=True)
+    parser.add_argument("--result_path", type=str)
+    parser.add_argument("--temp_dir", type=str, default="output/temp")
+    args = parser.parse_args()
+
+    if args.result_path is None:
+        args.result_path = args.generation_path.replace(".jsonl", "_result.jsonl")
+    
+    evaluate_main(args.generation_path, args.result_path, temp_dir=args.temp_dir)
+    pass

Evaluation/LeetCode/human_eval/data.py

+from typing import Iterable, Dict
+import gzip
+import json
+import os
+
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+HUMAN_EVAL = os.path.join(ROOT, "..", "data", "HumanEval.jsonl.gz")
+
+
+def read_problems(evalset_file: str = HUMAN_EVAL) -> Dict[str, Dict]:
+    return {task["task_id"]: task for task in stream_jsonl(evalset_file)}
+
+
+def stream_jsonl(filename: str) -> Iterable[Dict]:
+    """
+    Parses each jsonl line and yields it as a dictionary
+    """
+    if filename.endswith(".gz"):
+        with open(filename, "rb") as gzfp:
+            with gzip.open(gzfp, 'rt') as fp:
+                for line in fp:
+                    if any(not x.isspace() for x in line):
+                        yield json.loads(line)
+    else:
+        with open(filename, "r") as fp:
+            for line in fp:
+                if any(not x.isspace() for x in line):
+                    yield json.loads(line)
+
+
+def write_jsonl(filename: str, data: Iterable[Dict], append: bool = False):
+    """
+    Writes an iterable of dictionaries to jsonl
+    """
+    if append:
+        mode = 'ab'
+    else:
+        mode = 'wb'
+    filename = os.path.expanduser(filename)
+    if filename.endswith(".gz"):
+        with open(filename, mode) as fp:
+            with gzip.GzipFile(fileobj=fp, mode='wb') as gzfp:
+                for x in data:
+                    gzfp.write((json.dumps(x) + "\n").encode('utf-8'))
+    else:
+        with open(filename, mode) as fp:
+            for x in data:
+                fp.write((json.dumps(x) + "\n").encode('utf-8'))

Evaluation/LeetCode/human_eval/evaluation.py

+import os
+import json
+import gzip
+import numpy as np
+import itertools
+
+from typing import *
+from tqdm.auto import tqdm
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from human_eval.data import stream_jsonl
+from human_eval.execution import check_correctness
+
+IMPORT_HELPER = {
+    "python": [
+        "import math",
+        "import re",
+        "import sys",
+        "import copy",
+        "import datetime",
+        "import itertools",
+        "import collections",
+        "import heapq",
+        "import functools",
+        "import hashlib",
+        "import numpy",
+        "import numpy as np",
+        "import string",
+        "from typing import *",
+        "from collections import *",
+        "from functools import *"
+    ],
+    "go"   : [
+        "math",
+        "strings",
+        "fmt",
+        "strconv",
+        "time",
+        "bytes",
+        "regexp",
+        "sort",
+        "math/rand",
+        "crypto/md5",
+    ],
+    "cpp"   : [
+        "#include<stdlib.h>",
+        "#include<algorithm>",
+        "#include<math.h>",
+        "#include<stdio.h>",
+        "#include<vector>",
+        "#include<string>",
+        "#include<climits>",
+        "#include<cstring>",
+        "#include<iostream>",
+    ],
+}
+
+
+LANGUAGE_NAME = {
+    "cpp"   : "CPP",
+    "go"    : "Go",
+    "java"  : "Java",
+    "js"    : "JavaScript",
+    "python": "Python",
+}
+
+
+def read_dataset(
+    data_file: str = None,
+    dataset_type: str = "humaneval",
+    num_shot=None,
+) -> Dict:
+    if num_shot is not None:
+        print(f"{num_shot}-shot setting...")
+    if "humaneval" in dataset_type.lower():
+        if data_file is None:
+            current_path = os.path.dirname(os.path.abspath(__file__))
+            data_file = os.path.join(current_path, "..", "humaneval-x", "python", "data", "humaneval_python.jsonl.gz")
+        dataset = {task["task_id"]: task for task in stream_jsonl(data_file)}
+    else:
+        raise f"Dataset: {dataset_type} not supported."
+
+    return dataset
+
+def estimate_pass_at_k(
+        num_samples: Union[int, List[int], np.ndarray],
+        num_correct: Union[List[int], np.ndarray],
+        k: int
+) -> np.ndarray:
+    """
+    Estimates pass@k of each problem and returns them in an array.
+    """
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """
+        Calculates 1 - comb(n - c, k) / comb(n, k).
+        """
+        if n - c < k:
+            return 1.0
+        return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
+
+def process_humaneval_test(sample, problems, example_test=False, is_mbpp=False, language="python"):
+    task_id = sample["task_id"]
+    
+    if is_mbpp:
+        return sample["generation"] + "\n" + "\n".join(problems[task_id]["test"])
+    #language = task_id.split("/")[0].lower()
+
+    prompt = sample.get("prompt", "")
+    if example_test and "example_test" in problems[task_id] and problems[task_id]["example_test"] != "":
+        test = problems[task_id]["example_test"]
+    else:
+        test = problems[task_id]["test"]
+    code = sample["generation"]
+
+    # Pre-process for different languages
+    if language == "python":
+        '''code_ = []
+        for line in code.split("\n"):
+            if (len(line.strip()) > 0 and line[0] != ' ' and line[0] != '\t'):
+                break
+            code_.append(line)
+        code = "\n".join(code_)'''
+        test_setup = "\n".join(IMPORT_HELPER["python"]) + "\n"
+        test_string = test_setup + code + "\n" + test + "\n"
+    elif language == "cpp":
+        test_set_up = ""
+        for s in IMPORT_HELPER["cpp"]:
+            if s not in prompt:
+                test_set_up += s + "\n"
+        test_string = test_set_up + "\n" + code + "\n" + test
+    elif language == "java":
+        test_string = code + "\n" + test
+    elif language in ["js", "javascript", "ts", "cs", "sh"]:
+        test_string = code + "\n" + test
+    elif language == "go":
+        import_string = problems[task_id]["import"]
+        prompt = prompt.replace(import_string, "")
+        if example_test and "example_test" in problems[task_id]:
+            test = problems[task_id]["example_test"]
+        else:
+            test = problems[task_id]["test"]
+        test_setup = problems[task_id]["test_setup"]
+        other_pkgs = []
+        for pkg in IMPORT_HELPER["go"]:
+            if pkg not in test_setup:
+                p = pkg.split("/")[-1]
+                if p + "." in code:
+                    other_pkgs.append(f"\"{pkg}\"")
+        if other_pkgs:
+            import_other_pkgs = "import (\n" + "    ".join([p + "\n" for p in other_pkgs]) + ")"
+            test_string = test_setup + "\n" + import_other_pkgs + "\n" + prompt + code + "\n" + test
+        else:
+            test_string = test_setup + "\n" + prompt + code + "\n" + test
+    elif language == "rust":
+        main = "\nfn main(){ \n } \n"
+        declaration = problems[task_id]["declaration"]
+        test_string = main + declaration + prompt + code + test
+    elif language == "php":
+        test_string = code + "\n" + test + "?>"
+    return test_string
+
+
+def stream_jsonl_all(filename: str) -> Iterable[Dict]:
+    results = []
+    if filename.endswith(".gz"):
+        fp = gzip.open(open(filename, "rb"), "rt")
+    else:
+        fp = open(filename, "r")
+    for line in fp:
+        if any(not x.isspace() for x in line):
+            results.append(json.loads(line))
+    fp.close()
+
+    return results
+
+
+def evaluate_functional_correctness(
+    input_file: str = None,
+    tmp_dir: str = "./",
+    n_workers: int = 32,
+    timeout: float = 10.0,
+    problem_file: str = "../data/humaneval_python.jsonl.gz",
+    result_path: str = None,
+    k: List[int] = [1, 10, 100],
+    test_groundtruth: bool = False,
+    example_test: bool = False,
+    is_mbpp: bool = False,
+    language: str = "python",
+):
+    if example_test:
+        print("Example test...")
+
+    problems = read_dataset(problem_file, dataset_type="humaneval")
+    sample_jsonl = stream_jsonl_all(input_file)
+    with ThreadPoolExecutor(max_workers=n_workers) as executor:
+        futures = []
+        completion_id = Counter()
+        n_samples = 0
+        results = defaultdict(list)
+
+        if test_groundtruth:
+            print("Testing ground truth...")
+            for sample in tqdm(problems.values()):
+                task_id = sample["task_id"]
+                lang = task_id.split("/")[0].lower()
+                if lang == "javascript":
+                    lang = "js"
+                tmp_dir_ = os.path.join(tmp_dir, lang, "evaluation")
+                sample["generation"] = sample["canonical_solution"]
+                sample["test_code"] = process_humaneval_test(sample, problems, example_test, language)
+                if sample["test_code"] is None:
+                    continue
+                args = (task_id, sample, lang, timeout, tmp_dir_, completion_id[task_id])
+                future = executor.submit(check_correctness, *args)
+                futures.append(future)
+                completion_id[task_id] += 1
+                n_samples += 1
+        else:
+            print("Reading Samples...")
+            id2samples = {}
+            for sample in tqdm(sample_jsonl):
+                task_id = sample["task_id"]
+
+                if not is_mbpp:
+                    lang = language
+                if not is_mbpp and lang == "javascript":
+                    lang = "js"
+                if is_mbpp:
+                    lang = "python"
+                tmp_dir_ = os.path.join(tmp_dir, lang, "evaluation")
+                sample["task_id"] = task_id
+                sample["test_code"] = process_humaneval_test(sample, problems, example_test, is_mbpp, language)
+                if sample["test_code"] is None:
+                    continue
+                if "completion_id" in sample:
+                    completion_id_ = sample["completion_id"]
+                else:
+                    completion_id_ = completion_id[task_id]
+                args = (task_id, sample, lang, timeout, tmp_dir_, completion_id_)
+                id2samples[(task_id, completion_id_)] = sample
+                future = executor.submit(check_correctness, *args)
+                futures.append(future)
+                completion_id[task_id] += 1
+                n_samples += 1
+
+        if len(completion_id) == len(problems):
+            evaluate_pass_at_k = True
+        else:
+            evaluate_pass_at_k = False
+
+        print("Running test suites...")
+        sample_with_results = []
+        for future in tqdm(as_completed(futures), total=len(futures)):
+            result = future.result()
+            results[result["task_id"]].append((result["completion_id"], result))
+
+            sample = id2samples[(result["task_id"], result["completion_id"])]
+            sample_with_results.append({
+                'task_id': result['task_id'],
+                'completion_id': result["completion_id"],
+                'passed': result['passed'],
+                'generation': sample['generation']
+            })
+
+            for key in sample:
+                if key not in sample_with_results[-1]:
+                    sample_with_results[-1][key] = sample[key]
+
+    # Calculate pass@k.
+    total, correct = [], []
+    for result in results.values():
+        passed = [r[1]["passed"] for r in result]
+        total.append(len(passed))
+        correct.append(sum(passed))
+
+    total = np.array(total)
+    correct = np.array(correct)
+    if evaluate_pass_at_k:
+        ks = k
+        pass_at_k = {
+            f"pass@{k}": estimate_pass_at_k(total, correct, k).mean()
+            for k in ks if (total >= k).all()
+        }
+        print(pass_at_k)
+    else:
+        print("Total:", np.sum(total))
+        print("Correct:", np.sum(correct))
+    
+    if result_path is not None:
+        with open(result_path, 'w', encoding='utf-8') as fw:
+            for sample_with_result in sample_with_results:
+                fw.write(json.dumps(sample_with_result) + '\n')
+            print("Save evaluation results to\n{}".format(result_path))
+
+    return pass_at_k

Evaluation/LeetCode/readme.md

+## 1. Introduction
+We construct the LeetCode Contest benchmark to to further validate the model's capability in real-world programming problems.
+[LeetCode](https://leetcode.com/) presents competition-level problems, offering significant challenges that test the model's problem understanding and code generation skills. We collected the latest problems from LeetCode Contests to prevent the appearance of both the problems or their solutions in our pre-training data. A total of `180` problems were collected from July 2023 to January 2024. For each problem, we collected `100` test cases. The data format is the same as human-eval. For more details, please refer to [leetcode_contest_data](./data/20240121-Jul.jsonl).
+
+## 2. Evaluation
+Please follow the following two steps to evaluate the model's performance on our LeetCode Contest benchmark:
+
+1. Run `vllm_inference.py` to get generation results.
+```bash
+cd Evaluation/LeetCode
+
+# Set the model or path here
+MODEL="deepseek-ai/deepseek-coder-7b-instruct"
+
+python vllm_inference.py --model_name_or_path $MODEL --saved_path output/20240121-Jul.deepseek-coder-7b-instruct.jsonl
+```
+
+If you want to evaluate the model with COT, please add `--cot` to the command:
+```bash
+python vllm_inference.py --model_name_or_path $MODEL --saved_path output/20240121-Jul.deepseek-coder-7b-instruct.jsonl --cot
+```
+
+2. Run `evaluate_leetcode.py` to get evaluation results.
+```bash
+python evaluate_leetcode.py --generation_path output/20240121-Jul.deepseek-coder-7b-instruct.jsonl --result_path output/20240121-Jul.deepseek-coder-7b-instruct.result.jsonl
+```
+
+## 3. Experimental Results
+We report experimental results here:
+
+| Model                       | Size | Easy (45) | Medium (91) | Hard (44) | Overall(180) |
+|-----------------------------|------|-----------|-------------|-----------|--------------|
+| WizardCoder-V1.0            | 15B  | 17.8%     | 1.1%        | 0.0%      | 5.0%         |
+| CodeLlama-Instruct          | 34B  | 24.4%     | 4.4%        | 4.5%      | 9.4%         |
+| Phind-CodeLlama-V2          | 34B  | 26.7%     | 8.8%        | 9.1%      | 13.3%        |
+| | | | |
+| GPT-3.5-Turbo               | -    | 46.7%     | 15.4 %      | 15.9%     | 23.3%        |
+| GPT-3.5-Turbo + CoT         | -    | 42.2%     | 15.4%       | 20.5%     | 23.3%        |
+| GPT-4-Turbo                 | -    | 73.3%     | 31.9%       | 25.0%     | 40.6%        |
+| GPT-4-Turbo + CoT           | -    | 71.1%     | 35.2%       | 25.0%     | 41.8%        |
+| | | | |
+| DeepSeek-Coder-Instruct     | 1.3B | 22.2%     | 1.1%        | 4.5%      | 7.2%         |
+| DeepSeek-Coder-Instruct + CoT | 1.3B | 22.2%   | 2.2%        | 2.3%      | 7.2%         |
+| DeepSeek-Coder-Instruct     | 6.7B | 44.4%     | 12.1%       | 9.1%      | 19.4%        |
+| DeepSeek-Coder-Instruct + CoT | 6.7B | 44.4%   | 17.6%       | 4.5%      | 21.1%        |
+| DeepSeek-Coder-Instruct     | 33B  | 57.8%     | 22.0%       | 9.1%      | 27.8%        |
+| DeepSeek-Coder-Instruct + CoT | 33B | 53.3%    | 25.3%       | 11.4%     | 28.9%        |
+

Evaluation/LeetCode/vllm_inference.py

+from vllm import LLM, SamplingParams
+import json
+from transformers import AutoTokenizer
+from pathlib import Path
+
+version = "20240121-Jul"
+
+def generate_batch(examples, tokenizer, llm, model: str):
+    stop = None
+    if model == 'deepseekcoder-instruct':
+        prompts = [
+            tokenizer.apply_chat_template([{'role': 'user', 'content': ex['prompt_sft'] }], tokenize=False, add_generation_prompt=True)
+            for ex in examples
+        ]
+    else:
+        raise NotImplementedError()
+
+    # Create a sampling params object.
+    sampling_params = SamplingParams(
+        temperature=0.0,
+        # top_p=0.95,
+        max_tokens=1024,
+        stop=stop
+    )
+
+    print("Sample prompt: {}".format(prompts[0]))
+    outputs = llm.generate(prompts, sampling_params)
+    for i in range(len(examples)):
+        examples[i]['output'] = outputs[i].outputs[0].text
+
+    return examples
+
+def generate_main(data_path: str, model_name_or_path: str, saved_path: str, model_type: str='deepseekcoder-instruct', cot: bool=False):
+    examples = [json.loads(x) for x in open(data_path).readlines()]
+    def _convert_for_sft(ex):
+        ex['prompt_sft'] = ex["prompt_sft"] + "\nYou need first write a step-by-step outline and then write the code."
+        return ex
+    
+    if cot:
+        examples = [_convert_for_sft(x) for x in examples]
+        saved_path = saved_path.replace(".jsonl", ".cot.jsonl")
+
+    print(model_type)
+    print("Model `{}`, COT = {}:{}".format(model_type, cot, model_name_or_path))
+    print("Saved path: {}".format(saved_path))
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
+    print("load tokenizer {} from {} over.".format(tokenizer.__class__, model_name_or_path))
+
+    # Create an LLM.
+    llm = LLM(
+        model=model_name_or_path,
+        pipeline_parallel_size=1,
+        tensor_parallel_size=8,
+        max_num_seqs=512,
+        max_num_batched_tokens=8192,
+        max_model_len=4096,
+        gpu_memory_utilization=0.85,
+        trust_remote_code=True
+    )
+    
+    generated_examples = generate_batch(examples, tokenizer, llm, model_type)    
+    print("Generate all over!!!")
+    with open(saved_path, 'w', encoding='utf-8') as fw:
+        for ex in generated_examples:
+            fw.write(json.dumps(ex) + '\n')
+        print("Save {} processed examples into {} over!".format(len(generated_examples), saved_path))
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--data_path', type=str, default=Path(__file__).parent.joinpath(f"data/{version}.jsonl").as_posix())
+    parser.add_argument('--model_name_or_path', type=str, default='deepseek-ai/deepseek-coder-7b-instruct')
+    parser.add_argument('--saved_path', type=str, default=f'output/{version}.deepseek-coder-7b-instruct.jsonl')
+    parser.add_argument('--cot', action='store_true', default=False)
+    args = parser.parse_args()
+
+    generate_main(
+        data_path=args.data_path,
+        model_name_or_path=args.model_name_or_path,
+        saved_path=args.saved_path,
+        cot=args.cot,
+    )

Evaluation/MBPP/README.md

+## 1. Introduction
+
+We provide a test script to evaluate the performance of the **deepseek-coder** model on code generation benchmarks, [**MBPP**](https://huggingface.co/datasets/mbpp), with 3-shot setting.
+
+
+
+## 2. Setup
+
+```
+pip install accelerate
+pip install attrdict
+pip install transformers
+pip install pytorch
+```
+
+
+
+## 3. Evaluation
+
+We've created a sample script, **eval.sh**, that demonstrates how to test the **deepseek-coder-1.3b-base** model on the MBPP dataset leveraging **8** GPUs.
+
+```bash
+MODEL_NAME_OR_PATH="deepseek-ai/deepseek-coder-1.3b-base"
+DATASET_ROOT="data/"
+LANGUAGE="python"
+python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --dataroot ${DATASET_ROOT} 
+```
+
+## 4. Experimental Results
+
+We report experimental results here for several models. We set the maximum input length to **4096** and the maximum output length to **500**, and employ the **greedy search strategy**.
+
+
+
+#### (1) Multilingual Base Models
+
+| Model             | Size | Pass@1 | 
+|-------------------|------|--------|
+| CodeShell         | 7B   | 38.6%  | 
+| CodeGeeX2         | 6B   | 36.2%  |
+| StarCoder     | 16B  | 42.8%  | 
+| CodeLLama-Base   | 7B   | 38.6%  | 
+| CodeLLama-Base    | 13B  | 47.0%  | 
+| CodeLLama-Base    | 34B  | 55.0%  | 
+| | | | |  |  |  |  |  |  | |
+| DeepSeek-Coder-Base| 1.3B   | 46.8%  |
+| DeepSeek-Coder-Base| 5.7B   | 57.2%  | 
+| DeepSeek-Coder-Base| 6.7B   | 60.6%  | 
+| DeepSeek-Coder-Base|33B  | **66.0%**  |
+
+#### (2) Instruction-Tuned Models
+| Model               | Size | Pass@1  |
+|---------------------|------|--------|
+| GPT-3.5-Turbo            | -    | 70.8%  | 
+| GPT-4               | -    | **80.0%**  |
+| | | | |  |  |  |  |  |  | |
+| DeepSeek-Coder-Instruct | 1.3B  | 49.4%      |
+| DeepSeek-Coder-Instruct  | 6.7B  | 65.4%     |
+| DeepSeek-Coder-Instruct  | 33B | **70.0%**     | 
+
+

Evaluation/MBPP/eval.sh

+MODEL_NAME_OR_PATH="deepseek-ai/deepseek-coder-1b-base/"
+DATASET_ROOT="data/"
+LANGUAGE="python"
+CUDA_VISIBLE_DEVICES=1,2,3 python -m accelerate.commands.launch --config_file test_config.yaml eval_pal.py --logdir ${MODEL_NAME_OR_PATH} --language ${LANGUAGE} --dataroot ${DATASET_ROOT}
\ No newline at end of file

Evaluation/MBPP/eval_instruct.py

+import argparse
+import json
+import os
+import torch
+import re
+from pathlib import Path
+from tqdm import tqdm
+
+data_abs_dir = Path(__file__).parent / "data"
+
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from human_eval.evaluation import evaluate_functional_correctness
+
+def read_test_examples(data_path: str):
+    def format_test_example(q, tests, code: str=None):
+        prompt = ">>> Problem:\n{}\n>>> Test Cases:\n{}\n".format(q.strip(), "\n".join(tests))
+        if code:
+            code = code.replace("\r", "").replace("\t", "    ")
+            prompt += "\n>>> Code:\n```python\n{}\n```".format(code)
+        return prompt
+
+    examples = [json.loads(x) for x in open(data_path)]
+    print("Read all {} examples from {} over!".format(len(examples), data_path))
+
+    # test_cases
+    examples_str = []
+    for i in range(1, 4):
+        ex = examples[i]
+        q, test, code = ex['text'], ex['test_list'], ex['code']
+        ex_prompt = format_test_example(q, test, code)
+        example_prompt = '- Example {}:\n{}'.format(i, ex_prompt)
+        examples_str += [example_prompt]
+
+    for i in range(10, 510):
+        ex = examples[i]
+        q, test, code = ex['text'], ex['test_list'], ex['code']
+        
+        prompt = format_test_example(q, test, code=None)
+
+        prompt_with_shots = '''
+Please refer the given examples and generate a python function for my problem.
+Examples are listed as follows:
+{}
+
+Here is my problem:
+{}
+'''.strip().format('\n\n'.join(examples_str), prompt)
+        yield {
+            'task_id': ex['task_id'],
+            'prompt': prompt_with_shots
+        }
+
+def convert_for_evaluation(example):
+    gpt_completion = example['gpt_completion']
+    generation = gpt_completion
+    try:
+        code_block: str = re.findall(f'```python\n(.*?)```', gpt_completion, re.DOTALL | re.IGNORECASE)[0]
+        generation = code_block
+    except Exception as ex:
+        print("Failed to extract codeblock:\n{}".format(gpt_completion))
+
+    example['generation'] = generation
+    return example
+
+def generate_one(example, tokenizer, model):
+    prompt = example['prompt']
+    inputs = tokenizer.apply_chat_template(
+        [{'role': 'user', 'content': prompt }],
+        return_tensors="pt", add_generation_prompt=True
+    ).to(model.device)
+
+    stop_id = tokenizer.convert_tokens_to_ids("<|EOT|>")
+    assert isinstance(stop_id, int), "Invalid tokenizer, EOT id not found"
+    outputs = model.generate(
+        inputs, 
+        max_new_tokens=512,
+        do_sample=False,
+        # top_p=0.95,
+        # temperature=temperature,
+        pad_token_id=stop_id,
+        eos_token_id=stop_id
+    )
+
+    output = tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True)
+    # print(output)
+    example['gpt_completion'] = output
+    return convert_for_evaluation(example)
+
+def generate_main(args):
+    model_name_or_path = args.model
+    saved_path = args.output_path
+    temp_dir = args.temp_dir
+    os.makedirs(temp_dir, exist_ok=True)
+    problem_file = os.path.join(data_abs_dir, f"mbpp.jsonl")
+
+    print("model", model_name_or_path)
+    tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
+    print("load tokenizer {} from {} over.".format(tokenizer.__class__, model_name_or_path))
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name_or_path,
+        torch_dtype=torch.bfloat16,
+        device_map="auto",
+    )
+    model.eval()
+
+    examples = list(read_test_examples(problem_file))
+    print("Read {} examples for evaluation over.".format(len(examples)))
+
+    generated_examples = []
+    for ex in tqdm(examples, desc='Generating'):
+        gen_example = generate_one(ex, tokenizer, model)
+        generated_examples.append(gen_example)
+        print("Generate {}/{} over...".format(len(generated_examples), len(examples)))
+
+    print("Generate all over!!!")
+    with open(saved_path, 'w', encoding='utf-8') as fw:
+        for ex in generated_examples:
+            fw.write(json.dumps(ex) + '\n')
+        print("Save {} processed examples into {} over!".format(len(generated_examples), saved_path))
+    
+    result = evaluate_functional_correctness(
+        input_file=saved_path,
+        tmp_dir=temp_dir,
+        problem_file=os.path.join(data_abs_dir, f"mbpp_test.jsonl"),
+        language='python',
+        is_mbpp=True
+    )
+    print(result, model_name_or_path)
+    pass
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--model', type=str, help="model name or path")
+    parser.add_argument('--output_path', type=str, help="output path of your generation")
+    parser.add_argument('--temp_dir', type=str, help="temp dir for evaluation", default="tmp")
+    args = parser.parse_args()
+
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+    generate_main(args)
+    pass
\ No newline at end of file

Evaluation/MBPP/eval_pal.py

+import os
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn.functional as F
+import json
+import torch.distributed as dist
+import subprocess
+import sys
+from accelerate import Accelerator
+from accelerate import DistributedDataParallelKwargs
+from pathlib import Path
+from argparse import ArgumentParser
+from mbpp import MBPP as evaltor
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+if __name__ == '__main__':
+    kwargs_handlers = [DistributedDataParallelKwargs(find_unused_parameters=True)]
+    accelerator = Accelerator(mixed_precision="bf16", kwargs_handlers=kwargs_handlers)   
+
+
+    parser = ArgumentParser()
+    parser.add_argument("--logdir", type=str, default="")
+    parser.add_argument("--dataroot", type=str, default="")
+    args = parser.parse_args()
+
+    logdir = args.logdir
+
+    if logdir == "":
+        logdir = "tmp/"
+    tokenizer = dict(
+        cls=AutoTokenizer,
+        model_path=logdir,)
+
+    dataroot = args.dataroot
+
+    evaluator = evaltor(data_root=dataroot, max_seq_len=4096, tokenizer_cfg=tokenizer, log_dir=logdir, n_sample=1, batch_size=1, max_gen_len=500)
+    model = AutoModelForCausalLM.from_pretrained(logdir, device_map=accelerator.device, trust_remote_code=True, torch_dtype=torch.bfloat16)
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+    evaluator.eval_model(model, accelerator)

Evaluation/MBPP/human_eval/data.py

+from typing import Iterable, Dict
+import gzip
+import json
+import os
+
+
+ROOT = os.path.dirname(os.path.abspath(__file__))
+HUMAN_EVAL = os.path.join(ROOT, "..", "data", "HumanEval.jsonl.gz")
+
+
+def read_problems(evalset_file: str = HUMAN_EVAL) -> Dict[str, Dict]:
+    return {task["task_id"]: task for task in stream_jsonl(evalset_file)}
+
+
+def stream_jsonl(filename: str) -> Iterable[Dict]:
+    """
+    Parses each jsonl line and yields it as a dictionary
+    """
+    if filename.endswith(".gz"):
+        with open(filename, "rb") as gzfp:
+            with gzip.open(gzfp, 'rt') as fp:
+                for line in fp:
+                    if any(not x.isspace() for x in line):
+                        yield json.loads(line)
+    else:
+        with open(filename, "r", encoding="utf-8") as fp:
+            for line in fp:
+                if any(not x.isspace() for x in line):
+                    yield json.loads(line)
+
+
+def write_jsonl(filename: str, data: Iterable[Dict], append: bool = False):
+    """
+    Writes an iterable of dictionaries to jsonl
+    """
+    if append:
+        mode = 'ab'
+    else:
+        mode = 'wb'
+    filename = os.path.expanduser(filename)
+    if filename.endswith(".gz"):
+        with open(filename, mode) as fp:
+            with gzip.GzipFile(fileobj=fp, mode='wb') as gzfp:
+                for x in data:
+                    gzfp.write((json.dumps(x) + "\n").encode('utf-8'))
+    else:
+        with open(filename, mode) as fp:
+            for x in data:
+                fp.write((json.dumps(x) + "\n").encode('utf-8'))

Evaluation/MBPP/human_eval/evaluate_functional_correctness.py

+import fire
+import sys
+
+from .data import HUMAN_EVAL
+from .evaluation import evaluate_functional_correctness
+
+
+def entry_point(
+    sample_file: str,
+    k: str = "1,10,100",
+    n_workers: int = 4,
+    timeout: float = 3.0,
+    problem_file: str = "",
+    is_mbpp: bool = False,
+):
+    """
+    Evaluates the functional correctness of generated samples, and writes
+    results to f"{sample_file}_results.jsonl.gz"
+    """
+    k = list(map(int, k.split(",")))
+    results = evaluate_functional_correctness(sample_file, k, n_workers, timeout, problem_file, is_mbpp)
+    print(results)
+
+
+def main():
+    fire.Fire(entry_point)
+
+
+sys.exit(main())