Initial commit

FinanceIQ/script/baichuan.sh

+#!/bin/bash
+
+cd ../src
+
+# Baichuan-7B
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --model_name_or_path baichuan-inc/Baichuan-7B \
+    --save_dir ../results/Baichuan-7B-Base \
+    --num_few_shot $i
+done
+
+# Baichuan-13B-Base
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --model_name_or_path baichuan-inc/Baichuan-13B-Base \
+    --save_dir ../results/Baichuan-13B-Base \
+    --num_few_shot $i
+done
+
+# Baichuan-13B-Chat
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --model_name_or_path baichuan-inc/Baichuan-13B-Chat \
+    --save_dir ../results/Baichuan-13B-Chat \
+    --num_few_shot $i
+done
+
+# Baichuan2-7B-Base
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --model_name_or_path baichuan-inc/Baichuan2-7B-Base \
+    --save_dir ../results/Baichuan2-7B-Base \
+    --num_few_shot $i
+done
+
+# Baichuan2-7B-Chat
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --model_name_or_path baichuan-inc/Baichuan2-7B-Chat \
+    --save_dir ../results/Baichuan2-7B-Chat \
+    --num_few_shot $i
+done
+
+# Baichuan2-13B-Base
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --model_name_or_path baichuan-inc/Baichuan2-13B-Base \
+    --save_dir ../results/Baichuan2-13B-Base \
+    --num_few_shot $i
+done
+
+# Baichuan2-13B-Chat
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --model_name_or_path baichuan-inc/Baichuan2-13B-Chat \
+    --save_dir ../results/Baichuan2-13B-Chat \
+    --num_few_shot $i
+done
+
+

FinanceIQ/script/chatglm.sh

+#!/bin/bash
+
+cd ../src
+
+# ChatGLM
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path THUDM/chatglm-6b \
+    --save_dir ../results/ChatGLM-6B \
+    --num_few_shot $i
+done
+
+
+# ChatGLM-2
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path THUDM/chatglm2-6b \
+    --save_dir ../results/ChatGLM2-6B \
+    --num_few_shot $i
+done

FinanceIQ/script/chatgpt.sh

+#!/bin/bash
+
+cd ../src
+
+for i in {0,5}; do
+python3 -u chatgpt.py \
+    --data_dir ../data \
+    --save_dir ../results/ChatGPT \
+    --num_few_shot $i
+done

FinanceIQ/script/chinese-alpaca-2.sh

+#!/bin/bash
+
+cd ../src
+
+# Chinese-Alpaca-2-7B
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path ziqingyang/chinese-alpaca-2-7b \
+    --save_dir ../results/Chinese-Alpaca-2-7B \
+    --num_few_shot $i
+done
+
+# Chinese-Alpaca-2-13B 
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path ziqingyang/chinese-alpaca-2-13b \
+    --save_dir ../results/Chinese-Alpaca-2-13B \
+    --num_few_shot $i
+done
+

FinanceIQ/script/chinese-llama-2.sh

+#!/bin/bash
+
+cd ../src
+
+# Chinese-Llama-2-7B
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path ziqingyang/chinese-llama-2-7b \
+    --save_dir ../results/Chinese-Llama-2-7B \
+    --num_few_shot $i
+done
+
+# Chinese-Llama-2-13B
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path ziqingyang/chinese-llama-2-13b \
+    --save_dir ../results/Chinese-Llama-2-13B \
+    --num_few_shot $i
+done

FinanceIQ/script/ernie_bot.sh

+
+cd ../src
+
+# ERNIE-Bot-turbo
+for i in {0,5}; do
+python3 -u ernie.py \
+    --version ERNIE-Bot-turbo \
+    --data_dir ../data \
+    --save_dir ../results/ErnieBot-turbo \
+    --num_few_shot $i
+done
+
+# ERNIE-Bot
+for i in {0,5}; do
+python3 -u ernie.py \
+    --version ERNIE-Bot \
+    --data_dir ../data \
+    --save_dir ../results/ErnieBot-turbo \
+    --num_few_shot 0
+done
+

FinanceIQ/script/gpt4.sh

+#!/bin/bash
+cd ../src
+
+for i in {0,5}; do
+python3 -u gpt4.py \
+    --data_dir ../data \
+    --save_dir ../results/GPT4\
+    --num_few_shot $i
+done

FinanceIQ/script/internlm-7b.sh

+#!/bin/bash
+
+cd ../src
+
+# InternLM-7B
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path internlm/internlm-7b \
+    --save_dir ../results/InternLM-7B \
+    --num_few_shot $i
+done
+
+# InternLM-Chat-7B
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path internlm/internlm-chat-7b \
+    --save_dir ../results/InternLM-7B-Chat \
+    --num_few_shot $i
+done

FinanceIQ/script/llama2-7b.sh

+#!/bin/bash
+
+cd ../src
+
+# LLaMA2-7B-Base
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path meta-llama/Llama-2-7b-base-hf \
+    --save_dir ../results/LLaMA2-7B-Base \
+    --num_few_shot $i
+done
+
+# LLaMA2-7B-Chat
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path meta-llama/Llama-2-7b-chat-hf \
+    --save_dir ../results/LLaMA2-7B-Chat \
+    --num_few_shot $i
+done
+
+# LLaMA2-13B-Base
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path meta-llama/Llama-2-13b-base-hf \
+    --save_dir ../results/LLaMA2-13B-Base \
+    --num_few_shot $i
+done
+
+# LLaMA2-13B-Chat
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path meta-llama/Llama-2-13b-chat-hf \
+    --save_dir ../results/LLaMA2-13B-Chat \
+    --num_few_shot $i
+done
+

FinanceIQ/script/qwen-7b.sh

+#!/bin/bash
+
+cd ../src
+
+# Qwen-7B (base)
+for i in {5,0}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path Qwen/Qwen-7B \
+    --save_dir ../results/Qwen-7B \
+    --num_few_shot $i
+done
+
+# Qwen-7B-Chat
+for i in {0,5}; do
+python -u hf_causal_model.py \
+    --data_dir ../data \
+    --model_name_or_path Qwen/Qwen-7B-Chat \
+    --save_dir ../results/Qwen-7B-Chat \
+    --num_few_shot $i
+done
+

FinanceIQ/src/categories.py

+
+
+subjects = [
+    '注册会计师（CPA）', 
+    '银行从业资格',
+    '证券从业资格', 
+    '基金从业资格', 
+    '保险从业资格CICE', 
+    '经济师', 
+    '税务师', 
+    '期货从业资格', 
+    '理财规划师',
+    '精算师-金融数学', 
+    ]

FinanceIQ/src/chatgpt.py

+import os
+import argparse
+import numpy as np
+import openai
+import tiktoken
+from tqdm import tqdm
+from time import sleep
+from utils import choices, format_example, gen_prompt, run_eval
+
+
+openai.api_key = "YOUR_API_KEY"
+encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")
+
+def get_response(inputs):
+    completion = None
+    retry = 0
+    while completion is None:
+        try:
+            completion = openai.ChatCompletion.create(
+                engine="gpt-3.5-turbo",
+                messages=[{"role": "user", "content": inputs}]
+                )
+            return completion.choices[0].message['content']
+        except Exception as msg:
+            print(msg)
+            retry += 1
+            if retry > 3:
+                return 'ChatGPT failed'
+
+def eval(subject, dev_df, test_df, num_few_shot, max_length, cot, **kwargs):
+    cors = []
+    all_preds = []
+    answers = choices[: test_df.shape[1] - 2]
+
+    for i in tqdm(range(test_df.shape[0])):
+        try:
+            prompt_end = format_example(test_df, i, subject, include_answer=False, cot=cot)
+            prompt = gen_prompt(dev_df, subject, prompt_end, num_few_shot, encoding, max_length, cot=cot)
+            label = test_df.iloc[i, test_df.shape[1] - 1]
+        except Exception as e:
+            print(f'failed to format example {i}')
+            continue
+
+        pred = get_response(prompt)
+        all_preds.append(pred.replace("\n", "") if pred is not None else "")
+        if pred and pred[0] in choices:
+            cors.append(pred[0] == label)
+
+    acc = np.mean(cors)
+    print(f"Average accuracy {acc:.3f} - {subject}")
+    print(f"{len(cors)} results, {len(all_preds)-len(cors)} inappropriate formated answers.")
+    return all_preds
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data_dir", "-d", type=str, default="../data")
+    parser.add_argument("--save_dir", "-s", type=str, default="../results/ChatGPT")
+    parser.add_argument("--num_few_shot", "-n", type=int, default=0)
+    parser.add_argument("--max_length", type=int, default=4096)
+    parser.add_argument("--cot", action='store_true')
+    args = parser.parse_args()
+    print('\n\n\n=========================')
+    print(f'args = {args}')
+    print('=========================')
+
+    run_eval(None, None, eval, args)
+

FinanceIQ/src/ernie.py

+import argparse
+import numpy as np
+from tqdm import tqdm
+from time import sleep
+import requests
+import json
+
+from utils import choices, format_example, gen_prompt, run_eval
+
+API_KEY = "YOUR-API-KEY"
+SECRET_KEY = "YOUR-SECRET-KEY"
+
+def get_access_token():
+    url = "https://aip.baidubce.com/oauth/2.0/token"
+    params = {"grant_type": "client_credentials", "client_id": API_KEY, "client_secret": SECRET_KEY}
+    return str(requests.post(url, params=params).json().get("access_token"))
+
+def get_response(inputs):
+    completion = None
+    retry = 0
+    while completion is None:
+        try:
+            message = {"messages": [{"role": "user","content": inputs}]}
+            payload = json.dumps(message)
+            headers = {'Content-Type': 'application/json'}
+            response = requests.request("POST", url, headers=headers, data=payload)
+            return response.json()['result']
+        except Exception as msg:
+            print(msg)
+            retry += 1
+            if retry > 3:
+                return '生成失败'
+            sleep(3)
+            continue
+
+def eval(subject, dev_df, test_df, num_few_shot, max_length, cot, **kwargs):
+    cors = []
+    all_preds = []
+
+    for i in tqdm(range(test_df.shape[0])):
+        try:
+            prompt_end = format_example(test_df, i, subject, include_answer=False, cot=cot)
+            prompt = gen_prompt(dev_df, subject, prompt_end, num_few_shot, None, max_length, cot=cot)
+            label = test_df.iloc[i, test_df.shape[1] - 1]
+        except Exception as e:
+            print(f'failed to format example {i}: {e}')
+            continue
+
+        pred = get_response(prompt)
+        if pred and pred[0] in choices:
+            cors.append(pred[0] == label)
+        all_preds.append(pred.replace("\n", "") if pred is not None else "")
+
+    acc = np.mean(cors)
+    print(f"Average accuracy {acc:.3f} - {subject}")
+    print(f"{len(cors)} results, {len(all_preds)-len(cors)} inappropriate formated answers.")
+    return all_preds
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data_dir", "-d", type=str, default="../data_modify")
+    parser.add_argument("--save_dir", "-s", type=str, default="../results_modify/ErnieBot-turbo")
+    parser.add_argument("--num_few_shot", "-n", type=int, default=0)
+    parser.add_argument("--max_length", type=int, default=4096)
+    parser.add_argument("--cot", action='store_true')
+    parser.add_argument("--version", "-v", type=str, default="ERNIE-Bot-turbo")
+    args = parser.parse_args()
+    print('\n\n\n=========================')
+    print(f'args = {args}')
+    print('=========================')
+
+    token = get_access_token()
+    if args.version == "ERNIE-Bot-turbo":
+        print('Using ERNIE-Bot-turbo') 
+        url = f"https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/eb-instant?access_token={token}"
+    else:
+        print('Using ERNIE-Bot')
+        url = f"https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions?access_token={token}"
+
+    run_eval(None, None, eval, args)

FinanceIQ/src/gpt4.py

+import os
+import argparse
+import numpy as np
+import openai
+from tqdm import tqdm
+from time import sleep
+import tiktoken
+from utils import choices, format_example, gen_prompt, run_eval
+
+
+openai.api_key = "YOUR_API_KEY"
+encoding = tiktoken.encoding_for_model("gpt-4-0613")
+
+def get_response(inputs):
+    timeout_counter = 0
+    completion = None
+    retry = 0
+    while completion is None:
+        try:
+            completion = openai.ChatCompletion.create(
+                engine='gpt-4',
+                messages=[{"role": "user", "content": inputs}]
+                )
+            return completion.choices[0].message['content']
+        except Exception as msg:
+            try:
+                sleep_time = int(str(msg).split("Please retry after ")[1].split(" second")[0]) + 0.5
+                print(f'sleep {sleep_time} seconds')
+                sleep(sleep_time)
+            except:
+                print(msg)
+                sleep(5)
+            retry += 1
+            if retry > 3:
+                return 'GPT4 failed'
+
+def eval(subject, dev_df, test_df, num_few_shot, max_length, cot, **kwargs):
+    cors = []
+    all_preds = []
+
+    for i in tqdm(range(test_df.shape[0])):
+        try:
+            prompt_end = format_example(test_df, i, subject, include_answer=False, cot=cot)
+            prompt = gen_prompt(dev_df, subject, prompt_end, num_few_shot, encoding, max_length, cot=cot)
+            label = test_df.iloc[i, test_df.shape[1] - 1]
+        except Exception as e:
+            print(e)
+            print(f'failed to format example {i}')
+            continue
+
+        pred = get_response(prompt)
+        if pred and pred[0] in choices:
+            cors.append(pred[0] == label)
+        all_preds.append(pred.replace("\n", "") if pred is not None else "")
+
+    acc = np.mean(cors)
+    print(f"Average accuracy {acc:.3f} - {subject}")
+    print(f"{len(cors)} results, {len(all_preds)-len(cors)} inappropriate formated answers.")
+    return all_preds
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data_dir", "-d", type=str, default="../data")
+    parser.add_argument("--save_dir", "-s", type=str, default="../results/GPT4")
+    parser.add_argument("--num_few_shot", "-n", type=int, default=0)
+    parser.add_argument("--max_length", type=int, default=4096)
+    parser.add_argument("--cot", action='store_true')
+    args = parser.parse_args()
+    print('\n\n\n=========================')
+    print(f'args = {args}')
+    print('=========================')
+
+    run_eval(None, None, eval, args)
+

FinanceIQ/src/hf_causal_model.py

+import os
+import torch
+import numpy as np
+import argparse
+import time
+from tqdm import tqdm
+from utils import choices, format_example, gen_prompt, run_eval
+
+from peft import PeftModel
+from transformers import LlamaForCausalLM, LlamaTokenizer, AutoModelForCausalLM, AutoTokenizer, AutoModel
+from transformers.generation.utils import GenerationConfig
+
+def eval(model, tokenizer, subject, dev_df, test_df, num_few_shot, max_length, cot):
+    cors = []
+    all_preds = []
+
+    for i in tqdm(range(test_df.shape[0])):
+        prompt_end = format_example(test_df, i, subject, include_answer=False, cot=cot)
+        prompt = gen_prompt(dev_df=dev_df,
+                            subject=subject,
+                            prompt_end=prompt_end,
+                            num_few_shot=num_few_shot,
+                            tokenizer=tokenizer,
+                            max_length=max_length)
+        label = test_df.iloc[i, test_df.shape[1] - 1]
+
+        with torch.no_grad():
+            if is_chat_history:
+                pred, history = model.chat(tokenizer, prompt, history=[]) # for ChatGLM and InternLM-Chat
+            else:
+                if is_chat_model:
+                    messages = [{"role": "user", "content": prompt}]
+                    pred = model.chat(tokenizer, messages) # for model with model.chat() function, e.g. Baichuan-13B-Chat
+                else:
+                    inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+                    outputs = model.generate(**inputs, max_new_tokens=512, repetition_penalty=1.1)
+                    pred = tokenizer.decode(outputs.cpu()[0][len(inputs.input_ids[0]):], skip_special_tokens=True)
+
+        all_preds.append(pred.replace("\n", ""))
+        if pred and pred[0] in choices:
+            cors.append(pred[0] == label)
+
+    acc = np.mean(cors)
+    print(f"Average accuracy {acc:.4f} - {subject}")
+    print(f"{len(cors)} results, {len(all_preds)-len(cors)} inappropriate formated answers.")
+    return all_preds
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_name_or_path", type=str, default="")
+    parser.add_argument("--lora_weights", type=str, default="")
+    parser.add_argument("--data_dir", type=str, default="../data")
+    parser.add_argument("--save_dir", type=str, default="../results/not_specified")
+    parser.add_argument("--num_few_shot", type=int, default=0)
+    parser.add_argument("--max_length", type=int, default=2048)
+    parser.add_argument("--load_in_8bit", action='store_true')
+    parser.add_argument("--with_conf", action='store_true')
+    parser.add_argument("--cot", action='store_true')
+    args = parser.parse_args()
+    print('\n\n\n=========================')
+    print(f'args = {args}')
+    print('=========================')
+    
+    # is LLaMA series model
+    is_llama = 'llama' in args.model_name_or_path.lower() \
+                or 'alpaca' in args.model_name_or_path.lower()
+    if is_llama:
+        is_chat_model = False # The LLaMA family models do not support model.chat() function
+    else:
+        is_chat_model = 'chat' in args.model_name_or_path.lower()
+
+    is_chat_history = 'chatglm' in args.model_name_or_path.lower() \
+        or 'internlm-chat' in args.model_name_or_path.lower() \
+        or 'qwen-7b-chat' in args.model_name_or_path.lower()
+    is_chatglm = 'chatglm' in args.model_name_or_path.lower()
+
+    print(f'model: {args.model_name_or_path}')
+    if is_llama:
+        tokenizer_class = LlamaTokenizer
+        model_class = LlamaForCausalLM
+    elif is_chatglm:
+        tokenizer_class = AutoTokenizer
+        model_class = AutoModel
+    else:
+        tokenizer_class = AutoTokenizer
+        model_class = AutoModelForCausalLM
+    tic = time.time()
+    tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path, trust_remote_code=True)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        trust_remote_code=True,
+                                        load_in_8bit=args.load_in_8bit,
+                                        device_map="auto"
+                                        )
+    if is_chatglm:
+        model = model.half().cuda()  # For ChatGLM
+    if is_chat_model and not is_chatglm:
+        model.generation_config = GenerationConfig.from_pretrained(args.model_name_or_path)
+    print(model.generation_config)
+    print(f'loaded model: {args.model_name_or_path}  costtime = {time.time()-tic:2f}s')
+    
+    if args.lora_weights != "":
+        model = PeftModel.from_pretrained(
+                        model,
+                        args.lora_weights,
+                        torch_dtype=torch.float16,
+                        )
+        
+    run_eval(model, tokenizer, eval, args)

FinanceIQ/src/utils.py

+import os
+import re
+import glob
+import random
+import os.path as osp
+import numpy as np
+import pandas as pd
+import json
+from collections import defaultdict
+choices = ["A", "B", "C", "D"]
+
+from categories import subjects
+
+
+def format_example(df, idx, subject, include_answer=True, cot=False):
+    question = df.iloc[idx, 0]
+    assert isinstance(question, str) and len(question) > 0, 'question is empty'
+    prompt = "题目：" + question
+    k = df.shape[1] - 2
+    for j in range(k):
+        option_choice = choices[j] # A, B, C, D
+        option_content = df.iloc[idx, j + 1]
+        prompt += f"\n{option_choice}. {option_content}"
+
+    # Chain-of-thought
+    if cot:
+        prompt += "\n逐步分析并给出答案选项。"
+    else:
+        prompt += "\n答案是："
+
+    if include_answer:
+        answer = df.iloc[idx, k + 1]
+        prompt += f"{answer}\n\n"
+    return prompt
+
+def gen_prompt(dev_df, subject, prompt_end, num_few_shot=0, tokenizer=None, max_length=2048, cot=False):
+    if cot: # Chain-of-thought
+        prompt = f"以下是关于{subject}的单项选择题，请分析并选出正确答案。\n\n"
+    else:
+        prompt = f"以下是关于{subject}的单项选择题，请直接给出正确答案的选项。\n\n"
+
+    # If no tokenizer, don't consider max length.
+    if tokenizer==None:
+        for i in range(num_few_shot):
+            example = format_example(dev_df, i, subject)
+            prompt += example
+        return prompt + prompt_end
+
+    start_end_token_len = len(tokenizer.encode(prompt)+tokenizer.encode(prompt_end))
+    # If cannot fit in model even without training data, remove the prompt at the beginning.
+    if start_end_token_len>max_length:
+        return prompt_end
+
+    prompt_list = []
+    if num_few_shot > 0:
+        for i in range(num_few_shot):
+            example = format_example(dev_df, i, subject)
+            prompt_list.append((example, tokenizer.encode(example)))
+
+        while prompt_list != [] and sum(len(e[1]) for e in prompt_list) >= max_length - start_end_token_len:
+            print(f"Warning: {len(prompt_list)} shot case exceeds max_input_length, remove 1 shot.")
+            longest_length = max([len(e[1]) for e in prompt_list])
+            prompt_list = [e for e in prompt_list if len(e[1]) != longest_length]
+        for p in prompt_list:
+            prompt += p[0]
+
+    return prompt + prompt_end
+
+
+def run_eval(model, tokenizer, eval, args):
+
+    if model:
+        model.eval()
+
+    args.save_dir = f"{args.save_dir}_{args.num_few_shot}_shot"
+    if not os.path.exists(args.save_dir):
+        os.mkdir(args.save_dir)
+
+    for i, subject in enumerate(subjects):
+        print(f'Evaluating {i+1}/{len(subjects)} {subject}')
+        out_file = os.path.join(args.save_dir, f"results_{subject}.csv")
+        if os.path.exists(out_file):  # If result file exist, skip this subject
+            print(f'  {subject} exists, skip')
+            continue
+        dev_df = pd.read_csv(os.path.join(args.data_dir, "dev", subject + ".csv"), header=0, index_col=0)
+        test_df = pd.read_csv(os.path.join(args.data_dir, "test", subject + ".csv"), header=0, index_col=0)
+
+        # Call the model to get the answer
+        preds = eval(model=model,
+                    tokenizer=tokenizer,
+                    subject=subject,
+                    dev_df=dev_df,
+                    test_df=test_df,
+                    num_few_shot=args.num_few_shot,
+                    max_length=args.max_length,
+                    cot=args.cot if 'cot' in args else False)
+        test_df['prediction'] = preds
+
+        # Save the generated result as a CSV file
+        test_df.to_csv(out_file, header=None)
+        print(f'result save to {out_file}')
+
+    # Calculate the accuracy from the generated results
+    compute_accuracy(args.save_dir)
+
+
+def extract_choice(response):
+    '''
+        Always return a choice, even cannot match by regex,
+        to ensure fair comparison to other models.
+    '''
+    response = str(response)
+    if response[0] in choices:
+        return response[0]
+    # 1. Single match
+    patterns = [
+        (r'答案(选项)?(是|为)：? ?([ABCD])', 3),
+        (r'答案(是|为)选项 ?([ABCD])', 2),
+        (r'故?选择?：? ?([ABCD])',1),
+        (r'([ABCD]) ?选?项(是|为)?正确',1),
+        (r'正确的?选项(是|为) ?([ABCD])',2),
+        (r'答案(应该)?(是|为)([ABCD])',3),
+        (r'选项 ?([ABCD]) ?(是|为)?正确',1),
+        (r'选择答案 ?([ABCD])',1),
+        (r'答案?：?([ABCD])',1),
+        (r'([ABCD])(选?项)?是?符合题意',1),
+        (r'答案选项：? ?([ABCD])', 1), # chatglm
+        (r'答案(选项)?为(.*?)([ABCD])', 3), # chatgpt
+
+    ]
+    for pattern,idx in patterns:
+        m = re.search(pattern, response, re.M)
+        if m:
+            answer = m.group(idx)
+            assert answer in choices
+            return answer
+
+    # 2. Recursive match
+    patterns = [
+        (r'([ABCD])(.*?)当选', 1),
+        (r'([ABCD])(.*?)正确', 1),
+    ]
+    for pattern,idx in patterns:
+        m = re.search(pattern, response, re.M)
+        if m:
+            while m:
+                answer = m.group(idx)
+                m = re.search(pattern, m.group(0)[1:], re.M)
+            assert answer in choices
+            return answer
+
+    # 3. Weak single match
+    patterns = [
+        (r'[^不]是：? ?([ABCD])', 1),
+    ]
+    for pattern,idx in patterns:
+        m = re.search(pattern, response, re.M)
+        if m:
+            answer = m.group(idx)
+            assert answer in choices
+            return answer
+
+    # 4. Check the only mentioend choices
+    pattern = r'^[^ABCD]*([ABCD])[^ABCD]*$'
+    m = re.match(pattern, response)
+    if m:
+        answer = m.group(1)
+        assert answer in choices
+        return answer
+
+    return choices[random.randint(0,3)]
+
+
+def compute_accuracy(result_dir=''):
+
+    all_acc = defaultdict(float)
+    result = {}
+    result['model'] = result_dir.split('/')[-1]
+    for subject in subjects:
+        try:
+            file = glob.glob(osp.join(result_dir, f"results_{subject}.csv"))[0]
+        except:
+            print(f"Warning, {subject} result file not found")
+            continue
+        df = pd.read_csv(file, names=['id','question','A','B','C','D','answer','response'], index_col=0)
+        if df.iloc[0]['question'] == '1':
+            df = df.drop(0)
+        df['pred'] = df['response'].apply(extract_choice)
+        df['acc'] = df['answer'] == df['pred']
+        acc = np.mean(df['acc']) * 100
+        all_acc[subject] = acc
+        result[subject] = round(acc,2)
+    
+    for subject in subjects:
+        print(f"{subject:40s} {all_acc[subject]:.2f}")
+    avg_all_acc = np.mean(list(all_acc.values()))
+    print(f"{'Overall':30s} {avg_all_acc:.2f}")
+
+    # Save result as result.json
+    result['Overall'] = round(avg_all_acc, 2)
+    filename = osp.join(result_dir, 'result.json')
+    with open(filename, 'w') as f:
+        json.dump(result, f, ensure_ascii=False, indent=4)
+        print(f'result save to {filename}')
+
+if __name__ == "__main__":
+    pass

LICENSE

+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2024 GLM-4-9B Model Team @ Zhipu AI
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.