Skip to content

GitLab

".github/vscode:/vscode.git/clone" did not exist on "a0935dec60eb635e7f8b3bf796aab9047cd6ee88"
Commit 7e2f06a3 authored by Rayyyyy's avatar Rayyyyy
Browse files

Update inference of llm

parent e958fb21
Show whitespace changes
Inline Side-by-side
Showing with 147 additions and 114 deletions
llm_service/inferencer.py
View file @ 7e2f06a3
......@@ -2,13 +2,13 @@ import time
import os
import configparser
import argparse
from multiprocessing import Value
from aiohttp import web
import torch
from loguru import logger
from aiohttp import web
from multiprocessing import Value
from fastllm_pytools import llm
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
from transformers import AutoModelForCausalLM, AutoTokenizer
COMMON = {
......@@ -29,8 +29,30 @@ COMMON = {
"<福昕阅读器补丁链接>": "补丁链接: https://pan.baidu.com/s/1QJQ1kHRplhhFly-vxJquFQ,提取码: aupx1",
"<W330-H35A_22DB4/W3335HA安装win7网盘链接>": "硬盘链接: https://pan.baidu.com/s/1fDdGPH15mXiw0J-fMmLt6Q提取码: k97i",
"<X680 G55服务器售后培训资料网盘链接>": "云盘连接下载:链接:https://pan.baidu.com/s/1gaok13DvNddtkmk6Q-qLYg?pwd=xyhb提取码:xyhb",
"<展厅管理员>": "北京-穆淑娟18001053012\n天津-马书跃15720934870\n昆山-关天琪15304169908\n成都-贾小芳18613216313\n重庆-李子艺17347743273\n安阳-郭永军15824623085\n桐乡-李梦瑶18086537055\n青岛-陶祉伊15318733259",
"<线上预约展厅>": "北京、天津、昆山、成都、重庆、安阳、桐乡、青岛",
"<马华>": "联系人:马华，电话:13761751980，邮箱:china@pinbang.com",
"<梁静>": "联系人:梁静，电话:18917566297，邮箱:ing.liang@omaten.com",
"<徐斌>": "联系人:徐斌，电话:13671166044，邮箱:244898943@qq.com",
"<俞晓枫>": "联系人:俞晓枫，电话13750869272，邮箱:857233013@qq.com",
"<刘广鹏>": "联系人:刘广鹏，电话13321992411，邮箱:liuguangpeng@pinbang.com",
"<马英伟>": "联系人:马英伟，电话:13260021849，邮箱:13260021849@163.com",
"<杨洋>": "联系人：杨洋，电话15801203938，邮箱bing523888@163.com",
"<展会合规要求>": "1.展品内容:展品内容需符合公司合规要求，展示内容需经过法务合规审查。\n2.文字材料内容:文字材料内容需符合公司合规要求，展示内容需经过法务合规审查。\n3.展品标签:展品标签内容需符合公司合规要求。\n4.礼品内容:礼品内容需符合公司合规要求。\n5.视频内容:视频内容需符合公司合规要求，展示内容需经过法务合规审查。\n6.讲解词内容:讲解词内容需符合公司合规要求，展示内容需经过法务合规审查。\n7.现场发放材料:现场发放的材料内容需符合公司合规要求。\n8.展示内容:整体展示内容需要经过法务合规审查。",
"<展会质量>": "1.了解展会的组织者背景、往届展会的评价以及提供的服务支持，确保展会的专业性和高效性。\n.了解展会的规模、参观人数、行业影响力等因素，以判断展会是否能够提供足够的曝光度和商机。\n3.关注同行业其他竞争对手是否参展，以及他们的展位布置、展示内容等信息，以便制定自己的参展策略。\n4.展会的日期是否与公司的其他重要活动冲突，以及举办地点是否便于客户和合作伙伴的参观。\n5.销售部门会询问展会方提供的宣传渠道和推广服务，以及如何利用这些资源来提升公司及产品的知名度。\n6.记录展会期间的重要领导参观、商机线索、合作洽谈、公司拜访预约等信息，跟进后续商业机会。",
"<摊位费规则>": "根据展位面积大小，支付相应费用。\n展位照明费:支付展位内的照明服务费。\n展位保安费:支付展位内的保安服务费。\n展位网络使用费:支付展位内网络使用的费用。\n展位电源使用费:支付展位内电源使用的费用。",
"<展会主题要求>": "展会主题的确定需要符合公司产品和服务业务范围，以确保能够吸引目标客户群体。因此，确定展会主题时，需要考虑以下因素:\n专业性:展会的主题应确保专业性，符合行业特点和目标客户的需求。\n目标客户群体:展会的主题定位应考虑目标客户群体，确保能够吸引他们的兴趣。\n业务重点:展会的主题应突出公司的业务重点和优势，以便更好地推广公司的核心产品或服务。\n行业影响力:展会的主题定位需要考虑行业的最新发展趋势，以凸显公司的行业地位和影响力。\n往届展会经验:可以参考往届展会的主题定位，总结经验教训，以确定本届展会的主题。\n市场部意见:在确定展会主题时，应听取市场部的意见，确保主题符合公司的整体市场战略。\n领导意见:还需要考虑公司领导的意见，以确保展会主题符合公司的战略发展方向。",
"<办理展商证注意事项>": "人员范围:除公司领导和同事需要办理展商证外，展会运营工作人员也需要办理。\n提前准备:展商证的办理需要提前进行，以确保摄影师、摄像师等工作人员可以提前入场进行布置。\n办理流程:需要熟悉展商证的办理流程，准备好相关材料，如身份证件等。\n数量需求:需要评估所需的展商证数量，避免数量不足或过多的情况。\n有效期限:展商证的有效期限需要注意，避免在展期内过期。\n存放安全:办理完的展商证需要妥善保管，避免丢失或被他人使用。\n使用规范:使用展商证时需要遵守展会相关规定，不得转让给他人使用。\n回收处理:展会结束后，需要及时回收展商证，避免泄露相关信息。",
"<项目单价要求>": "请注意:无论是否年框供应商，项目单价都不得超过采购部制定的“2024常见活动项目标准单价”，此报价仅可内部使用，严禁外传",
"<年框供应商细节表格>": "在线表格https://kdocs.cn/l/camwZE63frNw",
"<年框供应商流程>": "1.需求方发出项目需求（大型项目需比稿）\n2.外协根据项目需求报价，提供需求方“预算单”（按照基准单价报价，如有发现不按单价情况，解除合同不再使用）\n3.需求方确认预算价格，并提交OA市场活动申请\n4.外协现场执行\n5.需求方现场验收，并签署验收单（物料、设备、人员等实际清单）\n6.外协出具结算单（金额与验收单一致，加盖公章）、结案报告、年框合同，作为报销凭证\n7.外协请需求方项目负责人填写“满意度调研表”（如无，会影响年度评价）\n8.需求方项目经理提交报销",
"<市场活动结案报告内容>": "1.项目简介（时间、地点、参与人数等）；2.最终会议安排；3.活动各环节现场图片；4.费用相关证明材料（如执行人员、物料照片）；5.活动成效汇总；6.活动原始照片/视频网络链接",
"<展板设计选择>": "1.去OA文档中心查找一些设计模板; 2. 联系专业的活动服务公司来协助设计",
"<餐费标准>": "一般地区的餐饮费用规定为不超过300元/人（一顿正餐），特殊地区则为不超过400元/人（一顿正餐），特殊地区的具体规定请参照公司的《差旅费管理制度》",
"":"",
}
def build_history_messages(prompt, history, system: str = None):
history_messages = []
if system is not None and len(system) > 0:
......@@ -42,39 +64,46 @@ def build_history_messages(prompt, history, system: str = None):
return history_messages
class InferenceWrapper:
class LLMInference:
def __init__(self, model_path: str, use_vllm: bool, stream_chat: bool, tensor_parallel_size: int):
def __init__(self,
model,
tokenzier,
device: str = 'cuda',
use_vllm: bool = False,
stream_chat: bool = False
) -> None:
self.device = device
self.model = model
self.tokenzier = tokenzier
self.use_vllm = use_vllm
self.stream_chat = stream_chat
# huggingface
self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True).half().cuda()
self.model = model.eval()
if self.use_vllm:
def generate_response(self, prompt, history=[]):
print("generate")
output_text = ''
error = ''
time_tokenizer = time.time()
try:
## vllm
# from vllm import LLM, SamplingParams
# self.sampling_params = SamplingParams(temperature=1, top_p=0.95)
# self.llm = LLM(model=model_path,
# trust_remote_code=True,
# enforce_eager=True,
# tensor_parallel_size=tensor_parallel_size)
## fastllm
if self.stream_chat:
# fastllm的流式初始化
self.model = llm.model(model_path)
else:
self.model = llm.from_hf(self.model, self.tokenizer, dtype="float16")
output_text = self.chat(prompt, history)
except Exception as e:
logger.error(f"fastllm initial failed, {e}")
error = str(e)
logger.error(error)
time_finish = time.time()
logger.debug('output_text：{} \ntimecost {} '.format(output_text,
time_finish - time_tokenizer))
return output_text, error
def substitution(self, output_text):
# 翻译特殊字符
import re
matchObj = re.split('.*(<.*>).*', output_text, re.M|re.I)
if matchObj:
if len(matchObj) > 1:
obj = matchObj[1]
replace_str = COMMON.get(obj)
if replace_str:
......@@ -84,44 +113,50 @@ class InferenceWrapper:
def chat(self, prompt: str, history=[]):
'''单轮问答'''
print("in chat")
output_text = ''
logger.info("****************** in chat ******************")
messages = [{"role": "user", "content": prompt}]
try:
if self.use_vllm:
## vllm
# output_text = []
# outputs = self.llm.generate(prompt, self.sampling_params)
# for output in outputs:
# prompt = output.prompt
# generated_text = output.outputs[0].text
# print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
# output_text.append(generated_text)
## fastllm
output_text = self.model.response(prompt)
prompt_token_ids = [self.tokenizer.apply_chat_template(messages, add_generation_prompt=True)]
outputs = self.model.generate(prompt_token_ids=prompt_token_ids, sampling_params=self.tokenzier)
output_text = []
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
generated_text_ = self.substitution(generated_text)
output_text.append(generated_text_)
logger.info(f"using vllm, output_text {output_text}")
return ''.join(output_text)
else:
output_text, _ = self.model.chat(self.tokenizer,
prompt,
history,
do_sample=False)
output_text = self.substitution(output_text)
print("output_text", output_text)
# transformers
output_text = ''
input_ids = self.tokenizer.apply_chat_template(
messages, add_generation_prompt=True, return_tensors="pt").to('cuda')
outputs = self.model.generate(
input_ids,
max_new_tokens=1024,
)
response = outputs[0][input_ids.shape[-1]:]
generated_text = self.tokenizer.decode(response, skip_special_tokens=True)
output_text = self.substitution(generated_text)
logger.info(f"using transformers, output_text {output_text}")
return output_text
except Exception as e:
logger.error(f"chat inference failed, {e}")
return output_text
def chat_stream(self, prompt: str, history=[]):
'''流式服务'''
import re
if self.use_vllm:
from fastllm_pytools import llm
# Fastllm
for response in self.model.stream_response(prompt, history=[]):
response = self.substitution(response)
yield response
else:
# HuggingFace
current_length = 0
for response, _, _ in self.model.stream_chat(self.tokenizer, prompt, history=history,
......@@ -133,40 +168,31 @@ class InferenceWrapper:
current_length = len(response)
class LLMInference:
def __init__(self,
model_path: str,
tensor_parallel_size: int,
device: str = 'cuda',
use_vllm: bool = False,
stream_chat: bool = False
) -> None:
self.device = device
self.inference = InferenceWrapper(model_path=model_path,
use_vllm=use_vllm,
stream_chat=stream_chat,
tensor_parallel_size=tensor_parallel_size)
def generate_response(self, prompt, history=[]):
print("generate")
output_text = ''
error = ''
time_tokenizer = time.time()
def init_model(model_path, use_vllm=False, tp_size=1):
## init models
# huggingface
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True,device_map="auto").half().cuda().eval()
if use_vllm:
try:
output_text = self.inference.chat(prompt, history)
# vllm
from vllm import LLM, SamplingParams
tokenizer = SamplingParams(temperature=1,
top_p=0.95,
max_tokens=1024,
stop_token_ids=[tokenizer.eos_token_id])
model = LLM(model=model_path,
trust_remote_code=True,
enforce_eager=True,
dtype="float16",
tensor_parallel_size=tp_size)
except Exception as e:
error = str(e)
logger.error(error)
time_finish = time.time()
logger.debug('output_text：{} \ntimecost {} '.format(output_text,
time_finish - time_tokenizer))
logger.error(f"fastllm initial failed, {e}")
return output_text, error
return model, tokenizer
def llm_inference(args):
......@@ -176,22 +202,26 @@ def llm_inference(args):
bind_port = int(config['default']['bind_port'])
model_path = config['llm']['local_llm_path']
tensor_parallel_size = config.getint('llm', 'tensor_parallel_size')
use_vllm = config.getboolean('llm', 'use_vllm')
print("inference")
inference_wrapper = InferenceWrapper(model_path,
model, tokenzier = init_model(model_path, use_vllm, tensor_parallel_size)
inference = LLMInference(model,
tokenzier,
use_vllm=use_vllm,
tensor_parallel_size=1,
tensor_parallel_size=tensor_parallel_size,
stream_chat=args.stream_chat)
async def inference(request):
start = time.time()
input_json = await request.json()
prompt = input_json['prompt']
prompt = input_json['query']
history = input_json['history']
if args.stream_chat:
text = inference_wrapper.stream_chat(prompt=prompt, history=history)
text = inference.stream_chat(prompt=prompt, history=history)
else:
text = inference_wrapper.chat(prompt=prompt, history=history)
text = inference.chat(prompt=prompt, history=history)
end = time.time()
logger.debug('问题:{} 回答:{} \ntimecost {} '.format(prompt, text, end - start))
return web.json_response({'text': text})
......@@ -208,15 +238,19 @@ def infer_test(args):
model_path = config['llm']['local_llm_path']
use_vllm = config.getboolean('llm', 'use_vllm')
tensor_parallel_size = config.getint('llm', 'tensor_parallel_size')
inference_wrapper = InferenceWrapper(model_path,
stream_chat = config.getboolean('llm', 'stream_chat')
model, tokenzier = init_model(model_path, use_vllm, tensor_parallel_size)
inference = LLMInference(model,
tokenzier,
use_vllm=use_vllm,
tensor_parallel_size=1,
stream_chat=args.stream_chat)
tensor_parallel_size=tensor_parallel_size,
stream_chat=stream_chat)
# prompt = "hello，please introduce yourself..."
prompt ='65N32-US主板清除CMOS配置的方法'
history = []
time_first = time.time()
output_text = inference_wrapper.chat(prompt)
output_text = inference.chat(prompt)
time_second = time.time()
logger.debug('问题:{} 回答:{} \ntimecost {} '.format(
prompt, output_text, time_second - time_first))
......@@ -246,7 +280,7 @@ def parse_args():
parser.add_argument(
'--DCU_ID',
type=str,
default='1',
default='0,1',
help='设置DCU卡号,卡号之间用英文逗号隔开,输入样例:"0,1,2"')
parser.add_argument(
'--stream_chat',
......@@ -259,8 +293,8 @@ def parse_args():
def main():
args = parse_args()
set_envs(args.DCU_ID)
llm_inference(args)
# infer_test(args)
# llm_inference(args)
infer_test(args)
if __name__ == '__main__':
......
llm_service/retriever.py
View file @ 7e2f06a3
......@@ -38,7 +38,7 @@ class Retriever:
self.retriever = self.vector_store.as_retriever(
search_type='similarity',
search_kwargs={
'score_threshold': 0.4,
'score_threshold': self.reject_throttle,
'k': 30
}
)
......
server.py
View file @ 7e2f06a3
......@@ -34,5 +34,4 @@ def parse_args():
if __name__ == '__main__':
args = parse_args()
reply, ref = start(args.query)
logger.debug('reply: {} \nref: {} '.format(reply,
ref))
logger.debug('reply: {} \nref: {} '.format(reply, ref))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment