Skip to content

Navigation Menu

Sign in

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 63ac012

Browse filesBrowse files
lightislostlightislost
committed
提交toollearning 评测代码与示例
1 parent 0baa886 commit 63ac012
Copy full SHA for 63ac012

File tree

4 files changed

+369
-3
lines changed
Filter options

4 files changed

+369
-3
lines changed

‎README_zh.md

Copy file name to clipboardExpand all lines: README_zh.md
+3-3Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -253,9 +253,9 @@ answer: D
253253
explanation: 根据分析，题目中的时间序列在12点出的值265要明显大于周围数据，存在着突增现象，因此选择D是正确的。
254254
```
255255
#### 🔧 ToolLearning样本示例
256-
工具学习样本的数据格式与OpenAI的函数调用格式兼容。<br>
257-
详情请参阅[tool_learning_info_zh.md](resources/tool_learning_info_zh.md)。<br>
258-
工具学习评测过程，详情请参阅见 [tool_learning_evalution.md](resources/tool_learning_evalution.md)。<br>
256+
工具学习样本的数据格式与OpenAI的函数调用格式兼容。
257+
详情请参阅[tool_learning_info_zh.md](resources/tool_learning_info_zh.md)。
258+
工具学习评测过程，详情请参阅见 [tool_learning_evalution.md](resources/tool_learning_evalution.md)。
259259
<br>
260260
261261
## 🚀 如何进行测试

‎resources/tool_learning_evalution.md

Copy file name to clipboard
+224Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
## tool learning 数据集评测教程
2+
3+
### chatml接入方式
4+
如果需要在自己的 huggingface 格式的模型上进行测试的话，总的步骤分为如下几步:
5+
1. 编写 ~/evals/FuncCallEvalution 的 create_prompts 函数
6+
2. 编写 ~/models/base_model 的 相关函数
7+
3. 注册模型和评估函数
8+
4. 执行测试脚本
9+
如果模型在加载进来后不需要特殊的处理，而且输入也不需要转换为特定的格式（e.g. chatml 格式或者其他的 human-bot 格式），请直接跳转到第四步直接发起测试。
10+
11+
#### 1. 编写 loader 函数
12+
如果模型在加载进来还需要做一些额外的处理（e.g. tokenizer 调整），需要去 `src.context_builder.context_builder_family.py` 中继承 `ModelAndTokenizerLoader` 类来覆写对应的 `load_model` 和 `load_tokenizer` 函数，具体可以参照以下示例：
13+
```python
14+
class FuncCallEvalution(ToolEvalution):
15+
16+
def create_prompts(self, func_call_datas):
17+
'''
18+
datas: [
19+
{
20+
"instruction": history[his_idx],
21+
"input": "",
22+
"output": output,
23+
"history": [(human_content, ai_content), (), ()],
24+
"functions": tools
25+
}
26+
]
27+
'''
28+
system_content = '''CodeFuse是一个面向研发领域的智能助手，旨在中立的、无害的帮助用户解决开发相关的问题，所有的回答均使用Markdown格式返回。
29+
你能利用许多工具和功能来完成给定的任务，在每一步中，你需要分析当前状态，并通过执行函数调用来确定下一步的行动方向。你可以进行多次尝试。如果你计划连续尝试不同的条件，请每次尝试一种条件。若给定了Finish函数,则以Finish调用结束，若没提供Finish函数，则以不带function_call的对话结束。'''
30+
function_format = '''You are ToolGPT, you have access to the following APIs:\n{tools}'''
31+
32+
func_call_train_datas = []
33+
history_error_cnt = 0
34+
funccall_error_cnt = 0
35+
36+
for data in func_call_datas:
37+
tools = data["functions"]
38+
chatrounds = data["chatrounds"]
39+
40+
function_content = ""
41+
if len(tools) > 0:
42+
function_content = function_format.format(tools=json.dumps(tools, ensure_ascii=False, sort_keys=True))
43+
44+
history = []
45+
for i in chatrounds:
46+
if i["role"]=="system":
47+
continue
48+
49+
if i["role"]=="user":
50+
history.append(("user", i["content"]))
51+
52+
if i["role"] == "assistant":
53+
if "function_call" in i:
54+
if not isinstance(i["function_call"], dict):
55+
funccall_error_cnt+=1
56+
continue
57+
content = "#function" + json.dumps({**{"content": i["content"]}, **i["function_call"]}, ensure_ascii=False)
58+
else:
59+
content = i["content"]
60+
history.append(("assistant", content))
61+
62+
63+
if i["role"] == "function":
64+
content = json.dumps({**{"content": i["content"]}, **{"name": i["name"]}}, ensure_ascii=False)
65+
history.append(("user", content))
66+
67+
68+
history = [i[1] for i in history]
69+
history[0] = "\n".join([system_content,function_content, history[0]])
70+
71+
for his_idx in range(0, len(history), 2):
72+
output = history[his_idx+1]
73+
74+
if "#function" in output:
75+
output = output.split("#function")[-1]
76+
77+
try:
78+
output = json.loads(output)
79+
except:
80+
output = {"content": output}
81+
82+
83+
func_call_train_datas.append(
84+
{
85+
"instruction": history[his_idx],
86+
"input": "",
87+
"output": output,
88+
"history": [history[:his_idx+2][i:i+2] for i in range(0, len(history[:his_idx]), 2)],
89+
"functions": tools
90+
},
91+
)
92+
return func_call_train_datas
93+
```
94+
95+
#### 2. 编写 Model 的 context_builder 函数
96+
如果输入需要转换为特定的格式（e.g. chatml 格式或者其他的 human-bot 格式），则需要去 `src.context_builder.context_builder_family` 中继承 ContextBuilder 类来覆写 make_context 函数，这个函数是用来将输入转换格式为对应需要的输出的，一个示例如下：
97+
```python
98+
class ToolModel:
99+
def __init__(self, model_path: str, template: str, trust_remote_code=True, tensor_parallel_size=1, gpu_memory_utilization=0.25):
100+
self.model_path = model_path
101+
self.trust_remote_code = trust_remote_code
102+
self.tensor_parallel_size = tensor_parallel_size
103+
self.gpu_memory_utilization = gpu_memory_utilization
104+
self.load_model(self.model_path, self.trust_remote_code, self.tensor_parallel_size, self.gpu_memory_utilization)
105+
106+
def generate(self, prompts: str, template: str = None, generate_configs: GenerateConfigs = None) -> list:
107+
'''产出对应结果'''
108+
pass
109+
110+
def generate_params(
111+
self, generate_configs: GenerateConfigs,
112+
):
113+
'''generate param'''
114+
kargs = generate_configs.dict()
115+
return kargs
116+
117+
def load_model(self, model_path, trust_remote_code=True, tensor_parallel_size=1, gpu_memory_utilization=0.25):
118+
'''加载模型'''
119+
self.tokenizer = AutoTokenizer.from_pretrained(self.model_path, trust_remote_code=trust_remote_code)
120+
self.model = AutoModelForCausalLM.from_pretrained(self.model_path, device_map="auto", trust_remote_code=trust_remote_code).eval()
121+
122+
# self.model = LLM(model=model_path, trust_remote_code=trust_remote_code, tensor_parallel_size=tensor_parallel_size, gpu_memory_utilization=gpu_memory_utilization)
123+
```
124+
125+
#### 3. 注册模型和eval函数即可
126+
在 ~/models/__init__.py 中注册即可
127+
```python
128+
from .base_model import ToolModel
129+
130+
__all__ = [
131+
"ToolModel",
132+
]
133+
```
134+
在 ~/evasl/__init__.py 中注册即可
135+
```python
136+
from .base_evalution import ToolEvalution
137+
from .toolfill_evalution import ToolFillEvalution
138+
from .toolparser_evalution import ToolParserEvalution
139+
from .toolsummary_evalution import ToolSummaryEvalution
140+
from .func_call_evalution import FuncCallEvalution
141+
142+
143+
__all__ = [
144+
"ToolEvalution", "ToolFillEvalution", "ToolParserEvalution", "ToolSummaryEvalution", "FuncCallEvalution"
145+
]
146+
```
147+
148+
149+
#### 4. 执行测试脚本
150+
修改 ~/src/qwen_eval_main.py# datainfos和model_infos
151+
```python
152+
model_infos = [
153+
{"model_name": "", "template": "chatml", "model_path": "",
154+
"peft_path": "", "model_class": QwenModel}]
155+
156+
datainfos = [
157+
{"dataset_path": "~/fcdata_luban_zh_test.jsonl", "dataset_name": "fcdata_luban_zh", "tool_task": "func_call"},
158+
{"dataset_path": "~/test_datas/fcdata_zh_test_v1.jsonl", "dataset_name": "fcdata_zh", "tool_task": "func_call"},
159+
]
160+
```
161+
162+
运行下述命令即可
163+
```Bash
164+
python qwen_eval_main.py
165+
```
166+
167+
<br>
168+
169+
### 非chatml接入
170+
如果需要在自己的 huggingface 格式的模型上进行测试的话，总的步骤分为如下几步:
171+
1. 编写 ~/getAssistantAns.py 相关代码
172+
2. 执行测试脚本
173+
174+
175+
#### 1、编写 getAssistantAns 示例
176+
```
177+
class GetAssistantAns():
178+
# 按照自己推理需求自己修改代码
179+
180+
def __init__(self, gpu_num=1):
181+
model = AutoModelForCausalLM.from_pretrained(model_name)
182+
device_list = []
183+
for gpu_idx in range(gpu_num):
184+
device_list.append(torch.device("cuda:0"))
185+
186+
# 将模型移动到指定的GPU设备
187+
model.to(device)
188+
189+
190+
def gen_answer(self, chat_dict, gpu_index):
191+
# 这里实际根据自己推理逻辑 然后转为标准格式返回
192+
# 以下仅仅是样例
193+
import time
194+
print(os.environ["CUDA_VISIBLE_DEVICES"])
195+
time.sleep(1)
196+
rtn_dict1 = {
197+
"role": "assistant",
198+
"content": None,
199+
"function_call":
200+
{
201+
"name": "get_fudan_university_scoreline",
202+
"arguments": "{\n \"year\": \"2020\"\n}"
203+
}
204+
}
205+
206+
rtn_dict2 = {
207+
"role": "assistant",
208+
"content": "2020年复旦大学的分数线如下：\n\n- 文科一批：630分\n- 文科二批：610分\n- 理科一批：650分\n- 理科二批：630分"
209+
}
210+
211+
return random.choice([rtn_dict1, rtn_dict2])
212+
```
213+
#### 2、执行测试脚本
214+
修改 ~/src/opensource_functioncall_evalution.py # test_ans_file_list
215+
```python
216+
test_ans_file_list = [
217+
"fcdata_zh_test.jsonl"
218+
]
219+
```
220+
221+
运行下述命令即可
222+
```Bash
223+
python opensource_functioncall_evalution.py
224+
```

‎src/getAssistantAns.py

Copy file name to clipboard
+141Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
import os
2+
import json
3+
import random
4+
import torch
5+
from transformers import AutoModelForCausalLM
6+
7+
class GetAssistantAns():
8+
# 按照自己推理需求自己修改代码
9+
10+
def __init__(self, gpu_num=1):
11+
model = AutoModelForCausalLM.from_pretrained(model_name)
12+
device_list = []
13+
for gpu_idx in range(gpu_num):
14+
device_list.append(torch.device("cuda:0"))
15+
16+
# 将模型移动到指定的GPU设备
17+
model.to(device)
18+
19+
20+
def gen_answer(self, chat_dict, gpu_index):
21+
# 这里实际根据自己推理逻辑 然后转为标准格式返回
22+
# 以下仅仅是样例
23+
import time
24+
print(os.environ["CUDA_VISIBLE_DEVICES"])
25+
time.sleep(1)
26+
rtn_dict1 = {
27+
"role": "assistant",
28+
"content": None,
29+
"function_call":
30+
{
31+
"name": "get_fudan_university_scoreline",
32+
"arguments": "{\n \"year\": \"2020\"\n}"
33+
}
34+
}
35+
36+
rtn_dict2 = {
37+
"role": "assistant",
38+
"content": "2020年复旦大学的分数线如下：\n\n- 文科一批：630分\n- 文科二批：610分\n- 理科一批：650分\n- 理科二批：630分"
39+
}
40+
41+
return random.choice([rtn_dict1, rtn_dict2])
42+
43+
# ======================================================================
44+
# 下面注释的部分是一个huggingface推理的多卡的demo
45+
# 备注 线程数量==2，也就是2卡效率最高 多个卡并不会提升效率，存在资源抢占的情况
46+
# 可以采用多卡部署服务，然后调用服务的方式提升效率
47+
48+
# import os
49+
# import re
50+
# import json
51+
# import random
52+
# import torch
53+
# import copy
54+
# import transformers
55+
# from transformers import AutoModelForCausalLM, CodeLlamaTokenizer, TextStreamer
56+
# end_token_id = 2
57+
# checkpoint = "<your_huggingface_model_dir>"
58+
# tokenizer = CodeLlamaTokenizer.from_pretrained(checkpoint)
59+
# torch.manual_seed(random.randint(0, 100000))
60+
# temperature = 0.2
61+
# top_p = 0.95
62+
# top_k = 40
63+
# repetition_penalty = 1.1
64+
# output_len = 2048
65+
# role_start = "[START]"
66+
# role_end = "[END]"
67+
68+
69+
# def change2traindata(fc_dict):
70+
# chatrounds_list = []
71+
# # insert your code
72+
# # eg
73+
# # chatrounds_list = [
74+
# # {
75+
# # "role": "system",
76+
# # "content":"你好，我是小助手，我能帮你做什么呢？"
77+
# # },
78+
# # {
79+
# # "role": "functionapis"
80+
# # "content": "You are tool gpt, you can using following apis []"
81+
# # },
82+
# # {
83+
# # "role": "user",
84+
# # "content": "我想知道复旦大学的分数线"
85+
# # },
86+
# # {
87+
# # "role": "bot",
88+
# # "content": "#function{*****}"
89+
# # },
90+
# # {
91+
# # "role": "function",
92+
# # "content": ******
93+
# # },
94+
# # {
95+
# # "role": "bot",
96+
# # "content": "复旦大学分数线640"
97+
# # }
98+
# # ]
99+
# return chatrounds_list
100+
101+
102+
# def get_chatrounds_ids(chatrounds_list):
103+
# input_ids = []
104+
# for chatround in chatrounds_list:
105+
# input_ids += tokenizer.encode(role_start + chatround["role"]+ role_end) + tokenizer.encode(chatround["content"], add_special_tokens=False) + [tokenizer.eos_token_id]
106+
# input_ids += tokenizer.encode(role_start + "bot" + role_end)
107+
# return input_ids
108+
109+
110+
# class GetAssistantAns():
111+
# # 按照自己推理需求自己修改代码
112+
113+
# def __init__(self, gpu_num=1):
114+
# print(checkpoint)
115+
# print("Loading model")
116+
# model = AutoModelForCausalLM.from_pretrained(checkpoint).half().eval()
117+
# device_list = [torch.device(f"cuda:%d"%(i)) for i in range(gpu_num)]
118+
# self.model_list = [copy.deepcopy(model.to(device)) for device in device_list]
119+
# print("Loading finish")
120+
121+
# def gen_answer(self, chat_dict, gpu_index=0):
122+
# chatrounds_list = change2traindata(chat_dict)
123+
# input_ids = get_chatrounds_ids(chatrounds_list)
124+
# output_ids = self.model_list[gpu_index].generate(torch.tensor([input_ids]).to(self.model_list[gpu_index].device), max_new_tokens=output_len, num_beams=1, num_return_sequences=1, do_sample=True, temperature=temperature, top_p=top_p, eos_token_id=end_token_id, top_k=top_k, streamer=None, repetition_penalty=repetition_penalty, pad_token_id=10000)[0]
125+
# res = tokenizer.decode(output_ids[len(input_ids):-1])
126+
# save_dict = {"role": "assistant"}
127+
# if res.startswith("#function"):
128+
# try:
129+
# res_dict = json.loads(re.sub("^#function", "", res))
130+
# save_dict["content"] = res_dict["content"]
131+
# save_dict["function_call"] = {}
132+
# save_dict["function_call"]["name"] = res_dict["name"]
133+
# save_dict["function_call"]["arguments"] = res_dict["arguments"]
134+
# except Exception as e:
135+
# print(e)
136+
# save_dict = {"role": "assistant"}
137+
# save_dict["content"] = res
138+
# else:
139+
# save_dict["content"] = res
140+
# # print(save_dict)
141+
# return save_dict

‎src/qwen_eval_main.py

Copy file name to clipboardExpand all lines: src/qwen_eval_main.py
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from src.models import QwenModel, ToolModel, InternlmModel
66
from src.models.generate_configs import GenerateConfigs
77
from src.prompts.base_prompts_config import TOOL_FILL_BASE_PROMPT, TOOL_PARSER_BASE_PROMPT, TOOL_SUMMARY_BASE_PROMPT
8+
from src.utils.jsonl_utils import save_to_jsonl_file
89

910
import warnings
1011
import re

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.