Skip to content

Navigation Menu

Sign in

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit f7e2dff

Browse filesBrowse files
lightislostlightislost
committed
add funccall evalution features
1 parent 338ed89 commit f7e2dff
Copy full SHA for f7e2dff

28 files changed

+1611
-0
lines changed

‎scripts/run_fc_example.sh

Copy file name to clipboard
+4Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
4+
python src/qwen_eval_main.py

‎src/datasets/__init__.py

Copy file name to clipboard
+9Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
from .base_dataset import ToolDataset
2+
from .toolfill_dataset import ToolFillDataset
3+
from .toolparser_dataset import ToolParserDataset
4+
from .toolsummary_dataset import ToolSummaryDataset
5+
from .funccall_dataset import FuncCallDataset
6+
7+
__all__ = [
8+
"ToolFillDataset", "ToolDataset", "ToolParserDataset", "ToolSummaryDataset", "FuncCallDataset"
9+
]

‎src/datasets/base_dataset.py

Copy file name to clipboard
+40Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
from src.utils.jsonl_utils import read_jsonl_file, save_to_jsonl_file
2+
from src.utils.json_utils import read_json_file, save_to_json_file
3+
4+
5+
6+
class ToolDataset:
7+
def __init__(self, dataset_name, tool_task, filepath):
8+
self.dataset_name = dataset_name
9+
self.tool_task = tool_task
10+
self.filepath = filepath
11+
self.datas = self.load_data()
12+
13+
def load_data(self, ) -> list:
14+
if self.filepath:
15+
return self.load_data_from_local(self.filepath)
16+
elif self.dataset_name and self.tool_task:
17+
return self.load_data_from_hf(self.tool_task)
18+
return []
19+
20+
def load_data_from_local(self, filepath):
21+
''''''
22+
pass
23+
24+
def load_data_from_hf(self, tool_task):
25+
pass
26+
27+
def __iter__(self):
28+
self.current_index = 0
29+
return self
30+
31+
def __next__(self):
32+
if self.current_index < len(self.datas):
33+
current_item = self.datas[self.current_index]
34+
self.current_index += 1
35+
return current_item
36+
else:
37+
raise StopIteration
38+
39+
def __len__(self):
40+
return len(self.datas)

‎src/datasets/funccall_dataset.py

Copy file name to clipboard
+41Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
from src.utils.jsonl_utils import read_jsonl_file, save_to_jsonl_file
2+
from src.utils.json_utils import read_json_file, save_to_json_file
3+
from .base_dataset import ToolDataset
4+
5+
import os
6+
7+
8+
9+
10+
class FuncCallDataset(ToolDataset):
11+
def __init__(self, dataset_name, tool_task, filepath):
12+
self.dataset_name = dataset_name
13+
self.tool_task = tool_task
14+
self.filepath = filepath
15+
self.datas = self.load_data()
16+
17+
def load_data(self, ) -> list:
18+
if self.filepath:
19+
return self.load_data_from_local(self.filepath)
20+
elif self.dataset_name and self.tool_task:
21+
return self.load_data_from_hf(self.tool_task)
22+
return []
23+
24+
def load_data_from_local(self, filepath):
25+
def _load_from_file(filename):
26+
if "jsonl" in filename:
27+
return read_jsonl_file(filename)
28+
elif "json" in filename:
29+
return read_json_file(filename)
30+
31+
datas = []
32+
if os.path.isdir(filepath):
33+
for filename in os.listdir(filepath):
34+
datas.extend(_load_from_file(os.path.join(filepath, filename)))
35+
else:
36+
datas = _load_from_file(filepath)
37+
38+
return datas
39+
40+
def load_data_from_hf(self, tool_task):
41+
pass

‎src/datasets/toolfill_dataset.py

Copy file name to clipboard
+30Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
from src.utils.jsonl_utils import read_jsonl_file, save_to_jsonl_file
2+
from src.utils.json_utils import read_json_file, save_to_json_file
3+
from .base_dataset import ToolDataset
4+
5+
6+
7+
8+
class ToolFillDataset(ToolDataset):
9+
def __init__(self, dataset_name, tool_task, filepath):
10+
self.dataset_name = dataset_name
11+
self.tool_task = tool_task
12+
self.filepath = filepath
13+
self.datas = self.load_data()
14+
15+
def load_data(self, ) -> list:
16+
if self.filepath:
17+
return self.load_data_from_local(self.filepath)
18+
elif self.dataset_name and self.tool_task:
19+
return self.load_data_from_hf(self.tool_task)
20+
return []
21+
22+
def load_data_from_local(self, filepath):
23+
if "jsonl" in filepath:
24+
return read_jsonl_file(filepath)
25+
elif "json" in filepath:
26+
return read_json_file(filepath)
27+
return []
28+
29+
def load_data_from_hf(self, tool_task):
30+
pass

‎src/datasets/toolparser_dataset.py

Copy file name to clipboard
+30Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
from src.utils.jsonl_utils import read_jsonl_file, save_to_jsonl_file
2+
from src.utils.json_utils import read_json_file, save_to_json_file
3+
from .base_dataset import ToolDataset
4+
5+
6+
7+
8+
class ToolParserDataset(ToolDataset):
9+
def __init__(self, dataset_name, tool_task, filepath):
10+
self.dataset_name = dataset_name
11+
self.tool_task = tool_task
12+
self.filepath = filepath
13+
self.datas = self.load_data()
14+
15+
def load_data(self, ) -> list:
16+
if self.filepath:
17+
return self.load_data_from_local(self.filepath)
18+
elif self.dataset_name and self.tool_task:
19+
return self.load_data_from_hf(self.tool_task)
20+
return []
21+
22+
def load_data_from_local(self, filepath):
23+
if "jsonl" in filepath:
24+
return read_jsonl_file(filepath)
25+
elif "json" in filepath:
26+
return read_json_file(filepath)
27+
return []
28+
29+
def load_data_from_hf(self, tool_task):
30+
pass

‎src/datasets/toolsummary_dataset.py

Copy file name to clipboard
+28Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
from src.utils.jsonl_utils import read_jsonl_file, save_to_jsonl_file
2+
from src.utils.json_utils import read_json_file, save_to_json_file
3+
from .base_dataset import ToolDataset
4+
5+
6+
class ToolSummaryDataset(ToolDataset):
7+
def __init__(self, dataset_name, tool_task, filepath):
8+
self.dataset_name = dataset_name
9+
self.tool_task = tool_task
10+
self.filepath = filepath
11+
self.datas = self.load_data()
12+
13+
def load_data(self, ) -> list:
14+
if self.filepath:
15+
return self.load_data_from_local(self.filepath)
16+
elif self.dataset_name and self.tool_task:
17+
return self.load_data_from_hf(self.tool_task)
18+
return []
19+
20+
def load_data_from_local(self, filepath):
21+
if "jsonl" in filepath:
22+
return read_jsonl_file(filepath)
23+
elif "json" in filepath:
24+
return read_json_file(filepath)
25+
return []
26+
27+
def load_data_from_hf(self, tool_task):
28+
pass

‎src/evals/__init__.py

Copy file name to clipboard
+10Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from .base_evalution import ToolEvalution
2+
from .toolfill_evalution import ToolFillEvalution
3+
from .toolparser_evalution import ToolParserEvalution
4+
from .toolsummary_evalution import ToolSummaryEvalution
5+
from .func_call_evalution import FuncCallEvalution
6+
7+
8+
__all__ = [
9+
"ToolEvalution", "ToolFillEvalution", "ToolParserEvalution", "ToolSummaryEvalution", "FuncCallEvalution"
10+
]

‎src/evals/base_evalution.py

Copy file name to clipboard
+43Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
from src.models.base_model import ToolModel
2+
from src.models.generate_configs import GenerateConfigs
3+
from src.datasets import ToolFillDataset
4+
5+
6+
7+
class ToolEvalution:
8+
def __init__(
9+
self,
10+
model: ToolModel,
11+
dataset: ToolFillDataset,
12+
base_prompt: str = '',
13+
generate_configs: GenerateConfigs = None,
14+
):
15+
self.model = model
16+
self.dataset = dataset
17+
self.base_prompt = base_prompt
18+
self.generate_configs = generate_configs
19+
20+
if not isinstance(model, ToolModel):
21+
raise BaseException(f"must be ToolModel Class! not {model}")
22+
23+
def calc(self):
24+
'''开始计算结果'''
25+
self.predicts = []
26+
for idx, data in enumerate(self.dataset):
27+
# if idx >= 5: break
28+
prompt = self.base_prompt.format(**data)
29+
answer = data["api_param"]
30+
predict = self.generate(prompt, self.generate_configs)
31+
self.predicts.append({"prompt": prompt, "predict": predict, "answer": answer})
32+
33+
metric = self.eval_metric(self.predicts)
34+
return metric
35+
36+
def generate(self, prompt, generate_configs):
37+
'''返回结果'''
38+
return self.model.generate(prompt, generate_configs)
39+
40+
def eval_metric(self, datas):
41+
'''calc custom metric'''
42+
pass
43+

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.