62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159 | class FaithfulnessEvaluator(BaseEvaluator):
"""忠实度评估器。
评估响应是否忠实于上下文（即响应是否得到上下文支持或是虚构的）。
此评估器仅考虑响应字符串和上下文字符串列表。
Args:
service_context（可选[ServiceContext]）：
用于评估的服务上下文。
raise_error（布尔型）：当响应无效时是否引发错误。默认为False。
eval_template（可选[Union[str, BasePromptTemplate]]）：
用于评估的模板。
refine_template（可选[Union[str, BasePromptTemplate]]）：
用于改进评估的模板。"""
def __init__(
self,
llm: Optional[LLM] = None,
raise_error: bool = False,
eval_template: Optional[Union[str, BasePromptTemplate]] = None,
refine_template: Optional[Union[str, BasePromptTemplate]] = None,
# deprecated
service_context: Optional[ServiceContext] = None,
) -> None:
"""初始化参数。"""
self._llm = llm or llm_from_settings_or_context(Settings, service_context)
self._raise_error = raise_error
self._eval_template: BasePromptTemplate
if isinstance(eval_template, str):
self._eval_template = PromptTemplate(eval_template)
else:
self._eval_template = eval_template or DEFAULT_EVAL_TEMPLATE
self._refine_template: BasePromptTemplate
if isinstance(refine_template, str):
self._refine_template = PromptTemplate(refine_template)
else:
self._refine_template = refine_template or DEFAULT_REFINE_TEMPLATE
def _get_prompts(self) -> PromptDictType:
"""获取提示。"""
return {
"eval_template": self._eval_template,
"refine_template": self._refine_template,
}
def _update_prompts(self, prompts: PromptDictType) -> None:
"""更新提示。"""
if "eval_template" in prompts:
self._eval_template = prompts["eval_template"]
if "refine_template" in prompts:
self._refine_template = prompts["refine_template"]
async def aevaluate(
self,
query: str | None = None,
response: str | None = None,
contexts: Sequence[str] | None = None,
sleep_time_in_seconds: int = 0,
**kwargs: Any,
) -> EvaluationResult:
"""评估响应是否忠实于上下文。"""
del kwargs # Unused
await asyncio.sleep(sleep_time_in_seconds)
if contexts is None or response is None:
raise ValueError("contexts and response must be provided")
docs = [Document(text=context) for context in contexts]
index = SummaryIndex.from_documents(docs)
query_engine = index.as_query_engine(
llm=self._llm,
text_qa_template=self._eval_template,
refine_template=self._refine_template,
)
response_obj = await query_engine.aquery(response)
raw_response_txt = str(response_obj)
if "yes" in raw_response_txt.lower():
passing = True
else:
passing = False
if self._raise_error:
raise ValueError("The response is invalid")
return EvaluationResult(
query=query,
response=response,
contexts=contexts,
passing=passing,
score=1.0 if passing else 0.0,
feedback=raw_response_txt,
)
|