Query response

评估模块。

QueryResponseEvaluator module-attribute

QueryResponseEvaluator = RelevancyEvaluator

RelevancyEvaluator

Bases: BaseEvaluator

相关性评估器。

评估检索到的上下文和对查询的响应的相关性。 该评估器考虑查询字符串、检索到的上下文和响应字符串。

Parameters:

Name	Type	Description	Default
service_context(Optional[ServiceContext])		用于评估的服务上下文。	required
raise_error(Optional[bool])		如果响应无效是否引发错误。 默认为False。	required
eval_template(Optional[Union[str,	BasePromptTemplate]]	用于评估的模板。	required
refine_template(Optional[Union[str,	BasePromptTemplate]]	用于细化的模板。	required

Source code in llama_index/core/evaluation/relevancy.py

class RelevancyEvaluator(BaseEvaluator):
    """相关性评估器。

    评估检索到的上下文和对查询的响应的相关性。
    该评估器考虑查询字符串、检索到的上下文和响应字符串。

    Args:
        service_context(Optional[ServiceContext]):
            用于评估的服务上下文。
        raise_error(Optional[bool]):
            如果响应无效是否引发错误。
            默认为False。
        eval_template(Optional[Union[str, BasePromptTemplate]]):
            用于评估的模板。
        refine_template(Optional[Union[str, BasePromptTemplate]]):
            用于细化的模板。"""

    def __init__(
        self,
        llm: Optional[LLM] = None,
        raise_error: bool = False,
        eval_template: Optional[Union[str, BasePromptTemplate]] = None,
        refine_template: Optional[Union[str, BasePromptTemplate]] = None,
        # deprecated
        service_context: Optional[ServiceContext] = None,
    ) -> None:
        """初始化参数。"""
        self._llm = llm or llm_from_settings_or_context(Settings, service_context)
        self._raise_error = raise_error

        self._eval_template: BasePromptTemplate
        if isinstance(eval_template, str):
            self._eval_template = PromptTemplate(eval_template)
        else:
            self._eval_template = eval_template or DEFAULT_EVAL_TEMPLATE

        self._refine_template: BasePromptTemplate
        if isinstance(refine_template, str):
            self._refine_template = PromptTemplate(refine_template)
        else:
            self._refine_template = refine_template or DEFAULT_REFINE_TEMPLATE

    def _get_prompts(self) -> PromptDictType:
        """获取提示。"""
        return {
            "eval_template": self._eval_template,
            "refine_template": self._refine_template,
        }

    def _update_prompts(self, prompts: PromptDictType) -> None:
        """更新提示。"""
        if "eval_template" in prompts:
            self._eval_template = prompts["eval_template"]
        if "refine_template" in prompts:
            self._refine_template = prompts["refine_template"]

    async def aevaluate(
        self,
        query: str | None = None,
        response: str | None = None,
        contexts: Sequence[str] | None = None,
        sleep_time_in_seconds: int = 0,
        **kwargs: Any,
    ) -> EvaluationResult:
        """评估上下文和回复是否与查询相关。"""
        del kwargs  # Unused

        if query is None or contexts is None or response is None:
            raise ValueError("query, contexts, and response must be provided")

        docs = [Document(text=context) for context in contexts]
        index = SummaryIndex.from_documents(docs)

        query_response = f"Question: {query}\nResponse: {response}"

        await asyncio.sleep(sleep_time_in_seconds)

        query_engine = index.as_query_engine(
            llm=self._llm,
            text_qa_template=self._eval_template,
            refine_template=self._refine_template,
        )
        response_obj = await query_engine.aquery(query_response)

        raw_response_txt = str(response_obj)

        if "yes" in raw_response_txt.lower():
            passing = True
        else:
            if self._raise_error:
                raise ValueError("The response is invalid")
            passing = False

        return EvaluationResult(
            query=query,
            response=response,
            passing=passing,
            score=1.0 if passing else 0.0,
            feedback=raw_response_txt,
            contexts=contexts,
        )

aevaluate async

aevaluate(
    query: str | None = None,
    response: str | None = None,
    contexts: Sequence[str] | None = None,
    sleep_time_in_seconds: int = 0,
    **kwargs: Any
) -> EvaluationResult

评估上下文和回复是否与查询相关。

Source code in llama_index/core/evaluation/relevancy.py

async def aevaluate(
    self,
    query: str | None = None,
    response: str | None = None,
    contexts: Sequence[str] | None = None,
    sleep_time_in_seconds: int = 0,
    **kwargs: Any,
) -> EvaluationResult:
    """评估上下文和回复是否与查询相关。"""
    del kwargs  # Unused

    if query is None or contexts is None or response is None:
        raise ValueError("query, contexts, and response must be provided")

    docs = [Document(text=context) for context in contexts]
    index = SummaryIndex.from_documents(docs)

    query_response = f"Question: {query}\nResponse: {response}"

    await asyncio.sleep(sleep_time_in_seconds)

    query_engine = index.as_query_engine(
        llm=self._llm,
        text_qa_template=self._eval_template,
        refine_template=self._refine_template,
    )
    response_obj = await query_engine.aquery(query_response)

    raw_response_txt = str(response_obj)

    if "yes" in raw_response_txt.lower():
        passing = True
    else:
        if self._raise_error:
            raise ValueError("The response is invalid")
        passing = False

    return EvaluationResult(
        query=query,
        response=response,
        passing=passing,
        score=1.0 if passing else 0.0,
        feedback=raw_response_txt,
        contexts=contexts,
    )