Index

数据集模块。

BaseLlamaDataExample

Bases: BaseModel

基础的羊驼数据集示例类。

Source code in llama_index/core/llama_dataset/base.py

class BaseLlamaDataExample(BaseModel):
    """基础的羊驼数据集示例类。"""

    @property
    @abstractmethod
    def class_name(self) -> str:
        """类名。"""
        return "BaseLlamaDataExample"

class_name abstractmethod property

class_name: str

类名。

BaseLlamaDataset

Bases: BaseModel, Generic[P]

Source code in llama_index/core/llama_dataset/base.py

class BaseLlamaDataset(BaseModel, Generic[P]):
    _example_type: Type[BaseLlamaDataExample] = BaseLlamaDataExample  # type: ignore[misc]
    examples: List[BaseLlamaDataExample] = Field(
        default=[], description="Data examples of this dataset."
    )
    _predictions_cache: List[BaseLlamaExamplePrediction] = PrivateAttr(
        default_factory=list
    )

    def __getitem__(self, val: Union[slice, int]) -> List[BaseLlamaDataExample]:
        """启用切片和索引。

返回`examples`上的所需切片。
"""
        return self.examples[val]

    @abstractmethod
    def to_pandas(self) -> PandasDataFrame:
        """创建pandas数据框。"""

    def save_json(self, path: str) -> None:
        """保存json。"""
        with open(path, "w") as f:
            examples = [self._example_type.dict(el) for el in self.examples]
            data = {
                "examples": examples,
            }

            json.dump(data, f, indent=4)

    @classmethod
    def from_json(cls, path: str) -> "BaseLlamaDataset":
        """加载json。"""
        with open(path) as f:
            data = json.load(f)

        examples = [cls._example_type.parse_obj(el) for el in data["examples"]]

        return cls(
            examples=examples,
        )

    @abstractmethod
    def _construct_prediction_dataset(
        self, predictions: List[BaseLlamaExamplePrediction]
    ) -> BaseLlamaPredictionDataset:
        """构建特定的预测数据集。

Args:
    predictions (List[BaseLlamaExamplePrediction]): 预测列表。

Returns:
    BaseLlamaPredictionDataset: 一个包含预测的数据集。
"""

    @abstractmethod
    def _predict_example(
        self,
        predictor: P,
        example: BaseLlamaDataExample,
        sleep_time_in_seconds: int = 0,
    ) -> BaseLlamaExamplePrediction:
        """在单个示例上进行预测。

注意：子类需要实现这个方法。

Args:
    predictor (PredictorType): 用于进行预测的预测器。
    example (BaseLlamaDataExample): 要进行预测的示例。

Returns:
    BaseLlamaExamplePrediction: 预测结果。
"""

    def make_predictions_with(
        self,
        predictor: P,
        show_progress: bool = False,
        batch_size: int = 20,
        sleep_time_in_seconds: int = 0,
    ) -> BaseLlamaPredictionDataset:
        """使用给定的查询引擎进行预测。

Args:
    predictor (PredictorType): 用于进行预测的预测器。
    show_progress (bool, optional): 显示预测进度。
    batch_size (int): 用于批量异步调用，特别是为了减少因openai而出现RateLimitError的机会。
    sleep_time_in_seconds (int): 在批量调用之间休眠的时间量，以减少因openai而出现RateLimitError的机会。

Returns:
    BaseLlamaPredictionDataset: 一个预测数据集。
"""
        if self._predictions_cache:
            start_example_position = len(self._predictions_cache)
        else:
            start_example_position = 0

        for batch in self._batch_examples(
            batch_size=batch_size, start_position=start_example_position
        ):
            if show_progress:
                example_iterator = tqdm.tqdm(batch)
            else:
                example_iterator = batch
            for example in example_iterator:
                self._predictions_cache.append(
                    self._predict_example(predictor, example, sleep_time_in_seconds)
                )

        return self._construct_prediction_dataset(predictions=self._predictions_cache)

    # async methods
    @abstractmethod
    async def _apredict_example(
        self,
        predictor: P,
        example: BaseLlamaDataExample,
        sleep_time_in_seconds: int,
    ) -> BaseLlamaExamplePrediction:
        """异步预测单个样本。

注意：子类需要实现这个方法。

Args:
    predictor (PredictorType): 用于进行预测的预测器。
    example (BaseLlamaDataExample): 要进行预测的样本。

Returns:
    BaseLlamaExamplePrediction: 预测结果。
"""

    def _batch_examples(
        self,
        batch_size: int = 20,
        start_position: int = 0,
    ) -> Generator[List[BaseLlamaDataExample], None, None]:
        """使用给定的batch_size对示例和预测进行分批处理。"""
        num_examples = len(self.examples)
        for ndx in range(start_position, num_examples, batch_size):
            yield self.examples[ndx : min(ndx + batch_size, num_examples)]

    async def amake_predictions_with(
        self,
        predictor: P,
        show_progress: bool = False,
        batch_size: int = 20,
        sleep_time_in_seconds: int = 1,
    ) -> BaseLlamaPredictionDataset:
        """使用给定的查询引擎进行异步预测。

Args:
    predictor (PredictorType): 用于进行预测的预测器。
    show_progress (bool, 可选): 显示预测进度。
    batch_size (int): 用于批量异步调用，特别是为了减少因openai而触发RateLimitError的机会。
    sleep_time_in_seconds (int): 在批量调用之间休眠的时间量，以减少因openai而触发RateLimitError的机会。

Returns:
    BaseLlamaPredictionDataset: 预测的数据集。
"""
        if self._predictions_cache:
            start_example_position = len(self._predictions_cache)
        else:
            start_example_position = 0

        for batch in self._batch_examples(
            batch_size=batch_size, start_position=start_example_position
        ):
            tasks = []
            for example in batch:
                tasks.append(
                    self._apredict_example(predictor, example, sleep_time_in_seconds)
                )
            asyncio_mod = asyncio_module(show_progress=show_progress)

            try:
                if show_progress:
                    batch_predictions = await asyncio_mod.gather(
                        *tasks, desc="Batch processing of predictions"
                    )
                else:
                    batch_predictions = await asyncio_mod.gather(*tasks)
            except RateLimitError as err:
                if show_progress:
                    asyncio_mod.close()
                raise ValueError(
                    "You've hit rate limits on your OpenAI subscription. This"
                    " class caches previous predictions after each successful"
                    " batch execution. Based off this cache, when executing this"
                    " command again it will attempt to predict on only the examples "
                    "that have not yet been predicted. Try reducing your batch_size."
                ) from err
            self._predictions_cache += batch_predictions
            # time.sleep(sleep_time_in_seconds)

        prediction_dataset = self._construct_prediction_dataset(
            predictions=self._predictions_cache
        )
        self._predictions_cache = []  # clear cache
        return prediction_dataset

    @property
    @abstractmethod
    def class_name(self) -> str:
        """类名。"""
        return "BaseLlamaDataset"

class_name abstractmethod property

class_name: str

类名。

to_pandas abstractmethod

to_pandas() -> DataFrame

创建pandas数据框。

Source code in llama_index/core/llama_dataset/base.py

@abstractmethod
def to_pandas(self) -> PandasDataFrame:
    """创建pandas数据框。"""

save_json

save_json(path: str) -> None

保存json。

Source code in llama_index/core/llama_dataset/base.py

def save_json(self, path: str) -> None:
    """保存json。"""
    with open(path, "w") as f:
        examples = [self._example_type.dict(el) for el in self.examples]
        data = {
            "examples": examples,
        }

        json.dump(data, f, indent=4)

from_json classmethod

from_json(path: str) -> BaseLlamaDataset

加载json。

Source code in llama_index/core/llama_dataset/base.py

@classmethod
def from_json(cls, path: str) -> "BaseLlamaDataset":
    """加载json。"""
    with open(path) as f:
        data = json.load(f)

    examples = [cls._example_type.parse_obj(el) for el in data["examples"]]

    return cls(
        examples=examples,
    )

make_predictions_with

make_predictions_with(
    predictor: P,
    show_progress: bool = False,
    batch_size: int = 20,
    sleep_time_in_seconds: int = 0,
) -> BaseLlamaPredictionDataset

使用给定的查询引擎进行预测。

Parameters:

Name	Type	Description	Default
predictor	PredictorType	用于进行预测的预测器。	required
show_progress	bool	显示预测进度。	False
batch_size	int	用于批量异步调用，特别是为了减少因openai而出现RateLimitError的机会。	20
sleep_time_in_seconds	int	在批量调用之间休眠的时间量，以减少因openai而出现RateLimitError的机会。	0

Returns:

Name	Type	Description
BaseLlamaPredictionDataset	BaseLlamaPredictionDataset	一个预测数据集。

Source code in llama_index/core/llama_dataset/base.py

    def make_predictions_with(
        self,
        predictor: P,
        show_progress: bool = False,
        batch_size: int = 20,
        sleep_time_in_seconds: int = 0,
    ) -> BaseLlamaPredictionDataset:
        """使用给定的查询引擎进行预测。

Args:
    predictor (PredictorType): 用于进行预测的预测器。
    show_progress (bool, optional): 显示预测进度。
    batch_size (int): 用于批量异步调用，特别是为了减少因openai而出现RateLimitError的机会。
    sleep_time_in_seconds (int): 在批量调用之间休眠的时间量，以减少因openai而出现RateLimitError的机会。

Returns:
    BaseLlamaPredictionDataset: 一个预测数据集。
"""
        if self._predictions_cache:
            start_example_position = len(self._predictions_cache)
        else:
            start_example_position = 0

        for batch in self._batch_examples(
            batch_size=batch_size, start_position=start_example_position
        ):
            if show_progress:
                example_iterator = tqdm.tqdm(batch)
            else:
                example_iterator = batch
            for example in example_iterator:
                self._predictions_cache.append(
                    self._predict_example(predictor, example, sleep_time_in_seconds)
                )

        return self._construct_prediction_dataset(predictions=self._predictions_cache)

amake_predictions_with async

amake_predictions_with(
    predictor: P,
    show_progress: bool = False,
    batch_size: int = 20,
    sleep_time_in_seconds: int = 1,
) -> BaseLlamaPredictionDataset

使用给定的查询引擎进行异步预测。

Parameters:

Name	Type	Description	Default
predictor	PredictorType	用于进行预测的预测器。	required
show_progress	(bool, 可选)	显示预测进度。	False
batch_size	int	用于批量异步调用，特别是为了减少因openai而触发RateLimitError的机会。	20
sleep_time_in_seconds	int	在批量调用之间休眠的时间量，以减少因openai而触发RateLimitError的机会。	1

Returns:

Name	Type	Description
BaseLlamaPredictionDataset	BaseLlamaPredictionDataset	预测的数据集。

Source code in llama_index/core/llama_dataset/base.py

    async def amake_predictions_with(
        self,
        predictor: P,
        show_progress: bool = False,
        batch_size: int = 20,
        sleep_time_in_seconds: int = 1,
    ) -> BaseLlamaPredictionDataset:
        """使用给定的查询引擎进行异步预测。

Args:
    predictor (PredictorType): 用于进行预测的预测器。
    show_progress (bool, 可选): 显示预测进度。
    batch_size (int): 用于批量异步调用，特别是为了减少因openai而触发RateLimitError的机会。
    sleep_time_in_seconds (int): 在批量调用之间休眠的时间量，以减少因openai而触发RateLimitError的机会。

Returns:
    BaseLlamaPredictionDataset: 预测的数据集。
"""
        if self._predictions_cache:
            start_example_position = len(self._predictions_cache)
        else:
            start_example_position = 0

        for batch in self._batch_examples(
            batch_size=batch_size, start_position=start_example_position
        ):
            tasks = []
            for example in batch:
                tasks.append(
                    self._apredict_example(predictor, example, sleep_time_in_seconds)
                )
            asyncio_mod = asyncio_module(show_progress=show_progress)

            try:
                if show_progress:
                    batch_predictions = await asyncio_mod.gather(
                        *tasks, desc="Batch processing of predictions"
                    )
                else:
                    batch_predictions = await asyncio_mod.gather(*tasks)
            except RateLimitError as err:
                if show_progress:
                    asyncio_mod.close()
                raise ValueError(
                    "You've hit rate limits on your OpenAI subscription. This"
                    " class caches previous predictions after each successful"
                    " batch execution. Based off this cache, when executing this"
                    " command again it will attempt to predict on only the examples "
                    "that have not yet been predicted. Try reducing your batch_size."
                ) from err
            self._predictions_cache += batch_predictions
            # time.sleep(sleep_time_in_seconds)

        prediction_dataset = self._construct_prediction_dataset(
            predictions=self._predictions_cache
        )
        self._predictions_cache = []  # clear cache
        return prediction_dataset

BaseLlamaExamplePrediction

Bases: BaseModel

基础的羊驼数据集示例类。

Source code in llama_index/core/llama_dataset/base.py

class BaseLlamaExamplePrediction(BaseModel):
    """基础的羊驼数据集示例类。"""

    @property
    @abstractmethod
    def class_name(self) -> str:
        """类名。"""
        return "BaseLlamaPrediction"

class_name abstractmethod property

class_name: str

类名。

BaseLlamaPredictionDataset

Bases: BaseModel

Source code in llama_index/core/llama_dataset/base.py

class BaseLlamaPredictionDataset(BaseModel):
    _prediction_type: Type[BaseLlamaExamplePrediction] = BaseLlamaExamplePrediction  # type: ignore[misc]
    predictions: List[BaseLlamaExamplePrediction] = Field(
        default=list, description="Predictions on train_examples."
    )

    def __getitem__(self, val: Union[slice, int]) -> List[BaseLlamaExamplePrediction]:
        """启用切片和索引。

返回`predictions`上的所需切片。
"""
        return self.predictions[val]

    @abstractmethod
    def to_pandas(self) -> PandasDataFrame:
        """创建pandas数据框。"""

    def save_json(self, path: str) -> None:
        """保存json。"""
        with open(path, "w") as f:
            predictions = None
            if self.predictions:
                predictions = [
                    self._prediction_type.dict(el) for el in self.predictions
                ]
            data = {
                "predictions": predictions,
            }

            json.dump(data, f, indent=4)

    @classmethod
    def from_json(cls, path: str) -> "BaseLlamaPredictionDataset":
        """加载json。"""
        with open(path) as f:
            data = json.load(f)

        predictions = [cls._prediction_type.parse_obj(el) for el in data["predictions"]]

        return cls(
            predictions=predictions,
        )

    @property
    @abstractmethod
    def class_name(self) -> str:
        """类名。"""
        return "BaseLlamaPredictionDataset"

class_name abstractmethod property

class_name: str

类名。

to_pandas abstractmethod

to_pandas() -> DataFrame

创建pandas数据框。

Source code in llama_index/core/llama_dataset/base.py

@abstractmethod
def to_pandas(self) -> PandasDataFrame:
    """创建pandas数据框。"""

save_json

save_json(path: str) -> None

保存json。

Source code in llama_index/core/llama_dataset/base.py

def save_json(self, path: str) -> None:
    """保存json。"""
    with open(path, "w") as f:
        predictions = None
        if self.predictions:
            predictions = [
                self._prediction_type.dict(el) for el in self.predictions
            ]
        data = {
            "predictions": predictions,
        }

        json.dump(data, f, indent=4)

from_json classmethod

from_json(path: str) -> BaseLlamaPredictionDataset

加载json。

Source code in llama_index/core/llama_dataset/base.py

@classmethod
def from_json(cls, path: str) -> "BaseLlamaPredictionDataset":
    """加载json。"""
    with open(path) as f:
        data = json.load(f)

    predictions = [cls._prediction_type.parse_obj(el) for el in data["predictions"]]

    return cls(
        predictions=predictions,
    )

CreatedByType

Bases: str, Enum

例子中的原始数据类型。

Source code in llama_index/core/llama_dataset/base.py

class CreatedByType(str, Enum):
    """例子中的原始数据类型。"""

    HUMAN = "human"
    AI = "ai"

    def __str__(self) -> str:
        return self.value

EvaluatorExamplePrediction

Bases: BaseLlamaExamplePrediction

评估示例预测类。

Parameters:

Name	Type	Description	Default
feedback	可选[str]	评估者的反馈。	required
score	可选[float]	评估者的分数。	required

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

class EvaluatorExamplePrediction(BaseLlamaExamplePrediction):
    """评估示例预测类。

    Args:
        feedback (可选[str]): 评估者的反馈。
        score (可选[float]): 评估者的分数。"""

    feedback: str = Field(
        default_factory=str,
        description="The generated (predicted) response that can be compared to a reference (ground-truth) answer.",
    )
    score: Optional[float] = Field(
        default=None,
        description="The generated (predicted) response that can be compared to a reference (ground-truth) answer.",
    )
    invalid_prediction: bool = Field(
        default=False, description="Whether or not the prediction is a valid one."
    )
    invalid_reason: Optional[str] = Field(
        default=None, description="Reason as to why prediction is invalid."
    )

    @property
    def class_name(self) -> str:
        """数据示例类名称。"""
        return "EvaluatorExamplePrediction"

class_name property

class_name: str

数据示例类名称。

EvaluatorPredictionDataset

Bases: BaseLlamaPredictionDataset

评估预测数据集类。

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

class EvaluatorPredictionDataset(BaseLlamaPredictionDataset):
    """评估预测数据集类。"""

    _prediction_type = EvaluatorExamplePrediction

    def to_pandas(self) -> PandasDataFrame:
        """创建pandas数据框。"""
        data = {}
        if self.predictions:
            data = {
                "feedback": [t.feedback for t in self.predictions],
                "score": [t.score for t in self.predictions],
            }

        return PandasDataFrame(data)

    @property
    def class_name(self) -> str:
        """类名。"""
        return "EvaluatorPredictionDataset"

class_name property

class_name: str

类名。

to_pandas

to_pandas() -> DataFrame

创建pandas数据框。

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

def to_pandas(self) -> PandasDataFrame:
    """创建pandas数据框。"""
    data = {}
    if self.predictions:
        data = {
            "feedback": [t.feedback for t in self.predictions],
            "score": [t.score for t in self.predictions],
        }

    return PandasDataFrame(data)

LabelledEvaluatorDataExample

Bases: BaseLlamaDataExample

评估示例类。

该数据类包含执行新的“预测”即评估所需的要素。在这里，评估器旨在评估响应与相关查询以及可选上下文。

Parameters:

Name	Type	Description	Default
query	str	用户查询	required
query_by	CreatedBy	由人类或AI生成的查询（模型名称）	required
contexts	Optional[List[str]]	用于响应的上下文	required
answer	str	要评估的查询的答案。	required
answer_by		由人类或AI生成的参考答案（模型名称）。	required
ground_truth_answer	Optional[str]		required
ground_truth_answer_by	Optional[CreatedBy]		required
reference_feedback	str	参考反馈评估。	required
reference_score	float	参考分数评估。	required
reference_evaluation_by	CreatedBy	由人类或AI生成的评估（模型名称）	required

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

class LabelledEvaluatorDataExample(BaseLlamaDataExample):
    """评估示例类。

    该数据类包含执行新的“预测”即评估所需的要素。在这里，评估器旨在评估响应与相关查询以及可选上下文。

    Args:
        query (str): 用户查询
        query_by (CreatedBy): 由人类或AI生成的查询（模型名称）
        contexts (Optional[List[str]]): 用于响应的上下文
        answer (str): 要评估的查询的答案。
        answer_by: 由人类或AI生成的参考答案（模型名称）。
        ground_truth_answer (Optional[str]):
        ground_truth_answer_by (Optional[CreatedBy]):
        reference_feedback (str): 参考反馈评估。
        reference_score (float): 参考分数评估。
        reference_evaluation_by (CreatedBy): 由人类或AI生成的评估（模型名称）"""

    query: str = Field(
        default_factory=str, description="The user query for the example."
    )
    query_by: Optional[CreatedBy] = Field(
        default=None, description="What generated the query."
    )
    contexts: Optional[List[str]] = Field(
        default_factory=None,
        description="The contexts used to generate the answer.",
    )
    answer: str = Field(
        default_factory=str,
        description="The provided answer to the example that is to be evaluated.",
    )
    answer_by: Optional[CreatedBy] = Field(
        default=None, description="What generated the answer."
    )
    ground_truth_answer: Optional[str] = Field(
        default=None,
        description="The ground truth answer to the example that is used to evaluate the provided `answer`.",
    )
    ground_truth_answer_by: Optional[CreatedBy] = Field(
        default=None, description="What generated the ground-truth answer."
    )
    reference_feedback: Optional[str] = Field(
        default=None,
        description="The reference feedback (ground-truth).",
    )
    reference_score: float = Field(
        default_factory=float, description="The reference score (ground-truth)."
    )
    reference_evaluation_by: Optional[CreatedBy] = Field(
        default=None, description="What generated the evaluation (feedback and score)."
    )

    @property
    def class_name(self) -> str:
        """数据示例类名称。"""
        return "LabelledEvaluatorDataExample"

class_name property

class_name: str

数据示例类名称。

LabelledEvaluatorDataset

Bases: BaseLlamaDataset[BaseEvaluator]

标记评估数据集类。

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

class LabelledEvaluatorDataset(BaseLlamaDataset[BaseEvaluator]):
    """标记评估数据集类。"""

    _example_type = LabelledEvaluatorDataExample

    def to_pandas(self) -> PandasDataFrame:
        """创建pandas数据框。"""
        data = {
            "query": [t.query for t in self.examples],
            "answer": [t.answer for t in self.examples],
            "contexts": [t.contexts for t in self.examples],
            "ground_truth_answer": [t.ground_truth_answer for t in self.examples],
            "query_by": [str(t.query_by) for t in self.examples],
            "answer_by": [str(t.answer_by) for t in self.examples],
            "ground_truth_answer_by": [
                str(t.ground_truth_answer_by) for t in self.examples
            ],
            "reference_feedback": [t.reference_feedback for t in self.examples],
            "reference_score": [t.reference_score for t in self.examples],
            "reference_evaluation_by": [
                t.reference_evaluation_by for t in self.examples
            ],
        }

        return PandasDataFrame(data)

    async def _apredict_example(
        self,
        predictor: BaseEvaluator,
        example: LabelledEvaluatorDataExample,
        sleep_time_in_seconds: int,
    ) -> EvaluatorExamplePrediction:
        """使用查询引擎进行异步预测RAG示例。"""
        await asyncio.sleep(sleep_time_in_seconds)
        try:
            eval_result: EvaluationResult = await predictor.aevaluate(
                query=example.query,
                response=example.answer,
                contexts=example.contexts,
                reference=example.ground_truth_answer,
                sleep_time_in_seconds=sleep_time_in_seconds,
            )
        except Exception as err:
            # TODO: raise warning here as well
            return EvaluatorExamplePrediction(
                invalid_prediction=True, invalid_reason=f"Caught error {err!s}"
            )

        if not eval_result.invalid_result:
            return EvaluatorExamplePrediction(
                feedback=eval_result.feedback, score=eval_result.score
            )
        else:
            return EvaluatorExamplePrediction(
                invalid_prediction=True, invalid_reason=eval_result.invalid_reason
            )

    def _predict_example(
        self,
        predictor: BaseEvaluator,
        example: LabelledEvaluatorDataExample,
        sleep_time_in_seconds: int = 0,
    ) -> EvaluatorExamplePrediction:
        """使用查询引擎预测RAG示例。"""
        time.sleep(sleep_time_in_seconds)
        try:
            eval_result: EvaluationResult = predictor.evaluate(
                query=example.query,
                response=example.answer,
                contexts=example.contexts,
                reference=example.ground_truth_answer,
                sleep_time_in_seconds=sleep_time_in_seconds,
            )
        except Exception as err:
            # TODO: raise warning here as well
            return EvaluatorExamplePrediction(
                invalid_prediction=True, invalid_reason=f"Caught error {err!s}"
            )

        if not eval_result.invalid_result:
            return EvaluatorExamplePrediction(
                feedback=eval_result.feedback, score=eval_result.score
            )
        else:
            return EvaluatorExamplePrediction(
                invalid_prediction=True, invalid_reason=eval_result.invalid_reason
            )

    def _construct_prediction_dataset(
        self, predictions: List[EvaluatorExamplePrediction]
    ) -> EvaluatorPredictionDataset:
        """构建预测数据集。"""
        return EvaluatorPredictionDataset(predictions=predictions)

    @property
    def class_name(self) -> str:
        """类名。"""
        return "LabelledEvaluatorDataset"

class_name property

class_name: str

类名。

to_pandas

to_pandas() -> DataFrame

创建pandas数据框。

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

def to_pandas(self) -> PandasDataFrame:
    """创建pandas数据框。"""
    data = {
        "query": [t.query for t in self.examples],
        "answer": [t.answer for t in self.examples],
        "contexts": [t.contexts for t in self.examples],
        "ground_truth_answer": [t.ground_truth_answer for t in self.examples],
        "query_by": [str(t.query_by) for t in self.examples],
        "answer_by": [str(t.answer_by) for t in self.examples],
        "ground_truth_answer_by": [
            str(t.ground_truth_answer_by) for t in self.examples
        ],
        "reference_feedback": [t.reference_feedback for t in self.examples],
        "reference_score": [t.reference_score for t in self.examples],
        "reference_evaluation_by": [
            t.reference_evaluation_by for t in self.examples
        ],
    }

    return PandasDataFrame(data)

LabelledPairwiseEvaluatorDataExample

Bases: LabelledEvaluatorDataExample

标记的成对评估数据示例类。

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

class LabelledPairwiseEvaluatorDataExample(LabelledEvaluatorDataExample):
    """标记的成对评估数据示例类。"""

    second_answer: str = Field(
        default_factory=str,
        description="The second answer to the example that is to be evaluated along versus `answer`.",
    )
    second_answer_by: Optional[CreatedBy] = Field(
        default=None, description="What generated the second answer."
    )

    @property
    def class_name(self) -> str:
        """数据示例类名称。"""
        return "LabelledPairwiseEvaluatorDataExample"

class_name property

class_name: str

数据示例类名称。

LabelledPairwiseEvaluatorDataset

Bases: BaseLlamaDataset[BaseEvaluator]

标记的成对评估数据集。用于评估执行成对评估的评估者。

Parameters:

Name	Type	Description	Default
BaseLlamaDataset	_type_	description	required

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

class LabelledPairwiseEvaluatorDataset(BaseLlamaDataset[BaseEvaluator]):
    """标记的成对评估数据集。用于评估执行成对评估的评估者。

    Args:
        BaseLlamaDataset (_type_): _description_"""

    _example_type = LabelledPairwiseEvaluatorDataExample

    def to_pandas(self) -> PandasDataFrame:
        """创建pandas数据框。"""
        data = {
            "query": [t.query for t in self.examples],
            "answer": [t.answer for t in self.examples],
            "second_answer": [t.second_answer for t in self.examples],
            "contexts": [t.contexts for t in self.examples],
            "ground_truth_answer": [t.ground_truth_answer for t in self.examples],
            "query_by": [str(t.query_by) for t in self.examples],
            "answer_by": [str(t.answer_by) for t in self.examples],
            "second_answer_by": [str(t.second_answer_by) for t in self.examples],
            "ground_truth_answer_by": [
                str(t.ground_truth_answer_by) for t in self.examples
            ],
            "reference_feedback": [t.reference_feedback for t in self.examples],
            "reference_score": [t.reference_score for t in self.examples],
            "reference_evaluation_by": [
                t.reference_evaluation_by for t in self.examples
            ],
        }

        return PandasDataFrame(data)

    async def _apredict_example(
        self,
        predictor: BaseEvaluator,
        example: LabelledPairwiseEvaluatorDataExample,
        sleep_time_in_seconds: int,
    ) -> PairwiseEvaluatorExamplePrediction:
        """使用Evaluator进行异步预测评估的示例。"""
        await asyncio.sleep(sleep_time_in_seconds)
        try:
            eval_result: EvaluationResult = await predictor.aevaluate(
                query=example.query,
                response=example.answer,
                second_response=example.second_answer,
                contexts=example.contexts,
                reference=example.ground_truth_answer,
                sleep_time_in_seconds=sleep_time_in_seconds,
            )
        except Exception as err:
            # TODO: raise warning here as well
            return PairwiseEvaluatorExamplePrediction(
                invalid_prediction=True, invalid_reason=f"Caught error {err!s}"
            )

        if not eval_result.invalid_result:
            return PairwiseEvaluatorExamplePrediction(
                feedback=eval_result.feedback,
                score=eval_result.score,
                evaluation_source=eval_result.pairwise_source,
            )
        else:
            return PairwiseEvaluatorExamplePrediction(
                invalid_prediction=True, invalid_reason=eval_result.invalid_reason
            )

    def _predict_example(
        self,
        predictor: BaseEvaluator,
        example: LabelledPairwiseEvaluatorDataExample,
        sleep_time_in_seconds: int = 0,
    ) -> PairwiseEvaluatorExamplePrediction:
        """使用查询引擎预测RAG示例。"""
        time.sleep(sleep_time_in_seconds)
        try:
            eval_result: EvaluationResult = predictor.evaluate(
                query=example.query,
                response=example.answer,
                second_response=example.second_answer,
                contexts=example.contexts,
                reference=example.ground_truth_answer,
                sleep_time_in_seconds=sleep_time_in_seconds,
            )
        except Exception as err:
            # TODO: raise warning here as well
            return PairwiseEvaluatorExamplePrediction(
                invalid_prediction=True, invalid_reason=f"Caught error {err!s}"
            )

        if not eval_result.invalid_result:
            return PairwiseEvaluatorExamplePrediction(
                feedback=eval_result.feedback,
                score=eval_result.score,
                evaluation_source=eval_result.pairwise_source,
            )
        else:
            return PairwiseEvaluatorExamplePrediction(
                invalid_prediction=True, invalid_reason=eval_result.invalid_reason
            )

    def _construct_prediction_dataset(
        self, predictions: List[PairwiseEvaluatorExamplePrediction]
    ) -> PairwiseEvaluatorPredictionDataset:
        """构建预测数据集。"""
        return PairwiseEvaluatorPredictionDataset(predictions=predictions)

    @property
    def class_name(self) -> str:
        """类名。"""
        return "LabelledPairwiseEvaluatorDataset"

class_name property

class_name: str

类名。

to_pandas

to_pandas() -> DataFrame

创建pandas数据框。

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

def to_pandas(self) -> PandasDataFrame:
    """创建pandas数据框。"""
    data = {
        "query": [t.query for t in self.examples],
        "answer": [t.answer for t in self.examples],
        "second_answer": [t.second_answer for t in self.examples],
        "contexts": [t.contexts for t in self.examples],
        "ground_truth_answer": [t.ground_truth_answer for t in self.examples],
        "query_by": [str(t.query_by) for t in self.examples],
        "answer_by": [str(t.answer_by) for t in self.examples],
        "second_answer_by": [str(t.second_answer_by) for t in self.examples],
        "ground_truth_answer_by": [
            str(t.ground_truth_answer_by) for t in self.examples
        ],
        "reference_feedback": [t.reference_feedback for t in self.examples],
        "reference_score": [t.reference_score for t in self.examples],
        "reference_evaluation_by": [
            t.reference_evaluation_by for t in self.examples
        ],
    }

    return PandasDataFrame(data)

PairwiseEvaluatorExamplePrediction

Bases: BaseLlamaExamplePrediction

Pairwise evaluation example prediction class. 一对一评估示例预测类。

Parameters:

Name	Type	Description	Default
feedback	Optional[str]	评估者的反馈。	required
score	Optional[float]	评估者的分数。	required
evaluation_source	EvaluationSource	评估来源，原始顺序、翻转或不确定。	required

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

class PairwiseEvaluatorExamplePrediction(BaseLlamaExamplePrediction):
    """Pairwise evaluation example prediction class.
一对一评估示例预测类。

Args:
    feedback (Optional[str]): 评估者的反馈。
    score (Optional[float]): 评估者的分数。
    evaluation_source (EvaluationSource): 评估来源，原始顺序、翻转或不确定。"""

    feedback: str = Field(
        default_factory=str,
        description="The generated (predicted) response that can be compared to a reference (ground-truth) answer.",
    )
    score: Optional[float] = Field(
        default=None,
        description="The generated (predicted) response that can be compared to a reference (ground-truth) answer.",
    )
    evaluation_source: Optional[EvaluationSource] = Field(
        default=None,
        description=(
            "Whether the evaluation comes from original, or flipped ordering. Can also be neither here indicating inconclusive judgement."
        ),
    )
    invalid_prediction: bool = Field(
        default=False, description="Whether or not the prediction is a valid one."
    )
    invalid_reason: Optional[str] = Field(
        default=None, description="Reason as to why prediction is invalid."
    )

    @property
    def class_name(self) -> str:
        """数据示例类名称。"""
        return "PairwiseEvaluatorExamplePrediction"

class_name property

class_name: str

数据示例类名称。

PairwiseEvaluatorPredictionDataset

Bases: BaseLlamaPredictionDataset

成对评估预测数据集类。

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

class PairwiseEvaluatorPredictionDataset(BaseLlamaPredictionDataset):
    """成对评估预测数据集类。"""

    _prediction_type = PairwiseEvaluatorExamplePrediction

    def to_pandas(self) -> PandasDataFrame:
        """创建pandas数据框。"""
        data = {}
        if self.predictions:
            data = {
                "feedback": [t.feedback for t in self.predictions],
                "score": [t.score for t in self.predictions],
                "ordering": [t.evaluation_source.value for t in self.predictions],
            }

        return PandasDataFrame(data)

    @property
    def class_name(self) -> str:
        """类名。"""
        return "PairwiseEvaluatorPredictionDataset"

class_name property

class_name: str

类名。

to_pandas

to_pandas() -> DataFrame

创建pandas数据框。

Source code in llama_index/core/llama_dataset/evaluator_evaluation.py

def to_pandas(self) -> PandasDataFrame:
    """创建pandas数据框。"""
    data = {}
    if self.predictions:
        data = {
            "feedback": [t.feedback for t in self.predictions],
            "score": [t.score for t in self.predictions],
            "ordering": [t.evaluation_source.value for t in self.predictions],
        }

    return PandasDataFrame(data)

LabelledRagDataExample

Bases: BaseLlamaDataExample

RAG示例类。类似于传统的ML数据集，该数据集包含用于进行预测的“特征”（即查询+上下文）和用于评估预测的“标签”（即响应）。

Source code in llama_index/core/llama_dataset/rag.py

class LabelledRagDataExample(BaseLlamaDataExample):
    """RAG示例类。类似于传统的ML数据集，该数据集包含用于进行预测的“特征”（即查询+上下文）和用于评估预测的“标签”（即响应）。

Args:
    query（str）：用户查询
    query_by（CreatedBy）：由人类或AI生成的查询（模型名称）
    reference_contexts（Optional[List[str]]）：用于响应的上下文
    reference_answer（[str]）：查询的参考答案。在评估时将获得满分的答案。
    reference_answer_by：由人类或AI生成的参考答案（模型名称）。"""

    query: str = Field(
        default_factory=str, description="The user query for the example."
    )
    query_by: Optional[CreatedBy] = Field(
        default=None, description="What generated the query."
    )
    reference_contexts: Optional[List[str]] = Field(
        default_factory=None,
        description="The contexts used to generate the reference answer.",
    )
    reference_answer: str = Field(
        default_factory=str,
        description="The reference (ground-truth) answer to the example.",
    )
    reference_answer_by: Optional[CreatedBy] = Field(
        default=None, description="What generated the reference answer."
    )

    @property
    def class_name(self) -> str:
        """数据示例类名称。"""
        return "LabelledRagDataExample"

class_name property

class_name: str

数据示例类名称。

LabelledRagDataset

Bases: BaseLlamaDataset[BaseQueryEngine]

RagDataset类。

Source code in llama_index/core/llama_dataset/rag.py

class LabelledRagDataset(BaseLlamaDataset[BaseQueryEngine]):
    """RagDataset类。"""

    _example_type = LabelledRagDataExample

    def to_pandas(self) -> PandasDataFrame:
        """创建pandas数据框。"""
        data = {
            "query": [t.query for t in self.examples],
            "reference_contexts": [t.reference_contexts for t in self.examples],
            "reference_answer": [t.reference_answer for t in self.examples],
            "reference_answer_by": [str(t.reference_answer_by) for t in self.examples],
            "query_by": [str(t.query_by) for t in self.examples],
        }

        return PandasDataFrame(data)

    async def _apredict_example(
        self,
        predictor: BaseQueryEngine,
        example: LabelledRagDataExample,
        sleep_time_in_seconds: int,
    ) -> RagExamplePrediction:
        """使用查询引擎进行异步预测RAG示例。"""
        await asyncio.sleep(sleep_time_in_seconds)
        response = await predictor.aquery(example.query)
        return RagExamplePrediction(
            response=str(response), contexts=[s.text for s in response.source_nodes]
        )

    def _predict_example(
        self,
        predictor: BaseQueryEngine,
        example: LabelledRagDataExample,
        sleep_time_in_seconds: int = 0,
    ) -> RagExamplePrediction:
        """使用查询引擎预测RAG示例。"""
        time.sleep(sleep_time_in_seconds)
        response = predictor.query(example.query)
        return RagExamplePrediction(
            response=str(response), contexts=[s.text for s in response.source_nodes]
        )

    def _construct_prediction_dataset(
        self, predictions: List[RagExamplePrediction]
    ) -> RagPredictionDataset:
        """构建预测数据集。"""
        return RagPredictionDataset(predictions=predictions)

    @property
    def class_name(self) -> str:
        """类名。"""
        return "LabelledRagDataset"

class_name property

class_name: str

类名。

to_pandas

to_pandas() -> DataFrame

创建pandas数据框。

Source code in llama_index/core/llama_dataset/rag.py

def to_pandas(self) -> PandasDataFrame:
    """创建pandas数据框。"""
    data = {
        "query": [t.query for t in self.examples],
        "reference_contexts": [t.reference_contexts for t in self.examples],
        "reference_answer": [t.reference_answer for t in self.examples],
        "reference_answer_by": [str(t.reference_answer_by) for t in self.examples],
        "query_by": [str(t.query_by) for t in self.examples],
    }

    return PandasDataFrame(data)

RagExamplePrediction

Bases: BaseLlamaExamplePrediction

RAG示例预测类。

Parameters:

Name	Type	Description	Default
response	str	LLM生成的响应。	required
contexts	Optional[List[str]]	用于生成响应的检索到的上下文（文本）。	required

Source code in llama_index/core/llama_dataset/rag.py

class RagExamplePrediction(BaseLlamaExamplePrediction):
    """RAG示例预测类。

    Args:
        response (str): LLM生成的响应。
        contexts (Optional[List[str]]): 用于生成响应的检索到的上下文（文本）。"""

    response: str = Field(
        default_factory=str,
        description="The generated (predicted) response that can be compared to a reference (ground-truth) answer.",
    )
    contexts: Optional[List[str]] = Field(
        default_factory=None,
        description="The contexts in raw text form used to generate the response.",
    )

    @property
    def class_name(self) -> str:
        """数据示例类名称。"""
        return "RagExamplePrediction"

class_name property

class_name: str

数据示例类名称。

RagPredictionDataset

Bases: BaseLlamaPredictionDataset

RagDataset类。

Source code in llama_index/core/llama_dataset/rag.py

class RagPredictionDataset(BaseLlamaPredictionDataset):
    """RagDataset类。"""

    _prediction_type = RagExamplePrediction

    def to_pandas(self) -> PandasDataFrame:
        """创建pandas数据框。"""
        data = {}
        if self.predictions:
            data = {
                "response": [t.response for t in self.predictions],
                "contexts": [t.contexts for t in self.predictions],
            }

        return PandasDataFrame(data)

    @property
    def class_name(self) -> str:
        """类名。"""
        return "RagPredictionDataset"

class_name property

class_name: str

类名。

to_pandas

to_pandas() -> DataFrame

创建pandas数据框。

Source code in llama_index/core/llama_dataset/rag.py

def to_pandas(self) -> PandasDataFrame:
    """创建pandas数据框。"""
    data = {}
    if self.predictions:
        data = {
            "response": [t.response for t in self.predictions],
            "contexts": [t.contexts for t in self.predictions],
        }

    return PandasDataFrame(data)

download_llama_dataset

download_llama_dataset(
    llama_dataset_class: str,
    download_dir: str,
    llama_datasets_url: str = LLAMA_DATASETS_URL,
    llama_datasets_lfs_url: str = LLAMA_DATASETS_LFS_URL,
    llama_datasets_source_files_tree_url: str = LLAMA_DATASETS_SOURCE_FILES_GITHUB_TREE_URL,
    show_progress: bool = False,
    load_documents: bool = True,
) -> Tuple[Type[BaseLlamaDataset], List[Document]]

从datasets-LFS和llamahub下载数据集。

Parameters:

Name	Type	Description	Default
dataset_class		您想要下载的llamadataset类的名称，例如PaulGrahamEssayDataset。	required
custom_dir		要下载加载器的自定义目录名称（在父文件夹下）。	required
custom_path		要下载加载器的自定义目录路径。	required
llama_datasets_url	str	从llama_datasets存储库获取普通文件的URL	LLAMA_DATASETS_URL
llama_datasets_lfs_url	str	从llama_datasets存储库获取lfs-traced文件的URL	LLAMA_DATASETS_LFS_URL
llama_datasets_source_files_tree_url	str	列出source_files内容的URL	LLAMA_DATASETS_SOURCE_FILES_GITHUB_TREE_URL
refresh_cache		如果为true，则将跳过本地缓存，并直接从远程存储库获取加载器。	required
source_files_dirpath		存储源文件的目录	required
library_path		库文件的文件名。	required
base_file_name		rag数据集的json文件	required
disable_library_cache		控制库缓存的布尔值	required
override_path		控制覆盖路径的布尔值	required
show_progress	bool	控制是否显示下载源文件的进度的布尔值	False
load_documents	bool	控制是否加载LabelledRagDataset的source_files的布尔值。	True

Returns:

Type	Description
Tuple[Type[BaseLlamaDataset], List[Document]]	BaseLlamaDataset和List[Document]

Source code in llama_index/core/llama_dataset/download.py

def download_llama_dataset(
    llama_dataset_class: str,
    download_dir: str,
    llama_datasets_url: str = LLAMA_DATASETS_URL,
    llama_datasets_lfs_url: str = LLAMA_DATASETS_LFS_URL,
    llama_datasets_source_files_tree_url: str = LLAMA_DATASETS_SOURCE_FILES_GITHUB_TREE_URL,
    show_progress: bool = False,
    load_documents: bool = True,
) -> Tuple[Type[BaseLlamaDataset], List[Document]]:
    """从datasets-LFS和llamahub下载数据集。

Args:
    dataset_class: 您想要下载的llamadataset类的名称，例如`PaulGrahamEssayDataset`。
    custom_dir: 要下载加载器的自定义目录名称（在父文件夹下）。
    custom_path: 要下载加载器的自定义目录路径。
    llama_datasets_url: 从llama_datasets存储库获取普通文件的URL
    llama_datasets_lfs_url: 从llama_datasets存储库获取lfs-traced文件的URL
    llama_datasets_source_files_tree_url: 列出source_files内容的URL
    refresh_cache: 如果为true，则将跳过本地缓存，并直接从远程存储库获取加载器。
    source_files_dirpath: 存储源文件的目录
    library_path: 库文件的文件名。
    base_file_name: rag数据集的json文件
    disable_library_cache: 控制库缓存的布尔值
    override_path: 控制覆盖路径的布尔值
    show_progress: 控制是否显示下载源文件的进度的布尔值
    load_documents: 控制是否加载LabelledRagDataset的source_files的布尔值。

Returns:
    `BaseLlamaDataset`和`List[Document]`
"""
    filenames: Tuple[str, str] = download(
        llama_dataset_class,
        llama_datasets_url=llama_datasets_url,
        llama_datasets_lfs_url=llama_datasets_lfs_url,
        llama_datasets_source_files_tree_url=llama_datasets_source_files_tree_url,
        refresh_cache=True,
        custom_path=download_dir,
        library_path="library.json",
        disable_library_cache=True,
        override_path=True,
        show_progress=show_progress,
    )
    dataset_filename, source_files_dir = filenames
    track_download(llama_dataset_class, MODULE_TYPE.DATASETS)

    dataset = _resolve_dataset_class(dataset_filename).from_json(dataset_filename)
    documents = []

    # for now only rag datasets need to provide the documents
    # in order to build an index over them
    if "rag_dataset.json" in dataset_filename and load_documents:
        documents = SimpleDirectoryReader(input_dir=source_files_dir).load_data(
            show_progress=show_progress
        )

    return (dataset, documents)