Dataset schema

BaseEvalSample

Bases: BaseModel

Base class for evaluation samples.

to_dict

to_dict() -> Dict

Get the dictionary representation of the sample without attributes that are None.

Source code in src/ragas/dataset_schema.py

def to_dict(self) -> t.Dict:
    """
    Get the dictionary representation of the sample without attributes that are None.
    """
    return self.model_dump(exclude_none=True)

get_features

get_features() -> List[str]

Get the features of the sample that are not None.

Source code in src/ragas/dataset_schema.py

def get_features(self) -> t.List[str]:
    """
    Get the features of the sample that are not None.
    """
    return list(self.to_dict().keys())

SingleTurnSample

Bases: BaseEvalSample

Represents evaluation samples for single-turn interactions.

Attributes:

Name	Type	Description
user_input	Optional[str]	The input query from the user.
retrieved_contexts	Optional[List[str]]	List of contexts retrieved for the query.
reference_contexts	Optional[List[str]]	List of reference contexts for the query.
response	Optional[str]	The generated response for the query.
multi_responses	Optional[List[str]]	List of multiple responses generated for the query.
reference	Optional[str]	The reference answer for the query.
rubric	Optional[Dict[str, str]]	Evaluation rubric for the sample.

MultiTurnSample

Bases: BaseEvalSample

Represents evaluation samples for multi-turn interactions.

Attributes:

Name	Type	Description
user_input	List[Union[HumanMessage, AIMessage, ToolMessage]]	A list of messages representing the conversation turns.
reference	(Optional[str], optional)	The reference answer or expected outcome for the conversation.
reference_tool_calls	(Optional[List[ToolCall]], optional)	A list of expected tool calls for the conversation.
rubrics	(Optional[Dict[str, str]], optional)	Evaluation rubrics for the conversation.
reference_topics	(Optional[List[str]], optional)	A list of reference topics for the conversation.

validate_user_input classmethod

validate_user_input(
    messages: List[
        Union[HumanMessage, AIMessage, ToolMessage]
    ]
) -> List[Union[HumanMessage, AIMessage, ToolMessage]]

Validates the user input messages.

Source code in src/ragas/dataset_schema.py

@field_validator("user_input")
@classmethod
def validate_user_input(
    cls,
    messages: t.List[t.Union[HumanMessage, AIMessage, ToolMessage]],
) -> t.List[t.Union[HumanMessage, AIMessage, ToolMessage]]:
    """Validates the user input messages."""
    if not (
        isinstance(m, (HumanMessage, AIMessage, ToolMessage)) for m in messages
    ):
        raise ValueError(
            "All inputs must be instances of HumanMessage, AIMessage, or ToolMessage."
        )

    prev_message = None
    for m in messages:
        if isinstance(m, ToolMessage):
            if not isinstance(prev_message, AIMessage):
                raise ValueError(
                    "ToolMessage instances must be preceded by an AIMessage instance."
                )
            if prev_message.tool_calls is None:
                raise ValueError(
                    f"ToolMessage instances must be preceded by an AIMessage instance with tool_calls. Got {prev_message}"
                )
        prev_message = m

    return messages

to_messages

to_messages()

Converts the user input messages to a list of dictionaries.

Source code in src/ragas/dataset_schema.py

def to_messages(self):
    """Converts the user input messages to a list of dictionaries."""
    return [m.model_dump() for m in self.user_input]

pretty_repr

pretty_repr()

Returns a pretty string representation of the conversation.

Source code in src/ragas/dataset_schema.py

def pretty_repr(self):
    """Returns a pretty string representation of the conversation."""
    lines = []
    for m in self.user_input:
        lines.append(m.pretty_repr())

    return "\n".join(lines)

EvaluationDataset

Bases: BaseModel

Represents a dataset of evaluation samples.

Parameters:

Name	Type	Description	Default
samples	List[BaseEvalSample]	A list of evaluation samples.	required

Attributes:

Name	Type	Description
samples	List[BaseEvalSample]	A list of evaluation samples.

Methods:

Name	Description
validate_samples	Validates that all samples are of the same type.
get_sample_type	Returns the type of the samples in the dataset.
to_hf_dataset	Converts the dataset to a Hugging Face Dataset.
to_pandas	Converts the dataset to a pandas DataFrame.
features	Returns the features of the samples.
from_list	Creates an EvaluationDataset from a list of dictionaries.
from_dict	Creates an EvaluationDataset from a dictionary.

validate_samples

validate_samples(
    samples: List[BaseEvalSample],
) -> List[BaseEvalSample]

Validates that all samples are of the same type.

Source code in src/ragas/dataset_schema.py

@field_validator("samples")
def validate_samples(
    cls, samples: t.List[BaseEvalSample]
) -> t.List[BaseEvalSample]:
    """Validates that all samples are of the same type."""
    if len(samples) == 0:
        return samples

    first_sample_type = type(samples[0])
    if not all(isinstance(sample, first_sample_type) for sample in samples):
        raise ValueError("All samples must be of the same type")

    return samples

get_sample_type

get_sample_type()

Returns the type of the samples in the dataset.

Source code in src/ragas/dataset_schema.py

def get_sample_type(self):
    """Returns the type of the samples in the dataset."""
    return type(self.samples[0])

to_hf_dataset

to_hf_dataset() -> Dataset

Converts the dataset to a Hugging Face Dataset.

Source code in src/ragas/dataset_schema.py

def to_hf_dataset(self) -> HFDataset:
    """Converts the dataset to a Hugging Face Dataset."""
    try:
        from datasets import Dataset as HFDataset
    except ImportError:
        raise ImportError(
            "datasets is not installed. Please install it to use this function."
        )

    return HFDataset.from_list(self._to_list())

to_pandas

to_pandas() -> DataFrame

Converts the dataset to a pandas DataFrame.

Source code in src/ragas/dataset_schema.py

def to_pandas(self) -> PandasDataframe:
    """Converts the dataset to a pandas DataFrame."""
    try:
        import pandas as pd
    except ImportError:
        raise ImportError(
            "pandas is not installed. Please install it to use this function."
        )

    data = self._to_list()
    return pd.DataFrame(data)

features

features()

Returns the features of the samples.

Source code in src/ragas/dataset_schema.py

def features(self):
    """Returns the features of the samples."""
    return self.samples[0].get_features()

from_list classmethod

from_list(mapping: List[Dict])

Creates an EvaluationDataset from a list of dictionaries.

Source code in src/ragas/dataset_schema.py

@classmethod
def from_list(cls, mapping: t.List[t.Dict]):
    """Creates an EvaluationDataset from a list of dictionaries."""
    samples = []
    if all(
        "user_input" in item and isinstance(mapping[0]["user_input"], list)
        for item in mapping
    ):
        samples.extend(MultiTurnSample(**sample) for sample in mapping)
    else:
        samples.extend(SingleTurnSample(**sample) for sample in mapping)
    return cls(samples=samples)

from_dict classmethod

from_dict(mapping: Dict)

Creates an EvaluationDataset from a dictionary.

Source code in src/ragas/dataset_schema.py

@classmethod
def from_dict(cls, mapping: t.Dict):
    """Creates an EvaluationDataset from a dictionary."""
    samples = []
    if all(
        "user_input" in item and isinstance(mapping[0]["user_input"], list)
        for item in mapping
    ):
        samples.extend(MultiTurnSample(**sample) for sample in mapping)
    else:
        samples.extend(SingleTurnSample(**sample) for sample in mapping)
    return cls(samples=samples)