Index

MultiModalLLM

Bases: ChainableMixin, BaseComponent

多模式LLM界面。

Source code in llama_index/core/multi_modal_llms/base.py

class MultiModalLLM(ChainableMixin, BaseComponent):
    """多模式LLM界面。"""

    callback_manager: CallbackManager = Field(
        default_factory=CallbackManager, exclude=True
    )

    class Config:
        arbitrary_types_allowed = True

    @validator("callback_manager", pre=True)
    def _validate_callback_manager(cls, v: CallbackManager) -> CallbackManager:
        if v is None:
            return CallbackManager([])
        return v

    @property
    @abstractmethod
    def metadata(self) -> MultiModalLLMMetadata:
        """多模态LLM元数据。"""

    @abstractmethod
    def complete(
        self, prompt: str, image_documents: Sequence[ImageDocument], **kwargs: Any
    ) -> CompletionResponse:
        """多模式LLM的完成端点。"""

    @abstractmethod
    def stream_complete(
        self, prompt: str, image_documents: Sequence[ImageDocument], **kwargs: Any
    ) -> CompletionResponseGen:
        """多模式LLM的流式完成端点。"""

    @abstractmethod
    def chat(
        self,
        messages: Sequence[ChatMessage],
        **kwargs: Any,
    ) -> ChatResponse:
        """多模式LLM的聊天端点。"""

    @abstractmethod
    def stream_chat(
        self,
        messages: Sequence[ChatMessage],
        **kwargs: Any,
    ) -> ChatResponseGen:
        """多模式LLM的流式聊天端点。"""

    # ===== Async Endpoints =====

    @abstractmethod
    async def acomplete(
        self, prompt: str, image_documents: Sequence[ImageDocument], **kwargs: Any
    ) -> CompletionResponse:
        """多模式LLM的异步完成端点。"""

    @abstractmethod
    async def astream_complete(
        self, prompt: str, image_documents: Sequence[ImageDocument], **kwargs: Any
    ) -> CompletionResponseAsyncGen:
        """多模式LLM的异步流完成端点。"""

    @abstractmethod
    async def achat(
        self,
        messages: Sequence[ChatMessage],
        **kwargs: Any,
    ) -> ChatResponse:
        """多模式LLM的异步聊天端点。"""

    @abstractmethod
    async def astream_chat(
        self,
        messages: Sequence[ChatMessage],
        **kwargs: Any,
    ) -> ChatResponseAsyncGen:
        """多模式LLM的异步流式聊天端点。"""

    def _as_query_component(self, **kwargs: Any) -> QueryComponent:
        """返回查询组件。"""
        if self.metadata.is_chat_model:
            # TODO: we don't have a separate chat component
            return MultiModalCompleteComponent(multi_modal_llm=self, **kwargs)
        else:
            return MultiModalCompleteComponent(multi_modal_llm=self, **kwargs)

metadata abstractmethod property

metadata: MultiModalLLMMetadata

多模态LLM元数据。

complete abstractmethod

complete(
    prompt: str,
    image_documents: Sequence[ImageDocument],
    **kwargs: Any
) -> CompletionResponse

多模式LLM的完成端点。

Source code in llama_index/core/multi_modal_llms/base.py

@abstractmethod
def complete(
    self, prompt: str, image_documents: Sequence[ImageDocument], **kwargs: Any
) -> CompletionResponse:
    """多模式LLM的完成端点。"""

stream_complete abstractmethod

stream_complete(
    prompt: str,
    image_documents: Sequence[ImageDocument],
    **kwargs: Any
) -> CompletionResponseGen

多模式LLM的流式完成端点。

Source code in llama_index/core/multi_modal_llms/base.py

@abstractmethod
def stream_complete(
    self, prompt: str, image_documents: Sequence[ImageDocument], **kwargs: Any
) -> CompletionResponseGen:
    """多模式LLM的流式完成端点。"""

chat abstractmethod

chat(
    messages: Sequence[ChatMessage], **kwargs: Any
) -> ChatResponse

多模式LLM的聊天端点。

Source code in llama_index/core/multi_modal_llms/base.py

@abstractmethod
def chat(
    self,
    messages: Sequence[ChatMessage],
    **kwargs: Any,
) -> ChatResponse:
    """多模式LLM的聊天端点。"""

stream_chat abstractmethod

stream_chat(
    messages: Sequence[ChatMessage], **kwargs: Any
) -> ChatResponseGen

多模式LLM的流式聊天端点。

Source code in llama_index/core/multi_modal_llms/base.py

@abstractmethod
def stream_chat(
    self,
    messages: Sequence[ChatMessage],
    **kwargs: Any,
) -> ChatResponseGen:
    """多模式LLM的流式聊天端点。"""

acomplete abstractmethod async

acomplete(
    prompt: str,
    image_documents: Sequence[ImageDocument],
    **kwargs: Any
) -> CompletionResponse

多模式LLM的异步完成端点。

Source code in llama_index/core/multi_modal_llms/base.py

@abstractmethod
async def acomplete(
    self, prompt: str, image_documents: Sequence[ImageDocument], **kwargs: Any
) -> CompletionResponse:
    """多模式LLM的异步完成端点。"""

astream_complete abstractmethod async

astream_complete(
    prompt: str,
    image_documents: Sequence[ImageDocument],
    **kwargs: Any
) -> CompletionResponseAsyncGen

多模式LLM的异步流完成端点。

Source code in llama_index/core/multi_modal_llms/base.py

@abstractmethod
async def astream_complete(
    self, prompt: str, image_documents: Sequence[ImageDocument], **kwargs: Any
) -> CompletionResponseAsyncGen:
    """多模式LLM的异步流完成端点。"""

achat abstractmethod async

achat(
    messages: Sequence[ChatMessage], **kwargs: Any
) -> ChatResponse

多模式LLM的异步聊天端点。

Source code in llama_index/core/multi_modal_llms/base.py

@abstractmethod
async def achat(
    self,
    messages: Sequence[ChatMessage],
    **kwargs: Any,
) -> ChatResponse:
    """多模式LLM的异步聊天端点。"""

astream_chat abstractmethod async

astream_chat(
    messages: Sequence[ChatMessage], **kwargs: Any
) -> ChatResponseAsyncGen

多模式LLM的异步流式聊天端点。

Source code in llama_index/core/multi_modal_llms/base.py

@abstractmethod
async def astream_chat(
    self,
    messages: Sequence[ChatMessage],
    **kwargs: Any,
) -> ChatResponseAsyncGen:
    """多模式LLM的异步流式聊天端点。"""

BaseMultiModalComponent

Bases: QueryComponent

基础LLM组件。

Source code in llama_index/core/multi_modal_llms/base.py

class BaseMultiModalComponent(QueryComponent):
    """基础LLM组件。"""

    multi_modal_llm: MultiModalLLM = Field(..., description="LLM")
    streaming: bool = Field(default=False, description="Streaming mode")

    class Config:
        arbitrary_types_allowed = True

    def set_callback_manager(self, callback_manager: Any) -> None:
        """设置回调管理器。"""

set_callback_manager

set_callback_manager(callback_manager: Any) -> None

设置回调管理器。

Source code in llama_index/core/multi_modal_llms/base.py

def set_callback_manager(self, callback_manager: Any) -> None:
    """设置回调管理器。"""

MultiModalCompleteComponent

Bases: BaseMultiModalComponent

多模态完成组件。

Source code in llama_index/core/multi_modal_llms/base.py

class MultiModalCompleteComponent(BaseMultiModalComponent):
    """多模态完成组件。"""

    def _validate_component_inputs(self, input: Dict[str, Any]) -> Dict[str, Any]:
        """在运行组件期间验证组件输入。"""
        if "prompt" not in input:
            raise ValueError("Prompt must be in input dict.")

        # do special check to see if prompt is a list of chat messages
        if isinstance(input["prompt"], get_args(List[ChatMessage])):
            raise NotImplementedError(
                "Chat messages not yet supported as input to multi-modal model."
            )
        else:
            input["prompt"] = validate_and_convert_stringable(input["prompt"])

        # make sure image documents are valid
        if "image_documents" in input:
            if not isinstance(input["image_documents"], list):
                raise ValueError("image_documents must be a list.")
            for doc in input["image_documents"]:
                if not isinstance(doc, ImageDocument):
                    raise ValueError(
                        "image_documents must be a list of ImageDocument objects."
                    )

        return input

    def _run_component(self, **kwargs: Any) -> Any:
        """运行组件。"""
        # TODO: support only complete for now
        prompt = kwargs["prompt"]
        image_documents = kwargs.get("image_documents", [])
        if self.streaming:
            response = self.multi_modal_llm.stream_complete(prompt, image_documents)
        else:
            response = self.multi_modal_llm.complete(prompt, image_documents)
        return {"output": response}

    async def _arun_component(self, **kwargs: Any) -> Any:
        """运行组件。"""
        # TODO: support only complete for now
        # non-trivial to figure how to support chat/complete/etc.
        prompt = kwargs["prompt"]
        image_documents = kwargs.get("image_documents", [])
        if self.streaming:
            response = await self.multi_modal_llm.astream_complete(
                prompt, image_documents
            )
        else:
            response = await self.multi_modal_llm.acomplete(prompt, image_documents)
        return {"output": response}

    @property
    def input_keys(self) -> InputKeys:
        """输入键。"""
        # TODO: support only complete for now
        return InputKeys.from_keys({"prompt", "image_documents"})

    @property
    def output_keys(self) -> OutputKeys:
        """输出键。"""
        return OutputKeys.from_keys({"output"})

input_keys property

input_keys: InputKeys

输入键。

output_keys property

output_keys: OutputKeys

输出键。