Source code for langchain_community.document_transformers.html2text
from typing import Any, Sequence
from langchain_core.documents import BaseDocumentTransformer, Document
[docs]class Html2TextTransformer(BaseDocumentTransformer):
"""用替换字符串替换特定搜索模式的出现
参数:
ignore_links:是否应忽略链接;默认为True。
ignore_images:是否应忽略图像;默认为True。
示例:
.. code-block:: python
from langchain_community.document_transformers import Html2TextTransformer
html2text = Html2TextTransformer()
docs_transform = html2text.transform_documents(docs)"""
[docs] def __init__(self, ignore_links: bool = True, ignore_images: bool = True) -> None:
self.ignore_links = ignore_links
self.ignore_images = ignore_images
[docs] def transform_documents(
self,
documents: Sequence[Document],
**kwargs: Any,
) -> Sequence[Document]:
try:
import html2text
except ImportError:
raise ImportError(
"""html2text package not found, please
install it with `pip install html2text`"""
)
# Create a html2text.HTML2Text object and override some properties
h = html2text.HTML2Text()
h.ignore_links = self.ignore_links
h.ignore_images = self.ignore_images
new_documents = []
for d in documents:
new_document = Document(
page_content=h.handle(d.page_content), metadata={**d.metadata}
)
new_documents.append(new_document)
return new_documents
[docs] async def atransform_documents(
self,
documents: Sequence[Document],
**kwargs: Any,
) -> Sequence[Document]:
raise NotImplementedError