Source code for langchain_community.document_loaders.hn
from typing import Any, List
from langchain_core.documents import Document
from langchain_community.document_loaders.web_base import WebBaseLoader
[docs]class HNLoader(WebBaseLoader):
"""加载`Hacker News`数据。
它可以加载主页结果或评论页面的数据。"""
[docs] def load(self) -> List[Document]:
"""获取重要的HN网页信息。
HN网页组成部分包括:
- 标题
- 内容
- 源URL
- 发布时间
- 帖子作者
- 评论数量
- 帖子排名
"""
soup_info = self.scrape()
if "item" in self.web_path:
return self.load_comments(soup_info)
else:
return self.load_results(soup_info)
[docs] def load_results(self, soup: Any) -> List[Document]:
"""从一个Hacker News页面加载项目。"""
items = soup.select("tr[class='athing']")
documents = []
for lineItem in items:
ranking = lineItem.select_one("span[class='rank']").text
link = lineItem.find("span", {"class": "titleline"}).find("a").get("href")
title = lineItem.find("span", {"class": "titleline"}).text.strip()
metadata = {
"source": self.web_path,
"title": title,
"link": link,
"ranking": ranking,
}
documents.append(
Document(
page_content=title, link=link, ranking=ranking, metadata=metadata
)
)
return documents