Dynamo DB Docstore 演示¶
本指南向您展示如何直接使用由DynamoDB支持的DocumentStore
抽象。通过将节点放入文档存储库,这使您能够在相同的基础文档存储库上定义多个索引,而不是在索引之间复制数据。
如果您在colab上打开这个笔记本,您可能需要安装LlamaIndex 🦙。
In [ ]:
Copied!
%pip install llama-index-storage-docstore-dynamodb
%pip install llama-index-storage-index-store-dynamodb
%pip install llama-index-vector-stores-dynamodb
%pip install llama-index-llms-openai
%pip install llama-index-storage-docstore-dynamodb
%pip install llama-index-storage-index-store-dynamodb
%pip install llama-index-vector-stores-dynamodb
%pip install llama-index-llms-openai
In [ ]:
Copied!
!pip install llama-index
!pip install llama-index
In [ ]:
Copied!
import nest_asyncio
nest_asyncio.apply()
import nest_asyncio
nest_asyncio.apply()
In [ ]:
Copied!
import logging
import sys
import os
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
import logging
import sys
import os
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
In [ ]:
Copied!
from llama_index.core import SimpleDirectoryReader, StorageContext
from llama_index.core import VectorStoreIndex, SimpleKeywordTableIndex
from llama_index.core import SummaryIndex
from llama_index.llms.openai import OpenAI
from llama_index.core.response.notebook_utils import display_response
from llama_index.core import Settings
from llama_index.core import SimpleDirectoryReader, StorageContext
from llama_index.core import VectorStoreIndex, SimpleKeywordTableIndex
from llama_index.core import SummaryIndex
from llama_index.llms.openai import OpenAI
from llama_index.core.response.notebook_utils import display_response
from llama_index.core import Settings
下载数据¶
In [ ]:
Copied!
!mkdir -p 'data/paul_graham/'
!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham/paul_graham_essay.txt'
!mkdir -p 'data/paul_graham/'
!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham/paul_graham_essay.txt'
加载文档¶
In [ ]:
Copied!
reader = SimpleDirectoryReader("./data/paul_graham/")
documents = reader.load_data()
reader = SimpleDirectoryReader("./data/paul_graham/")
documents = reader.load_data()
解析到节点¶
In [ ]:
Copied!
from llama_index.core.node_parser import SentenceSplitter
nodes = SentenceSplitter().get_nodes_from_documents(documents)
from llama_index.core.node_parser import SentenceSplitter
nodes = SentenceSplitter().get_nodes_from_documents(documents)
加入Docstore¶
In [ ]:
Copied!
TABLE_NAME = os.environ["DYNAMODB_TABLE_NAME"]
TABLE_NAME = os.environ["DYNAMODB_TABLE_NAME"]
In [ ]:
Copied!
from llama_index.storage.docstore.dynamodb import DynamoDBDocumentStore
from llama_index.storage.index_store.dynamodb import DynamoDBIndexStore
from llama_index.vector_stores.dynamodb import DynamoDBVectorStore
from llama_index.storage.docstore.dynamodb import DynamoDBDocumentStore
from llama_index.storage.index_store.dynamodb import DynamoDBIndexStore
from llama_index.vector_stores.dynamodb import DynamoDBVectorStore
In [ ]:
Copied!
storage_context = StorageContext.from_defaults(
docstore=DynamoDBDocumentStore.from_table_name(table_name=TABLE_NAME),
index_store=DynamoDBIndexStore.from_table_name(table_name=TABLE_NAME),
vector_store=DynamoDBVectorStore.from_table_name(table_name=TABLE_NAME),
)
storage_context = StorageContext.from_defaults(
docstore=DynamoDBDocumentStore.from_table_name(table_name=TABLE_NAME),
index_store=DynamoDBIndexStore.from_table_name(table_name=TABLE_NAME),
vector_store=DynamoDBVectorStore.from_table_name(table_name=TABLE_NAME),
)
In [ ]:
Copied!
storage_context.docstore.add_documents(nodes)
storage_context.docstore.add_documents(nodes)
定义和添加多个索引¶
每个索引使用相同的基础节点。
In [ ]:
Copied!
# https://gpt-index.readthedocs.io/en/latest/api_reference/indices/list.html
# 创建一个SummaryIndex对象,传入节点和存储上下文作为参数
summary_index = SummaryIndex(nodes, storage_context=storage_context)
# https://gpt-index.readthedocs.io/en/latest/api_reference/indices/list.html
# 创建一个SummaryIndex对象,传入节点和存储上下文作为参数
summary_index = SummaryIndex(nodes, storage_context=storage_context)
In [ ]:
Copied!
# https://gpt-index.readthedocs.io/en/latest/api_reference/indices/vector_store.html
vector_index = VectorStoreIndex(nodes, storage_context=storage_context)
# https://gpt-index.readthedocs.io/en/latest/api_reference/indices/vector_store.html
vector_index = VectorStoreIndex(nodes, storage_context=storage_context)
In [ ]:
Copied!
# https://gpt-index.readthedocs.io/en/latest/api_reference/indices/table.html
# 创建一个简单的关键词表索引
keyword_table_index = SimpleKeywordTableIndex(
nodes, storage_context=storage_context
)
# https://gpt-index.readthedocs.io/en/latest/api_reference/indices/table.html
# 创建一个简单的关键词表索引
keyword_table_index = SimpleKeywordTableIndex(
nodes, storage_context=storage_context
)
In [ ]:
Copied!
# 注意:docstore仍然具有相同的节点
len(storage_context.docstore.docs)
# 注意:docstore仍然具有相同的节点
len(storage_context.docstore.docs)
测试保存和加载¶
In [ ]:
Copied!
# 注意:默认情况下,docstore、index_store和vector_index在创建时会持久化到DynamoDB中
# 注意:您也可以使用以下命令将简单的向量存储持久化到磁盘
storage_context.persist()
# 注意:默认情况下,docstore、index_store和vector_index在创建时会持久化到DynamoDB中
# 注意:您也可以使用以下命令将简单的向量存储持久化到磁盘
storage_context.persist()
In [ ]:
Copied!
# 记录索引ID
list_id = summary_index.index_id # 摘要索引ID
vector_id = vector_index.index_id # 向量索引ID
keyword_id = keyword_table_index.index_id # 关键词表索引ID
# 记录索引ID
list_id = summary_index.index_id # 摘要索引ID
vector_id = vector_index.index_id # 向量索引ID
keyword_id = keyword_table_index.index_id # 关键词表索引ID
In [ ]:
Copied!
from llama_index.core import load_index_from_storage
# 重新创建存储上下文
storage_context = StorageContext.from_defaults(
docstore=DynamoDBDocumentStore.from_table_name(table_name=TABLE_NAME),
index_store=DynamoDBIndexStore.from_table_name(table_name=TABLE_NAME),
vector_store=DynamoDBVectorStore.from_table_name(table_name=TABLE_NAME),
)
summary_index = load_index_from_storage(
storage_context=storage_context, index_id=list_id
)
keyword_table_index = load_index_from_storage(
storage_context=storage_context, index_id=keyword_id
)
# 需要在StorageContext中添加"vector_store=DynamoDBVectorStore.from_table_name(table_name=TABLE_NAME)",以从DynamoDB加载向量索引
vector_index = load_index_from_storage(
storage_context=storage_context, index_id=vector_id
)
from llama_index.core import load_index_from_storage
# 重新创建存储上下文
storage_context = StorageContext.from_defaults(
docstore=DynamoDBDocumentStore.from_table_name(table_name=TABLE_NAME),
index_store=DynamoDBIndexStore.from_table_name(table_name=TABLE_NAME),
vector_store=DynamoDBVectorStore.from_table_name(table_name=TABLE_NAME),
)
summary_index = load_index_from_storage(
storage_context=storage_context, index_id=list_id
)
keyword_table_index = load_index_from_storage(
storage_context=storage_context, index_id=keyword_id
)
# 需要在StorageContext中添加"vector_store=DynamoDBVectorStore.from_table_name(table_name=TABLE_NAME)",以从DynamoDB加载向量索引
vector_index = load_index_from_storage(
storage_context=storage_context, index_id=vector_id
)
测试一些查询语句¶
In [ ]:
Copied!
chatgpt = OpenAI(temperature=0, model="gpt-3.5-turbo")
Settings.llm = chatgpt
Settings.chunk_size = 1024
chatgpt = OpenAI(temperature=0, model="gpt-3.5-turbo")
Settings.llm = chatgpt
Settings.chunk_size = 1024
In [ ]:
Copied!
query_engine = summary_index.as_query_engine()
list_response = query_engine.query("What is a summary of this document?")
query_engine = summary_index.as_query_engine()
list_response = query_engine.query("What is a summary of this document?")
In [ ]:
Copied!
display_response(list_response)
display_response(list_response)
In [ ]:
Copied!
query_engine = vector_index.as_query_engine()
vector_response = query_engine.query("What did the author do growing up?")
query_engine = vector_index.as_query_engine()
vector_response = query_engine.query("What did the author do growing up?")
In [ ]:
Copied!
display_response(vector_response)
display_response(vector_response)
In [ ]:
Copied!
query_engine = keyword_table_index.as_query_engine()
keyword_response = query_engine.query(
"What did the author do after his time at YC?"
)
query_engine = keyword_table_index.as_query_engine()
keyword_response = query_engine.query(
"What did the author do after his time at YC?"
)
In [ ]:
Copied!
display_response(keyword_response)
display_response(keyword_response)