Skip to content

使用向量存储#

LlamaIndex 提供多种与向量存储/向量数据库集成的方式:

  1. LlamaIndex 可以将向量存储本身用作索引。与任何其他索引一样,该索引可以存储文档并用于回答查询。
  2. LlamaIndex 可以从向量存储加载数据,类似于任何其他数据连接器。然后可以在 LlamaIndex 数据结构中使用这些数据。

将向量存储用作索引#

LlamaIndex 还支持不同的向量存储作为 VectorStoreIndex 的存储后端。

详细的 API 参考可以在这里找到。 与 LlamaIndex 中的任何其他索引(树、关键词表、列表)类似,VectorStoreIndex 可以基于任何文档集合构建。我们使用索引中的向量存储来存储输入文本块的嵌入。

构建完成后,该索引可用于查询。

默认向量存储索引构建/查询

默认情况下,VectorStoreIndex 使用内存中的 SimpleVectorStore,作为默认存储上下文的一部分进行初始化。

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader

# 加载文档并构建索引
documents = SimpleDirectoryReader("../paul_graham_essay/data").load_data()
index = VectorStoreIndex.from_documents(documents)

# 查询索引
query_engine = index.as_query_engine()
response = query_engine.query("作者在成长过程中做了什么?")

自定义向量存储索引构建/查询

我们可以按以下方式查询自定义向量存储:

from llama_index.core import (
    VectorStoreIndex,
    SimpleDirectoryReader,
    StorageContext,
)
from llama_index.vector_stores.deeplake import DeepLakeVectorStore

# 构建向量存储并自定义存储上下文
storage_context = StorageContext.from_defaults(
    vector_store=DeepLakeVectorStore(dataset_path="<dataset_path>")
)

# 加载文档并构建索引
documents = SimpleDirectoryReader("../paul_graham_essay/data").load_data()
index = VectorStoreIndex.from_documents(
    documents, storage_context=storage_context
)

# 查询索引
query_engine = index.as_query_engine()
response = query_engine.query("作者在成长过程中做了什么?")

下面我们展示了如何构建我们支持的各种向量存储的更多示例。

阿里云 OpenSearch

from llama_index.vector_stores.alibabacloud_opensearch import (
    AlibabaCloudOpenSearchStore,
    AlibabaCloudOpenSearchConfig,
)

config = AlibabaCloudOpenSearchConfig(
    endpoint="***",
    instance_id="***",
    username="your_username",
    password="your_password",
    table_name="llama",
)

vector_store = AlibabaCloudOpenSearchStore(config)

Amazon Neptune - Neptune Analytics

from llama_index.vector_stores.neptune import NeptuneAnalyticsVectorStore

graph_identifier = ""
embed_dim = 1536

neptune_vector_store = NeptuneAnalyticsVectorStore(
    graph_identifier=graph_identifier, embedding_dimension=1536
)

Apache Cassandra®

from llama_index.vector_stores.cassandra import CassandraVectorStore
import cassio

# 要通过 CQL 使用 Astra DB 云实例:
cassio.init(database_id="1234abcd-...", token="AstraCS:...")

# 对于 Cassandra 集群:
from cassandra.cluster import Cluster

cluster = Cluster(["127.0.0.1"])
cassio.init(session=cluster.connect(), keyspace="my_keyspace")

# 在上述 `cassio.init(...)` 之后,创建一个向量存储:
vector_store = CassandraVectorStore(
    table="cass_v_table", embedding_dimension=1536
)

Astra DB

from llama_index.vector_stores.astra_db import AstraDBVectorStore

astra_db_store = AstraDBVectorStore(
    token="AstraCS:xY3b...",  # 你的 Astra DB 令牌
    api_endpoint="https://012...abc-us-east1.apps.astra.datastax.com",  # 你的 Astra DB API 端点
    collection_name="astra_v_table",  # 你选择的表名
    embedding_dimension=1536,  # 使用的嵌入模型的嵌入维度
)

Azure Cognitive Search

from azure.core.credentials import AzureKeyCredential
from llama_index.vector_stores.azureaisearch import AzureAISearchVectorStore

search_service_api_key = "YOUR-AZURE-SEARCH-SERVICE-ADMIN-KEY"
search_service_endpoint = "YOUR-AZURE-SEARCH-SERVICE-ENDPOINT"
search_service_api_version = "2023-11-01"
credential = AzureKeyCredential(search_service_api_key)

# 要使用的索引名称
index_name = "llamaindex-vector-demo"

client = SearchIndexClient(
    endpoint=search_service_endpoint,
    credential=credential,
)

vector_store = AzureAISearchVectorStore(
    search_or_index_client=client,
    index_name=index_name,
    embedding_dimensionality=1536,
)

Chroma

import chromadb
from llama_index.vector_stores.chroma import ChromaVectorStore

# 创建一个 Chroma 客户端
# EphemeralClient 仅在内存中操作,PersistentClient 还会保存到磁盘
chroma_client = chromadb.EphemeralClient()
chroma_collection = chroma_client.create_collection("quickstart")

# 构建向量存储
vector_store = ChromaVectorStore(
    chroma_collection=chroma_collection,
)

ClickHouse

import clickhouse_connect
from llama_index.vector_stores import ClickHouseVectorStore

# 创建一个 ClickHouse 客户端
client = clickhouse_connect.get_client(
    host="YOUR_CLUSTER_HOST",
    port=8123,
    username="YOUR_USERNAME",
    password="YOUR_CLUSTER_PASSWORD",
)

# 构建向量存储
vector_store = ClickHouseVectorStore(clickhouse_client=client)

Couchbase

from datetime import timedelta

from couchbase.auth import PasswordAuthenticator
from couchbase.cluster import Cluster
from couchbase.options import ClusterOptions

# 创建一个 Couchbase Cluster 对象
auth = PasswordAuthenticator("数据库用户名", "数据库密码")
options = ClusterOptions(auth)
cluster = Cluster("集群连接字符串", options)

# 等待集群准备就绪。
cluster.wait_until_ready(timedelta(seconds=5))

# 创建向量存储
vector_store = CouchbaseVectorStore(
    cluster=cluster,
    bucket_name="桶名称",
    scope_name="作用域名称",
    collection_name="集合名称",
    index_name="搜索索引名称",
)

DashVector#

import dashvector
from llama_index.vector_stores.dashvector import DashVectorStore

# 初始化 dashvector 客户端
client = dashvector.Client(
    api_key="your-dashvector-api-key",
    endpoint="your-dashvector-cluster-endpoint",
)

# 创建一个 DashVector 集合
client.create("quickstart", dimension=1536)
collection = client.get("quickstart")

# 构建向量存储
vector_store = DashVectorStore(collection)

DeepLake#

import os
import getpath
from llama_index.vector_stores.deeplake import DeepLakeVectorStore

os.environ["OPENAI_API_KEY"] = getpath.getpath("OPENAI_API_KEY: ")
os.environ["ACTIVELOOP_TOKEN"] = getpath.getpath("ACTIVELOOP_TOKEN: ")
dataset_path = "hub://adilkhan/paul_graham_essay"

# 构建向量存储
vector_store = DeepLakeVectorStore(dataset_path=dataset_path, overwrite=True)

DocArray#

from llama_index.vector_stores.docarray import (
    DocArrayHnswVectorStore,
    DocArrayInMemoryVectorStore,
)

# 构建向量存储
vector_store = DocArrayHnswVectorStore(work_dir="hnsw_index")

# 或者,构建内存中的向量存储
vector_store = DocArrayInMemoryVectorStore()

Elasticsearch#

首先,您可以在本地或Elastic Cloud上启动 Elasticsearch。

要在本地使用 Docker 启动 Elasticsearch,请运行以下命令:

docker run -p 9200:9200 \
  -e "discovery.type=single-node" \
  -e "xpack.security.enabled=false" \
  -e "xpack.security.http.ssl.enabled=false" \
  -e "xpack.license.self_generated.type=trial" \
  docker.elastic.co/elasticsearch/elasticsearch:8.9.0

然后,使用 LlamaIndex 连接并使用 Elasticsearch 作为向量数据库

from llama_index.vector_stores.elasticsearch import ElasticsearchStore

vector_store = ElasticsearchStore(
    index_name="llm-project",
    es_url="http://localhost:9200",
    # 云连接选项:
    # es_cloud_id="<cloud_id>",
    # es_user="elastic",
    # es_password="<password>",
)

这可以与 VectorStoreIndex 一起使用,以提供检索、查询、删除、持久化索引等功能。

Epsilla#

from pyepsilla import vectordb
from llama_index.vector_stores.epsilla import EpsillaVectorStore

# 创建 Epsilla 客户端
epsilla_client = vectordb.Client()

# 构建向量存储
vector_store = EpsillaVectorStore(client=epsilla_client)

注意: EpsillaVectorStore 依赖于 pyepsilla 库和正在运行的 Epsilla 向量数据库。 如果尚未安装,请使用 pip/pip3 install pyepsilla 安装。 可以通过 Docker 镜像找到正在运行的 Epsilla 向量数据库。 有关完整说明,请参阅以下文档: https://epsilla-inc.gitbook.io/epsilladb/quick-start

Faiss#

import faiss
from llama_index.vector_stores.faiss import FaissVectorStore

# 创建 faiss 索引
d = 1536
faiss_index = faiss.IndexFlatL2(d)

# 构建向量存储
vector_store = FaissVectorStore(faiss_index)

...

# 注意: 由于 faiss 索引是内存中的,我们需要显式调用
#       vector_store.persist() 或 storage_context.persist() 将其保存到磁盘。
#       persist() 接受可选参数 persist_path。如果未给出,则使用默认路径。
storage_context.persist()

txtai#

import txtai
from llama_index.vector_stores.txtai import TxtaiVectorStore

# 创建 txtai 索引
txtai_index = txtai.ann.ANNFactory.create(
    {"backend": "numpy", "dimension": 512}
)

# 构建向量存储
vector_store = TxtaiVectorStore(txtai_index)

Jaguar#

from llama_index.core.schema import TextNode
from llama_index.core.vector_stores import VectorStoreQuery
from jaguardb_http_client.JaguarHttpClient import JaguarHttpClient
from llama_index.vector_stores.jaguar import JaguarVectorStore


# 构建向量存储客户端
url = "http://127.0.0.1:8080/fwww/"
pod = "vdb"
store = "llamaindex_rag_store"
vector_index = "v"
vector_type = "cosine_fraction_float"
vector_dimension = 3

# 需要 JAGUAR_API_KEY 环境变量或文件 $HOME/.jagrc 来保存 jaguar API 密钥以连接到 jaguar 存储服务器
vector_store = JaguarVectorStore(
    pod, store, vector_index, vector_type, vector_dimension, url
)

# 登录 jaguar 服务器进行安全认证
vector_store.login()

# 在后端服务器上创建一个向量存储
metadata_fields = "author char(32), category char(16)"
text_size = 1024
vector_store.create(metadata_fields, text_size)

# 存储一些文本
node = TextNode(
    text="Return of King Lear",
    metadata={"author": "William", "category": "Tragedy"},
    embedding=[0.9, 0.1, 0.4],
)
vector_store.add(nodes=[node], use_node_metadata=True)

# 进行查询
qembedding = [0.4, 0.2, 0.8]
vsquery = VectorStoreQuery(query_embedding=qembedding, similarity_top_k=1)
query_result = vector_store.query(vsquery)

# 使用元数据过滤器进行查询(where 条件)
qembedding = [0.6, 0.1, 0.4]
vsquery = VectorStoreQuery(query_embedding=qembedding, similarity_top_k=3)
where = "author='Eve' or (author='Adam' and category='History')"
query_result = vector_store.query(vsquery, where=where)

# 进行查询时忽略旧数据(带有时间截止)
qembedding = [0.3, 0.3, 0.8]
vsquery = VectorStoreQuery(query_embedding=qembedding, similarity_top_k=3)
args = "day_cutoff=180"  # 仅搜索最近 180 天的数据
query_result = vector_store.query(vsquery, args=args)

# 检查向量是否异常
text = ("Gone With The Wind",)
embed_of_text = [0.7, 0.1, 0.2]
node = TextNode(text=text, embedding=embed_of_text)
true_or_false = vector_store.is_anomalous(node)

# llama_index RAG 应用
from llama_index.embeddings.openai import OpenAIEmbedding
from llama_index.core import StorageContext
from llama_index.core import VectorStoreIndex

question = "What did the author do growing up?"

storage_context = StorageContext.from_defaults(vector_store=vector_store)
embed_model = OpenAIEmbedding()
embed_of_question = [0.7, 0.1, 0.2]

db_documents = vector_store.load_documents(embed_of_question, 10)
index = VectorStoreIndex.from_documents(
    db_documents,
    embed_model=embed_model,
    storage_context=storage_context,
)

query_engine = index.as_query_engine()
print(f"Question: {question}")
response = query_engine.query(question)
print(f"Answer: {str(response)}")

# 注销以清理资源
vector_store.logout()
注意: 客户端(需要 jaguardb-http-client) <--> HTTP 网关 <--> JaguarDB 服务器 客户端端需要运行: "pip install -U jaguardb-http-client"

Milvus

  • Milvus 索引提供了存储文档及其嵌入向量的功能。
import pymilvus
from llama_index.vector_stores.milvus import MilvusVectorStore

# 构建向量存储
vector_store = MilvusVectorStore(
    uri="https://localhost:19530", overwrite="True"
)

注意: MilvusVectorStore 依赖于 pymilvus 库。 如果尚未安装,请使用 pip install pymilvus 进行安装。 如果在构建 grpcio 时遇到问题,请检查是否使用的是 Python 3.11 (已知问题:https://github.com/milvus-io/pymilvus/issues/1308) 并尝试降级版本。

MongoDBAtlas

# 向构造函数提供 URI,或使用环境变量
import pymongo
from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch
from llama_index.core import VectorStoreIndex
from llama_index.core import StorageContext
from llama_index.core import SimpleDirectoryReader

# mongo_uri = os.environ["MONGO_URI"]
mongo_uri = (
    "mongodb+srv://<username>:<password>@<host>?retryWrites=true&w=majority"
)
mongodb_client = pymongo.MongoClient(mongo_uri)

# 构建存储
store = MongoDBAtlasVectorSearch(mongodb_client)
storage_context = StorageContext.from_defaults(vector_store=store)
uber_docs = SimpleDirectoryReader(
    input_files=["../data/10k/uber_2021.pdf"]
).load_data()

# 构建索引
index = VectorStoreIndex.from_documents(
    uber_docs, storage_context=storage_context
)

MyScale

import clickhouse_connect
from llama_index.vector_stores.myscale import MyScaleVectorStore

# 创建 MyScale 客户端
client = clickhouse_connect.get_client(
    host="YOUR_CLUSTER_HOST",
    port=8443,
    username="YOUR_USERNAME",
    password="YOUR_CLUSTER_PASSWORD",
)

# 构建向量存储
vector_store = MyScaleVectorStore(myscale_client=client)

Neo4j

  • Neo4j 存储文本、元数据和嵌入向量,并可定制返回图数据形式的元数据。
from llama_index.vector_stores.neo4jvector import Neo4jVectorStore

# 构建向量存储
neo4j_vector = Neo4jVectorStore(
    username="neo4j",
    password="pleaseletmein",
    url="bolt://localhost:7687",
    embed_dim=1536,
)

Pinecone

import pinecone
from llama_index.vector_stores.pinecone import PineconeVectorStore

# 创建 Pinecone 索引
api_key = "api_key"
pinecone.init(api_key=api_key, environment="us-west1-gcp")
pinecone.create_index(
    "quickstart", dimension=1536, metric="euclidean", pod_type="p1"
)
index = pinecone.Index("quickstart")

# 构建向量存储
vector_store = PineconeVectorStore(pinecone_index=index)

Qdrant

import qdrant_client
from llama_index.vector_stores.qdrant import QdrantVectorStore

# 创建 Qdrant 向量存储
client = qdrant_client.QdrantClient(
    host="<qdrant-host>", api_key="<qdrant-api-key>", https=True
)
collection_name = "paul_graham"

# 构建向量存储
vector_store = QdrantVectorStore(
    client=client,
    collection_name=collection_name,
)

Redis

首先,启动 Redis-Stack(或从 Redis 供应商获取 URL)

docker run --name redis-vecdb -d -p 6379:6379 -p 8001:8001 redis/redis-stack:latest

然后连接并使用 Redis 作为向量数据库与 LlamaIndex

from llama_index.vector_stores.redis import RedisVectorStore

vector_store = RedisVectorStore(
    index_name="llm-project",
    redis_url="redis://localhost:6379",
    overwrite=True,
)

可以与 VectorStoreIndex 一起使用,提供用于检索、查询、删除、持久化索引等的查询接口。

SingleStore

from llama_index.vector_stores.singlestoredb import SingleStoreVectorStore
import os

# 可以在环境变量中设置 SingleStore 数据库 URL
# 或将其作为参数传递给 SingleStoreVectorStore 构造函数
os.environ["SINGLESTOREDB_URL"] = "PLACEHOLDER URL"
vector_store = SingleStoreVectorStore(
    table_name="embeddings",
    content_field="content",
    metadata_field="metadata",
    vector_field="vector",
    timeout=30,
)

TiDB

from llama_index.vector_stores.tidbvector import TiDBVectorStore

tidbvec = TiDBVectorStore(
    # 连接 URL 格式
    # - mysql+pymysql://root@34.212.137.91:4000/test
    connection_string="PLACEHOLDER URL",
    table_name="llama_index_vectorstore",
    distance_strategy="cosine",
    vector_dimension=1536,
)

Timescale

from llama_index.vector_stores.timescalevector import TimescaleVectorStore

vector_store = TimescaleVectorStore.from_params(
    service_url="YOUR TIMESCALE SERVICE URL",
    table_name="paul_graham_essay",
)

Upstash

from llama_index.vector_stores.upstash import UpstashVectorStore

vector_store = UpstashVectorStore(url="YOUR_URL", token="YOUR_TOKEN")

Vertex AI Vector Search

from llama_index.vector_stores.vertexaivectorsearch import VertexAIVectorStore

vector_store = VertexAIVectorStore(
    project_id="[your-google-cloud-project-id]",
    region="[your-google-cloud-region]",
    index_id="[your-index-resource-name]",
    endpoint_id="[your-index-endpoint-name]",
)

Weaviate

import weaviate
from llama_index.vector_stores.weaviate import WeaviateVectorStore

# 创建 Weaviate 客户端
resource_owner_config = weaviate.AuthClientPassword(
    username="<username>",
    password="<password>",
)
client = weaviate.Client(
    "https://<cluster-id>.semi.network/",
    auth_client_secret=resource_owner_config,
)

# 构建向量存储
vector_store = WeaviateVectorStore(weaviate_client=client)

Zep

Zep 存储文本、元数据和嵌入。所有这些都会在搜索结果中返回。

from llama_index.vector_stores.zep import ZepVectorStore

vector_store = ZepVectorStore(
    api_url="<api_url>",
    api_key="<api_key>",
    collection_name="<unique_collection_name>",  # 可以是现有集合,也可以是新集合
    embedding_dimensions=1536,  # 可选,如果创建新集合则需要
)

storage_context = StorageContext.from_defaults(vector_store=vector_store)

index = VectorStoreIndex.from_documents(
    documents, storage_context=storage_context
)

# 使用文本查询和元数据过滤器查询索引
filters = MetadataFilters(
    filters=[ExactMatchFilter(key="theme", value="Mafia")]
)
retriever = index.as_retriever(filters=filters)
result = retriever.retrieve("Inception 是关于什么的?")

Zilliz

  • Zilliz Cloud(Milvus 的托管版本)使用 Milvus Index 并带有一些额外的参数。
import pymilvus
from llama_index.vector_stores.milvus import MilvusVectorStore

# 构建向量存储
vector_store = MilvusVectorStore(
    uri="foo.vectordb.zillizcloud.com",
    token="your_token_here",
    overwrite="True",
)

示例笔记本可以在此处找到

使用数据连接器从向量存储加载数据#

LlamaIndex 支持从大量来源加载数据。有关更多详细信息和 API 文档,请参阅数据连接器

Chroma 同时存储文档和向量。以下是使用 Chroma 的示例:

from llama_index.readers.chroma import ChromaReader
from llama_index.core import SummaryIndex

# Chroma 读取器从持久化的 Chroma 集合加载数据。
# 这需要集合名称和持久化目录。
reader = ChromaReader(
    collection_name="chroma_collection",
    persist_directory="examples/data_connectors/chroma_collection",
)

query_vector = [n1, n2, n3, ...]

documents = reader.load_data(
    collection_name="demo", query_vector=query_vector, limit=5
)
index = SummaryIndex.from_documents(documents)

query_engine = index.as_query_engine()
response = query_engine.query("<query_text>")
display(Markdown(f"<b>{response}</b>"))

Qdrant 同样存储文档和向量。以下是使用 Qdrant 的示例:

from llama_index.readers.qdrant import QdrantReader

reader = QdrantReader(host="localhost")

# query_vector 是查询向量的嵌入表示
# 示例查询向量
# query_vector = [0.3, 0.3, 0.3, 0.3, ...]

query_vector = [n1, n2, n3, ...]

# 注意:必需的参数是 collection_name 和 query_vector。
# 有关更多详细信息,请参阅 Python 客户端:https;//github.com/qdrant/qdrant_client
# 文档

documents = reader.load_data(
    collection_name="demo", query_vector=query_vector, limit=5
)

注意:由于 Weaviate 可以存储文档和向量对象的混合体,用户可以选择显式指定 class_nameproperties 来查询文档,或者可以选择指定原始的 GraphQL 查询。请参阅下文的用法。

# 选项 1:指定 class_name 和 properties

# 1) 使用 class_name 和 properties 加载数据
documents = reader.load_data(
    class_name="<class_name>",
    properties=["property1", "property2", "..."],
    separate_documents=True,
)

# 2) 示例 GraphQL 查询
query = """
{
    Get {
        <class_name> {
            <property1>
            <property2>
        }
    }
}
"""

documents = reader.load_data(graphql_query=query, separate_documents=True)

注意:Pinecone 和 Faiss 数据加载器假定各自的数据源仅存储向量;文本内容存储在其他位置。因此,这两个数据加载器要求用户在 load_data 调用中指定 id_to_text_map

例如,这是 Pinecone 数据加载器 PineconeReader 的示例用法:

```python
from llama_index.readers.pinecone import PineconeReader

reader = PineconeReader(api_key=api_key, environment="us-west1-gcp")

id_to_text_map = {
    "id1": "text blob 1",
    "id2": "text blob 2",
}

query_vector = [n1, n2, n3, ...]

documents = reader.load_data(
    index_name="quickstart",
    id_to_text_map=id_to_text_map,
    top_k=3,
    vector=query_vector,
    separate_documents=True,
)
示例笔记本可以在此处找到

向量存储示例#